├── Awesome Oscillator backtest.py
├── Bollinger Bands Pattern Recognition backtest.py
├── Dual Thrust backtest.py
├── Heikin-Ashi backtest.py
├── LICENSE
├── London Breakout backtest.py
├── MACD Oscillator backtest.py
├── Monte Carlo project
├── Monte Carlo backtest.py
├── README.md
└── preview
│ ├── ge accuracy.png
│ ├── ge simulation.png
│ ├── ge simulation2.png
│ ├── ge versus.png
│ ├── nvda simulation.png
│ ├── nvda versus.png
│ └── xkcd_curve_fitting.png
├── Oil Money project
├── Oil Money CAD.py
├── Oil Money COP.py
├── Oil Money NOK.py
├── Oil Money RUB.py
├── Oil Money Trading backtest.py
├── README.md
├── data
│ ├── brent crude nokjpy.csv
│ ├── urals crude rubaud.csv
│ ├── vas crude copaud.csv
│ └── wcs crude cadaud.csv
├── oil production
│ ├── oil production choropleth.csv
│ ├── oil production choropleth.py
│ ├── oil production cost curve.csv
│ ├── oil production cost curve.py
│ └── worldmapshape.json
└── preview
│ ├── cad after.png
│ ├── cad before.png
│ ├── cad crude.png
│ ├── cad currency.png
│ ├── cad elbow.png
│ ├── cad groups.png
│ ├── cad kmeans.gif
│ ├── cad model.png
│ ├── cad silhouette.png
│ ├── cad wcs in aud.png
│ ├── cad wcs in usd.png
│ ├── cop after 2017.png
│ ├── cop before 2017.png
│ ├── cop groups.png
│ ├── cop model.png
│ ├── cop profit distribution.png
│ ├── cop profit heatmap.png
│ ├── cop trading asset.png
│ ├── cop trading positions.png
│ ├── cop vs brl.png
│ ├── cop vs crude.png
│ ├── cop vs gold.png
│ ├── cop vs mxn.png
│ ├── cop vs usd vs mxn.png
│ ├── cop vs usd.png
│ ├── cop vs vas.png
│ ├── nok EG failed.png
│ ├── nok asset value.png
│ ├── nok brent crude.png
│ ├── nok correlation.png
│ ├── nok fitted vs actual.png
│ ├── nok model summary.png
│ ├── nok oil money positions.png
│ ├── nok profit distribution.png
│ ├── nok profit heatmap.png
│ ├── nok trading asset.png
│ ├── nok trading positions.png
│ ├── nok trend.png
│ ├── nok vs brent.png
│ ├── nok vs gdp.png
│ ├── nok vs ir.png
│ ├── oil production bubble map.png
│ ├── oil production choropleth.PNG
│ ├── oil production cost curve.png
│ ├── rub 2015 positions.png
│ ├── rub 2016 positions.png
│ ├── rub 2017 positions.png
│ ├── rub 2017- trend.png
│ ├── rub 2018 positions.png
│ ├── rub model -2016.png
│ ├── rub model 2017-.png
│ ├── rub ols -2016.PNG
│ ├── rub ols 2017-.PNG
│ ├── rub stepwise1.png
│ └── rub stepwise2.png
├── Options Straddle backtest.py
├── Ore Money project
├── README.md
├── iron ore audeur.csv
├── iron ore brlaud.csv
├── iron ore production
│ ├── iron ore production bubble map.csv
│ └── iron ore production bubble map.py
├── iron ore uahusd.csv
└── preview
│ └── iron ore production bubble map.png
├── Pair trading backtest.py
├── Parabolic SAR backtest.py
├── README.md
├── RSI Pattern Recognition backtest.py
├── Shooting Star backtest.py
├── Smart Farmers project
├── README.md
├── check consistency.py
├── cleanse data.py
├── country selection.py
├── data
│ ├── capita.csv
│ ├── cme.csv
│ ├── forecast.csv
│ ├── grand.csv
│ ├── malay_gdp.csv
│ ├── malay_land.csv
│ ├── malay_pop.csv
│ ├── malay_prix.csv
│ ├── malay_prod.csv
│ ├── mapping.csv
│ ├── palm.csv
│ └── tres_grand.csv
├── estimate demand.py
├── forecast.py
└── preview
│ ├── cabbage price.png
│ ├── cabbage production.png
│ ├── cabbage regression.png
│ ├── cocoa price.png
│ ├── cocoa production.png
│ ├── cocoa regression.png
│ ├── coconut price.png
│ ├── coconut production.png
│ ├── coconut regression.png
│ ├── demand model.PNG
│ ├── mango price.png
│ ├── mango production.png
│ ├── mango regression.png
│ ├── naive model matrix.PNG
│ ├── naive model.PNG
│ ├── oil palm price.png
│ ├── oil palm production.png
│ ├── oil palm regression.png
│ ├── oil palm vs palm oil.png
│ ├── overall production.png
│ ├── price change derivation.PNG
│ ├── pricing mechanism.PNG
│ ├── ricardian model.PNG
│ ├── rubber price.png
│ ├── rubber production.png
│ ├── rubber regression.png
│ └── workflow.png
├── VIX Calculator.py
├── data
├── bitcoin.csv
├── cme holidays.csv
├── gbpusd.csv
├── henry hub european options.csv
├── stoxx50.xlsx
└── treasury yield curve rates.csv
└── preview
├── awesome asset.png
├── awesome ma.png
├── awesome oscillator.png
├── awesome positions.png
├── bollinger bands bottom w pattern.png
├── bollinger bands positions.png
├── dual thrust positions.png
├── heikin-ashi asset value.png
├── heikin-ashi positions.png
├── london breakout positions.png
├── london breakout thresholds.png
├── macd oscillator.png
├── macd positions.png
├── options straddle payoff diagram.png
├── pair trading asset.png
├── pair trading eg two step.PNG
├── pair trading positions.png
├── parabolic sar positions.png
├── quantitative-trading-profile.png
├── rsi head-shoulder pattern.png
├── rsi oscillator.png
├── rsi pattern oscillator.png
├── rsi pattern positions.png
├── rsi positions.png
├── shooting star positions.png
└── vix calculator.PNG
/Awesome Oscillator backtest.py:
--------------------------------------------------------------------------------
1 | # coding: utf-8
2 |
3 | #details of awesome oscillator can be found here
4 | # https://www.tradingview.com/wiki/Awesome_Oscillator_(AO)
5 | #basically i use awesome oscillator to compare with macd oscillator
6 | #lets see which one makes more money
7 | #there is not much difference between two of em
8 | #this time i use exponential smoothing on macd
9 | #for awesome oscillator, i use simple moving average instead
10 | #the rules are quite simple
11 | #these two are momentum trading strategy
12 | #they compare the short moving average with long moving average
13 | #if the difference is positive
14 | #we long the asset, vice versa
15 | #awesome oscillator has slightly more conditions for signals
16 | #we will see about it later
17 | #for more details about macd
18 | # https://github.com/je-suis-tm/quant-trading/blob/master/MACD%20oscillator%20backtest.py
19 |
20 |
21 | # In[1]:
22 | #need to get fix yahoo finance package first
23 | import matplotlib.pyplot as plt
24 | import numpy as np
25 | import pandas as pd
26 | import fix_yahoo_finance as yf
27 |
28 |
29 | # In[2]:
30 |
31 | #this part is macd
32 | #i will not go into details as i have another session called macd
33 | #the only difference is that i use ewma function to apply exponential smoothing technique
34 | def ewmacd(signals,ma1,ma2):
35 |
36 | signals['macd ma1']=signals['Close'].ewm(span=ma1).mean()
37 | signals['macd ma2']=signals['Close'].ewm(span=ma2).mean()
38 |
39 | return signals
40 |
41 | def signal_generation(df,method,ma1,ma2):
42 |
43 | signals=method(df,ma1,ma2)
44 | signals['macd positions']=0
45 | signals['macd positions'][ma1:]=np.where(signals['macd ma1'][ma1:]>=signals['macd ma2'][ma1:],1,0)
46 | signals['macd signals']=signals['macd positions'].diff()
47 | signals['macd oscillator']=signals['macd ma1']-signals['macd ma2']
48 |
49 | return signals
50 |
51 |
52 | # In[3]:
53 |
54 | #for awesome oscillator
55 | #moving average is based on the mean of high and low instead of close price
56 | def awesome_ma(signals):
57 |
58 | signals['awesome ma1'],signals['awesome ma2']=0,0
59 | signals['awesome ma1']=((signals['High']+signals['Low'])/2).rolling(window=5).mean()
60 | signals['awesome ma2']=((signals['High']+signals['Low'])/2).rolling(window=34).mean()
61 |
62 | return signals
63 |
64 |
65 | #awesome signal generation,AWESOME!
66 | def awesome_signal_generation(df,method):
67 |
68 | signals=method(df)
69 | signals.reset_index(inplace=True)
70 | signals['awesome signals']=0
71 | signals['awesome oscillator']=signals['awesome ma1']-signals['awesome ma2']
72 | signals['cumsum']=0
73 |
74 |
75 | for i in range(2,len(signals)):
76 |
77 | #awesome oscillator has an extra way to generate signals
78 | #its called saucer
79 | #A Bearish Saucer setup occurs when the AO is below the Zero Line
80 | #in another word, awesome oscillator is negative
81 | #A Bearish Saucer entails two consecutive green bars (with the second bar being higher than the first bar) being followed by a red bar.
82 | #in another word, green bar refers to open price is higher than close price
83 |
84 | if (signals['Open'][i]>signals['Close'][i] and
85 | signals['Open'][i-1]<signals['Close'][i-1] and
86 | signals['Open'][i-2]<signals['Close'][i-2] and
87 | signals['awesome oscillator'][i-1]>signals['awesome oscillator'][i-2] and
88 | signals['awesome oscillator'][i-1]<0 and
89 | signals['awesome oscillator'][i]<0):
90 | signals.at[i,'awesome signals']=1
91 |
92 |
93 | #this is bullish saucer
94 | #vice versa
95 |
96 | if (signals['Open'][i]<signals['Close'][i] and
97 | signals['Open'][i-1]>signals['Close'][i-1] and
98 | signals['Open'][i-2]>signals['Close'][i-2] and
99 | signals['awesome oscillator'][i-1]<signals['awesome oscillator'][i-2] and
100 | signals['awesome oscillator'][i-1]>0 and
101 | signals['awesome oscillator'][i]>0):
102 | signals.at[i,'awesome signals']=-1
103 |
104 |
105 | #this part is the same as macd signal generation
106 | #nevertheless, we have extra rules to get signals ahead of moving average
107 | #if we get signals before moving average generate any signal
108 | #we will ignore signals generated by moving average then
109 | #as it is delayed and probably deliver fewer profit than previous signals
110 | #we use cumulated sum to see if there has been created any open positions
111 | #if so, we will take a pass
112 |
113 | if signals['awesome ma1'][i]>signals['awesome ma2'][i]:
114 | signals.at[i,'awesome signals']=1
115 | signals['cumsum']=signals['awesome signals'].cumsum()
116 | if signals['cumsum'][i]>1:
117 | signals.at[i,'awesome signals']=0
118 |
119 | if signals['awesome ma1'][i]<signals['awesome ma2'][i]:
120 | signals.at[i,'awesome signals']=-1
121 | signals['cumsum']=signals['awesome signals'].cumsum()
122 | if signals['cumsum'][i]<0:
123 | signals.at[i,'awesome signals']=0
124 |
125 | signals['cumsum']=signals['awesome signals'].cumsum()
126 |
127 | return signals
128 |
129 |
130 | # In[4]:
131 |
132 | #we plot the results to compare
133 | #basically the same as macd
134 | #im not gonna explain much
135 | def plot(new,ticker):
136 |
137 | #positions
138 | fig=plt.figure()
139 | ax=fig.add_subplot(211)
140 |
141 | new['Close'].plot(label=ticker)
142 | ax.plot(new.loc[new['awesome signals']==1].index,new['Close'][new['awesome signals']==1],label='AWESOME LONG',lw=0,marker='^',c='g')
143 | ax.plot(new.loc[new['awesome signals']==-1].index,new['Close'][new['awesome signals']==-1],label='AWESOME SHORT',lw=0,marker='v',c='r')
144 |
145 | plt.legend(loc='best')
146 | plt.grid(True)
147 | plt.title('Positions')
148 |
149 | bx=fig.add_subplot(212,sharex=ax)
150 | new['Close'].plot(label=ticker)
151 | bx.plot(new.loc[new['macd signals']==1].index,new['Close'][new['macd signals']==1],label='MACD LONG',lw=0,marker='^',c='g')
152 | bx.plot(new.loc[new['macd signals']==-1].index,new['Close'][new['macd signals']==-1],label='MACD SHORT',lw=0,marker='v',c='r')
153 |
154 | plt.legend(loc='best')
155 | plt.grid(True)
156 | plt.show()
157 |
158 |
159 | #oscillator
160 | fig=plt.figure()
161 | cx=fig.add_subplot(211)
162 |
163 | c=np.where(new['Open']>new['Close'],'r','g')
164 | cx.bar(range(len(new)),new['awesome oscillator'],color=c,label='awesome oscillator')
165 |
166 | plt.grid(True)
167 | plt.legend(loc='best')
168 | plt.title('Oscillator')
169 |
170 | dx=fig.add_subplot(212,sharex=cx)
171 |
172 | new['macd oscillator'].plot(kind='bar',label='macd oscillator')
173 |
174 | plt.grid(True)
175 | plt.legend(loc='best')
176 | plt.xlabel('')
177 | plt.xticks([])
178 | plt.show()
179 |
180 |
181 |
182 | #moving average
183 | fig=plt.figure()
184 | ex=fig.add_subplot(211)
185 |
186 | new['awesome ma1'].plot(label='awesome ma1')
187 | new['awesome ma2'].plot(label='awesome ma2',linestyle=':')
188 |
189 | plt.legend(loc='best')
190 | plt.grid(True)
191 | plt.xticks([])
192 | plt.xlabel('')
193 | plt.title('Moving Average')
194 |
195 | fig=plt.figure()
196 | fx=fig.add_subplot(212,sharex=bx)
197 |
198 | new['macd ma1'].plot(label='macd ma1')
199 | new['macd ma2'].plot(label='macd ma2',linestyle=':')
200 |
201 | plt.legend(loc='best')
202 | plt.grid(True)
203 | plt.show()
204 |
205 |
206 | # In[5]:
207 |
208 | #normally i dont include backtesting stats
209 | #for the comparison, i am willing to make an exception
210 | #capital0 is intial capital
211 | #positions defines how much shares we buy for every single trade
212 | def portfolio(signals):
213 |
214 | capital0=5000
215 | positions=100
216 |
217 | portfolio=pd.DataFrame()
218 | portfolio['Close']=signals['Close']
219 |
220 | #cumsum is used to calculate the change of value while holding shares
221 | portfolio['awesome holding']=signals['cumsum']*portfolio['Close']*positions
222 | portfolio['macd holding']=signals['macd positions']*portfolio['Close']*positions
223 |
224 | #basically cash is initial capital minus the profit we make from every trade
225 | #note that we have to use cumulated sum to add every profit into our cash
226 | portfolio['awesome cash']=capital0-(signals['awesome signals']*portfolio['Close']*positions).cumsum()
227 | portfolio['macd cash']=capital0-(signals['macd signals']*portfolio['Close']*positions).cumsum()
228 |
229 | portfolio['awesome asset']=portfolio['awesome holding']+portfolio['awesome cash']
230 | portfolio['macd asset']=portfolio['macd holding']+portfolio['macd cash']
231 |
232 | portfolio['awesome return']=portfolio['awesome asset'].pct_change()
233 | portfolio['macd return']=portfolio['macd asset'].pct_change()
234 |
235 | return portfolio
236 |
237 |
238 | # In[6]:
239 |
240 | #lets plot how two strategies increase our asset value
241 | def profit(portfolio):
242 |
243 | gx=plt.figure()
244 | gx.add_subplot(111)
245 |
246 | portfolio['awesome asset'].plot()
247 | portfolio['macd asset'].plot()
248 |
249 | plt.legend(loc='best')
250 | plt.grid(True)
251 | plt.title('Awesome VS MACD')
252 | plt.show()
253 |
254 |
255 | # In[7]:
256 |
257 | #i use a function to calculate maximum drawdown
258 | #the idea is simple
259 | #for every day, we take the current asset value
260 | #to compare with the previous highest asset value
261 | #we get our daily drawdown
262 | #it is supposed to be negative if it is not the maximum for this period so far
263 | #we implement a temporary variable to store the minimum value
264 | #which is called maximum drawdown
265 | #for each daily drawdown that is smaller than our temporary value
266 | #we update the temp until we finish our traversal
267 | #in the end we return the maximum drawdown
268 | def mdd(series):
269 |
270 | temp=0
271 | for i in range(1,len(series)):
272 | if temp>(series[i]/max(series[:i])-1):
273 | temp=(series[i]/max(series[:i])-1)
274 |
275 | return temp
276 |
277 |
278 | def stats(portfolio):
279 |
280 | stats=pd.DataFrame([0])
281 |
282 | #lets calculate some sharpe ratios
283 | #note that i set risk free return at 0 for simplicity
284 | #alternatively we can use snp500 as a benchmark
285 | stats['awesome sharpe']=(portfolio['awesome asset'].iloc[-1]/5000-1)/np.std(portfolio['awesome return'])
286 | stats['macd sharpe']=(portfolio['macd asset'].iloc[-1]/5000-1)/np.std(portfolio['macd return'])
287 |
288 | stats['awesome mdd']=mdd(portfolio['awesome asset'])
289 | stats['macd mdd']=mdd(portfolio['macd asset'])
290 |
291 | #ta-da!
292 | print(stats)
293 |
294 |
295 | # In[8]:
296 |
297 | def main():
298 |
299 | #awesome oscillator uses 5 lags as short ma
300 | #34 lags as long ma
301 | #for the consistent comparison
302 | #i apply the same to macd oscillator
303 | ma1=5
304 | ma2=34
305 |
306 | #downloading
307 | stdate=input('start date in format yyyy-mm-dd:')
308 | eddate=input('end date in format yyyy-mm-dd:')
309 | ticker=input('ticker:')
310 | df=yf.download(ticker,start=stdate,end=eddate)
311 |
312 | #slicing the downloaded dataset
313 | #if the dataset is too large
314 | #backtesting plot would look messy
315 | slicer=int(input('slicing:'))
316 | signals=signal_generation(df,ewmacd,ma1,ma2)
317 | sig=awesome_signal_generation(signals,awesome_ma)
318 | new=sig[slicer:]
319 | plot(new,ticker)
320 |
321 | portfo=portfolio(sig)
322 | profit(portfo)
323 |
324 | stats(portfo)
325 |
326 | #from my tests
327 | #macd has demonstrated a higher sharpe ratio
328 | #it executes fewer trades but brings more profits
329 | #however its maximum drawdown is higher than awesome oscillator
330 | #which one is better?
331 | #it depends on your risk averse level
332 |
333 | if __name__ == '__main__':
334 | main()
335 |
--------------------------------------------------------------------------------
/Bollinger Bands Pattern Recognition backtest.py:
--------------------------------------------------------------------------------
1 |
2 | # coding: utf-8
3 |
4 | # In[1]:
5 |
6 | #bollinger bands is a simple indicator
7 | #just moving average plus moving standard deviation
8 | #but pattern recognition is a differenct case
9 | #visualization is easy for human to identify the pattern
10 | #but for the machines, we gotta find a different approach
11 | #when we talk about pattern recognition these days
12 | #people always respond with machine learning
13 | #why machine learning when u can use arithmetic approach
14 | #which is much faster and simpler?
15 |
16 | #there are many patterns for recognition
17 | #top m, bottom w, head-shoulder top, head-shoulder bottom, elliott waves
18 | #in this content, we only discuss bottom w
19 | #top m is just the reverse of bottom w
20 | #rules of bollinger bands and bottom w can be found in the following link:
21 | # https://www.tradingview.com/wiki/Bollinger_Bands_(BB)
22 |
23 | import os
24 | import pandas as pd
25 | import matplotlib.pyplot as plt
26 | import copy
27 | import numpy as np
28 |
29 |
30 | # In[2]:
31 | os.chdir('d:/')
32 |
33 |
34 | # In[3]:
35 |
36 | #first step is to calculate moving average and moving standard deviation
37 | #we plus/minus two standard deviations on moving average
38 | #we get our upper, mid, lower bands
39 | def bollinger_bands(df):
40 |
41 | data=copy.deepcopy(df)
42 | data['std']=data['price'].rolling(window=20,min_periods=20).std()
43 | data['mid band']=data['price'].rolling(window=20,min_periods=20).mean()
44 | data['upper band']=data['mid band']+2*data['std']
45 | data['lower band']=data['mid band']-2*data['std']
46 |
47 | return data
48 |
49 |
50 | # In[4]:
51 |
52 |
53 | #the signal generation is a bit tricky
54 | #there are four conditions to satisfy
55 | #for the shape of w, there are five nodes
56 | #from left to right, top to bottom, l,k,j,m,i
57 | #when we generate signals
58 | #the iteration node is the top right node i, condition 4
59 | #first, we find the middle node j, condition 2
60 | #next, we identify the first bottom node k, condition 1
61 | #after that, we point out the first top node l
62 | #l is not any of those four conditions
63 | #we just use it for pattern visualization
64 | #finally, we locate the second bottom node m, condition 3
65 | #plz refer to the following link for my poor visualization
66 | # https://github.com/je-suis-tm/quant-trading/blob/master/preview/bollinger%20bands%20bottom%20w%20pattern.png
67 | def signal_generation(data,method):
68 |
69 | #according to investopedia
70 | #for a double bottom pattern
71 | #we should use 3-month horizon which is 75
72 | period=75
73 |
74 | #alpha denotes the difference between price and bollinger bands
75 | #if alpha is too small, its unlikely to trigger a signal
76 | #if alpha is too large, its too easy to trigger a signal
77 | #which gives us a higher probability to lose money
78 | #beta denotes the scale of bandwidth
79 | #when bandwidth is larger than beta, it is expansion period
80 | #when bandwidth is smaller than beta, it is contraction period
81 | alpha=0.0001
82 | beta=0.0001
83 |
84 | df=method(data)
85 | df['signals']=0
86 |
87 | #as usual, cumsum denotes the holding position
88 | #coordinates store five nodes of w shape
89 | #later we would use these coordinates to draw a w shape
90 | df['cumsum']=0
91 | df['coordinates']=''
92 |
93 | for i in range(period,len(df)):
94 |
95 | #moveon is a process control
96 | #if moveon==true, we move on to verify the next condition
97 | #if false, we move on to the next iteration
98 | #threshold denotes the value of node k
99 | #we would use it for the comparison with node m
100 | #plz refer to condition 3
101 | moveon=False
102 | threshold=0.0
103 |
104 | #bottom w pattern recognition
105 | #there is another signal generation method called walking the bands
106 | #i personally think its too late for following the trend
107 | #after confirmation of several breakthroughs
108 | #maybe its good for stop and reverse
109 | #condition 4
110 | if (df['price'][i]>df['upper band'][i]) and \
111 | (df['cumsum'][i]==0):
112 |
113 | for j in range(i,i-period,-1):
114 |
115 | #condition 2
116 | if (np.abs(df['mid band'][j]-df['price'][j])<alpha) and \
117 | (np.abs(df['mid band'][j]-df['upper band'][i])<alpha):
118 | moveon=True
119 | break
120 |
121 | if moveon==True:
122 | moveon=False
123 | for k in range(j,i-period,-1):
124 |
125 | #condition 1
126 | if (np.abs(df['lower band'][k]-df['price'][k])<alpha):
127 | threshold=df['price'][k]
128 | moveon=True
129 | break
130 |
131 | if moveon==True:
132 | moveon=False
133 | for l in range(k,i-period,-1):
134 |
135 | #this one is for plotting w shape
136 | if (df['mid band'][l]<df['price'][l]):
137 | moveon=True
138 | break
139 |
140 | if moveon==True:
141 | moveon=False
142 | for m in range(i,j,-1):
143 |
144 | #condition 3
145 | if (df['price'][m]-df['lower band'][m]<alpha) and \
146 | (df['price'][m]>df['lower band'][m]) and \
147 | (df['price'][m]<threshold):
148 | df.at[i,'signals']=1
149 | df.at[i,'coordinates']='%s,%s,%s,%s,%s'%(l,k,j,m,i)
150 | df['cumsum']=df['signals'].cumsum()
151 | moveon=True
152 | break
153 |
154 | #clear our positions when there is contraction on bollinger bands
155 | #contraction on the bandwidth is easy to understand
156 | #when price momentum exists, the price would move dramatically for either direction
157 | #which greatly increases the standard deviation
158 | #when the momentum vanishes, we clear our positions
159 |
160 | #note that we put moveon in the condition
161 | #just in case our signal generation time is contraction period
162 | #but we dont wanna clear positions right now
163 | if (df['cumsum'][i]!=0) and \
164 | (df['std'][i]<beta) and \
165 | (moveon==False):
166 | df.at[i,'signals']=-1
167 | df['cumsum']=df['signals'].cumsum()
168 |
169 | return df
170 |
171 |
172 | # In[5]:
173 |
174 | #visualization
175 | def plot(new):
176 |
177 | #as usual we could cut the dataframe into a small slice
178 | #for a tight and neat figure
179 | #a and b denotes entry and exit of a trade
180 | a,b=list(new[new['signals']!=0].iloc[:2].index)
181 |
182 | newbie=new[a-85:b+30]
183 | newbie.set_index(pd.to_datetime(newbie['date'],format='%Y-%m-%d %H:%M:%S'),inplace=True)
184 |
185 |
186 | fig=plt.figure(figsize=(10,5))
187 | ax=fig.add_subplot(111)
188 |
189 | #plotting positions on price series and bollinger bands
190 | ax.plot(newbie['price'],label='price')
191 | ax.fill_between(newbie.index,newbie['lower band'],newbie['upper band'],alpha=0.2,color='#45ADA8')
192 | ax.plot(newbie['mid band'],linestyle='--',label='moving average',c='#132226')
193 | ax.plot(newbie['price'][newbie['signals']==1],marker='^',markersize=12, \
194 | lw=0,c='g',label='LONG')
195 | ax.plot(newbie['price'][newbie['signals']==-1],marker='v',markersize=12, \
196 | lw=0,c='r',label='SHORT')
197 |
198 | #plotting w shape
199 | #we locate the coordinates then find the exact date as index
200 | temp=newbie['coordinates'][newbie['signals']==1]
201 | indexlist=list(map(int,temp[temp.index[0]].split(',')))
202 | ax.plot(newbie['price'][pd.to_datetime(new['date'].iloc[indexlist])], \
203 | lw=5,alpha=0.7,c='#FE4365',label='double bottom pattern')
204 |
205 | #add some captions
206 | plt.text((newbie.loc[newbie['signals']==1].index[0]), \
207 | newbie['lower band'][newbie['signals']==1],'Expansion',fontsize=15,color='#563838')
208 | plt.text((newbie.loc[newbie['signals']==-1].index[0]), \
209 | newbie['lower band'][newbie['signals']==-1],'Contraction',fontsize=15,color='#563838')
210 |
211 | plt.legend(loc='best')
212 | plt.title('Bollinger Bands Pattern Recognition')
213 | plt.ylabel('price')
214 | plt.grid(True)
215 | plt.show()
216 |
217 |
218 | # In[6]:
219 |
220 | #ta-da
221 | def main():
222 |
223 | #again, i download data from histdata.com
224 | #and i take the average of bid and ask price
225 | df=pd.read_csv('gbpusd.csv')
226 |
227 | signals=signal_generation(df,bollinger_bands)
228 |
229 | new=copy.deepcopy(signals)
230 | plot(new)
231 |
232 | #how to calculate stats could be found from my other code called Heikin-Ashi
233 | # https://github.com/je-suis-tm/quant-trading/blob/master/heikin%20ashi%20backtest.py
234 |
235 |
236 | if __name__ == '__main__':
237 | main()
238 |
--------------------------------------------------------------------------------
/Dual Thrust backtest.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | """
3 | Created on Mon Mar 19 15:22:38 2018
4 | @author: Administrator
5 |
6 | """
7 | # In[1]:
8 |
9 | #dual thrust is an opening range breakout strategy
10 | #it is very similar to London Breakout
11 | #please check London Breakout if u have any questions
12 | # https://github.com/je-suis-tm/quant-trading/blob/master/London%20Breakout%20backtest.py
13 | #Initially we set up upper and lower thresholds based on previous days open, close, high and low
14 | #When the market opens and the price exceeds thresholds, we would take long/short positions prior to upper/lower thresholds
15 | #However, there is no stop long/short position in this strategy
16 | #We clear all positions at the end of the day
17 | #rules of dual thrust can be found in the following link
18 | # https://www.quantconnect.com/tutorials/dual-thrust-trading-algorithm/
19 |
20 | import os
21 | import matplotlib.pyplot as plt
22 | import numpy as np
23 | import pandas as pd
24 |
25 | # In[2]:
26 |
27 | os.chdir('D:/')
28 |
29 |
30 | # In[3]:
31 |
32 |
33 | #data frequency convertion from minute to intra daily
34 | #as we are doing backtesting, we have already got all the datasets we need
35 | #we can create a table to store all open, close, high and low prices
36 | #and calculate the range before we get to signal generation
37 | #otherwise, we would have to put this part inside the loop
38 | #it would greatly increase the time complexity
39 | #however, in real time trading, we do not have futures price
40 | #we have to store all past information in sql db
41 | #we have to calculate the range from db before the market opens
42 |
43 | def min2day(df,column,year,month,rg):
44 |
45 | #lets create a dictionary
46 | #we use keys to classify different info we need
47 | memo={'date':[],'open':[],'close':[],'high':[],'low':[]}
48 |
49 | #no matter which month
50 | #the maximum we can get is 31 days
51 | #thus, we only need to run a traversal on 31 days
52 | #nevertheless, not everyday is a workday
53 | #assuming our raw data doesnt contain weekend prices
54 | #we use try function to make sure we get the info of workdays without errors
55 | #note that i put date at the end of the loop
56 | #the date appendix doesnt depend on our raw data
57 | #it only relies on the range function above
58 | #we could accidentally append weekend date if we put it at the beginning of try function
59 | #not until the program cant find price in raw data will the program stop
60 | #by that time, we have already appended weekend date
61 | #we wanna make sure the length of all lists in dictionary are the same
62 | #so that we can construct a structured table in the next step
63 | for i in range(1,32):
64 |
65 | try:
66 | temp=df['%s-%s-%s 3:00:00'%(year,month,i):'%s-%s-%s 12:00:00'%(year,month,i)][column]
67 |
68 | memo['open'].append(temp[0])
69 | memo['close'].append(temp[-1])
70 | memo['high'].append(max(temp))
71 | memo['low'].append(min(temp))
72 | memo['date'].append('%s-%s-%s'%(year,month,i))
73 |
74 |
75 | except Exception:
76 | pass
77 |
78 | intraday=pd.DataFrame(memo)
79 | intraday.set_index(pd.to_datetime(intraday['date']),inplace=True)
80 |
81 |
82 | #preparation
83 | intraday['range1']=intraday['high'].rolling(rg).max()-intraday['close'].rolling(rg).min()
84 | intraday['range2']=intraday['close'].rolling(rg).max()-intraday['low'].rolling(rg).min()
85 | intraday['range']=np.where(intraday['range1']>intraday['range2'],intraday['range1'],intraday['range2'])
86 |
87 | return intraday
88 |
89 |
90 | #signal generation
91 | #even replace assignment with pandas.at
92 | #it still takes a while for us to get the result
93 | #any optimization suggestion besides using numpy array?
94 | def signal_generation(df,intraday,param,column,rg):
95 |
96 | #as the lags of days have been set to 5
97 | #we should start our backtesting after 4 workdays of current month
98 | #cumsum is to control the holding of underlying asset
99 | #sigup and siglo are the variables to store the upper/lower threshold
100 | #upper and lower are for the purpose of tracking sigup and siglo
101 | signals=df[df.index>=intraday['date'].iloc[rg-1]]
102 | signals['signals']=0
103 | signals['cumsum']=0
104 | signals['upper']=0.0
105 | signals['lower']=0.0
106 | sigup=float(0)
107 | siglo=float(0)
108 |
109 | #for traversal on time series
110 | #the tricky part is the slicing
111 | #we have to either use [i:i] or pd.Series
112 | #first we set up thresholds at the beginning of london market
113 | #which is est 3am
114 | #if the price exceeds either threshold
115 | #we will take long/short positions
116 |
117 | for i in signals.index:
118 |
119 | #note that intraday and dataframe have different frequencies
120 | #obviously different metrics for indexes
121 | #we use variable date for index convertion
122 | date='%s-%s-%s'%(i.year,i.month,i.day)
123 |
124 |
125 | #market opening
126 | #set up thresholds
127 | if (i.hour==3 and i.minute==0):
128 | sigup=float(param*intraday['range'][date]+pd.Series(signals[column])[i])
129 | siglo=float(-(1-param)*intraday['range'][date]+pd.Series(signals[column])[i])
130 |
131 | #thresholds got breached
132 | #signals generating
133 | if (sigup!=0 and pd.Series(signals[column])[i]>sigup):
134 | signals.at[i,'signals']=1
135 | if (siglo!=0 and pd.Series(signals[column])[i]<siglo):
136 | signals.at[i,'signals']=-1
137 |
138 |
139 | #check if signal has been generated
140 | #if so, use cumsum to verify that we only generate one signal for each situation
141 | if pd.Series(signals['signals'])[i]!=0:
142 | signals['cumsum']=signals['signals'].cumsum()
143 | if (pd.Series(signals['cumsum'])[i]>1 or pd.Series(signals['cumsum'])[i]<-1):
144 | signals.at[i,'signals']=0
145 |
146 | #if the price goes from below the lower threshold to above the upper threshold during the day
147 | #we reverse our positions from short to long
148 | if (pd.Series(signals['cumsum'])[i]==0):
149 | if (pd.Series(signals[column])[i]>sigup):
150 | signals.at[i,'signals']=2
151 | if (pd.Series(signals[column])[i]<siglo):
152 | signals.at[i,'signals']=-2
153 |
154 | #by the end of london market, which is est 12pm
155 | #we clear all opening positions
156 | #the whole part is very similar to London Breakout strategy
157 | if i.hour==12 and i.minute==0:
158 | sigup,siglo=float(0),float(0)
159 | signals['cumsum']=signals['signals'].cumsum()
160 | signals.at[i,'signals']=-signals['cumsum'][i:i]
161 |
162 | #keep track of trigger levels
163 | signals.at[i,'upper']=sigup
164 | signals.at[i,'lower']=siglo
165 |
166 | return signals
167 |
168 | #plotting the positions
169 | def plot(signals,intraday,column):
170 |
171 | #we have to do a lil bit slicing to make sure we can see the plot clearly
172 | #the only reason i go to -3 is that day we execute a trade
173 | #give one hour before and after market trading hour for as x axis
174 | date=pd.to_datetime(intraday['date']).iloc[-3]
175 | signew=signals['%s-%s-%s 02:00:00'%(date.year,date.month,date.day):'%s-%s-%s 13:00:00'%(date.year,date.month,date.day)]
176 |
177 | fig=plt.figure(figsize=(10,5))
178 | ax=fig.add_subplot(111)
179 |
180 | #mostly the same as other py files
181 | #the only difference is to create an interval for signal generation
182 | ax.plot(signew.index,signew[column],label=column)
183 | ax.fill_between(signew.loc[signew['upper']!=0].index,signew['upper'][signew['upper']!=0],signew['lower'][signew['upper']!=0],alpha=0.2,color='#355c7d')
184 | ax.plot(signew.loc[signew['signals']==1].index,signew[column][signew['signals']==1],lw=0,marker='^',markersize=10,c='g',label='LONG')
185 | ax.plot(signew.loc[signew['signals']==-1].index,signew[column][signew['signals']==-1],lw=0,marker='v',markersize=10,c='r',label='SHORT')
186 |
187 | #change legend text color
188 | lgd=plt.legend(loc='best').get_texts()
189 | for text in lgd:
190 | text.set_color('#6C5B7B')
191 |
192 | #add some captions
193 | plt.text('%s-%s-%s 03:00:00'%(date.year,date.month,date.day),signew['upper']['%s-%s-%s 03:00:00'%(date.year,date.month,date.day)],'Upper Bound',color='#C06C84')
194 | plt.text('%s-%s-%s 03:00:00'%(date.year,date.month,date.day),signew['lower']['%s-%s-%s 03:00:00'%(date.year,date.month,date.day)],'Lower Bound',color='#C06C84')
195 |
196 | plt.ylabel(column)
197 | plt.xlabel('Date')
198 | plt.title('Dual Thrust')
199 | plt.grid(True)
200 | plt.show()
201 |
202 |
203 |
204 | # In[4]:
205 | def main():
206 |
207 | #similar to London Breakout
208 | #my raw data comes from the same website
209 | # http://www.histdata.com/download-free-forex-data/?/excel/1-minute-bar-quotes
210 | #just take the mid price of whatever currency pair you want
211 |
212 | df=pd.read_csv('gbpusd.csv')
213 | df.set_index(pd.to_datetime(df['date']),inplace=True)
214 |
215 | #rg is the lags of days
216 | #param is the parameter of trigger range, it should be smaller than one
217 | #normally ppl use 0.5 to give long and short 50/50 chance to trigger
218 | rg=5
219 | param=0.5
220 |
221 | #these three variables are for the frequency convertion from minute to intra daily
222 | year=df.index[0].year
223 | month=df.index[0].month
224 | column='price'
225 |
226 | intraday=min2day(df,column,year,month,rg)
227 | signals=signal_generation(df,intraday,param,column,rg)
228 | plot(signals,intraday,column)
229 |
230 | #how to calculate stats could be found from my other code called Heikin-Ashi
231 | # https://github.com/je-suis-tm/quant-trading/blob/master/heikin%20ashi%20backtest.py
232 |
233 | if __name__ == '__main__':
234 | main()
235 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | Apache License
2 | Version 2.0, January 2004
3 | http://www.apache.org/licenses/
4 |
5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
6 |
7 | 1. Definitions.
8 |
9 | "License" shall mean the terms and conditions for use, reproduction,
10 | and distribution as defined by Sections 1 through 9 of this document.
11 |
12 | "Licensor" shall mean the copyright owner or entity authorized by
13 | the copyright owner that is granting the License.
14 |
15 | "Legal Entity" shall mean the union of the acting entity and all
16 | other entities that control, are controlled by, or are under common
17 | control with that entity. For the purposes of this definition,
18 | "control" means (i) the power, direct or indirect, to cause the
19 | direction or management of such entity, whether by contract or
20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the
21 | outstanding shares, or (iii) beneficial ownership of such entity.
22 |
23 | "You" (or "Your") shall mean an individual or Legal Entity
24 | exercising permissions granted by this License.
25 |
26 | "Source" form shall mean the preferred form for making modifications,
27 | including but not limited to software source code, documentation
28 | source, and configuration files.
29 |
30 | "Object" form shall mean any form resulting from mechanical
31 | transformation or translation of a Source form, including but
32 | not limited to compiled object code, generated documentation,
33 | and conversions to other media types.
34 |
35 | "Work" shall mean the work of authorship, whether in Source or
36 | Object form, made available under the License, as indicated by a
37 | copyright notice that is included in or attached to the work
38 | (an example is provided in the Appendix below).
39 |
40 | "Derivative Works" shall mean any work, whether in Source or Object
41 | form, that is based on (or derived from) the Work and for which the
42 | editorial revisions, annotations, elaborations, or other modifications
43 | represent, as a whole, an original work of authorship. For the purposes
44 | of this License, Derivative Works shall not include works that remain
45 | separable from, or merely link (or bind by name) to the interfaces of,
46 | the Work and Derivative Works thereof.
47 |
48 | "Contribution" shall mean any work of authorship, including
49 | the original version of the Work and any modifications or additions
50 | to that Work or Derivative Works thereof, that is intentionally
51 | submitted to Licensor for inclusion in the Work by the copyright owner
52 | or by an individual or Legal Entity authorized to submit on behalf of
53 | the copyright owner. For the purposes of this definition, "submitted"
54 | means any form of electronic, verbal, or written communication sent
55 | to the Licensor or its representatives, including but not limited to
56 | communication on electronic mailing lists, source code control systems,
57 | and issue tracking systems that are managed by, or on behalf of, the
58 | Licensor for the purpose of discussing and improving the Work, but
59 | excluding communication that is conspicuously marked or otherwise
60 | designated in writing by the copyright owner as "Not a Contribution."
61 |
62 | "Contributor" shall mean Licensor and any individual or Legal Entity
63 | on behalf of whom a Contribution has been received by Licensor and
64 | subsequently incorporated within the Work.
65 |
66 | 2. Grant of Copyright License. Subject to the terms and conditions of
67 | this License, each Contributor hereby grants to You a perpetual,
68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
69 | copyright license to reproduce, prepare Derivative Works of,
70 | publicly display, publicly perform, sublicense, and distribute the
71 | Work and such Derivative Works in Source or Object form.
72 |
73 | 3. Grant of Patent License. Subject to the terms and conditions of
74 | this License, each Contributor hereby grants to You a perpetual,
75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
76 | (except as stated in this section) patent license to make, have made,
77 | use, offer to sell, sell, import, and otherwise transfer the Work,
78 | where such license applies only to those patent claims licensable
79 | by such Contributor that are necessarily infringed by their
80 | Contribution(s) alone or by combination of their Contribution(s)
81 | with the Work to which such Contribution(s) was submitted. If You
82 | institute patent litigation against any entity (including a
83 | cross-claim or counterclaim in a lawsuit) alleging that the Work
84 | or a Contribution incorporated within the Work constitutes direct
85 | or contributory patent infringement, then any patent licenses
86 | granted to You under this License for that Work shall terminate
87 | as of the date such litigation is filed.
88 |
89 | 4. Redistribution. You may reproduce and distribute copies of the
90 | Work or Derivative Works thereof in any medium, with or without
91 | modifications, and in Source or Object form, provided that You
92 | meet the following conditions:
93 |
94 | (a) You must give any other recipients of the Work or
95 | Derivative Works a copy of this License; and
96 |
97 | (b) You must cause any modified files to carry prominent notices
98 | stating that You changed the files; and
99 |
100 | (c) You must retain, in the Source form of any Derivative Works
101 | that You distribute, all copyright, patent, trademark, and
102 | attribution notices from the Source form of the Work,
103 | excluding those notices that do not pertain to any part of
104 | the Derivative Works; and
105 |
106 | (d) If the Work includes a "NOTICE" text file as part of its
107 | distribution, then any Derivative Works that You distribute must
108 | include a readable copy of the attribution notices contained
109 | within such NOTICE file, excluding those notices that do not
110 | pertain to any part of the Derivative Works, in at least one
111 | of the following places: within a NOTICE text file distributed
112 | as part of the Derivative Works; within the Source form or
113 | documentation, if provided along with the Derivative Works; or,
114 | within a display generated by the Derivative Works, if and
115 | wherever such third-party notices normally appear. The contents
116 | of the NOTICE file are for informational purposes only and
117 | do not modify the License. You may add Your own attribution
118 | notices within Derivative Works that You distribute, alongside
119 | or as an addendum to the NOTICE text from the Work, provided
120 | that such additional attribution notices cannot be construed
121 | as modifying the License.
122 |
123 | You may add Your own copyright statement to Your modifications and
124 | may provide additional or different license terms and conditions
125 | for use, reproduction, or distribution of Your modifications, or
126 | for any such Derivative Works as a whole, provided Your use,
127 | reproduction, and distribution of the Work otherwise complies with
128 | the conditions stated in this License.
129 |
130 | 5. Submission of Contributions. Unless You explicitly state otherwise,
131 | any Contribution intentionally submitted for inclusion in the Work
132 | by You to the Licensor shall be under the terms and conditions of
133 | this License, without any additional terms or conditions.
134 | Notwithstanding the above, nothing herein shall supersede or modify
135 | the terms of any separate license agreement you may have executed
136 | with Licensor regarding such Contributions.
137 |
138 | 6. Trademarks. This License does not grant permission to use the trade
139 | names, trademarks, service marks, or product names of the Licensor,
140 | except as required for reasonable and customary use in describing the
141 | origin of the Work and reproducing the content of the NOTICE file.
142 |
143 | 7. Disclaimer of Warranty. Unless required by applicable law or
144 | agreed to in writing, Licensor provides the Work (and each
145 | Contributor provides its Contributions) on an "AS IS" BASIS,
146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147 | implied, including, without limitation, any warranties or conditions
148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149 | PARTICULAR PURPOSE. You are solely responsible for determining the
150 | appropriateness of using or redistributing the Work and assume any
151 | risks associated with Your exercise of permissions under this License.
152 |
153 | 8. Limitation of Liability. In no event and under no legal theory,
154 | whether in tort (including negligence), contract, or otherwise,
155 | unless required by applicable law (such as deliberate and grossly
156 | negligent acts) or agreed to in writing, shall any Contributor be
157 | liable to You for damages, including any direct, indirect, special,
158 | incidental, or consequential damages of any character arising as a
159 | result of this License or out of the use or inability to use the
160 | Work (including but not limited to damages for loss of goodwill,
161 | work stoppage, computer failure or malfunction, or any and all
162 | other commercial damages or losses), even if such Contributor
163 | has been advised of the possibility of such damages.
164 |
165 | 9. Accepting Warranty or Additional Liability. While redistributing
166 | the Work or Derivative Works thereof, You may choose to offer,
167 | and charge a fee for, acceptance of support, warranty, indemnity,
168 | or other liability obligations and/or rights consistent with this
169 | License. However, in accepting such obligations, You may act only
170 | on Your own behalf and on Your sole responsibility, not on behalf
171 | of any other Contributor, and only if You agree to indemnify,
172 | defend, and hold each Contributor harmless for any liability
173 | incurred by, or claims asserted against, such Contributor by reason
174 | of your accepting any such warranty or additional liability.
175 |
176 | END OF TERMS AND CONDITIONS
177 |
178 | APPENDIX: How to apply the Apache License to your work.
179 |
180 | To apply the Apache License to your work, attach the following
181 | boilerplate notice, with the fields enclosed by brackets "[]"
182 | replaced with your own identifying information. (Don't include
183 | the brackets!) The text should be enclosed in the appropriate
184 | comment syntax for the file format. We also recommend that a
185 | file or class name and description of purpose be included on the
186 | same "printed page" as the copyright notice for easier
187 | identification within third-party archives.
188 |
189 | Copyright [yyyy] [name of copyright owner]
190 |
191 | Licensed under the Apache License, Version 2.0 (the "License");
192 | you may not use this file except in compliance with the License.
193 | You may obtain a copy of the License at
194 |
195 | http://www.apache.org/licenses/LICENSE-2.0
196 |
197 | Unless required by applicable law or agreed to in writing, software
198 | distributed under the License is distributed on an "AS IS" BASIS,
199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200 | See the License for the specific language governing permissions and
201 | limitations under the License.
202 |
--------------------------------------------------------------------------------
/London Breakout backtest.py:
--------------------------------------------------------------------------------
1 | # coding: utf-8
2 |
3 | # In[1]:
4 |
5 | #this is to London, the greatest city in the world
6 | #i was a Londoner, proud of being Londoner, how i love the city!
7 | #to St Paul, Tate Modern, Millennium Bridge and so much more!
8 |
9 | #okay, lets get down to business
10 | #the idea of london break out strategy is to take advantage of fx trading hour
11 | #basically fx trading is 24 hour non stop for weekdays
12 | #u got tokyo, before tokyo closes, u got london
13 | #in the afternoon, u got new york, when new york closes, its sydney
14 | #and several hours later, tokyo starts again
15 | #however, among these three major players
16 | #london is where the majority trades are executed
17 | #not sure if it will stay the same after brexit actually takes place
18 | #what we intend to do is look at the last trading hour before london starts
19 | #we set up our thresholds based on that hours high and low
20 | #when london market opens, we examine the first 30 minutes
21 | #if it goes way above or below thresholds
22 | #we long or short certain currency pairs
23 | #and we clear our positions based on target and stop loss we set
24 | #if they havent reach the trigger condition by the end of trading hour
25 | #we exit our trades and close all positions
26 |
27 | #it sounds easy in practise
28 | #just a simple prediction of london fx market based on tokyo market
29 | #but the code of london breakout is extremely time consuming
30 | #first, we need to get one minute frequency dataset for backtest
31 | #i would recommend this website
32 | # http://www.histdata.com/download-free-forex-data/?/excel/1-minute-bar-quotes
33 | #we can get as many as free datasets of various currency pairs we want
34 | #before our backtesting, we should cleanse the raw data
35 | #what we get from the website is one minute frequency bid-ask price
36 | #i take the average of em and add a header called price
37 | #i save it on local disk then read it via python
38 | #please note that this website uses new york time zone utc -5
39 | #for non summer daylight saving time
40 | #london market starts at gmt 8 am
41 | #which is est 3 am
42 | #daylight saving time is another story
43 | #what a stupid idea it is
44 | import os
45 | os.chdir('d:/')
46 | import matplotlib.pyplot as plt
47 | import pandas as pd
48 |
49 | # In[2]:
50 |
51 | def london_breakout(df):
52 |
53 | df['signals']=0
54 |
55 | #cumsum is the cumulated sum of signals
56 | #later we would use it to control our positions
57 | df['cumsum']=0
58 |
59 | #upper and lower are our thresholds
60 | df['upper']=0.0
61 | df['lower']=0.0
62 |
63 | return df
64 |
65 |
66 | def signal_generation(df,method):
67 |
68 | #tokyo_price is a list to store average price of
69 | #the last trading hour of tokyo market
70 | #we use max, min to define the real threshold later
71 | tokyo_price=[]
72 |
73 | #risky_stop is a parameter set by us
74 | #it is to reduce the risk exposure to volatility
75 | #i am using 100 basis points
76 | #for instance, we have defined our upper and lower thresholds
77 | #however, when london market opens
78 | #the price goes skyrocketing
79 | #say 200 basis points above upper threshold
80 | #i personally wouldnt get in the market as its too risky
81 | #also, my stop loss and target is 50 basis points
82 | #just half of my risk interval
83 | #i will use this variable for later stop loss set up
84 | risky_stop=0.01
85 |
86 | #this is another parameter set by us
87 | #it is about how long opening volatility would wear off
88 | #for me, 30 minutes after the market opening is the boundary
89 | #as long as its under 30 minutes after the market opening
90 | #if the price reaches threshold level, i will trade on signals
91 | open_minutes=30
92 |
93 | #this is the price when we execute a trade
94 | #we need to save it to set up the stop loss
95 | executed_price=float(0)
96 |
97 | signals=method(df)
98 | signals['date']=pd.to_datetime(signals['date'])
99 |
100 | #this is the core part
101 | #the time complexity for this part is extremely high
102 | #as there are too many constraints
103 | #if u have a better idea to optimize it
104 | #plz let me know
105 |
106 | for i in range(len(signals)):
107 |
108 | #as mentioned before
109 | #the dataset use eastern standard time
110 | #so est 2am is the last hour before london starts
111 | #we try to append all the price into the list called threshold
112 | if signals['date'][i].hour==2:
113 | tokyo_price.append(signals['price'][i])
114 |
115 | #est 3am which is gmt 8am
116 | #thats when london market starts
117 | #good morning city of london and canary wharf!
118 | #right at this moment
119 | #we get max and min of the price of tokyo trading hour
120 | #we set up the threshold as the way it is
121 | #alternatively, we can put 10 basis points above and below thresholds
122 | #we also use upper and lower list to keep track of our thresholds
123 | #and now we clear the list called threshold
124 | elif signals['date'][i].hour==3 and signals['date'][i].minute==0:
125 |
126 | upper=max(tokyo_price)
127 | lower=min(tokyo_price)
128 |
129 | signals.at[i,'upper']=upper
130 | signals.at[i,'lower']=lower
131 |
132 | tokyo_price=[]
133 |
134 | #prior to 30 minutes i have mentioned before
135 | #as long as its under 30 minutes after market opening
136 | #signals will be generated once conditions are met
137 | #this is a relatively risky way
138 | #alternatively, we can set the signal generation time at a fixed point
139 | #when its gmt 8 30 am, we check the conditions to see if there is any signal
140 | elif signals['date'][i].hour==3 and signals['date'][i].minute<open_minutes:
141 |
142 | #again, we wanna keep track of thresholds during signal generation periods
143 | signals.at[i,'upper']=upper
144 | signals.at[i,'lower']=lower
145 |
146 | #this is the condition of signals generation
147 | #when the price is above upper threshold
148 | #we set signals to 1 which implies long
149 | if signals['price'][i]-upper>0:
150 | signals.at[i,'signals']=1
151 |
152 | #we use cumsum to check the cumulated sum of signals
153 | #we wanna make sure that
154 | #only the first price above upper threshold triggers the signal
155 | #also, if it goes skyrocketing
156 | #say 200 basis points above, which is 100 above our risk tolerance
157 | #we set it as a false alarm
158 | signals['cumsum']=signals['signals'].cumsum()
159 |
160 | if signals['price'][i]-upper>risky_stop:
161 | signals.at[i,'signals']=0
162 |
163 | elif signals['cumsum'][i]>1:
164 | signals.at[i,'signals']=0
165 |
166 | else:
167 |
168 | #we also need to store the price when we execute a trade
169 | #its for stop loss calculation
170 | executed_price=signals['price'][i]
171 |
172 | #vice versa
173 | if signals['price'][i]-lower<0:
174 | signals.at[i,'signals']=-1
175 |
176 | signals['cumsum']=signals['signals'].cumsum()
177 |
178 | if lower-signals['price'][i]>risky_stop:
179 | signals.at[i,'signals']=0
180 |
181 | elif signals['cumsum'][i]<-1:
182 | signals.at[i,'signals']=0
183 |
184 | else:
185 | executed_price=signals['price'][i]
186 |
187 | #when its gmt 5 pm, london market closes
188 | #we use cumsum to see if there is any position left open
189 | #we take -cumsum as a signal
190 | #if there is no open position, -0 is still 0
191 | elif signals['date'][i].hour==12:
192 | signals['cumsum']=signals['signals'].cumsum()
193 | signals.at[i,'signals']=-signals['cumsum'][i]
194 |
195 | #during london trading hour after opening but before closing
196 | #we still use cumsum to check our open positions
197 | #if there is any open position
198 | #we set our condition at original executed price +/- half of the risk interval
199 | #when it goes above or below our risk tolerance
200 | #we clear positions to claim profit or loss
201 | else:
202 | signals['cumsum']=signals['signals'].cumsum()
203 |
204 | if signals['cumsum'][i]!=0:
205 | if signals['price'][i]>executed_price+risky_stop/2:
206 | signals.at[i,'signals']=-signals['cumsum'][i]
207 |
208 | if signals['price'][i]<executed_price-risky_stop/2:
209 | signals.at[i,'signals']=-signals['cumsum'][i]
210 |
211 | return signals
212 |
213 |
214 |
215 | def plot(new):
216 |
217 | #the first plot is price with LONG/SHORT positions
218 | fig=plt.figure()
219 | ax=fig.add_subplot(111)
220 |
221 | new['price'].plot(label='price')
222 |
223 | ax.plot(new.loc[new['signals']==1].index,new['price'][new['signals']==1],lw=0,marker='^',c='g',label='LONG')
224 | ax.plot(new.loc[new['signals']==-1].index,new['price'][new['signals']==-1],lw=0,marker='v',c='r',label='SHORT')
225 |
226 | #this is the part where i add some vertical line to indicate market beginning and ending
227 | date=new.index[0].strftime('%Y-%m-%d')
228 | plt.axvline('%s 03:00:00'%(date),linestyle=':',c='k')
229 | plt.axvline('%s 12:00:00'%(date),linestyle=':',c='k')
230 |
231 |
232 | plt.legend(loc='best')
233 | plt.title('London Breakout')
234 | plt.ylabel('price')
235 | plt.xlabel('Date')
236 | plt.grid(True)
237 | plt.show()
238 |
239 |
240 | #lets look at the market opening and break it down into 110 minutes
241 | #we wanna observe how the price goes beyond the threshold
242 |
243 | f='%s 02:50:00'%(date)
244 | l='%s 03:30:00'%(date)
245 | news=signals[f:l]
246 | fig=plt.figure()
247 | bx=fig.add_subplot(111)
248 |
249 | bx.plot(news.loc[news['signals']==1].index,news['price'][news['signals']==1],lw=0,marker='^',markersize=10,c='g',label='LONG')
250 | bx.plot(news.loc[news['signals']==-1].index,news['price'][news['signals']==-1],lw=0,marker='v',markersize=10,c='r',label='SHORT')
251 |
252 | #i only need to plot non zero thresholds
253 | #zero is the value outta market opening period
254 | bx.plot(news.loc[news['upper']!=0].index,news['upper'][news['upper']!=0],lw=0,marker='.',markersize=7,c='#BC8F8F',label='upper threshold')
255 | bx.plot(news.loc[news['lower']!=0].index,news['lower'][news['lower']!=0],lw=0,marker='.',markersize=5,c='#FF4500',label='lower threshold')
256 | bx.plot(news['price'],label='price')
257 |
258 |
259 | plt.grid(True)
260 | plt.ylabel('price')
261 | plt.xlabel('time interval')
262 | plt.xticks([])
263 | plt.title('%s Market Opening'%date)
264 | plt.legend(loc='best')
265 | plt.show()
266 |
267 |
268 | # In[3]:
269 | def main():
270 |
271 | df=pd.read_csv('gbpusd.csv')
272 |
273 | signals=signal_generation(df,london_breakout)
274 |
275 | new=signals
276 | new.set_index(pd.to_datetime(signals['date']),inplace=True)
277 | date=new.index[0].strftime('%Y-%m-%d')
278 | new=new['%s'%date]
279 |
280 | plot(new)
281 |
282 | #how to calculate stats could be found from my other code called Heikin-Ashi
283 | # https://github.com/je-suis-tm/quant-trading/blob/master/heikin%20ashi%20backtest.py
284 |
285 | if __name__ == '__main__':
286 | main()
287 |
--------------------------------------------------------------------------------
/MACD Oscillator backtest.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | """
3 | Created on Tue Feb 6 11:57:46 2018
4 |
5 | @author: Administrator
6 | """
7 |
8 | # In[1]:
9 |
10 | #need to get fix yahoo finance package first
11 |
12 | import matplotlib.pyplot as plt
13 | import numpy as np
14 | import pandas as pd
15 | import fix_yahoo_finance as yf
16 |
17 |
18 |
19 | # In[2]:
20 |
21 | #simple moving average
22 | def macd(signals):
23 |
24 |
25 | signals['ma1']=signals['Close'].rolling(window=ma1,min_periods=1,center=False).mean()
26 | signals['ma2']=signals['Close'].rolling(window=ma2,min_periods=1,center=False).mean()
27 |
28 | return signals
29 |
30 |
31 |
32 | # In[3]:
33 |
34 | #signal generation
35 | #when the short moving average is larger than long moving average, we long and hold
36 | #when the short moving average is smaller than long moving average, we clear positions
37 | #the logic behind this is that the momentum has more impact on short moving average
38 | #we can subtract short moving average from long moving average
39 | #the difference between is sometimes positive, it sometimes becomes negative
40 | #thats why it is named as moving average converge/diverge oscillator
41 | def signal_generation(df,method):
42 |
43 | signals=method(df)
44 | signals['positions']=0
45 |
46 | #positions becomes and stays one once the short moving average is above long moving average
47 | signals['positions'][ma1:]=np.where(signals['ma1'][ma1:]>=signals['ma2'][ma1:],1,0)
48 |
49 | #as positions only imply the holding
50 | #we take the difference to generate real trade signal
51 | signals['signals']=signals['positions'].diff()
52 |
53 | #oscillator is the difference between two moving average
54 | #when it is positive, we long, vice versa
55 | signals['oscillator']=signals['ma1']-signals['ma2']
56 |
57 | return signals
58 |
59 |
60 |
61 | # In[4]:
62 |
63 | #plotting the backtesting result
64 | def plot(new, ticker):
65 |
66 | #the first plot is the actual close price with long/short positions
67 | fig=plt.figure()
68 | ax=fig.add_subplot(111)
69 |
70 | new['Close'].plot(label=ticker)
71 | ax.plot(new.loc[new['signals']==1].index,new['Close'][new['signals']==1],label='LONG',lw=0,marker='^',c='g')
72 | ax.plot(new.loc[new['signals']==-1].index,new['Close'][new['signals']==-1],label='SHORT',lw=0,marker='v',c='r')
73 |
74 | plt.legend(loc='best')
75 | plt.grid(True)
76 | plt.title('Positions')
77 |
78 | plt.show()
79 |
80 | #the second plot is long/short moving average with oscillator
81 | #note that i use bar chart for oscillator
82 | fig=plt.figure()
83 | cx=fig.add_subplot(211)
84 |
85 | new['oscillator'].plot(kind='bar',color='r')
86 |
87 | plt.legend(loc='best')
88 | plt.grid(True)
89 | plt.xticks([])
90 | plt.xlabel('')
91 | plt.title('MACD Oscillator')
92 |
93 | bx=fig.add_subplot(212)
94 |
95 | new['ma1'].plot(label='ma1')
96 | new['ma2'].plot(label='ma2',linestyle=':')
97 |
98 | plt.legend(loc='best')
99 | plt.grid(True)
100 | plt.show()
101 |
102 |
103 | # In[5]:
104 |
105 | def main():
106 |
107 | #input the long moving average and short moving average period
108 | #for the classic MACD, it is 12 and 26
109 | #once a upon a time you got six trading days in a week
110 | #so it is two week moving average versus one month moving average
111 | #for now, the ideal choice would be 10 and 21
112 |
113 | global ma1,ma2,stdate,eddate,ticker,slicer
114 |
115 | #macd is easy and effective
116 | #there is just one issue
117 | #entry signal is always late
118 | #watch out for downward EMA spirals!
119 | ma1=int(input('ma1:'))
120 | ma2=int(input('ma2:'))
121 | stdate=input('start date in format yyyy-mm-dd:')
122 | eddate=input('end date in format yyyy-mm-dd:')
123 | ticker=input('ticker:')
124 |
125 | #slicing the downloaded dataset
126 | #if the dataset is too large, backtesting plot would look messy
127 | #you get too many markers cluster together
128 | slicer=int(input('slicing:'))
129 |
130 | #downloading data
131 | df=yf.download(ticker,start=stdate,end=eddate)
132 |
133 | new=signal_generation(df,macd)
134 | new=new[slicer:]
135 | plot(new, ticker)
136 |
137 |
138 | #how to calculate stats could be found from my other code called Heikin-Ashi
139 | # https://github.com/je-suis-tm/quant-trading/blob/master/heikin%20ashi%20backtest.py
140 |
141 |
142 | if __name__ == '__main__':
143 | main()
144 |
--------------------------------------------------------------------------------
/Monte Carlo project/README.md:
--------------------------------------------------------------------------------
1 | ## Monte Carlo simulation in trading is nothing but house of cards
2 |
3 | 
4 |
5 | Why do I put this picture (see <a href=https://www.explainxkcd.com/wiki/index.php/2048:_Curve-Fitting>here</a> for explanation) right in front of everything? Have I caught your attention? Hopefully I have. These days, blogs on data science topic are dime a dozen. Some blogs teach you how to use decision tree/random forrest to predict the stock price movement, others illustrate the possibility of back-propagation neural network to forecast a bond price. The brutal truth is, most of them are nothing but house of cards. So far, I have not heard any quant shops hit the jackpot via machine learning. Most applications of machine learning in trading are leaned towards analytics rather than prediction or execution. One quant estimates the failure rate of machine learning in live tests of trading is at about <a href=https://www.bloomberg.com/news/articles/2019-07-11/why-machine-learning-hasn-t-made-investors-smarter-quicktake>90%</a>.
6 |
7 | Monte Carlo, my first thought on these two words is the grand casino, where you dress up in tuxedo, meet Famke Janssen after car chase and introduce yourself in a deep sexy voice, 'Bond, James Bond'. Indeed, the simulation is named after the infamous casino in Monaco near Côte d'Azur. It actually refers to the computer simulation of massive amount of random events. This unconventional mathematical method is extremely powerful in the study of stochastic process. Here comes the argument on Linkedin that caught my eyes the other day. "Stock price can be seemed as a <a href=https://en.wikipedia.org/wiki/Wiener_process>Wiener Process</a>. Hence, we can use Monte Carlo simulation to predict the stock price." said a data science blog. Well, in order to be a Wiener Process, we have to assume the stock price is continuous in time. In reality, the market closes. The overnight volatility exists. But that is not the biggest issue here. The biggest issue is, can we really use Monte Carlo simulation to predict the stock price, even a range or its direction?
8 |
9 | The author offered a quite interesting idea. As he suggested, the first step was to run as many simulations as possible on <a href=https://en.wikipedia.org/wiki/Stochastic_differential_equation#Use_in_probability_and_mathematical_finance>stochastic differential equations</a> to predict stock price. The goal of simulations was to pick a best fitted curve (in translation, the smallest standard deviation) compared to the historical data. There might exist a hidden pattern in the historical log return. The best fitted curve had the potential to replicate the hidden pattern and reflect it in the future forecast. The idea sounds neat, doesn't it? Inspired by his idea, we can build up the simulation accordingly. To fully unlock the potential of Monte Carlo simulation on fat tail events, the ticker we pick is General Electric, one of the worst performing stock in 2018. The share price of GE plunged 57.9% in 2018 thanks to its long history of M&A failures. The time horizon of the data series is 2016/1/15-2019/1/15. We split the series into halves, the first half as training horizon and the second half as testing horizon. Monte Carlo simulation will only justify its power if it can predict an extreme event like this.
10 |
11 | Let's take a look at the figure below. Wow, what a great fit! The best fit is almost like running a cool linear regression with valid input. As you can see, it smoothly fits the curve in training horizon.
12 |
13 | 
14 |
15 | If we extend it to testing horizon...
16 |
17 | 
18 |
19 | Oops, house of cards collapses. The real momentum is completely the opposite to the forecast, let alone the actual price. The forecast looks quite okay for the first 50 days of testing horizon. After that, the real momentum falls like a stone down through the deep sea while the forecast is gently climbing up. You may argue the number of the iterations is too small so we cannot make a case. Not exactly, let's look at the figure below.
20 |
21 | 
22 |
23 | We start from 500 times of simulation to 1500 times of simulation. Each round we increase the number by 50. We don't look at the actual price forecast, just the direction. If the end price of testing horizon is larger than the end price of training horizon, we define it as gain. Vice versa. Only when both actual and forecast directions align, we consider the forecast is accurate. As the result shows, the prediction accuracy is irrelavant to the numbers of simulation. The accuracy is more sort of tossing a coin to guess heads or tails regardless of the times of simulation. If you think 1500 is still not large enough, you can definitely try 150000, be my guest. We don't have so much computing power as an individual user (frankly no patience is the real reason) but I can assure you the result is gonna stay the same. <a href=https://en.wikipedia.org/wiki/Law_of_large_numbers>Law of Large Numbers</a> theorem would not work here.
24 |
25 | Now that the prophet of Monte Carlo turns out to be a con artist. Does Monte Carlo simulation have any application in risk management? Unless you are drinking the Kool-Aid. Let's extend the first figure a little bit longer to the end of testing horizon.
26 |
27 | 
28 |
29 | Obviously, out of 500 times of simulation, none of them successfully predict the scale of the downslide. The lowest price of 500 predictions is 10.99 per share. It is still way higher than the real bottom price at 6.71 per share (no wonder GE got its ass kicked out of DJIA). The so-called fat tail event simulation is merely a mirage. If you think GE in 2018 is an extreme case, you'd be so wrong. The next ticker we test is Nvidia from 2006/1/15 to 2009/1/15. In 2008, the share price of Nvidia dropped 75.6%!! The financial crisis is the true playground for risk quants. By default, we continue to split the time horizon of NVDA into two parts by 50:50. The result is in the figure below.
30 |
31 | 
32 |
33 | Undoubtedly, the best forecast fails the expectation again. It cannot accurately predict the direction of the momentum. For the extreme event, Monte Carlo simulation cannot predict the scale of downslide, again! 6.086 per share is the lowest price we achieve from our 500 times of simulation, yet the actual lowest is at 5.9 per share.
34 |
35 | 
36 |
37 | Still thinking about Monte Carlo for your next big project?
38 |
39 |
--------------------------------------------------------------------------------
/Monte Carlo project/preview/ge accuracy.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Monte Carlo project/preview/ge accuracy.png
--------------------------------------------------------------------------------
/Monte Carlo project/preview/ge simulation.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Monte Carlo project/preview/ge simulation.png
--------------------------------------------------------------------------------
/Monte Carlo project/preview/ge simulation2.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Monte Carlo project/preview/ge simulation2.png
--------------------------------------------------------------------------------
/Monte Carlo project/preview/ge versus.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Monte Carlo project/preview/ge versus.png
--------------------------------------------------------------------------------
/Monte Carlo project/preview/nvda simulation.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Monte Carlo project/preview/nvda simulation.png
--------------------------------------------------------------------------------
/Monte Carlo project/preview/nvda versus.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Monte Carlo project/preview/nvda versus.png
--------------------------------------------------------------------------------
/Monte Carlo project/preview/xkcd_curve_fitting.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Monte Carlo project/preview/xkcd_curve_fitting.png
--------------------------------------------------------------------------------
/Oil Money project/Oil Money CAD.py:
--------------------------------------------------------------------------------
1 |
2 | # coding: utf-8
3 |
4 | # In[1]:
5 |
6 |
7 | import pandas as pd
8 | import os
9 | import matplotlib.pyplot as plt
10 | import copy
11 | import matplotlib.patches as mpatches
12 | from mpl_toolkits import mplot3d
13 | import matplotlib.cm as cm
14 | import statsmodels.api as sm
15 | import numpy as np
16 | from sklearn.cluster import KMeans
17 | from sklearn.metrics import silhouette_score,silhouette_samples
18 | from sklearn.model_selection import train_test_split
19 | os.chdir('d:/')
20 |
21 |
22 | # In[2]:
23 |
24 | #plot two curves on same x axis, different y axis
25 | def dual_axis_plot(xaxis,data1,data2,fst_color='r',
26 | sec_color='b',fig_size=(10,5),
27 | x_label='',y_label1='',y_label2='',
28 | legend1='',legend2='',grid=False,title=''):
29 |
30 | fig=plt.figure(figsize=fig_size)
31 | ax=fig.add_subplot(111)
32 |
33 |
34 | ax.set_xlabel(x_label)
35 | ax.set_ylabel(y_label1, color=fst_color)
36 | ax.plot(xaxis, data1, color=fst_color,label=legend1)
37 | ax.tick_params(axis='y',labelcolor=fst_color)
38 | ax.yaxis.labelpad=15
39 |
40 | plt.legend(loc=3)
41 | ax2=ax.twinx()
42 |
43 | ax2.set_ylabel(y_label2, color=sec_color,rotation=270)
44 | ax2.plot(xaxis, data2, color=sec_color,label=legend2)
45 | ax2.tick_params(axis='y',labelcolor=sec_color)
46 | ax2.yaxis.labelpad=15
47 |
48 | fig.tight_layout()
49 | plt.legend(loc=4)
50 | plt.grid(grid)
51 | plt.title(title)
52 | plt.show()
53 |
54 | #get distance between a point and a line
55 | def get_distance(x,y,a,b):
56 |
57 | temp1=y-x*a-b
58 | temp2=(a**2+1)**0.5
59 |
60 | return np.abs(temp1/temp2)
61 |
62 | #create line equation from two points
63 | def get_line_params(x1,y1,x2,y2):
64 |
65 | a=(y1-y2)/(x1-x2)
66 | b=y1-a*x1
67 |
68 | return a,b
69 |
70 |
71 | # In[3]:
72 |
73 |
74 | df=pd.read_csv('wcs crude cadaud.csv',encoding='utf-8')
75 | df.set_index('date',inplace=True)
76 |
77 |
78 | # In[4]:
79 |
80 |
81 | df.index=pd.to_datetime(df.index,format='%m/%d/%Y')
82 |
83 |
84 | # In[5]:
85 |
86 |
87 | df=df.reindex(columns=
88 | ['cny',
89 | 'gbp',
90 | 'usd',
91 | 'eur',
92 | 'krw',
93 | 'mxn',
94 | 'gas',
95 | 'wcs',
96 | 'edmonton',
97 | 'wti',
98 | 'gold',
99 | 'jpy',
100 | 'cad'])
101 |
102 |
103 | # In[6]:
104 |
105 | #create r squared bar charts
106 |
107 | var=locals()
108 |
109 | for i in df.columns:
110 | if i!='cad':
111 | x=sm.add_constant(df[i])
112 | y=df['cad']
113 | m=sm.OLS(y,x).fit()
114 | var[str(i)]=m.rsquared
115 |
116 | ax=plt.figure(figsize=(10,5)).add_subplot(111)
117 | ax.spines['top'].set_visible(False)
118 | ax.spines['right'].set_visible(False)
119 |
120 | width=0.7
121 | colorlist=['#9499a6','#9499a6','#9499a6','#9499a6',
122 | '#9499a6','#9499a6','#9499a6','#582a20',
123 | '#be7052','#f2c083','#9499a6','#9499a6']
124 |
125 | temp=list(df.columns)
126 | for i in temp:
127 | if i!='cad':
128 | plt.bar(temp.index(i)+width,
129 | var[str(i)],width=width,label=i,
130 | color=colorlist[temp.index(i)])
131 |
132 | plt.title('Regressions on Loonie')
133 | plt.ylabel('R Squared\n')
134 | plt.xlabel('\nRegressors')
135 | plt.xticks(np.arange(len(temp))+width,
136 | ['Yuan', 'Sterling', 'Dollar', 'Euro', 'KRW',
137 | 'MXN', 'Gas', 'WCS', 'Edmonton',
138 | 'WTI', 'Gold', 'Yen'],fontsize=10)
139 | plt.show()
140 |
141 |
142 | # In[7]:
143 |
144 | #normalized value of loonie,yuan and sterling
145 |
146 | ax=plt.figure(figsize=(10,5)).add_subplot(111)
147 | ax.spines['top'].set_visible(False)
148 | ax.spines['right'].set_visible(False)
149 |
150 | (df['cny']/df['cny'].iloc[0]).plot(c='#77c9d4',label='Yuan')
151 | (df['gbp']/df['gbp'].iloc[0]).plot(c='#57bc90',label='Sterling')
152 | (df['cad']/df['cad'].iloc[0]).plot(c='#015249',label='Loonie')
153 |
154 | plt.legend(loc=0)
155 | plt.xlabel('Date')
156 | plt.ylabel('Normalized Value by 100')
157 | plt.title('Loonie vs Yuan vs Sterling')
158 | plt.show()
159 |
160 |
161 | # In[8]:
162 |
163 | #normalized value of wti,wcs and edmonton
164 |
165 | ax=plt.figure(figsize=(10,5)).add_subplot(111)
166 | ax.spines['top'].set_visible(False)
167 | ax.spines['right'].set_visible(False)
168 |
169 | (df['wti']/df['wti'].iloc[0]).plot(c='#2a78b2',
170 | label='WTI',alpha=0.5)
171 | (df['wcs']/df['wcs'].iloc[0]).plot(c='#7b68ee',
172 | label='WCS',alpha=0.5)
173 | (df['edmonton']/df['edmonton'].iloc[0]).plot(c='#110b3c',
174 | label='Edmonton',alpha=0.5)
175 | plt.legend(loc=0)
176 | plt.xlabel('Date')
177 | plt.ylabel('Normalized Value by 100')
178 | plt.title('Crude Oil Blends')
179 | plt.show()
180 |
181 |
182 | # In[9]:
183 |
184 |
185 | dual_axis_plot(df.index,df['cad'],df['wcs'],
186 | x_label='Date',y_label1='Canadian Dollar',
187 | y_label2='Western Canadian Select',
188 | legend1='Loonie',
189 | legend2='WCS',
190 | title='Loonie VS WCS in AUD',
191 | fst_color='#a5a77f',sec_color='#d8dc2c')
192 |
193 | dual_axis_plot(df.index,
194 | np.divide(df['cad'],df['usd']),
195 | np.divide(df['wcs'],df['usd']),
196 | x_label='Date',y_label1='Canadian Dollar',
197 | y_label2='Western Canadian Select',
198 | legend1='Loonie',
199 | legend2='WCS',
200 | title='Loonie VS WCS in USD',
201 | fst_color='#eb712f',sec_color='#91371b')
202 |
203 |
204 | # In[10]:
205 |
206 | #using elbow method to find optimal number of clusters
207 |
208 | df['date']=[i for i in range(len(df.index))]
209 |
210 | x=df[['cad','wcs','date']].reset_index(drop=True)
211 |
212 | sse=[]
213 | for i in range(1, 8):
214 | kmeans = KMeans(n_clusters = i)
215 | kmeans.fit(x)
216 | sse.append(kmeans.inertia_/10000)
217 |
218 | a,b=get_line_params(0,sse[0],len(sse)-1,sse[-1])
219 |
220 | distance=[]
221 | for i in range(len(sse)):
222 | distance.append(get_distance(i,sse[i],a,b))
223 |
224 | dual_axis_plot(np.arange(1,len(distance)+1),sse,distance,
225 | x_label='Numbers of Cluster',y_label1='Sum of Squared Error',
226 | y_label2='Perpendicular Distance',legend1='SSE',legend2='Distance',
227 | title='Elbow Method for K Means',fst_color='#116466',sec_color='#e85a4f')
228 |
229 |
230 |
231 | # In[11]:
232 |
233 | #using silhouette score to find optimal number of clusters
234 |
235 | sil=[]
236 | for n in range(2,8):
237 |
238 | clf=KMeans(n).fit(x)
239 | projection=clf.predict(x)
240 |
241 | sil.append(silhouette_score(x,projection))
242 |
243 | ax=plt.figure(figsize=(10,5)).add_subplot(111)
244 | ax.spines['top'].set_visible(False)
245 | ax.spines['right'].set_visible(False)
246 |
247 | ax.plot(np.arange(2,len(sil)+2),sil,
248 | label='Silhouette',c='#5c0811',
249 | drawstyle='steps-mid')
250 | ax.plot(sil.index(max(sil))+2,max(sil),
251 | marker='*',markersize=20,lw=0,
252 | label='Max Score',c='#d94330')
253 |
254 | plt.ylabel('Silhouette Score')
255 | plt.xlabel('Numbers of Cluster')
256 | plt.title('Silhouette Analysis for K Means')
257 | plt.legend(loc=0)
258 | plt.show()
259 |
260 |
261 | # In[12]:
262 |
263 | #k means
264 |
265 | clf=KMeans(n_clusters=2).fit(x)
266 | df['class']=clf.predict(x)
267 | threshold=df[df['class']==0].index[-1]
268 |
269 |
270 | # In[13]:
271 |
272 | #plot clusters in 3d figure
273 |
274 | ax=plt.figure(figsize=(10,7)).add_subplot(111, projection='3d')
275 |
276 |
277 | xdata=df['wcs']
278 | ydata=df['cad']
279 | zdata=df['date']
280 | ax.scatter3D(xdata[df['class']==0],ydata[df['class']==0],
281 | zdata[df['class']==0],c='#faed26',s=10,alpha=0.5,
282 | label='Before {}'.format(threshold.strftime('%Y-%m-%d')))
283 | ax.scatter3D(xdata[df['class']==1],ydata[df['class']==1],
284 | zdata[df['class']==1],c='#46344e',s=10,alpha=0.5,
285 | label='After {}'.format(threshold.strftime('%Y-%m-%d')))
286 | ax.grid(False)
287 | for i in ax.w_xaxis, ax.w_yaxis, ax.w_zaxis:
288 | i.pane.set_visible(False)
289 |
290 | ax.set_xlabel('WCS')
291 | ax.set_ylabel('Loonie')
292 | ax.set_zlabel('Date')
293 | ax.set_title('K Means on Loonie')
294 | ax.legend(loc=6,bbox_to_anchor=(0.12, -0.1), ncol=4)
295 |
296 | plt.show()
297 |
298 | #to generate gif, u can use the following code
299 | #it generates 3d figures from different angles
300 | #use imageio to concatenate
301 | """
302 | for ii in range(0,360,10):
303 | ax.view_init(elev=10., azim=ii)
304 | plt.savefig("cad kmeans%d.png" % (ii))
305 |
306 | import imageio
307 |
308 | filenames=["movie%d.png" % (ii) for ii in range(0,360,10)]
309 |
310 | images=list(map(lambda filename:imageio.imread(filename),
311 | filenames))
312 |
313 | imageio.mimsave('cad kmeans.gif',images,duration = 0.2)
314 | """
315 |
316 | # In[14]:
317 |
318 | #create before/after regression comparison
319 | #the threshold is based upon the finding of k means
320 |
321 | m=sm.OLS(df['cad'][df['class']==0],sm.add_constant(df['wcs'][df['class']==0])).fit()
322 | before=m.rsquared
323 | m=sm.OLS(df['cad'][df['class']==1],sm.add_constant(df['wcs'][df['class']==1])).fit()
324 | after=m.rsquared
325 |
326 | ax=plt.figure(figsize=(10,5)).add_subplot(111)
327 | ax.spines['top'].set_visible(False)
328 | ax.spines['right'].set_visible(False)
329 | plt.bar(['Before {}'.format(threshold.strftime('%Y-%m-%d')),
330 | 'After {}'.format(threshold.strftime('%Y-%m-%d'))],
331 | [before,after],color=['#f172a1','#a1c3d1'])
332 | plt.ylabel('R Squared')
333 | plt.title('Cluster + Regression')
334 | plt.show()
335 |
336 |
337 |
338 | # In[15]:
339 |
340 | #create 1 std, 2 std band
341 |
342 | for i in range(2):
343 |
344 | x_train,x_test,y_train,y_test=train_test_split(
345 | sm.add_constant(df['wcs'][df['class']==i]),
346 | df['cad'][df['class']==i],test_size=0.5,shuffle=False)
347 |
348 | m=sm.OLS(y_test,x_test).fit()
349 |
350 | forecast=m.predict(x_test)
351 |
352 | ax=plt.figure(figsize=(10,5)).add_subplot(111)
353 | ax.spines['top'].set_visible(False)
354 | ax.spines['right'].set_visible(False)
355 | forecast.plot(label='Fitted',c='#ab987a')
356 | y_test.plot(label='Actual',c='#ff533d')
357 | ax.fill_between(y_test.index,
358 | forecast+np.std(m.resid),
359 | forecast-np.std(m.resid),
360 | color='#0f1626', \
361 | alpha=0.6, \
362 | label='1 Sigma')
363 |
364 | ax.fill_between(y_test.index,
365 | forecast+2*np.std(m.resid),
366 | forecast-2*np.std(m.resid),
367 | color='#0f1626', \
368 | alpha=0.8, \
369 | label='2 Sigma')
370 |
371 | plt.legend(loc=0)
372 | title='Before '+threshold.strftime('%Y-%m-%d') if i==0 else 'After '+threshold.strftime('%Y-%m-%d')
373 | plt.title(f'{title}\nCanadian Dollar Positions\nR Squared {round(m.rsquared*100,2)}%\n')
374 | plt.xlabel('\nDate')
375 | plt.ylabel('CADAUD')
376 | plt.show()
377 |
378 |
379 |
380 |
381 |
--------------------------------------------------------------------------------
/Oil Money project/Oil Money COP.py:
--------------------------------------------------------------------------------
1 |
2 | # coding: utf-8
3 |
4 | # In[1]:
5 |
6 |
7 | import pandas as pd
8 | import os
9 | import matplotlib.pyplot as plt
10 | import statsmodels.api as sm
11 | import seaborn as sns
12 | import numpy as np
13 | from sklearn.model_selection import train_test_split
14 | os.chdir('k:/')
15 |
16 |
17 | # In[2]:
18 |
19 | #plot two curves on same x axis, different y axis
20 | def dual_axis_plot(xaxis,data1,data2,fst_color='r',
21 | sec_color='b',fig_size=(10,5),
22 | x_label='',y_label1='',y_label2='',
23 | legend1='',legend2='',grid=False,title=''):
24 |
25 | fig=plt.figure(figsize=fig_size)
26 | ax=fig.add_subplot(111)
27 |
28 |
29 | ax.set_xlabel(x_label)
30 | ax.set_ylabel(y_label1, color=fst_color)
31 | ax.plot(xaxis, data1, color=fst_color,label=legend1)
32 | ax.tick_params(axis='y',labelcolor=fst_color)
33 | ax.yaxis.labelpad=15
34 |
35 | plt.legend(loc=3)
36 | ax2=ax.twinx()
37 |
38 | ax2.set_ylabel(y_label2, color=sec_color,rotation=270)
39 | ax2.plot(xaxis, data2, color=sec_color,label=legend2)
40 | ax2.tick_params(axis='y',labelcolor=sec_color)
41 | ax2.yaxis.labelpad=15
42 |
43 | fig.tight_layout()
44 | plt.legend(loc=4)
45 | plt.grid(grid)
46 | plt.title(title)
47 | plt.show()
48 |
49 |
50 |
51 | # In[3]:
52 |
53 | #read dataframe
54 | df=pd.read_csv('vas crude copaud.csv',encoding='utf-8')
55 | df.set_index('date',inplace=True)
56 | df.index=pd.to_datetime(df.index)
57 |
58 |
59 | # In[4]:
60 |
61 | #run regression on each input
62 |
63 | D={}
64 |
65 | for i in df.columns:
66 | if i!='cop':
67 | x=sm.add_constant(df[i])
68 | y=df['cop']
69 | m=sm.OLS(y,x).fit()
70 | D[i]=m.rsquared
71 |
72 | D=dict(sorted(D.items(),key=lambda x:x[1],reverse=True))
73 |
74 |
75 | # In[5]:
76 |
77 | #create r squared bar charts
78 |
79 | colorlist=[]
80 |
81 | for i in D:
82 | if i =='wti':
83 | colorlist.append('#447294')
84 | elif i=='brent':
85 | colorlist.append('#8fbcdb')
86 | elif i=='vasconia':
87 | colorlist.append('#f4d6bc')
88 | else:
89 | colorlist.append('#cdc8c8')
90 |
91 | ax=plt.figure(figsize=(10,5)).add_subplot(111)
92 | ax.spines['top'].set_visible(False)
93 | ax.spines['right'].set_visible(False)
94 |
95 | width=0.7
96 |
97 | for i in D:
98 | plt.bar(list(D.keys()).index(i)+width,
99 | D[i],width=width,label=i,
100 | color=colorlist[list(D.keys()).index(i)])
101 |
102 | plt.title('Regressions on COP')
103 | plt.ylabel('R Squared\n')
104 | plt.xlabel('\nRegressors')
105 | plt.xticks(np.arange(len(D))+width,
106 | [i.upper() for i in D.keys()],fontsize=8)
107 | plt.show()
108 |
109 |
110 | # In[6]:
111 |
112 | #normalized value of wti,brent and vasconia
113 | ax=plt.figure(figsize=(10,5)).add_subplot(111)
114 | ax.spines['top'].set_visible(False)
115 | ax.spines['right'].set_visible(False)
116 |
117 | (df['vasconia']/df['vasconia'].iloc[0]).plot(c='#6f6ff4',
118 | label='Vasconia',alpha=0.5)
119 | (df['brent']/df['brent'].iloc[0]).plot(c='#e264c0',
120 | label='Brent',alpha=0.5)
121 | (df['wti']/df['wti'].iloc[0]).plot(c='#fb6630',
122 | label='WTI',alpha=0.5)
123 | plt.legend(loc=0)
124 | plt.xlabel('Date')
125 | plt.ylabel('Normalized Value by 100')
126 | plt.title('Crude Oil Blends')
127 | plt.show()
128 |
129 |
130 | # In[7]:
131 |
132 | #cop vs gold
133 | dual_axis_plot(df.index,df['cop'],df['gold'],
134 | x_label='Date',y_label1='Colombian Peso',
135 | y_label2='Gold LBMA',
136 | legend1='COP',
137 | legend2='Gold',
138 | title='COP VS Gold',
139 | fst_color='#96CEB4',sec_color='#FFA633')
140 |
141 | #cop vs usd
142 | dual_axis_plot(df.index,df['cop'],df['usd'],
143 | x_label='Date',y_label1='Colombian Peso',
144 | y_label2='US Dollar',
145 | legend1='COP',
146 | legend2='USD',
147 | title='COP VS USD',
148 | fst_color='#9DE0AD',sec_color='#5C4E5F')
149 |
150 |
151 | # In[8]:
152 |
153 | #cop vs brl
154 | dual_axis_plot(df.index,df['cop'],df['brl'],
155 | x_label='Date',y_label1='Colombian Peso',
156 | y_label2='Brazilian Real',
157 | legend1='COP',
158 | legend2='BRL',
159 | title='COP VS BRL',
160 | fst_color='#a4c100',sec_color='#f7db4f')
161 |
162 | #usd vs mxn
163 | dual_axis_plot(df.index,df['usd'],df['mxn'],
164 | x_label='Date',y_label1='US Dollar',
165 | y_label2='Mexican Peso',
166 | legend1='USD',
167 | legend2='MXN',
168 | title='USD VS MXN',
169 | fst_color='#F4A688',sec_color='#A2836E')
170 |
171 | #cop vs mxn
172 | dual_axis_plot(df.index,df['cop'],df['mxn'],
173 | x_label='Date',y_label1='Colombian Peso',
174 | y_label2='Mexican Peso',
175 | legend1='COP',
176 | legend2='MXN',
177 | title='COP VS MXN',
178 | fst_color='#F26B38',sec_color='#B2AD7F')
179 |
180 |
181 | # In[9]:
182 |
183 | #cop vs vasconia
184 | dual_axis_plot(df.index,df['cop'],df['vasconia'],
185 | x_label='Date',y_label1='Colombian Peso',
186 | y_label2='Vasconia Crude',
187 | legend1='COP',
188 | legend2='Vasconia',
189 | title='COP VS Vasconia',
190 | fst_color='#346830',sec_color='#BBAB9B')
191 |
192 |
193 | # In[10]:
194 |
195 | #create before/after regression comparison
196 | m=sm.OLS(df['cop'][:'2016'],sm.add_constant(df['vasconia'][:'2016'])).fit()
197 | before=m.rsquared
198 | m=sm.OLS(df['cop']['2017':],sm.add_constant(df['vasconia']['2017':])).fit()
199 | after=m.rsquared
200 |
201 | ax=plt.figure(figsize=(10,5)).add_subplot(111)
202 | ax.spines['top'].set_visible(False)
203 | ax.spines['right'].set_visible(False)
204 | plt.bar(['Before 2017',
205 | 'After 2017'],
206 | [before,after],color=['#82b74b', '#5DD39E'])
207 | plt.ylabel('R Squared')
208 | plt.title('Before/After Regression')
209 | plt.show()
210 |
211 |
212 | # In[11]:
213 |
214 |
215 | #create 1 std, 2 std band before 2017
216 | x_train,x_test,y_train,y_test=train_test_split(
217 | sm.add_constant(df['vasconia'][:'2016']),
218 | df['cop'][:'2016'],test_size=0.5,shuffle=False)
219 |
220 | m=sm.OLS(y_test,x_test).fit()
221 |
222 | forecast=m.predict(x_test)
223 |
224 | ax=plt.figure(figsize=(10,5)).add_subplot(111)
225 | ax.spines['top'].set_visible(False)
226 | ax.spines['right'].set_visible(False)
227 | forecast.plot(label='Fitted',c='#FEFBD8')
228 | y_test.plot(label='Actual',c='#ffd604')
229 | ax.fill_between(y_test.index,
230 | forecast+np.std(m.resid),
231 | forecast-np.std(m.resid),
232 | color='#F4A688',
233 | alpha=0.6,
234 | label='1 Sigma')
235 |
236 | ax.fill_between(y_test.index,
237 | forecast+2*np.std(m.resid),
238 | forecast-2*np.std(m.resid),
239 | color='#8c7544',
240 | alpha=0.8,
241 | label='2 Sigma')
242 |
243 | plt.legend(loc=0)
244 | plt.title(f'Colombian Peso Positions\nR Squared {round(m.rsquared*100,2)}%\n')
245 | plt.xlabel('\nDate')
246 | plt.ylabel('COPAUD')
247 | plt.show()
248 |
249 |
250 | # In[12]:
251 |
252 |
253 | #create 1 std, 2 std band after 2017
254 | x_train,x_test,y_train,y_test=train_test_split(
255 | sm.add_constant(df['vasconia']['2017':]),
256 | df['cop']['2017':],test_size=0.5,shuffle=False)
257 |
258 | m=sm.OLS(y_test,x_test).fit()
259 |
260 | forecast=m.predict(x_test)
261 |
262 | ax=plt.figure(figsize=(10,5)).add_subplot(111)
263 | ax.spines['top'].set_visible(False)
264 | ax.spines['right'].set_visible(False)
265 | forecast.plot(label='Fitted',c='#FEFBD8')
266 | y_test.plot(label='Actual',c='#ffd604')
267 | ax.fill_between(y_test.index,
268 | forecast+np.std(m.resid),
269 | forecast-np.std(m.resid),
270 | color='#F4A688',
271 | alpha=0.6,
272 | label='1 Sigma')
273 |
274 | ax.fill_between(y_test.index,
275 | forecast+2*np.std(m.resid),
276 | forecast-2*np.std(m.resid),
277 | color='#8c7544', \
278 | alpha=0.8, \
279 | label='2 Sigma')
280 |
281 | plt.legend(loc=0)
282 | plt.title(f'Colombian Peso Positions\nR Squared {round(m.rsquared*100,2)}%\n')
283 | plt.xlabel('\nDate')
284 | plt.ylabel('COPAUD')
285 | plt.show()
286 |
287 |
288 | # In[13]:
289 |
290 | #shrink data size for better viz
291 | dataset=df['2016':]
292 | dataset.reset_index(inplace=True)
293 |
294 |
295 | # In[14]:
296 |
297 | #import the strategy script as this is a script for analytics and visualization
298 | #the official trading strategy script is in the following link
299 | # https://github.com/je-suis-tm/quant-trading/blob/master/Oil%20Money%20project/Oil%20Money%20Trading%20backtest.py
300 | import oil_money_trading_backtest as om
301 |
302 | #generate signals,monitor portfolio performance
303 | #plot positions and total asset
304 | signals=om.signal_generation(dataset,'vasconia','cop',om.oil_money,stop=0.001)
305 | p=om.portfolio(signals,'cop')
306 | om.plot(signals,'cop')
307 | om.profit(p,'cop')
308 |
309 |
310 | # In[15]:
311 |
312 |
313 | #try different holding period and stop loss/profit point
314 | dic={}
315 | for holdingt in range(5,20):
316 | for stopp in np.arange(0.001,0.005,0.0005):
317 | signals=om.signal_generation(dataset,'vasconia','cop',om.oil_money,
318 | holding_threshold=holdingt,
319 | stop=round(stopp,4))
320 |
321 | p=om.portfolio(signals,'cop')
322 | dic[holdingt,round(stopp,4)]=p['asset'].iloc[-1]/p['asset'].iloc[0]-1
323 |
324 | profile=pd.DataFrame({'params':list(dic.keys()),'return':list(dic.values())})
325 |
326 |
327 | # In[16]:
328 |
329 |
330 | #plotting the distribution of return
331 | ax=plt.figure(figsize=(10,5)).add_subplot(111)
332 | ax.spines['top'].set_visible(False)
333 | ax.spines['right'].set_visible(False)
334 | profile['return'].apply(lambda x:x*100).hist(histtype='bar',
335 | color='#b2660e',
336 | width=0.45,bins=20)
337 | plt.title('Distribution of Return on COP Trading')
338 | plt.grid(False)
339 | plt.ylabel('Frequency')
340 | plt.xlabel('Return (%)')
341 | plt.show()
342 |
343 |
344 | # In[17]:
345 |
346 |
347 | #plotting the heatmap of return under different parameters
348 | #try to find the optimal parameters to maximize the return
349 |
350 | #convert the dataframe into a matrix format first
351 | matrix=pd.DataFrame(columns=[round(i,4) for i in np.arange(0.001,0.005,0.0005)])
352 |
353 | matrix['index']=np.arange(5,20)
354 | matrix.set_index('index',inplace=True)
355 |
356 | for i,j in profile['params']:
357 | matrix.at[i,round(j,4)]= profile['return'][profile['params']==(i,j)].item()*100
358 |
359 | for i in matrix.columns:
360 | matrix[i]=matrix[i].apply(float)
361 |
362 |
363 | #plotting
364 | fig=plt.figure(figsize=(10,5))
365 | ax=fig.add_subplot(111)
366 | sns.heatmap(matrix,cmap=plt.cm.viridis,
367 | xticklabels=3,yticklabels=3)
368 | ax.collections[0].colorbar.set_label('Return(%)\n',
369 | rotation=270)
370 | plt.xlabel('\nStop Loss/Profit (points)')
371 | plt.ylabel('Position Holding Period (days)\n')
372 | plt.title('Profit Heatmap\n',fontsize=10)
373 | plt.style.use('default')
374 |
375 | #it seems like the return doesnt depend on the stop profit/loss point
376 | #it is correlated with the length of holding period
377 | #the ideal one should be 17 trading days
378 | #as for stop loss/profit point could range from 0.002 to 0.005
379 |
380 |
--------------------------------------------------------------------------------
/Oil Money project/Oil Money RUB.py:
--------------------------------------------------------------------------------
1 | # coding: utf-8
2 |
3 | # In[1]:
4 |
5 |
6 | import os
7 | import pandas as pd
8 | import numpy as np
9 | import statsmodels.api as sm
10 | import matplotlib.pyplot as plt
11 | import re
12 |
13 | os.chdir('d:/')
14 | df=pd.read_csv('urals crude rubaud.csv')
15 |
16 |
17 | # In[2]:
18 |
19 |
20 | df.set_index('date',inplace=True)
21 | df.index=pd.to_datetime(df.index)
22 | df.dropna(inplace=True)
23 |
24 |
25 | # In[3]:
26 |
27 | #this is the part to create r squared of different regressors
28 | #in different years for stepwise regression
29 | #we can use locals to create lists for different currency
30 | #each list contains r squared of different years
31 | year=df.index.year.drop_duplicates().tolist()
32 | var=locals()
33 | for i in df.columns:
34 | if i!='rub':
35 | var[i]=[]
36 | for j in year:
37 | x=sm.add_constant(df[i][str(j):str(j)])
38 | y=df['rub'][str(j):str(j)]
39 | m=sm.OLS(y,x).fit()
40 | var[i].append(m.rsquared)
41 |
42 |
43 | ax=plt.figure(figsize=(10,5)).add_subplot(111)
44 | ax.spines['top'].set_visible(False)
45 | ax.spines['right'].set_visible(False)
46 |
47 | #to save you from the hassle
48 | #just use these codes to generate the bar chart
49 | width=0.3
50 | colorlist=['#c0334d','#d6618f','#f3d4a0','#f1931b']
51 | bar=locals()
52 | for j in range(len(year)):
53 | bar[j]=[var[i][j] for i in df.columns if i!='rub']
54 | plt.bar(np.arange(1,len([i for i in df.columns if i!='rub'])*2+1,2)+j*width,
55 | bar[j],width=width,label=year[j],color=colorlist[j])
56 |
57 | plt.legend(loc=0)
58 | plt.title('Stepwise Regression Year by Year')
59 | plt.ylabel('R Squared')
60 | plt.xlabel('Regressors')
61 | plt.xticks(np.arange(1,len([i for i in df.columns if i!='rub'])*2+1,2)+(len(year)-1)*width/2,
62 | ['Urals Crude', 'Japanese\nYen',
63 | 'Euro', 'Henry Hub', 'Chinese\nYuan',
64 | 'Korean\nWon', 'Ukrainian\nHryvnia'],fontsize=10)
65 | plt.show()
66 |
67 |
68 | # In[4]:
69 |
70 | #this is similar to In[3]
71 | #In[3] is r squared of each regressor in each year
72 | #In[4] is r squared of each regressor of years cumulated
73 | var=locals()
74 | for i in df.columns:
75 | if i!='rub':
76 | var[i]=[]
77 | for j in year:
78 | x=sm.add_constant(df[i][:str(j)])
79 | y=df['rub'][:str(j)]
80 | m=sm.OLS(y,x).fit()
81 | var[i].append(m.rsquared)
82 |
83 |
84 | ax=plt.figure(figsize=(10,5)).add_subplot(111)
85 | ax.spines['top'].set_visible(False)
86 | ax.spines['right'].set_visible(False)
87 |
88 | width=0.3
89 | colorlist=['#04060f','#03353e','#0294a5','#a79c93']
90 | bar=locals()
91 | for j in range(len(year)):
92 | bar[j]=[var[i][j] for i in df.columns if i!='rub']
93 | plt.bar(np.arange(1,len([i for i in df.columns if i!='rub'])*2+1,2)+j*width,
94 | bar[j],width=width,label=year[j],color=colorlist[j])
95 |
96 | plt.legend(loc=0)
97 | plt.title('Stepwise Regression Year Cumulated')
98 | plt.ylabel('R Squared')
99 | plt.xlabel('Regressors')
100 | plt.xticks(np.arange(1,len([i for i in df.columns if i!='rub'])*2+1,2)+(len(year)-1)*width/2,
101 | ['Urals Crude', 'Japanese\nYen',
102 | 'Euro', 'Henry Hub', 'Chinese\nYuan',
103 | 'Korean\nWon', 'Ukrainian\nHryvnia'],fontsize=10)
104 | plt.show()
105 |
106 |
107 | # In[5]:
108 |
109 | #print model summary and actual vs fitted line chart
110 | x=sm.add_constant(pd.concat([df['urals']],axis=1))
111 | y=df['rub']
112 | m=sm.OLS(y['2017':'2018'],x['2017':'2018']).fit()
113 | print(m.summary())
114 |
115 | ax=plt.figure(figsize=(10,5)).add_subplot(111)
116 | ax.spines['top'].set_visible(False)
117 | ax.spines['right'].set_visible(False)
118 | ax.plot(df.loc['2017':'2018'].index,m.predict(), \
119 | c='#0abda0',label='Fitted')
120 | ax.plot(df.loc['2017':'2018'].index, \
121 | df['rub']['2017':'2018'],c='#132226',label='Actual')
122 | plt.legend(loc=0)
123 | plt.title('Russian Ruble 2017-2018')
124 | plt.ylabel('RUBAUD')
125 | plt.xlabel('Date')
126 | plt.show()
127 |
128 |
129 | # In[6]:
130 |
131 | #print model summary and actual vs fitted line chart
132 | x=sm.add_constant(pd.concat([df['urals'],df['eur']],axis=1))
133 | y=df['rub']
134 | m=sm.OLS(y[:'2016'],x[:'2016']).fit()
135 | print(m.summary())
136 | ax=plt.figure(figsize=(10,5)).add_subplot(111)
137 | ax.spines['top'].set_visible(False)
138 | ax.spines['right'].set_visible(False)
139 | plt.plot(df.loc[:'2016'].index,m.predict(), \
140 | c='#c05640',label='Fitted')
141 | plt.plot(df.loc[:'2016'].index,df['rub'][:'2016'], \
142 | c='#edd170',label='Actual')
143 | plt.legend(loc=0)
144 | plt.title('Russian Ruble Before 2017')
145 | plt.ylabel('RUBAUD')
146 | plt.xlabel('Date')
147 | plt.show()
148 |
149 |
150 | # In[7]:
151 |
152 | #normalize different regressors by 100 as the initial value
153 | #so that we can observe the trend of different regressors in the same scale
154 | ax=plt.figure(figsize=(10,5)).add_subplot(111)
155 | ax.spines['top'].set_visible(False)
156 | ax.spines['right'].set_visible(False)
157 |
158 | (df['urals']['2017':'2018']/df['urals']['2017':'2018'].iloc[0]*100).plot(label='Urals Crude',c='#728ca3',alpha=0.5)
159 | (df['jpy']['2017':'2018']/df['jpy']['2017':'2018'].iloc[0]*100).plot(label='Japanese Yen',c='#99bfaa')
160 | (df['eur']['2017':'2018']/df['eur']['2017':'2018'].iloc[0]*100).plot(label='Euro',c='#5c868d')
161 | (df['rub']['2017':'2018']/df['rub']['2017':'2018'].iloc[0]*100).plot(label='Russian Ruble',c='#000000')
162 | plt.legend(loc=0)
163 | plt.title('2017-2018 Trend')
164 | plt.ylabel('Normalized Value by 100')
165 | plt.xlabel('Date')
166 | plt.show()
167 |
168 |
169 | # In[8]:
170 |
171 | #plot actual vs fitted line chart for each year
172 | #including one sigma and two sigma confidence interval
173 | for i in df.index.year.drop_duplicates():
174 |
175 | temp=df.loc[str(i):str(i)]
176 | train=temp.iloc[:int(len(temp)/3)]
177 | test=temp.iloc[int(len(temp)/3):]
178 | x=sm.add_constant(train['urals'])
179 | y=train['rub']
180 | m=sm.OLS(y,x).fit()
181 | forecast=m.predict(sm.add_constant(test['urals']))
182 | resid=np.std(train['rub']-m.predict())
183 |
184 | ax=plt.figure(figsize=(10,5)).add_subplot(111)
185 | ax.spines['top'].set_visible(False)
186 | ax.spines['right'].set_visible(False)
187 |
188 | ax.plot(test.index, \
189 | forecast, \
190 | label='Fitted',c='#f5ca99')
191 | test['rub'].plot(label='Actual',c='#ed5752')
192 |
193 |
194 | ax.fill_between(test.index, \
195 | forecast+resid, \
196 | forecast-resid, \
197 | color='#1e1f26', \
198 | alpha=0.8, \
199 | label='1 Sigma')
200 |
201 | ax.fill_between(test.index, \
202 | forecast+2*resid, \
203 | forecast-2*resid, \
204 | color='#d0e1f9', \
205 | alpha=0.7, \
206 | label='2 Sigma')
207 |
208 | plt.legend(loc='best')
209 | plt.title(f'{i} Russian Ruble Positions\nR Squared {round(m.rsquared*100,2)}%\n')
210 | plt.ylabel('RUBAUD')
211 | plt.xlabel('Date')
212 | plt.legend()
213 | plt.show()
214 |
215 |
--------------------------------------------------------------------------------
/Oil Money project/Oil Money Trading backtest.py:
--------------------------------------------------------------------------------
1 |
2 | # coding: utf-8
3 |
4 | # In[1]:
5 |
6 |
7 | #here is the official trading strategy script for this lil project
8 | #the details could be found in the readme of the repo, section norwegian krone and brent crude
9 | # https://github.com/je-suis-tm/quant-trading/blob/master/Oil%20Money%20project/README.md
10 | import statsmodels.api as sm
11 | import copy
12 | import pandas as pd
13 | import numpy as np
14 | import matplotlib.pyplot as plt
15 | import os
16 | os.chdir('d:/')
17 |
18 |
19 | # In[2]:
20 |
21 | #theoratically we only need two sigma to trigger the trading signals
22 | #i only add one sigma to make it look better in visualization
23 | def oil_money(dataset):
24 |
25 | df=copy.deepcopy(dataset)
26 |
27 | df['signals']=0
28 | df['pos2 sigma']=0.0
29 | df['neg2 sigma']=0.0
30 | df['pos1 sigma']=0.0
31 | df['neg1 sigma']=0.0
32 | df['forecast']=0.0
33 |
34 | return df
35 |
36 |
37 | # In[3]:
38 |
39 | #the trading idea is straight forward
40 | #we run regression on nok and brent of the past 50 data points by default
41 | #if the rsquared exceeds 0.7 by default
42 | #the regression model is deemed valid
43 | #we calculate the standard deviation of the residual
44 | #and use +/- two sigma as the threshold to trigger the trading signals
45 | #once the trade is executed
46 | #we would start a counter to count the period of position holding
47 | #if the holding period exceeds 10 days by default
48 | #we clear our positions
49 | #meanwhile, if the spread between current price and entry price exceeds stop limit
50 | #which is 0.5 points by default in both ways
51 | #we clear our positions to claim profit/loss
52 | #once our positions are cleared
53 | #we recalibrate our regression model based on the latest 50 data points
54 | #we keep doing this on and on
55 | def signal_generation(dataset,x,y,method, \
56 | holding_threshold=10, \
57 | stop=0.5,rsquared_threshold=0.7, \
58 | train_len=50):
59 |
60 | df=method(dataset)
61 |
62 | #variable holding takes 3 values, -1,0,1
63 | #0 implies no holding positions
64 | #1 implies long, -1 implies short
65 | #when we wanna clear our positions
66 | #we just reverse the sign of holding
67 | #which is quite convenient
68 | holding=0
69 |
70 | #trained is a boolean value
71 | #it indicates whether the current model is valid
72 | #in another word,when trained==True, r squared is over 0.7 by default
73 | #and the regressand is within two sigma range from the fitted value
74 | trained=False
75 |
76 | #counter counts the days of position holding
77 | counter=0
78 |
79 |
80 | for i in range(train_len,len(df)):
81 |
82 | #when we have uncleared positions
83 | if holding!=0:
84 |
85 | #when counter exceeds holding threshold
86 | #we clear our positions and reset all the parameters
87 | if counter>holding_threshold:
88 | df.at[i,'signals']=-holding
89 | holding=0
90 | trained=False
91 | counter=0
92 |
93 | #we use continue to skip this round of iteration
94 | #only if the clearing condition gets triggered
95 | continue
96 |
97 | #plz note i make stop loss and stop profit symmetric
98 | #thats why we use absolute value of the spread between current price and entry price
99 | #usually stop loss and stop profit are asymmetric
100 | #as ppl cannot take as much loss as profit
101 | if np.abs( \
102 | df[y].iloc[i]-df[y][df['signals']!=0].iloc[-1] \
103 | )>=stop:
104 | df.at[i,'signals']=-holding
105 | holding=0
106 | trained=False
107 | counter=0
108 |
109 | continue
110 |
111 | counter+=1
112 |
113 | else:
114 |
115 | #if we do not have a valid model yet
116 | #we would keep trying the latest 50 data points
117 | if not trained:
118 | X=sm.add_constant(df[x].iloc[i-train_len:i])
119 | Y=df[y].iloc[i-train_len:i]
120 | m=sm.OLS(Y,X).fit()
121 |
122 | #if r squared meets the statistical request
123 | #which is 0.7 by default
124 | #we can start to build up confidence intervals
125 | if m.rsquared>rsquared_threshold:
126 | trained=True
127 | sigma=np.std(Y-m.predict(X))
128 |
129 | #plz note that we set the forecast and confidence intervals
130 | #for every data point after the current one
131 | #this would fill in the blank once our model turns invalid
132 | #when we have a new valid model
133 | #the new forecast and confidence intervals would cover the former one
134 | df.at[i:,'forecast']= \
135 | m.predict(sm.add_constant(df[x].iloc[i:]))
136 |
137 | df.at[i:,'pos2 sigma']= \
138 | df['forecast'].iloc[i:]+2*sigma
139 |
140 | df.at[i:,'neg2 sigma']= \
141 | df['forecast'].iloc[i:]-2*sigma
142 |
143 | df.at[i:,'pos1 sigma']= \
144 | df['forecast'].iloc[i:]+sigma
145 |
146 | df.at[i:,'neg1 sigma']= \
147 | df['forecast'].iloc[i:]-sigma
148 |
149 | #once we have a valid model
150 | #we can feel free to generate trading signals
151 | if trained:
152 | if df[y].iloc[i]>df['pos2 sigma'].iloc[i]:
153 | df.at[i,'signals']=1
154 | holding=1
155 |
156 | #once the positions are entered
157 | #we set confidence intervals back to the fitted value
158 | #so we could avoid the confusion in our visualization
159 | #for instance. if we dont do that,
160 | #there would be confidence intervals even when the model is broken
161 | #we could have been asking why no trade has been executed,
162 | #even when actual price falls out of the confidence intervals?
163 | df.at[i:,'pos2 sigma']=df['forecast']
164 | df.at[i:,'neg2 sigma']=df['forecast']
165 | df.at[i:,'pos1 sigma']=df['forecast']
166 | df.at[i:,'neg1 sigma']=df['forecast']
167 |
168 | if df[y].iloc[i]<df['neg2 sigma'].iloc[i]:
169 | df.at[i,'signals']=-1
170 | holding=-1
171 |
172 | df.at[i:,'pos2 sigma']=df['forecast']
173 | df.at[i:,'neg2 sigma']=df['forecast']
174 | df.at[i:,'pos1 sigma']=df['forecast']
175 | df.at[i:,'neg1 sigma']=df['forecast']
176 |
177 |
178 | return df
179 |
180 |
181 |
182 | # In[4]:
183 |
184 | #this part is to monitor how our portfolio performs over time
185 | #details can be found from heiki ashi
186 | # https://github.com/je-suis-tm/quant-trading/blob/master/Heikin-Ashi%20backtest.py
187 | def portfolio(signals,close_price,capital0=5000):
188 |
189 | positions=capital0//max(signals[close_price])
190 | portfolio=pd.DataFrame()
191 | portfolio['close']=signals[close_price]
192 | portfolio['signals']=signals['signals']
193 |
194 | portfolio['holding']=portfolio['signals'].cumsum()* \
195 | portfolio['close']*positions
196 |
197 | portfolio['cash']=capital0-(portfolio['signals']* \
198 | portfolio['close']*positions).cumsum()
199 |
200 | portfolio['asset']=portfolio['holding']+portfolio['cash']
201 |
202 |
203 | return portfolio
204 |
205 |
206 | # In[5]:
207 |
208 | #plotting fitted vs actual price with confidence intervals and positions
209 | def plot(signals,close_price):
210 |
211 | data=copy.deepcopy(signals[signals['forecast']!=0])
212 | ax=plt.figure(figsize=(10,5)).add_subplot(111)
213 | ax.spines['top'].set_visible(False)
214 | ax.spines['right'].set_visible(False)
215 |
216 | data['forecast'].plot(label='Fitted',color='#f4f4f8',alpha=0.7)
217 | data[close_price].plot(label='Actual',color='#3c2f2f',alpha=0.7)
218 |
219 | ax.fill_between(data.index,data['pos1 sigma'], \
220 | data['neg1 sigma'],alpha=0.3, \
221 | color='#011f4b', label='1 Sigma')
222 | ax.fill_between(data.index,data['pos2 sigma'], \
223 | data['neg2 sigma'],alpha=0.3, \
224 | color='#ffc425', label='2 Sigma')
225 |
226 | ax.plot(data.loc[data['signals']==1].index, \
227 | data[close_price][data['signals']==1],marker='^', \
228 | c='#00b159',linewidth=0,label='LONG',markersize=11, \
229 | alpha=1)
230 | ax.plot(data.loc[data['signals']==-1].index, \
231 | data[close_price][data['signals']==-1],marker='v', \
232 | c='#ff6f69',linewidth=0,label='SHORT',markersize=11, \
233 | alpha=1)
234 |
235 | plt.title(f'Oil Money Project\n{close_price.upper()} Positions')
236 | plt.legend(loc='best')
237 | plt.xlabel('Date')
238 | plt.ylabel('Price')
239 | plt.show()
240 |
241 |
242 | # In[6]:
243 |
244 | #plotting portfolio performance over time with positions
245 | def profit(portfolio,close_price):
246 |
247 | data=copy.deepcopy(portfolio)
248 | ax=plt.figure(figsize=(10,5)).add_subplot(111)
249 | ax.spines['top'].set_visible(False)
250 | ax.spines['right'].set_visible(False)
251 |
252 | data['asset'].plot(label='Total Asset',color='#58668b')
253 |
254 | ax.plot(data.loc[data['signals']==1].index, \
255 | data['asset'][data['signals']==1],marker='^', \
256 | c='#00b159',linewidth=0,label='LONG',markersize=11, \
257 | alpha=1)
258 | ax.plot(data.loc[data['signals']==-1].index, \
259 | data['asset'][data['signals']==-1],marker='v', \
260 | c='#ff6f69',linewidth=0,label='SHORT',markersize=11, \
261 | alpha=1)
262 |
263 | plt.title(f'Oil Money Project\n{close_price.upper()} Total Asset')
264 | plt.legend(loc='best')
265 | plt.xlabel('Date')
266 | plt.ylabel('Asset Value')
267 | plt.show()
268 |
269 |
270 | # In[7]:
271 |
272 |
273 | def main():
274 |
275 | df=pd.read_csv('brent crude nokjpy.csv')
276 | signals=signal_generation(df,'brent','nok',oil_money)
277 | p=portfolio(signals,'nok')
278 |
279 | #pandas.at[] is the fastest but it doesnt support datetime index
280 | #so we have to set datetime index after iteration but before visualization
281 | signals.set_index('date',inplace=True)
282 | signals.index=pd.to_datetime(signals.index,format='%m/%d/%Y')
283 | p.set_index(signals.index,inplace=True)
284 |
285 | #we only visualize data point from 387 to 600
286 | #becuz the visualization of 5 years data could be too messy
287 | plot(signals.iloc[387:600],'nok')
288 | profit(p.iloc[387:600],'nok')
289 |
290 |
291 | if __name__ == '__main__':
292 | main()
293 |
294 |
--------------------------------------------------------------------------------
/Oil Money project/oil production/oil production choropleth.csv:
--------------------------------------------------------------------------------
1 | Country,Oil Production
2 | Russia,10551497
3 | Saudi Arabia,10460710
4 | United States of America,8875817
5 | Iraq,4451516
6 | Iran,3990956
7 | China,3980650
8 | Canada,3662694
9 | United Arab Emirates,3106077
10 | Kuwait,2923825
11 | Brazil,2515459
12 | Venezuela,2276967
13 | Mexico,2186877
14 | Nigeria,1999885
15 | Angola,1769615
16 | Norway,1647975
17 | Kazakhstan,1595199
18 | Qatar,1522902
19 | Algeria,1348361
20 | Oman,1006841
21 | Libya,1003000
22 | United Kingdom,939760
23 | Colombia,897784
24 | Indonesia,833667
25 | Azerbaijan,833538
26 | India,734180
27 | Malaysia,661240
28 | Ecuador,548421
29 | Argentina,510560
30 | Romania,504000
31 | Egypt,494325
32 | Vietnam,301850
33 | Australia,289749
34 | Thailand,257525
35 | Sudan,255000
36 | South Sudan,255000
37 | Turkmenistan,230779
38 | Equatorial Guinea,227000
39 | Gabon,210820
40 | Denmark,140637
41 | Chad,110156
42 | Brunei,109117
43 | Ghana,100549
44 | Cameroon,93205
45 | Pakistan,84746
46 | Italy,70675
47 | East Timor,60661
48 | Trinidad and Tobago,60090
49 | Bolivia,58077
50 | Papua New Guinea,56667
51 | Uzbekistan,52913
52 | Cuba,50000
53 | Turkey,49497
54 | Tunisia,48757
55 | Germany,46839
56 | Peru,40266
57 | New Zealand,35574
58 | Ukraine,31989
59 | Ivory Coast,30000
60 | Syria,30000
61 | Belarus,25000
62 | Mongolia,23426
63 | Albania,22915
64 | Yemen,22000
65 | Poland,20104
66 | Democratic Republic of the Congo,20000
67 | Philippines,20000
68 | Republic of Serbia,20000
69 | Netherlands,18087
70 | Suriname,17000
71 | France,16418
72 | Austria,15161
73 | Myanmar,15000
74 | Hungary,13833
75 | Croatia,13582
76 | Niger,13000
77 | Guatemala,8977
78 | Mauritania,5000
79 | Chile,4423
80 | Bangladesh,4189
81 | Japan,3918
82 | Greece,3172
83 | Spain,2667
84 | Czech Republic,2333
85 | Belize,2000
86 | Lithuania,2000
87 | South Africa,2000
88 | Bulgaria,1000
89 | Kyrgyzstan,1000
90 | Georgia,400
91 | Israel,390
92 | Slovakia,200
93 | Taiwan,196
94 | Tajikistan,180
95 | Morocco,160
96 | Jordan,22
97 | Slovenia,5
98 | Afghanistan,0
99 | Armenia,0
100 | Antarctica,0
101 | French Southern and Antarctic Lands,0
102 | Burundi,0
103 | Belgium,0
104 | Benin,0
105 | Burkina Faso,0
106 | The Bahamas,0
107 | Bosnia and Herzegovina,0
108 | Bermuda,0
109 | Bhutan,0
110 | Botswana,0
111 | Central African Republic,0
112 | Switzerland,0
113 | Republic of the Congo,0
114 | Costa Rica,0
115 | Northern Cyprus,0
116 | Cyprus,0
117 | Djibouti,0
118 | Dominican Republic,0
119 | Eritrea,0
120 | Estonia,0
121 | Ethiopia,0
122 | Finland,0
123 | Fiji,0
124 | Falkland Islands,0
125 | Guinea,0
126 | Gambia,0
127 | Guinea Bissau,0
128 | Greenland,0
129 | French Guiana,0
130 | Guyana,0
131 | Honduras,0
132 | Haiti,0
133 | Ireland,0
134 | Iceland,0
135 | Jamaica,0
136 | Kenya,0
137 | Cambodia,0
138 | South Korea,0
139 | Kosovo,0
140 | Laos,0
141 | Lebanon,0
142 | Liberia,0
143 | Sri Lanka,0
144 | Lesotho,0
145 | Luxembourg,0
146 | Latvia,0
147 | Moldova,0
148 | Madagascar,0
149 | Macedonia,0
150 | Mali,0
151 | Malta,0
152 | Montenegro,0
153 | Mozambique,0
154 | Malawi,0
155 | Namibia,0
156 | New Caledonia,0
157 | Nicaragua,0
158 | Nepal,0
159 | Panama,0
160 | Puerto Rico,0
161 | North Korea,0
162 | Portugal,0
163 | Paraguay,0
164 | Rwanda,0
165 | Western Sahara,0
166 | Senegal,0
167 | Solomon Islands,0
168 | Sierra Leone,0
169 | El Salvador,0
170 | Somaliland,0
171 | Somalia,0
172 | Sweden,0
173 | Swaziland,0
174 | Togo,0
175 | United Republic of Tanzania,0
176 | Uganda,0
177 | Uruguay,0
178 | Vanuatu,0
179 | West Bank,0
180 | Zambia,0
181 | Zimbabwe,0
--------------------------------------------------------------------------------
/Oil Money project/oil production/oil production choropleth.py:
--------------------------------------------------------------------------------
1 |
2 | # coding: utf-8
3 |
4 | # In[1]:
5 |
6 |
7 | import folium
8 | import os
9 | os.chdir('h:/')
10 | import pandas as pd
11 |
12 |
13 | # In[2]:
14 |
15 |
16 | #this table comes from wikipedia
17 | # https://en.wikipedia.org/wiki/List_of_countries_by_oil_production
18 | #but i have changed the names of some countries
19 | #try to keep names consistent with geojson file
20 | df=pd.read_csv('oil production choropleth.csv')
21 |
22 |
23 | # In[3]:
24 |
25 |
26 | df['Oil Production']=df['Oil Production'].apply(lambda x: x/1000)
27 |
28 |
29 | # In[4]:
30 |
31 |
32 | #location takes two arguments, latitude and longitude
33 | #zoom_start implies zoom level
34 | #1 is world map, 2 is continent map, 3 is region map
35 | #4 is country map, 5 is county map etc
36 | m=folium.Map(location=(30,50), zoom_start=4)
37 |
38 | #geo_data is a geojson file
39 | #its a file that indicates country shape on a map
40 | #we can download country and even more detailed level from the first link
41 | #the second link converts all the files in first link to geojson
42 | #https://gadm.org/download_country_v3.html
43 | #https://mapshaper.org/
44 | #here i found the map shape from github
45 | #i just cannot find the original link
46 | #so i just upload it to the repo
47 |
48 | #data is the dataframe
49 | #columns would be the columns we use in that dataframe
50 | #we need one column for the region name and the other one for value
51 | #the region name should be consistent with the region name in geojson
52 | #and key_on denotes the key in geojson for region names
53 | #fill_color is just matplotlib cmap
54 | #fill_opacity,line_opacity are plotting options
55 | #legend_name is just the name of the label in matplotlib
56 | #threshold_scale can only take up to six values in a list
57 | #for simplicity, we can use from branca.utilities import split_six
58 | #to get the quantile data equally divided into six parts
59 | m.choropleth(
60 | geo_data=(open("worldmapshape.json",encoding = "utf_8_sig").read()),
61 | name='choropleth',
62 | data=df,
63 | columns=['Country', 'Oil Production'],
64 | key_on='properties.name',
65 | fill_color='YlOrRd',
66 | fill_opacity=0.7,
67 | line_opacity=0.2,
68 | legend_name='Oil Production Thousand Barrels/Day',
69 | threshold_scale=[0.0,1.0, 150.0, 800.0, 2000.0, 4000.0]
70 | )
71 |
72 | #layout control is just a map filter
73 | #we can unselect choropleth any time
74 | folium.LayerControl().add_to(m)
75 | display(m)
76 |
77 | #in general, folium is a really good wrap up for leaflet.js
78 | #it saves me a lot of time from learning javascript
79 | #it is very straight forward, a very flat learning curve
80 | #there is only one thing i hate about it
81 | #which is the location name is always in local language
82 | #at least google map provides english plus local language
83 | #this is quite annoying, other than that, its pretty cool
84 |
--------------------------------------------------------------------------------
/Oil Money project/oil production/oil production cost curve.csv:
--------------------------------------------------------------------------------
1 | Country,Operational cost dollar per barrel,Capital cost dollar per barrel,Total cost dollar per barrel,Reserve k mil barrels,Daily production mil barrels,2015 average price
2 | United Kingdom,21.8,30.7,52.5,2.5423575,0.963415531,52.4
3 | Brazil,17.3,31.5,48.8,12.99978129,2.524976918,52.4
4 | Canada,18.7,22.4,41,171.5123079,4.388135578,52.4
5 | US,21.5,14.8,36.2,47.987,12.77291501,52.4
6 | Norway,24,12.1,36.1,8.005041226,1.9400792,52.4
7 | Angola,18.8,16.6,35.4,9.524,1.795646188,52.4
8 | Colombia,15.5,19.8,35.3,2.308,1.005574436,52.4
9 | Nigeria,16.2,15.3,31.6,37.062,2.201408745,52.4
10 | China,15.6,14.3,29.9,25.62646431,4.308835068,52.4
11 | Mexico,18.3,10.7,29.1,7.9765,2.586540847,52.4
12 | Kazakhstan,16.3,11.5,27.8,30,1.694757886,52.4
13 | Libya,16.6,7.2,23.8,48.363,0.436643836,52.4
14 | Venezuela,9.6,13.9,23.5,300.8783,2.630863,52.4
15 | Algeria,13.2,7.2,20.4,12.2,1.557665233,52.4
16 | Russian Federation,8.9,8.4,17.2,102.3752,11.00667742,52.4
17 | Iran,6.9,5.7,12.6,158.4,3.852759885,52.4
18 | United Arab Emirates,6.6,5.7,12.3,97.8,3.897972489,52.4
19 | Iraq,5.6,5.1,10.7,142.503,3.985940667,52.4
20 | Saudi Arabia,4.5,5.4,9.9,266.455,11.99793798,52.4
21 | Kuwait,3.7,4.8,8.5,101.5,3.060775981,52.4
22 |
--------------------------------------------------------------------------------
/Oil Money project/oil production/oil production cost curve.py:
--------------------------------------------------------------------------------
1 |
2 | # coding: utf-8
3 |
4 | # In[1]:
5 |
6 |
7 | import numpy as np
8 | import matplotlib.pyplot as plt
9 | import os
10 | os.chdir('d:/')
11 | import pandas as pd
12 |
13 |
14 | # In[2]:
15 |
16 |
17 | #traditional commodity cost curve
18 | #input two pandas series, the third one is optional
19 | def cost_curve(x,y1,y2=None,
20 | hline_var=0,hline_color='k',hline_name='',
21 | colormap='tab20c',legends=None,notes=None,
22 | ylabel='',xlabel='',title='',fig_size=(10,5)):
23 | y2 = [] if y2 is None else y2
24 | legends = [] if legends is None else legends
25 | notes = [] if notes is None else notes
26 |
27 | ax=plt.figure(figsize=fig_size).add_subplot(111)
28 | ax.spines['top'].set_visible(False)
29 | ax.spines['right'].set_visible(False)
30 |
31 | #set bar location on x axis
32 | wid=x
33 | cumwid=[0]
34 | for i in range(1,len(wid)):
35 | cumwid.append(wid[i-1]+cumwid[-1])
36 |
37 | #assign colors to different bars from cmap
38 | cmap=plt.cm.get_cmap(colormap)
39 | colors=[cmap(i) for i in np.linspace(0,1,len(y1))]
40 | colors2=[tuple([i/1.3 for i in j if i!=1]+[1.0]) for j in colors]
41 |
42 | for i in range(len(y1)):
43 | plt.bar(cumwid[i],
44 | y1[i],width=wid[i],label=legends[i] if len(legends)>0 else '',
45 | color=colors[i],align='edge')
46 |
47 | if len(y2)>0:
48 | plt.bar(cumwid[i],
49 | y2[i],width=wid[i],
50 | color=colors2[i],bottom=y1[i],align='edge'
51 | )
52 |
53 | #plot percentile line if needed
54 | plt.axhline(y=hline_var, linestyle='--',
55 | c=hline_color,label=hline_name)
56 |
57 | plt.title(title,pad=20)
58 | ax.yaxis.labelpad=10
59 | ax.xaxis.labelpad=10
60 | plt.ylabel(ylabel)
61 | plt.xlabel(xlabel)
62 |
63 | #slightly expand x axis to look nicer
64 | plt.xlim(min(cumwid),max(cumwid)+list(wid)[-1])
65 | plt.xticks([min(cumwid),
66 | np.mean([min(cumwid),
67 | max(cumwid)+list(wid)[-1]]),
68 | 1.01*max(cumwid)+list(wid)[-1]])
69 |
70 | #if cost curve breakdown is provided
71 | #add legends to the right
72 | if len(y2)>0:
73 | plt.text(1.1*max(cumwid)+list(wid)[-1],
74 | list(y1)[-1]/2,notes[0],
75 | verticalalignment='center', horizontalalignment='center')
76 | plt.text(1.1*max(cumwid)+list(wid)[-1],
77 | list(y1)[-1]+list(y2)[-1]/2,notes[1],
78 | verticalalignment='center', horizontalalignment='center')
79 |
80 | #legends of cost curve for different entities is plotted below the chart
81 | plt.legend(loc=6,bbox_to_anchor=(0.12, -0.4), ncol=4)
82 |
83 | plt.show()
84 |
85 |
86 |
87 | # In[3]:
88 |
89 | #why there is only 2015 data?
90 | #well, they said data is the new oil
91 | #especially true when it comes to oil data
92 |
93 | #i can only find oil production cost data of 2015 from the below address
94 | # https://www.statista.com/statistics/597669/cost-breakdown-of-producing-one-barrel-of-oil-in-the-worlds-leading-oil-producing-countries/
95 | #and i have to do some scraping to actually get the data
96 | #if u dont know scraping, that will be a problem
97 | #data scientists cant wait for engineers to feed you data
98 | #maybe its time for you to learn
99 | # https://github.com/je-suis-tm/web-scraping
100 |
101 | #also you can use oil breakeven price to replace cost data
102 | #the following two links contain quite a lot of information
103 | #sadly, fiscal breakeven price is widely used by opec countries
104 | #if u have other countries in mind, you will be disappointed
105 | # https://www.cfr.org/report/interactive-oil-exporters-external-breakeven-prices
106 | # http://graphics.wsj.com/oil-barrel-breakdown/
107 |
108 | #daily production data and proven reserve comes from bp
109 | # https://www.bp.com/en/global/corporate/energy-economics/statistical-review-of-world-energy.html
110 | #production capacity is a typical x axis for cost curve
111 | #you can find them from imf or eia, but again, only for opec countries
112 | #therefore, i used daily production data as alternative
113 | # https://www.imf.org/~/media/Files/Publications/REO/MCD-CCA/2018/May/English/mreo0518-statisticalappendix-elsx.ashx
114 | # https://www.eia.gov/opendata/qb.php?sdid=STEO.COPC_AG.A
115 |
116 | df=pd.read_csv('global oil cost curve.csv')
117 |
118 |
119 | # In[4]:
120 |
121 |
122 | cost_curve(df['Daily production mil barrels'],
123 | df['Operational cost dollar per barrel'],
124 | df['Capital cost dollar per barrel'],
125 | legends=df['Country'],
126 | notes=['Operational Cost','Capital Cost'],
127 | hline_var=np.percentile(df['Total cost dollar per barrel'],90),
128 | hline_color='#e08314',
129 | hline_name='90% Percentile',
130 | xlabel='Daily Production, Million Barrels',
131 | ylabel='US Dollar per Barrel',
132 | title='2015 Global Oil Cost Curve')
133 |
134 |
--------------------------------------------------------------------------------
/Oil Money project/preview/cad after.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Oil Money project/preview/cad after.png
--------------------------------------------------------------------------------
/Oil Money project/preview/cad before.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Oil Money project/preview/cad before.png
--------------------------------------------------------------------------------
/Oil Money project/preview/cad crude.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Oil Money project/preview/cad crude.png
--------------------------------------------------------------------------------
/Oil Money project/preview/cad currency.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Oil Money project/preview/cad currency.png
--------------------------------------------------------------------------------
/Oil Money project/preview/cad elbow.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Oil Money project/preview/cad elbow.png
--------------------------------------------------------------------------------
/Oil Money project/preview/cad groups.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Oil Money project/preview/cad groups.png
--------------------------------------------------------------------------------
/Oil Money project/preview/cad kmeans.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Oil Money project/preview/cad kmeans.gif
--------------------------------------------------------------------------------
/Oil Money project/preview/cad model.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Oil Money project/preview/cad model.png
--------------------------------------------------------------------------------
/Oil Money project/preview/cad silhouette.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Oil Money project/preview/cad silhouette.png
--------------------------------------------------------------------------------
/Oil Money project/preview/cad wcs in aud.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Oil Money project/preview/cad wcs in aud.png
--------------------------------------------------------------------------------
/Oil Money project/preview/cad wcs in usd.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Oil Money project/preview/cad wcs in usd.png
--------------------------------------------------------------------------------
/Oil Money project/preview/cop after 2017.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Oil Money project/preview/cop after 2017.png
--------------------------------------------------------------------------------
/Oil Money project/preview/cop before 2017.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Oil Money project/preview/cop before 2017.png
--------------------------------------------------------------------------------
/Oil Money project/preview/cop groups.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Oil Money project/preview/cop groups.png
--------------------------------------------------------------------------------
/Oil Money project/preview/cop model.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Oil Money project/preview/cop model.png
--------------------------------------------------------------------------------
/Oil Money project/preview/cop profit distribution.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Oil Money project/preview/cop profit distribution.png
--------------------------------------------------------------------------------
/Oil Money project/preview/cop profit heatmap.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Oil Money project/preview/cop profit heatmap.png
--------------------------------------------------------------------------------
/Oil Money project/preview/cop trading asset.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Oil Money project/preview/cop trading asset.png
--------------------------------------------------------------------------------
/Oil Money project/preview/cop trading positions.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Oil Money project/preview/cop trading positions.png
--------------------------------------------------------------------------------
/Oil Money project/preview/cop vs brl.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Oil Money project/preview/cop vs brl.png
--------------------------------------------------------------------------------
/Oil Money project/preview/cop vs crude.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Oil Money project/preview/cop vs crude.png
--------------------------------------------------------------------------------
/Oil Money project/preview/cop vs gold.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Oil Money project/preview/cop vs gold.png
--------------------------------------------------------------------------------
/Oil Money project/preview/cop vs mxn.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Oil Money project/preview/cop vs mxn.png
--------------------------------------------------------------------------------
/Oil Money project/preview/cop vs usd vs mxn.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Oil Money project/preview/cop vs usd vs mxn.png
--------------------------------------------------------------------------------
/Oil Money project/preview/cop vs usd.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Oil Money project/preview/cop vs usd.png
--------------------------------------------------------------------------------
/Oil Money project/preview/cop vs vas.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Oil Money project/preview/cop vs vas.png
--------------------------------------------------------------------------------
/Oil Money project/preview/nok EG failed.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Oil Money project/preview/nok EG failed.png
--------------------------------------------------------------------------------
/Oil Money project/preview/nok asset value.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Oil Money project/preview/nok asset value.png
--------------------------------------------------------------------------------
/Oil Money project/preview/nok brent crude.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Oil Money project/preview/nok brent crude.png
--------------------------------------------------------------------------------
/Oil Money project/preview/nok correlation.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Oil Money project/preview/nok correlation.png
--------------------------------------------------------------------------------
/Oil Money project/preview/nok fitted vs actual.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Oil Money project/preview/nok fitted vs actual.png
--------------------------------------------------------------------------------
/Oil Money project/preview/nok model summary.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Oil Money project/preview/nok model summary.png
--------------------------------------------------------------------------------
/Oil Money project/preview/nok oil money positions.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Oil Money project/preview/nok oil money positions.png
--------------------------------------------------------------------------------
/Oil Money project/preview/nok profit distribution.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Oil Money project/preview/nok profit distribution.png
--------------------------------------------------------------------------------
/Oil Money project/preview/nok profit heatmap.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Oil Money project/preview/nok profit heatmap.png
--------------------------------------------------------------------------------
/Oil Money project/preview/nok trading asset.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Oil Money project/preview/nok trading asset.png
--------------------------------------------------------------------------------
/Oil Money project/preview/nok trading positions.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Oil Money project/preview/nok trading positions.png
--------------------------------------------------------------------------------
/Oil Money project/preview/nok trend.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Oil Money project/preview/nok trend.png
--------------------------------------------------------------------------------
/Oil Money project/preview/nok vs brent.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Oil Money project/preview/nok vs brent.png
--------------------------------------------------------------------------------
/Oil Money project/preview/nok vs gdp.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Oil Money project/preview/nok vs gdp.png
--------------------------------------------------------------------------------
/Oil Money project/preview/nok vs ir.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Oil Money project/preview/nok vs ir.png
--------------------------------------------------------------------------------
/Oil Money project/preview/oil production bubble map.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Oil Money project/preview/oil production bubble map.png
--------------------------------------------------------------------------------
/Oil Money project/preview/oil production choropleth.PNG:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Oil Money project/preview/oil production choropleth.PNG
--------------------------------------------------------------------------------
/Oil Money project/preview/oil production cost curve.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Oil Money project/preview/oil production cost curve.png
--------------------------------------------------------------------------------
/Oil Money project/preview/rub 2015 positions.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Oil Money project/preview/rub 2015 positions.png
--------------------------------------------------------------------------------
/Oil Money project/preview/rub 2016 positions.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Oil Money project/preview/rub 2016 positions.png
--------------------------------------------------------------------------------
/Oil Money project/preview/rub 2017 positions.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Oil Money project/preview/rub 2017 positions.png
--------------------------------------------------------------------------------
/Oil Money project/preview/rub 2017- trend.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Oil Money project/preview/rub 2017- trend.png
--------------------------------------------------------------------------------
/Oil Money project/preview/rub 2018 positions.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Oil Money project/preview/rub 2018 positions.png
--------------------------------------------------------------------------------
/Oil Money project/preview/rub model -2016.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Oil Money project/preview/rub model -2016.png
--------------------------------------------------------------------------------
/Oil Money project/preview/rub model 2017-.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Oil Money project/preview/rub model 2017-.png
--------------------------------------------------------------------------------
/Oil Money project/preview/rub ols -2016.PNG:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Oil Money project/preview/rub ols -2016.PNG
--------------------------------------------------------------------------------
/Oil Money project/preview/rub ols 2017-.PNG:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Oil Money project/preview/rub ols 2017-.PNG
--------------------------------------------------------------------------------
/Oil Money project/preview/rub stepwise1.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Oil Money project/preview/rub stepwise1.png
--------------------------------------------------------------------------------
/Oil Money project/preview/rub stepwise2.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Oil Money project/preview/rub stepwise2.png
--------------------------------------------------------------------------------
/Ore Money project/README.md:
--------------------------------------------------------------------------------
1 | This is an upcoming project similar to Quant Trading - Oil Money. It is designed to be an upgraded version of oil money trading strategy with introduction of more sophisticated models and alternative datasets. The project will examine the causality between iron ore spot price and forex of iron ore exporting countries. If the project can replicate the success of oil money trading strategy, we shall be able to apply it to the currency of any major natural resource exporter and correlated commodity contracts.
2 |
3 | The following figure is the global iron ore production bubble map (check <a href=https://github.com/je-suis-tm/quant-trading/blob/master/Ore%20Money%20project/iron%20ore%20production/iron%20ore%20production%20bubble%20map.py>subfolder</a> for how it is visualized, this can also be done in choropleth, check this <a href=https://github.com/je-suis-tm/quant-trading/blob/master/Oil%20Money%20project/oil%20production/oil%20production%20choropleth.py>link</a> for how to create a choropleth). It lists out a few iron ore and currency arbitrage opportunities.
4 |
5 | 
6 |
--------------------------------------------------------------------------------
/Ore Money project/iron ore production/iron ore production bubble map.csv:
--------------------------------------------------------------------------------
1 | region,iron ore production,latitude,longitude
2 | Australia,817000,-24.15,133.08
3 | Brazil,397000,-10.47,-52.55
4 | China,375000,31.55,108.2
5 | India,156000,22.37,79.13
6 | Russia,101000,65.45,97.35
7 | South Africa,73000,-30.44,25.12
8 | Ukraine,67000,49.3,32.28
9 | United States,46000,37.91,-97.02
10 | Canada,46000,58.27,-105.42
11 | Iran,27000,32.44,53.3
12 | Sweden,25000,64.2,18.03
13 | Kazakhstan,21000,47.1,67.3
14 | Mexico,18840,20.2,-100.1
15 | Chile,17109,-33.24,-70.9
16 | Venezuela,16800,7.3,-65.55
17 | Sierra Leone,11895,7.3,-11.17
18 | Malaysia,11588,3.59,101.91
19 | Peru,10126,-11,-75
20 | Turkey,8589,39.57,35.54
21 | Mongolia,6736,46,105
22 | Vietnam,4708,17.05,107.55
23 | Indonesia,4000,-1.09,115.49
24 | Norway,3409,60.55,7.45
25 | Egypt,3320,26.01,30.14
26 | North Korea,3054,39.09,126.3
27 | Algeria,1067,29.42,3.08
28 | Philippines,1057,11.4,124.03
29 |
--------------------------------------------------------------------------------
/Ore Money project/iron ore production/iron ore production bubble map.py:
--------------------------------------------------------------------------------
1 |
2 | # coding: utf-8
3 |
4 | # In[1]:
5 |
6 |
7 | #installing basemap is pretty painful for anaconda
8 | #try conda install -c conda-forge basemap
9 | #if there is filenotfounderror
10 | #try conda install -c conda-forge proj4
11 | import pandas as pd
12 | import numpy as np
13 | import matplotlib.pyplot as plt
14 | from mpl_toolkits.basemap import Basemap
15 | import os
16 | os.chdir('d:/')
17 |
18 |
19 | # In[2]:
20 |
21 |
22 | #need data with value and coordinates to plot
23 | #the iron ore production data of 2015 comes from wikipedia
24 | # https://en.wikipedia.org/wiki/List_of_countries_by_iron_ore_production
25 | #the coordinates of each country's capital come from someone's personal blog
26 | # https://lab.lmnixon.org/4th/worldcapitals.html
27 | #note that the capital doesnt necessarily locate at the centre of a country
28 | #to make the figure look better, i slightly change the coordinates
29 | df=pd.read_csv('iron ore production bubble map.csv')
30 |
31 |
32 | # In[3]:
33 |
34 |
35 | ax=plt.figure(figsize=(50,20)).add_subplot(111)
36 |
37 | #draw up a world map
38 | #fill continents, lakes and country boundaries
39 | m = Basemap()
40 | m.drawmapboundary(fill_color='white', linewidth=0)
41 | m.fillcontinents(color='#c0c0c0',alpha=0.5, lake_color='white')
42 | m.drawcountries(linewidth=1,color='#a79c93')
43 |
44 | size=df['iron ore production']/max(df['iron ore production'])*40000
45 |
46 | #this is crucial if we use a different map projection
47 | x,y=m(df['longitude'].tolist(),df['latitude'].tolist())
48 |
49 | m.scatter(x,y, s=size, linewidths=2, edgecolors="#6c6b74",
50 | alpha=0.8,c=df['iron ore production'],cmap='autumn_r')
51 |
52 | for i in range(len(x)):
53 | plt.text(x[i],y[i],
54 | '%s '%(df['region'][i]),
55 | horizontalalignment='center',
56 | verticalalignment='center',
57 | size=25)
58 |
59 | cb=plt.colorbar(ticks=[10000,800000])
60 | cb.ax.set_yticklabels(cb.ax.get_yticklabels(), fontsize=20)
61 | cb.ax.set_ylabel('Iron Ore Production 1000 Tonnes per Year',fontsize=20,rotation=270)
62 | plt.title('Iron Ore Production by Countries', fontsize=42)
63 |
64 | plt.show()
65 |
66 |
--------------------------------------------------------------------------------
/Ore Money project/preview/iron ore production bubble map.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Ore Money project/preview/iron ore production bubble map.png
--------------------------------------------------------------------------------
/Parabolic SAR backtest.py:
--------------------------------------------------------------------------------
1 | # coding: utf-8
2 |
3 | # In[1]:
4 |
5 |
6 | #parabolic stop and reverse is very useful for trend following
7 | #sar is an indicator below the price when its an uptrend
8 | #and above the price when its a downtrend
9 | #it is very painful to calculate sar, though
10 | #and many explanations online including wiki cannot clearly explain the process
11 | #hence, the good idea would be to read info on wikipedia
12 | #and download an excel spreadsheet made by joeu2004
13 | #formulas are always more straight forward than descriptions
14 | #links are shown below
15 | # https://en.wikipedia.org/wiki/Parabolic_SAR
16 | # https://www.box.com/s/gbtrjuoktgyag56j6lv0
17 |
18 | import matplotlib.pyplot as plt
19 | import numpy as np
20 | import fix_yahoo_finance as yf
21 | import pandas as pd
22 |
23 |
24 | # In[2]:
25 |
26 | #the calculation of sar
27 | #as rules are very complicated
28 | #plz check the links above to understand more about it
29 |
30 | def parabolic_sar(new):
31 |
32 | #this is common accelerating factors for forex and commodity
33 | #for equity, af for each step could be set to 0.01
34 | initial_af=0.02
35 | step_af=0.02
36 | end_af=0.2
37 |
38 |
39 | new['trend']=0
40 | new['sar']=0.0
41 | new['real sar']=0.0
42 | new['ep']=0.0
43 | new['af']=0.0
44 |
45 | #initial values for recursive calculation
46 | new['trend'][1]=1 if new['Close'][1]>new['Close'][0] else -1
47 | new['sar'][1]=new['High'][0] if new['trend'][1]>0 else new['Low'][0]
48 | new.at[1,'real sar']=new['sar'][1]
49 | new['ep'][1]=new['High'][1] if new['trend'][1]>0 else new['Low'][1]
50 | new['af'][1]=initial_af
51 |
52 | #calculation
53 | for i in range(2,len(new)):
54 |
55 | temp=new['sar'][i-1]+new['af'][i-1]*(new['ep'][i-1]-new['sar'][i-1])
56 | if new['trend'][i-1]<0:
57 | new.at[i,'sar']=max(temp,new['High'][i-1],new['High'][i-2])
58 | temp=1 if new['sar'][i]<new['High'][i] else new['trend'][i-1]-1
59 | else:
60 | new.at[i,'sar']=min(temp,new['Low'][i-1],new['Low'][i-2])
61 | temp=-1 if new['sar'][i]>new['Low'][i] else new['trend'][i-1]+1
62 | new.at[i,'trend']=temp
63 |
64 |
65 | if new['trend'][i]<0:
66 | temp=min(new['Low'][i],new['ep'][i-1]) if new['trend'][i]!=-1 else new['Low'][i]
67 | else:
68 | temp=max(new['High'][i],new['ep'][i-1]) if new['trend'][i]!=1 else new['High'][i]
69 | new.at[i,'ep']=temp
70 |
71 |
72 | if np.abs(new['trend'][i])==1:
73 | temp=new['ep'][i-1]
74 | new.at[i,'af']=initial_af
75 | else:
76 | temp=new['sar'][i]
77 | if new['ep'][i]==new['ep'][i-1]:
78 | new.at[i,'af']=new['af'][i-1]
79 | else:
80 | new.at[i,'af']=min(end_af,new['af'][i-1]+step_af)
81 | new.at[i,'real sar']=temp
82 |
83 |
84 | return new
85 |
86 | # In[3]:
87 |
88 | #generating signals
89 | #idea is the same as macd oscillator
90 | #check the website below to learn more
91 | # https://github.com/je-suis-tm/quant-trading/blob/master/MACD%20oscillator%20backtest.py
92 |
93 | def signal_generation(df,method):
94 |
95 | new=method(df)
96 |
97 | new['positions'],new['signals']=0,0
98 | new['positions']=np.where(new['real sar']<new['Close'],1,0)
99 | new['signals']=new['positions'].diff()
100 |
101 | return new
102 |
103 |
104 |
105 |
106 |
107 | # In[4]:
108 |
109 | #plotting of sar and trading positions
110 | #still similar to macd
111 |
112 | def plot(new,ticker):
113 |
114 | fig=plt.figure()
115 | ax=fig.add_subplot(111)
116 |
117 | new['Close'].plot(lw=3,label='%s'%ticker)
118 | new['real sar'].plot(linestyle=':',label='Parabolic SAR',color='k')
119 | ax.plot(new.loc[new['signals']==1].index,new['Close'][new['signals']==1],marker='^',color='g',label='LONG',lw=0,markersize=10)
120 | ax.plot(new.loc[new['signals']==-1].index,new['Close'][new['signals']==-1],marker='v',color='r',label='SHORT',lw=0,markersize=10)
121 |
122 | plt.legend()
123 | plt.grid(True)
124 | plt.title('Parabolic SAR')
125 | plt.ylabel('price')
126 | plt.show()
127 |
128 |
129 | # In[5]:
130 |
131 | def main():
132 |
133 | #download data via fix yahoo finance library
134 | stdate=('2016-01-01')
135 | eddate=('2018-01-01')
136 | ticker=('EA')
137 |
138 | #slice is used for plotting
139 | #a two year dataset with 500 variables would be too much for a figure
140 | slicer=450
141 |
142 | df=yf.download(ticker,start=stdate,end=eddate)
143 |
144 | #delete adj close and volume
145 | #as we dont need them
146 | del df['Adj Close']
147 | del df['Volume']
148 |
149 | #no need to iterate over timestamp index
150 | df.reset_index(inplace=True)
151 |
152 | new=signal_generation(df,parabolic_sar)
153 |
154 | #convert back to time series for plotting
155 | #so that we get a date x axis
156 | new.set_index(new['date'],inplace=True)
157 |
158 | #shorten our plotting horizon and plot
159 | new=new[slicer:]
160 | plot(new,ticker)
161 |
162 | #how to calculate stats could be found from my other code called Heikin-Ashi
163 | # https://github.com/je-suis-tm/quant-trading/blob/master/heikin%20ashi%20backtest.py
164 |
165 |
166 | # In[6]:
167 |
168 | if __name__ == '__main__':
169 | main()
170 |
--------------------------------------------------------------------------------
/Shooting Star backtest.py:
--------------------------------------------------------------------------------
1 |
2 | # coding: utf-8
3 |
4 | # In[1]:
5 |
6 |
7 | #shooting star is my friend's fav indicator
8 | #the name is poetic and romantic
9 | #it is merely a vertical flipped hammer
10 | #hammer and shooting star could be confusing
11 | #since both of them can be inverted
12 | #i memorize them via a simple tune
13 | #if u see thor (with hammer),price shall soar
14 | #if u see star (shooting star),price shall fall
15 | #details of shooting star can be found in investopedia
16 | # https://www.investopedia.com/terms/s/shootingstar.asp
17 | import pandas as pd
18 | import matplotlib.pyplot as plt
19 | import numpy as np
20 | import yfinance
21 |
22 |
23 | # In[2]:
24 |
25 |
26 | #criteria of shooting star
27 | def shooting_star(data,lower_bound,body_size):
28 |
29 | df=data.copy()
30 |
31 | #open>close,red color
32 | df['condition1']=np.where(df['Open']>=df['Close'],1,0)
33 |
34 | #a candle with little or no lower wick
35 | df['condition2']=np.where(
36 | (df['Close']-df['Low'])<lower_bound*abs(
37 | df['Close']-df['Open']),1,0)
38 |
39 | #a candle with a small lower body
40 | df['condition3']=np.where(abs(
41 | df['Open']-df['Close'])<abs(
42 | np.mean(df['Open']-df['Close']))*body_size,1,0)
43 |
44 | #a long upper wick that is at least two times the size of the lower body
45 | df['condition4']=np.where(
46 | (df['High']-df['Open'])>=2*(
47 | df['Open']-df['Close']),1,0)
48 |
49 | #price uptrend
50 | df['condition5']=np.where(
51 | df['Close']>=df['Close'].shift(1),1,0)
52 | df['condition6']=np.where(
53 | df['Close'].shift(1)>=df['Close'].shift(2),1,0)
54 |
55 | #the next candle's high must stay
56 | #below the high of the shooting star
57 | df['condition7']=np.where(
58 | df['High'].shift(-1)<=df['High'],1,0)
59 |
60 | #the next candle's close below
61 | #the close of the shooting star
62 | df['condition8']=np.where(
63 | df['Close'].shift(-1)<=df['Close'],1,0)
64 |
65 | return df
66 |
67 |
68 | # In[3]:
69 |
70 |
71 | #signal generation
72 | #there are eight criteria according to investopedia
73 | def signal_generation(df,method,
74 | lower_bound=0.2,body_size=0.5,
75 | stop_threshold=0.05,
76 | holding_period=7):
77 |
78 | #get shooting star conditions
79 | data=method(df,lower_bound,body_size)
80 |
81 | #shooting star should suffice all conditions
82 | #in practise,you may find the definition too rigid
83 | #its important to relax a bit on the body size
84 | data['signals']=data['condition1']*data[
85 | 'condition2']*data['condition3']*data[
86 | 'condition4']*data['condition5']*data[
87 | 'condition6']*data['condition7']*data[
88 | 'condition8']
89 |
90 | #shooting star is a short signal
91 | data['signals']=-data['signals']
92 |
93 | #find exit position
94 | idxlist=data[data['signals']==-1].index
95 | for ind in idxlist:
96 |
97 | #entry point
98 | entry_pos=data['Close'].loc[ind]
99 |
100 | stop=False
101 | counter=0
102 | while not stop:
103 | ind+=1
104 | counter+=1
105 |
106 | #set stop loss/profit at +-5%
107 | if abs(data['Close'].loc[
108 | ind]/entry_pos-1)>stop_threshold:
109 | stop=True
110 | data['signals'].loc[ind]=1
111 |
112 | #set maximum holding period at 7 workdays
113 | if counter>=holding_period:
114 | stop=True
115 | data['signals'].loc[ind]=1
116 |
117 | #create positions
118 | data['positions']=data['signals'].cumsum()
119 |
120 | return data
121 |
122 |
123 | # In[4]:
124 |
125 |
126 | #since matplotlib remove the candlestick
127 | #plus we dont wanna install mpl_finance
128 | #we implement our own version
129 | #simply use fill_between to construct the bar
130 | #use line plot to construct high and low
131 | def candlestick(df,ax=None,highlight=None,titlename='',
132 | highcol='High',lowcol='Low',
133 | opencol='Open',closecol='Close',xcol='Date',
134 | colorup='r',colordown='g',highlightcolor='y',
135 | **kwargs):
136 |
137 | #bar width
138 | #use 0.6 by default
139 | dif=[(-3+i)/10 for i in range(7)]
140 |
141 | if not ax:
142 | ax=plt.figure(figsize=(10,5)).add_subplot(111)
143 |
144 | #construct the bars one by one
145 | for i in range(len(df)):
146 |
147 | #width is 0.6 by default
148 | #so 7 data points required for each bar
149 | x=[i+j for j in dif]
150 | y1=[df[opencol].iloc[i]]*7
151 | y2=[df[closecol].iloc[i]]*7
152 |
153 | barcolor=colorup if y1[0]>y2[0] else colordown
154 |
155 | #no high line plot if open/close is high
156 | if df[highcol].iloc[i]!=max(df[opencol].iloc[i],df[closecol].iloc[i]):
157 |
158 | #use generic plot to viz high and low
159 | #use 1.001 as a scaling factor
160 | #to prevent high line from crossing into the bar
161 | plt.plot([i,i],
162 | [df[highcol].iloc[i],
163 | max(df[opencol].iloc[i],
164 | df[closecol].iloc[i])*1.001],c='k',**kwargs)
165 |
166 | #same as high
167 | if df[lowcol].iloc[i]!=min(df[opencol].iloc[i],df[closecol].iloc[i]):
168 |
169 | plt.plot([i,i],
170 | [df[lowcol].iloc[i],
171 | min(df[opencol].iloc[i],
172 | df[closecol].iloc[i])*0.999],c='k',**kwargs)
173 |
174 | #treat the bar as fill between
175 | plt.fill_between(x,y1,y2,
176 | edgecolor='k',
177 | facecolor=barcolor,**kwargs)
178 |
179 | if highlight:
180 | if df[highlight].iloc[i]==-1:
181 | plt.fill_between(x,y1,y2,
182 | edgecolor='k',
183 | facecolor=highlightcolor,**kwargs)
184 |
185 | #only show 5 xticks
186 | plt.xticks([])
187 | plt.grid(True)
188 | plt.title(titlename)
189 |
190 |
191 | # In[5]:
192 |
193 |
194 | #plotting the backtesting result
195 | def plot(data,name):
196 |
197 | #first plot is candlestick to showcase
198 | ax1=plt.subplot2grid((250,1),(0,0),
199 | rowspan=120,
200 | ylabel='Candlestick')
201 | candlestick(data,ax1,
202 | highlight='signals',
203 | highlightcolor='#FFFF00')
204 |
205 | #the second plot is the actual price
206 | #with long/short positions as up/down arrows
207 | ax2=plt.subplot2grid((250,1),(130,0),
208 | rowspan=120,
209 | ylabel='£ per share',
210 | xlabel='Date')
211 | ax2.plot(data.index,
212 | data['Close'],
213 | label=name)
214 |
215 | #long/short positions are attached to
216 | #the real close price of the stock
217 | #set the line width to zero
218 | #thats why we only observe markers
219 | ax2.plot(data.loc[data['signals']==-1].index,
220 | data['Close'].loc[data['signals']==-1],
221 | marker='v',lw=0,c='r',label='short',
222 | markersize=10)
223 | ax2.plot(data.loc[data['signals']==1].index,
224 | data['Close'].loc[data['signals']==1],
225 | marker='^',lw=0,c='g',label='long',
226 | markersize=10)
227 |
228 | #only show five tickers
229 | plt.xticks(range(0,len(data),len(data)//5),
230 | data['Date'][0::len(data)//5].dt.date)
231 |
232 | plt.grid(True)
233 | plt.legend(loc='lower left')
234 | plt.tight_layout(pad=0.1)
235 | plt.show()
236 |
237 |
238 | # In[6]:
239 |
240 |
241 | def main():
242 |
243 | #initializing
244 | stdate='2000-01-01'
245 | eddate='2021-11-04'
246 | name='Vodafone'
247 | ticker='VOD.L'
248 |
249 | df=yfinance.download(ticker,start=stdate,end=eddate)
250 | df.reset_index(inplace=True)
251 | df['Date']=pd.to_datetime(df['Date'])
252 |
253 | #signal generation
254 | new=signal_generation(df,shooting_star)
255 |
256 | #get subset for better viz to highlight shooting star
257 | subset=new.loc[5268:5283].copy()
258 | subset.reset_index(inplace=True,drop=True)
259 |
260 | #viz
261 | plot(subset,name)
262 |
263 |
264 | # In[7]:
265 |
266 |
267 | if __name__ == '__main__':
268 | main()
269 |
270 |
--------------------------------------------------------------------------------
/Smart Farmers project/check consistency.py:
--------------------------------------------------------------------------------
1 |
2 | # coding: utf-8
3 |
4 | # In[1]:
5 |
6 |
7 | import os
8 | os.chdir('H:/')
9 | import pandas as pd
10 |
11 |
12 | # In[2]:
13 |
14 |
15 | prod=pd.read_csv('Production_Crops_E_All_Data_(Normalized).csv',
16 | encoding='latin-1')
17 |
18 | prix=pd.read_csv('Prices_E_All_Data_(Normalized).csv',
19 | encoding='latin-1')
20 |
21 | land=pd.read_csv('Inputs_LandUse_E_All_Data_(Normalized).csv',
22 | encoding='latin-1')
23 |
24 |
25 | # In[3]:
26 |
27 |
28 | global beginyear,endyear
29 | beginyear=2012;
30 | endyear=2019
31 |
32 |
33 | # In[4]:
34 |
35 |
36 | mapping=pd.read_csv('mapping.csv')
37 |
38 |
39 | # In[5]:
40 |
41 |
42 | #select malaysia from 2012-2018
43 | malay_land=land[land['Year'].isin(range(beginyear,endyear))][land['Area']=='Malaysia'][land['Element'].isin(['Area'])][land['Item'].isin(['Cropland'])]
44 |
45 | malay_prod=prod[prod['Year'].isin(range(beginyear,endyear))][prod['Area']=='Malaysia'][prod['Element'].isin(['Area harvested','Production'])]
46 |
47 | malay_prod=malay_prod.merge(mapping,on=['Item Code', 'Item'],how='left')
48 |
49 |
50 | # In[6]:
51 |
52 |
53 | #remove redundant cols
54 | for i in ['Area Code','Element Code','Year Code',
55 | 'Flag','COMMODITY','Item Code',
56 | 'subclass code','class code',
57 | 'DEFINITIONS, COVERAGE, REMARKS',]:
58 | del malay_prod[i]
59 |
60 |
61 | # In[7]:
62 |
63 |
64 | #select crops with available data
65 | a=set(malay_prod['Item'][malay_prod['Element']=='Area harvested'])
66 |
67 | b=set(malay_prod['Item'][malay_prod['Element']=='Production'])
68 |
69 | target_crops=a.intersection(b)
70 |
71 |
72 | # In[8]:
73 |
74 |
75 | #exclude land usage<1% without price data
76 | exclude=['Areca nuts',
77 | 'Bastfibres, other',
78 | 'Cashew nuts, with shell',
79 | 'Cereals, Total',
80 | 'Chillies and peppers, dry',
81 | 'Citrus Fruit, Total',
82 | 'Cloves',
83 | 'Coarse Grain, Total',
84 | 'Coffee, green',
85 | 'Coir',
86 | 'Fibre Crops Primary',
87 | 'Fruit Primary',
88 | 'Fruit, citrus nes',
89 | 'Fruit, fresh nes',
90 | 'Fruit, tropical fresh nes',
91 | 'Groundnuts, with shell',
92 | 'Manila fibre (abaca)',
93 | 'Nutmeg, mace and cardamoms',
94 | 'Oilcrops',
95 | 'Oilcrops, Cake Equivalent',
96 | 'Oilcrops, Oil Equivalent',
97 | 'Roots and Tubers, Total',
98 | 'Roots and tubers nes',
99 | 'Soybeans',
100 | 'Spices nes',
101 | 'Tea',
102 | 'Treenuts, Total',
103 | 'Vegetables Primary',
104 | 'Vegetables, fresh nes']
105 |
106 |
107 | # In[9]:
108 |
109 |
110 | #finalize the target
111 | targets=[i for i in target_crops if i not in exclude]
112 |
113 |
114 | # In[10]:
115 |
116 |
117 | #cleanse
118 | malay_crops=malay_prod[malay_prod['Item'].isin(targets)]
119 |
120 |
121 | # In[11]:
122 |
123 |
124 | #subtotal
125 | total=malay_prod[malay_prod['class'].isnull()]
126 |
127 |
128 | # In[12]:
129 |
130 |
131 | #compare sum of area by crops with sum of area by subclass
132 | sss=total[total['Element']=='Area harvested'].groupby(['Item','Year']).sum()
133 |
134 | ttt=malay_crops[malay_crops['Element']=='Area harvested'].groupby(['class','Year']).sum()
135 |
136 |
137 | # In[13]:
138 |
139 |
140 | #compare sum of area by crops
141 | malay_crops[malay_crops['Element']=='Area harvested'].groupby(['Year']).sum()
142 |
143 |
144 | # In[14]:
145 |
146 |
147 | #with cropland
148 | malay_land
149 |
150 |
--------------------------------------------------------------------------------
/Smart Farmers project/cleanse data.py:
--------------------------------------------------------------------------------
1 |
2 | # coding: utf-8
3 |
4 | # In[1]:
5 |
6 |
7 | import os
8 | os.chdir('H:/')
9 | import pandas as pd
10 | import numpy as np
11 |
12 |
13 | # ### define functions
14 |
15 | # In[2]:
16 |
17 |
18 | #etl
19 | def prepare(target_land,target_prod,target_prix):
20 |
21 | #land clean up
22 | target_land=target_land[['Year','Value']].copy()
23 |
24 | #prix clean up
25 | target_prix=target_prix[['Item','Year','Value']].copy()
26 |
27 | #create area and clean up
28 | target_area=target_prod[target_prod['Element']=='Area harvested'].copy()
29 |
30 | target_area=target_area[['Item','Year','Value','type','lifespan']].copy()
31 |
32 | #production clean up
33 | target_prod=target_prod[target_prod['Element']=='Production'].copy()
34 |
35 | target_prod=target_prod[['Item','Year','Value','class']].copy()
36 |
37 | #compute yield inverse and clean up
38 | target_yield_inverse=target_prod.merge(target_area,on=['Item','Year'],how='left')
39 |
40 | target_yield_inverse['Value']=np.divide(target_yield_inverse['Value_y'],target_yield_inverse['Value_x'])
41 |
42 | target_yield_inverse=target_yield_inverse[['Item', 'Year','Value','type', 'lifespan']]
43 |
44 | #sort by item and year
45 | for data in [target_prod,target_area,target_prix,target_yield_inverse]:
46 | data.sort_values(['Item','Year'],inplace=True)
47 |
48 | #concatenate
49 | global D
50 | D={}
51 |
52 | for currentyear in range(beginyear,endyear):
53 |
54 | temp1=target_prod[target_prod['Year']==currentyear].merge(
55 | target_area[target_area['Year']==currentyear],on=['Item', 'Year'],how='outer')
56 | temp2=target_prix[target_prix['Year']==currentyear].merge(
57 | target_yield_inverse[target_yield_inverse['Year']==currentyear],on=['Item', 'Year'],how='outer')
58 |
59 | temp1.columns=temp1.columns.str.replace('Value_x','production')
60 | temp1.columns=temp1.columns.str.replace('Value_y','area')
61 | temp2.columns=temp2.columns.str.replace('Value_x','price')
62 | temp2.columns=temp2.columns.str.replace('Value_y','yield_i')
63 |
64 | data=temp1.merge(temp2,on=['Item', 'Year', 'type', 'lifespan'],how='outer')
65 |
66 | D[currentyear]=data
67 |
68 | #compute eco lifespan for perennials
69 | for currentyear in range(beginyear,endyear):
70 |
71 | eco_lifespan=[default_discount for _ in range(len(D[currentyear]))]
72 | indices=D[currentyear]['lifespan'].dropna().index
73 | perennial=D[currentyear]['lifespan'].dropna().apply(lambda x:(x*eco_coeff-1)/(x*eco_coeff))
74 |
75 | for i in indices:
76 | eco_lifespan[i]=round(perennial.loc[i],4)
77 |
78 | D[currentyear]['eco lifespan']=eco_lifespan
79 |
80 | return D
81 |
82 |
83 | # ### execution
84 |
85 | # In[3]:
86 |
87 |
88 | global beginyear,endyear
89 | beginyear=2012
90 | endyear=2019
91 |
92 |
93 | # In[4]:
94 |
95 |
96 | global eco_coeff,default_discount
97 | eco_coeff=0.7
98 | default_discount=0.8
99 |
100 |
101 | # In[5]:
102 |
103 |
104 | prod=pd.read_csv('Production_Crops_E_All_Data_(Normalized).csv',
105 | encoding='latin-1')
106 |
107 | prix=pd.read_csv('Prices_E_All_Data_(Normalized).csv',
108 | encoding='latin-1')
109 |
110 | land=pd.read_csv('Inputs_LandUse_E_All_Data_(Normalized).csv',
111 | encoding='latin-1')
112 |
113 | population=pd.read_csv('Population_E_All_Data_(Normalized).csv',
114 | encoding='latin-1')
115 |
116 | gdp=pd.read_csv('Macro-Statistics_Key_Indicators_E_All_Data_(Normalized).csv',
117 | encoding='latin-1')
118 |
119 |
120 | # In[6]:
121 |
122 |
123 | os.chdir('H:/data')
124 |
125 |
126 | # In[7]:
127 |
128 |
129 | mapping=pd.read_csv('mapping.csv')
130 |
131 |
132 | # In[8]:
133 |
134 |
135 | #select malaysia from beginyear-endyear
136 | malay_land=land[land['Year'].isin(range(beginyear,endyear))][land['Area']=='Malaysia'][land['Element'].isin(['Area'])][land['Item'].isin(['Cropland'])]
137 |
138 | malay_prix=prix[prix['Year'].isin(range(beginyear,endyear))][prix['Area']=='Malaysia'][prix['Element']=='Producer Price (USD/tonne)'][prix['Months']=='Annual value']
139 |
140 | malay_prod=prod[prod['Year'].isin(range(beginyear,endyear))][prod['Area']=='Malaysia'][prod['Element'].isin(['Area harvested','Production'])]
141 |
142 | malay_pop=population[population['Element']=='Total Population - Both sexes'][population['Area']=='Malaysia'][population['Year']>=beginyear]
143 |
144 | malay_gdp=gdp[gdp['Element']=='Value US#39;][gdp['Item']=='Gross Domestic Product per capita'][gdp['Area']=='Malaysia'][gdp['Year'].isin(range(beginyear,endyear))]
145 |
146 |
147 | # In[9]:
148 |
149 |
150 | #exclude land usage<1%
151 | exclude=[813,236,809,1717,
152 | 512, 782, 656, 149, 667, 809,
153 | 813, 689, 698, 702, 723, 217,
154 | 226, 236, 242,463, 603, 619,
155 | 1720, 1729, 1731, 1732, 1735,
156 | 1738, 1753, 1804, 1841,1814]
157 |
158 |
159 | # In[10]:
160 |
161 |
162 | #find items without price
163 | a=set(malay_prod['Item Code'][malay_prod['Element']=='Area harvested'])
164 |
165 | b=set(malay_prod['Item Code'][malay_prod['Element']=='Production'])
166 |
167 | c=set(malay_prix['Item Code'])
168 |
169 | sans_prix=[i for i in a.intersection(b) if i not in c]
170 |
171 | sans_prix=[i for i in sans_prix if i not in exclude]
172 |
173 | print(sans_prix)
174 |
175 |
176 | # In[11]:
177 |
178 |
179 | #use austria oilseeds to replace
180 | oilseeds_prix=prix[prix['Area']=='Austria'][prix['Element']=='Producer Price (USD/tonne)'][prix['Item Code']==339][prix['Year'].isin(range(beginyear,endyear))]
181 |
182 |
183 | # In[12]:
184 |
185 |
186 | #use avg of maize and rice to replace cereals
187 | cereal_mean=malay_prix[malay_prix['Item'].isin(['Maize','Rice, paddy'])].groupby('Year').mean()['Value'].tolist()
188 |
189 | cereal_prix=oilseeds_prix.copy()
190 |
191 | cereal_prix['Value']=cereal_mean
192 |
193 | cereal_prix['Item']='Cereals (Rice Milled Eqv)'
194 |
195 | cereal_prix['Item Code']=1817
196 |
197 |
198 | # In[13]:
199 |
200 |
201 | #add missing
202 | malay_prix=malay_prix.append(oilseeds_prix).append(cereal_prix)
203 |
204 | malay_prix.reset_index(inplace=True,drop=True)
205 |
206 |
207 | # In[14]:
208 |
209 |
210 | #concat
211 | malay_prod=malay_prod.merge(mapping,on=['Item Code', 'Item'],how='left')
212 |
213 |
214 | # In[15]:
215 |
216 |
217 | #find inner join of crops
218 | a=set(malay_prod['Item Code'][malay_prod['Element']=='Area harvested'])
219 |
220 | b=set(malay_prod['Item Code'][malay_prod['Element']=='Production'])
221 |
222 | c=set(malay_prix['Item Code'])
223 |
224 | target_crops=a.intersection(b).intersection(c)
225 |
226 | targets=[i for i in target_crops if i not in exclude]
227 |
228 |
229 | # In[16]:
230 |
231 |
232 | #select crops
233 | malay_prod=malay_prod[malay_prod['Item Code'].isin(targets)]
234 | malay_prix=malay_prix[malay_prix['Item Code'].isin(targets)]
235 |
236 |
237 | # In[17]:
238 |
239 |
240 | #add 2018 land
241 | land_2018=malay_land.iloc[-1:].copy()
242 | land_2018.reset_index(inplace=True,drop=True)
243 | land_2018.at[0,'Year']=2018
244 | land_2018.at[0,'Year Code']=2018
245 |
246 | malay_land=malay_land.append(land_2018)
247 |
248 |
249 | # In[18]:
250 |
251 |
252 | #export to csv
253 | malay_land.to_csv('malay_land.csv',index=False)
254 |
255 | malay_prod.to_csv('malay_prod.csv',index=False)
256 |
257 | malay_prix.to_csv('malay_prix.csv',index=False)
258 |
259 | malay_pop.to_csv('malay_pop.csv',index=False)
260 |
261 | malay_gdp.to_csv('malay_gdp.csv',index=False)
262 |
263 |
264 | # In[19]:
265 |
266 |
267 | #concat
268 | D=prepare(malay_land,malay_prod,malay_prix)
269 |
270 | grand=pd.concat([D[i] for i in D])
271 |
272 | grand.reset_index(inplace=True,drop=True)
273 |
274 |
275 | # In[20]:
276 |
277 |
278 | #create from synthetic control unit
279 | malay_ginger=[1711.631924118213, 1918.0617150999785, 1926.352662122375, 1904.0526853161766]
280 |
281 | malay_lettuce=[808.9104313117714, 809.2907573824596, 880.0718990821069, 801.194381901839]
282 |
283 | malay_maize=[176.02653721679235, 181.61446377135653, 185.21444467527138]
284 |
285 | malay_orange=[385.20868947337976, 303.6319254759596, 277.51641822220614, 376.92771860265077]
286 |
287 | malay_sugarcane=[246.78089608441445]
288 |
289 | malay_tobacco=[4345.956117624693,
290 | 4091.368210599639,
291 | 3638.8980233955585,
292 | 3829.529414768218,
293 | 3758.1754705530975,
294 | 3889.6001454970665]
295 |
296 |
297 | # In[21]:
298 |
299 |
300 | #fill price
301 | fillnull=dict(zip(grand['Item'][grand['price'].isnull()].unique(),
302 | [malay_tobacco,malay_ginger,malay_lettuce,
303 | malay_orange,malay_maize,malay_sugarcane,]))
304 |
305 | for i in fillnull:
306 | indices=grand[grand['Item']==i][grand['price'].isnull()].index.tolist()
307 | for j in indices:
308 | grand.at[j,'price']=fillnull[i][indices.index(j)]
309 |
310 |
311 | # In[22]:
312 |
313 |
314 | grand.to_csv('grand.csv',index=False)
315 |
316 |
317 | # ### synthetic control unit
318 |
319 | # In[23]:
320 |
321 |
322 | # import statsmodels.api as sm
323 |
324 | # #show missing items
325 | # null=set(grand['Item'][grand['price'].isnull()])
326 |
327 | # ss=grand[grand['Item'].isin(null)].sort_values(['Item','Year'])
328 |
329 | # ss
330 |
331 | # #split here
332 |
333 | # #create synthetic control unit
334 | # for ii in ss['Item'].unique():
335 |
336 | # print(ii)
337 |
338 | # #cleanse data
339 | # temp=prix[prix['Item']==ii][prix['Year'].isin(range(beginyear,endyear))][prix['Element']=='Producer Price (USD/tonne)'][prix['Months']=='Annual value']
340 | # temp=temp.pivot(index='Year',columns='Area',values='Value')
341 |
342 | # #delete null data
343 | # for i in temp:
344 | # if temp[i].isnull().any() and i!='Malaysia':
345 | # del temp[i]
346 |
347 | # #train
348 | # x=temp.loc[temp['Malaysia'].dropna().index]
349 | # del x['Malaysia']
350 | # y=temp['Malaysia'].dropna()
351 | # x.reset_index(inplace=True,drop=True)
352 | # y.reset_index(inplace=True,drop=True)
353 |
354 | # m=sm.OLS(y,x).fit()
355 |
356 | # #train result
357 | # print(m.predict(),y.tolist())
358 |
359 | # #test result
360 | # test=temp[temp['Malaysia'].isnull()]
361 | # test=test[[i for i in test.columns if i!='Malaysia']]
362 | # print(m.predict(test).tolist())
363 |
364 |
--------------------------------------------------------------------------------
/Smart Farmers project/country selection.py:
--------------------------------------------------------------------------------
1 |
2 | # coding: utf-8
3 |
4 | # In[1]:
5 |
6 |
7 | target_country=['Australia','Spain',
8 | 'Morocco',
9 | 'United Kingdom',
10 | 'Poland',
11 | 'France',
12 | 'Mexico',
13 | 'Bangladesh',
14 | 'Canada',
15 | 'Viet Nam',
16 | 'Thailand',
17 | 'Guatemala',
18 | 'Colombia',
19 | 'Germany',
20 | 'China',
21 | 'Brazil',
22 | 'India',
23 | 'United States of America',
24 | 'Malaysia',
25 | 'Indonesia']
26 |
27 |
28 | # In[2]:
29 |
30 |
31 | target_year=[2000,
32 | 2001,
33 | 2002,
34 | 2003,
35 | 2004,
36 | 2005,
37 | 2006,
38 | 2007,
39 | 2008,
40 | 2009,
41 | 2010,
42 | 2011,
43 | 2012,
44 | 2013,
45 | 2014,
46 | 2015,
47 | 2016,
48 | 2017]
49 |
50 |
51 | # In[3]:
52 |
53 |
54 | import pandas as pd
55 | import numpy as np
56 | import os
57 | os.chdir('H:/')
58 |
59 |
60 | # In[4]:
61 |
62 |
63 | prod=pd.read_csv('Production_Crops_E_All_Data_(Normalized).csv',encoding='latin-1')
64 |
65 |
66 | # In[5]:
67 |
68 |
69 | prix=pd.read_csv('Prices_E_All_Data_(Normalized).csv',encoding='latin-1')
70 |
71 |
72 | # In[6]:
73 |
74 |
75 | echanger=pd.read_csv('Trade_Crops_Livestock_E_All_Data_(Normalized).csv',encoding='latin-1')
76 |
77 |
78 | # In[7]:
79 |
80 |
81 | #only compare crops with available data
82 | target_crops=set(prod['Item']).intersection(set(prix['Item'])).intersection(set(echanger['Item']))
83 |
84 |
85 | # # volume
86 |
87 | # In[8]:
88 |
89 |
90 | #extract export volume
91 | trade=echanger[echanger['Element']=='Export Quantity']
92 |
93 |
94 | # In[9]:
95 |
96 |
97 | #cleanse
98 | trade=trade[trade['Item'].isin(target_crops)]
99 |
100 | trade=trade[trade['Area'].isin(target_country)]
101 |
102 | trade=trade[trade['Year'].isin(target_year)]
103 |
104 | trade=trade[['Area','Year','Item','Value']]
105 |
106 |
107 | # In[10]:
108 |
109 |
110 | #extract production data
111 | prod=prod[prod['Element']=='Production']
112 |
113 |
114 | # In[11]:
115 |
116 |
117 | #cleanse
118 | prod=prod[prod['Item'].isin(target_crops)]
119 |
120 | prod=prod[prod['Area'].isin(target_country)]
121 |
122 | prod=prod[prod['Year'].isin(target_year)]
123 |
124 | prod=prod[['Area','Year','Item','Value']]
125 |
126 |
127 | # In[12]:
128 |
129 |
130 | #group by sum
131 | export=trade.groupby(['Area','Year']).sum()
132 |
133 |
134 | # In[13]:
135 |
136 |
137 | #group by sum
138 | supply=prod.groupby(['Area','Year']).sum()
139 |
140 |
141 | # In[14]:
142 |
143 |
144 | #create new dataframe to compute export volume percentage
145 | pourcent=pd.DataFrame(index=export.index)
146 |
147 |
148 | # In[15]:
149 |
150 |
151 | #compute percentage
152 | pourcent['value']=np.divide(export['Value'].tolist(),supply['Value'].tolist())
153 |
154 |
155 | # In[16]:
156 |
157 |
158 | #clean up index
159 | pourcent.reset_index(inplace=True)
160 |
161 |
162 | # In[17]:
163 |
164 |
165 | #historical average
166 | mean_pourcent=pourcent[['Area','value']].groupby('Area').mean()
167 |
168 |
169 | # In[18]:
170 |
171 |
172 | #sort by percentage
173 | mean_pourcent=mean_pourcent.sort_values('value')
174 |
175 |
176 | # In[19]:
177 |
178 |
179 | #export percentage for a particular year
180 | pourcent[pourcent['Year'].isin([2017])].sort_values('value')
181 |
182 |
183 | # # price
184 |
185 | # In[20]:
186 |
187 |
188 | #get usd price
189 | prix=prix[prix['Element']=='Producer Price (USD/tonne)']
190 |
191 |
192 | # In[21]:
193 |
194 |
195 | #cleanse
196 | prix=prix[prix['Item'].isin(target_crops)]
197 |
198 | prix=prix[prix['Area'].isin(target_country)]
199 |
200 | prix=prix[prix['Year'].isin(target_year)]
201 |
202 | prix=prix[['Area','Year','Item','Value']]
203 |
204 |
205 | # In[22]:
206 |
207 |
208 | #previously we examine the volume
209 | #now we focus on value
210 | value=prod.merge(prix,on=['Area','Year','Item'],how='inner')
211 |
212 |
213 | # In[23]:
214 |
215 |
216 | #volume times unit price equals to total value
217 | value['Value']=np.multiply(value['Value_x'],value['Value_y'])
218 |
219 |
220 | # In[24]:
221 |
222 |
223 | #cleanse
224 | value=value[['Area','Year','Item','Value']]
225 |
226 |
227 | # In[25]:
228 |
229 |
230 | #get export price
231 | exchange=echanger[echanger['Element']=='Export Value']
232 |
233 |
234 | # In[26]:
235 |
236 |
237 | #cleanse
238 | exchange=exchange[exchange['Item'].isin(target_crops)]
239 |
240 | exchange=exchange[exchange['Area'].isin(target_country)]
241 |
242 | exchange=exchange[exchange['Year'].isin(target_year)]
243 |
244 | exchange=exchange[['Area','Year','Item','Value']]
245 |
246 |
247 | # In[27]:
248 |
249 |
250 | #group by sum
251 | temp1=exchange.groupby(['Area','Year']).sum()
252 |
253 |
254 | # In[28]:
255 |
256 |
257 | #group by sum
258 | temp2=value.groupby(['Area','Year']).sum()
259 |
260 |
261 | # In[29]:
262 |
263 |
264 | #inner join
265 | temp=temp1.merge(temp2,on=['Area','Year'],how='inner')
266 |
267 |
268 | # In[30]:
269 |
270 |
271 | #create dataframe to compute export value percentage
272 | percentage=pd.DataFrame(index=temp.index)
273 |
274 |
275 | # In[31]:
276 |
277 |
278 | #compute export value percentage
279 | percentage['value']=np.divide(temp['Value_x'],temp['Value_y'])
280 |
281 |
282 | # In[32]:
283 |
284 |
285 | #clean up
286 | percentage.reset_index(inplace=True)
287 |
288 |
289 | # In[33]:
290 |
291 |
292 | #historical average
293 | mean_percentage=percentage[['Area','value']].groupby('Area').mean()
294 |
295 |
296 | # In[34]:
297 |
298 |
299 | #sort by percentage
300 | mean_percentage=mean_percentage.sort_values('value')
301 |
302 |
303 | # In[35]:
304 |
305 |
306 | #export percentage for a particular year
307 | percentage[percentage['Year'].isin([2016])].sort_values('value')
308 |
309 |
--------------------------------------------------------------------------------
/Smart Farmers project/data/capita.csv:
--------------------------------------------------------------------------------
1 | Date,Mid Price
2 | 12/31/1980,1926.963
3 | 12/31/1981,1920.127
4 | 12/31/1982,2006.4821
5 | 12/31/1983,2189.553
6 | 12/31/1984,2419.501
7 | 12/31/1985,2154.458
8 | 12/31/1986,1864.031
9 | 12/31/1987,2070.009
10 | 12/31/1988,2213.866
11 | 12/31/1989,2380.512
12 | 12/31/1990,2585.824
13 | 12/31/1991,2885.3059
14 | 12/31/1992,3378.7209
15 | 12/31/1993,3716.9519
16 | 12/31/1994,4027.509
17 | 12/31/1995,4678.0239
18 | 12/31/1996,5175.5562
19 | 12/31/1997,5011.5601
20 | 12/31/1998,3519.7661
21 | 12/31/1999,3762.769
22 | 12/31/2000,4347.73
23 | 12/31/2001,4189.0571
24 | 12/31/2002,4441.8472
25 | 12/31/2003,4740.3159
26 | 12/31/2004,5244.874
27 | 12/31/2005,5678.5972
28 | 12/31/2006,6353.4141
29 | 12/31/2007,7483.4092
30 | 12/31/2008,8769.3945
31 | 12/31/2009,7545.1118
32 | 12/31/2010,9047.2021
33 | 12/31/2011,10398.2363
34 | 12/31/2012,10806.8398
35 | 12/31/2013,10851.6641
36 | 12/31/2014,11165.2588
37 | 12/31/2015,9663.1113
38 | 12/31/2016,9523.2949
39 | 12/31/2017,9960.3232
40 | 12/31/2018,11072.3867
41 | 12/31/2019,11136.8115
42 | 12/31/2020,11484.5049
43 | 12/31/2021,12172.1621
44 | 12/31/2022,12875.417
45 | 12/31/2023,13629.3301
46 | 12/31/2024,14468.3174
47 |
--------------------------------------------------------------------------------
/Smart Farmers project/data/malay_gdp.csv:
--------------------------------------------------------------------------------
1 | Area Code,Area,Item Code,Item,Element Code,Element,Year Code,Year,Unit,Value,Flag,Note
2 | 131,Malaysia,22014,Gross Domestic Product per capita,6119,Value US$,2012,2012,,10779.504483,Fc,
3 | 131,Malaysia,22014,Gross Domestic Product per capita,6119,Value US$,2013,2013,,10882.259575,Fc,
4 | 131,Malaysia,22014,Gross Domestic Product per capita,6119,Value US$,2014,2014,,11183.869684000001,Fc,
5 | 131,Malaysia,22014,Gross Domestic Product per capita,6119,Value US$,2015,2015,,9808.7171,Fc,
6 | 131,Malaysia,22014,Gross Domestic Product per capita,6119,Value US$,2016,2016,,9659.564758,Fc,
7 | 131,Malaysia,22014,Gross Domestic Product per capita,6119,Value US$,2017,2017,,10085.774790000001,Fc,
8 | 131,Malaysia,22014,Gross Domestic Product per capita,6119,Value US$,2018,2018,,11190.754009,Fc,
9 |
--------------------------------------------------------------------------------
/Smart Farmers project/data/malay_land.csv:
--------------------------------------------------------------------------------
1 | Area Code,Area,Item Code,Item,Element Code,Element,Year Code,Year,Unit,Value,Flag
2 | 131,Malaysia,6620,Cropland,5110,Area,2012,2012,1000 ha,7544.2,Fm
3 | 131,Malaysia,6620,Cropland,5110,Area,2013,2013,1000 ha,7774.3,Q
4 | 131,Malaysia,6620,Cropland,5110,Area,2014,2014,1000 ha,7804.0,Fm
5 | 131,Malaysia,6620,Cropland,5110,Area,2015,2015,1000 ha,8284.97,Q
6 | 131,Malaysia,6620,Cropland,5110,Area,2016,2016,1000 ha,8305.7,Fm
7 | 131,Malaysia,6620,Cropland,5110,Area,2017,2017,1000 ha,8305.2,Fm
8 | 131,Malaysia,6620,Cropland,5110,Area,2018,2018,1000 ha,8305.2,Fm
9 |
--------------------------------------------------------------------------------
/Smart Farmers project/data/malay_pop.csv:
--------------------------------------------------------------------------------
1 | Area Code,Area,Item Code,Item,Element Code,Element,Year Code,Year,Unit,Value,Flag,Note
2 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2012,2012,1000 persons,29068.189,X,
3 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2013,2013,1000 persons,29468.923,X,
4 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2014,2014,1000 persons,29866.603,X,
5 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2015,2015,1000 persons,30270.962,X,
6 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2016,2016,1000 persons,30684.654,X,
7 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2017,2017,1000 persons,31104.646,X,
8 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2018,2018,1000 persons,31528.033,X,
9 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2019,2019,1000 persons,31949.777,X,
10 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2020,2020,1000 persons,32365.999,X,
11 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2021,2021,1000 persons,32776.194,X,
12 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2022,2022,1000 persons,33181.072,X,
13 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2023,2023,1000 persons,33579.265,X,
14 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2024,2024,1000 persons,33969.29,X,
15 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2025,2025,1000 persons,34349.936,X,
16 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2026,2026,1000 persons,34720.347,X,
17 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2027,2027,1000 persons,35080.112,X,
18 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2028,2028,1000 persons,35429.087,X,
19 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2029,2029,1000 persons,35767.388,X,
20 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2030,2030,1000 persons,36095.054,X,
21 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2031,2031,1000 persons,36411.996,X,
22 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2032,2032,1000 persons,36717.9,X,
23 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2033,2033,1000 persons,37012.445,X,
24 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2034,2034,1000 persons,37295.251,X,
25 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2035,2035,1000 persons,37566.148,X,
26 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2036,2036,1000 persons,37825.112,X,
27 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2037,2037,1000 persons,38072.527,X,
28 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2038,2038,1000 persons,38309.22,X,
29 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2039,2039,1000 persons,38536.264,X,
30 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2040,2040,1000 persons,38754.574,X,
31 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2041,2041,1000 persons,38964.542,X,
32 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2042,2042,1000 persons,39166.396,X,
33 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2043,2043,1000 persons,39360.773,X,
34 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2044,2044,1000 persons,39548.354,X,
35 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2045,2045,1000 persons,39729.699,X,
36 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2046,2046,1000 persons,39905.062,X,
37 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2047,2047,1000 persons,40074.631,X,
38 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2048,2048,1000 persons,40238.61,X,
39 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2049,2049,1000 persons,40397.168,X,
40 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2050,2050,1000 persons,40550.365,X,
41 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2051,2051,1000 persons,40698.294,X,
42 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2052,2052,1000 persons,40840.8,X,
43 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2053,2053,1000 persons,40977.445,X,
44 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2054,2054,1000 persons,41107.613,X,
45 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2055,2055,1000 persons,41230.759,X,
46 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2056,2056,1000 persons,41346.675,X,
47 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2057,2057,1000 persons,41455.194,X,
48 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2058,2058,1000 persons,41555.888,X,
49 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2059,2059,1000 persons,41648.244,X,
50 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2060,2060,1000 persons,41731.877,X,
51 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2061,2061,1000 persons,41806.614,X,
52 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2062,2062,1000 persons,41872.362,X,
53 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2063,2063,1000 persons,41928.871,X,
54 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2064,2064,1000 persons,41975.9,X,
55 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2065,2065,1000 persons,42013.361,X,
56 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2066,2066,1000 persons,42041.225,X,
57 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2067,2067,1000 persons,42059.669,X,
58 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2068,2068,1000 persons,42068.988,X,
59 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2069,2069,1000 persons,42069.605,X,
60 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2070,2070,1000 persons,42061.944,X,
61 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2071,2071,1000 persons,42046.292,X,
62 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2072,2072,1000 persons,42022.971,X,
63 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2073,2073,1000 persons,41992.517,X,
64 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2074,2074,1000 persons,41955.499,X,
65 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2075,2075,1000 persons,41912.491,X,
66 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2076,2076,1000 persons,41863.892,X,
67 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2077,2077,1000 persons,41810.176,X,
68 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2078,2078,1000 persons,41752.0,X,
69 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2079,2079,1000 persons,41690.087,X,
70 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2080,2080,1000 persons,41625.092,X,
71 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2081,2081,1000 persons,41557.426,X,
72 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2082,2082,1000 persons,41487.446,X,
73 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2083,2083,1000 persons,41415.604,X,
74 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2084,2084,1000 persons,41342.328,X,
75 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2085,2085,1000 persons,41267.997,X,
76 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2086,2086,1000 persons,41192.896,X,
77 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2087,2087,1000 persons,41117.199,X,
78 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2088,2088,1000 persons,41040.987,X,
79 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2089,2089,1000 persons,40964.235,X,
80 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2090,2090,1000 persons,40886.939,X,
81 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2091,2091,1000 persons,40809.104,X,
82 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2092,2092,1000 persons,40730.761,X,
83 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2093,2093,1000 persons,40651.951,X,
84 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2094,2094,1000 persons,40572.639,X,
85 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2095,2095,1000 persons,40492.785,X,
86 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2096,2096,1000 persons,40412.247,X,
87 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2097,2097,1000 persons,40330.825,X,
88 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2098,2098,1000 persons,40248.246,X,
89 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2099,2099,1000 persons,40164.17,X,
90 | 131,Malaysia,3010,Population - Est. & Proj.,511,Total Population - Both sexes,2100,2100,1000 persons,40078.168,X,
91 |
--------------------------------------------------------------------------------
/Smart Farmers project/estimate demand.py:
--------------------------------------------------------------------------------
1 |
2 | # coding: utf-8
3 |
4 | # In[1]:
5 |
6 |
7 | import numpy as np
8 | import os
9 | import pandas as pd
10 | import statsmodels.api as sm
11 | import matplotlib.pyplot as plt
12 | import cvxopt
13 | os.chdir('H:/')
14 |
15 |
16 | # ### define functions
17 |
18 | # In[2]:
19 |
20 |
21 | #create xy for least square
22 | def create_xy(target_crop,grande,malay_gdp,malay_pop):
23 |
24 | y=grande['price'][target_crop]
25 |
26 | x=pd.concat([malay_gdp['Value'],malay_pop['Value'],
27 | grande['production'][target_crop]],axis=1)
28 |
29 | x=sm.add_constant(x)
30 |
31 | #set production negative
32 | x[target_crop.lower()]=x[target_crop].apply(lambda x:-1*x)
33 | del x[target_crop]
34 |
35 | return x,y
36 |
37 |
38 | # In[3]:
39 |
40 |
41 | #linear regression
42 | def lin_reg(crops,grande,malay_gdp,malay_pop,viz=False):
43 |
44 | D={}
45 |
46 | #run regression
47 | for target_crop in crops:
48 |
49 | #create xy
50 | x,y=create_xy(target_crop,grande,malay_gdp,malay_pop)
51 |
52 | m=sm.OLS(y,sm.add_constant(x)).fit()
53 |
54 | D[target_crop]=(m.rsquared,m.params.tolist())
55 |
56 | #viz
57 | if viz:
58 | fig=plt.figure(figsize=(10,5))
59 | ax=fig.add_subplot(111)
60 | ax.spines['top'].set_visible(False)
61 | ax.spines['right'].set_visible(False)
62 | plt.ylabel('USD/Tonnes')
63 | plt.xlabel('Year')
64 | plt.plot(range(beginyear,endyear),m.predict(),label='Est',color='#C8C8A9')
65 | plt.plot(range(beginyear,endyear),y,label='Act',color='#EDE574')
66 | plt.title(target_crop+' Price')
67 | plt.legend()
68 | plt.show()
69 |
70 | return D
71 |
72 |
73 | # In[4]:
74 |
75 |
76 | def constrained_ols(x,y):
77 |
78 | linear_coeff=cvxopt.matrix(-1*np.mat(y.tolist())*np.mat(x)).T
79 | quadratic_coeff=cvxopt.matrix(np.mat(x).T*np.mat(x))
80 |
81 | #inequality constraint
82 | inequality_coeff=cvxopt.matrix(0.0,(len(x.columns),len(x.columns)))
83 |
84 | #diagonal matrix
85 | #use -1 to achieve larger than
86 | inequality_coeff[::len(x.columns)+1]=-1
87 |
88 | #no constraint for the constant
89 | inequality_coeff[0,0]=0
90 | inequality_value=cvxopt.matrix([0.0 for _ in range(len(x.columns))])
91 |
92 | cvxopt.solvers.options['show_progress']=False
93 |
94 | ans=cvxopt.solvers.qp(P=quadratic_coeff,q=linear_coeff,
95 | G=inequality_coeff,h=inequality_value)['x']
96 |
97 | return ans
98 |
99 |
100 | # In[5]:
101 |
102 |
103 | #create params by using constrained ols
104 | def get_params(crops,grande,malay_gdp,malay_pop,viz=False):
105 |
106 | D={}
107 |
108 | for target_crop in crops:
109 |
110 | #create xy
111 | x,y=create_xy(target_crop,grande,malay_gdp,malay_pop)
112 |
113 | #constrained ols
114 | ans=constrained_ols(x,y)
115 |
116 | #viz
117 | if viz:
118 |
119 | #get forecast and convert to list
120 | forecast=np.mat(ans).T*np.mat(x).T
121 | forecast=forecast.ravel().tolist()[0]
122 | fig=plt.figure(figsize=(10,5))
123 | ax=fig.add_subplot(111)
124 | ax.spines['top'].set_visible(False)
125 | ax.spines['right'].set_visible(False)
126 | plt.ylabel('USD/Tonnes')
127 | plt.xlabel('Year')
128 | plt.plot(range(beginyear,endyear),forecast,label='Est',color='#C8C8A9',)
129 | plt.plot(range(beginyear,endyear),y,label='Act',color='#EDE574')
130 | plt.legend()
131 | plt.title(target_crop+' Price')
132 | plt.show()
133 |
134 | D[target_crop]=list(ans)
135 |
136 | return D
137 |
138 |
139 | # ### execution
140 |
141 | # In[6]:
142 |
143 |
144 | global beginyear,endyear
145 | beginyear=2012
146 | endyear=2019
147 |
148 |
149 | # In[7]:
150 |
151 |
152 | grand=pd.read_csv('grand.csv')
153 |
154 | malay_pop=pd.read_csv('malay_pop.csv')
155 |
156 | malay_gdp=pd.read_csv('malay_gdp.csv')
157 |
158 |
159 | # In[8]:
160 |
161 |
162 | #cleanse
163 | malay_pop=malay_pop[malay_pop['Year'].isin(range(beginyear,endyear))]
164 | malay_pop.reset_index(inplace=True,drop=True)
165 |
166 |
167 | # In[9]:
168 |
169 |
170 | #pivot
171 | grande=grand.pivot(index='Year',columns='Item',values=['price','production'])
172 | grande.reset_index(inplace=True,drop=True)
173 |
174 |
175 | # In[10]:
176 |
177 |
178 | #find crops
179 | crops=grande.columns.levels[1]
180 |
181 |
182 | # ### linear regression
183 |
184 | # In[11]:
185 |
186 |
187 | D1=lin_reg(crops,grande,malay_gdp,malay_pop,viz=True)
188 |
189 |
190 | # In[12]:
191 |
192 |
193 | D1
194 |
195 |
196 | # ### constrained regression
197 |
198 | # In[13]:
199 |
200 |
201 | D2=get_params(crops,grande,malay_gdp,malay_pop,viz=True)
202 |
203 |
204 | # In[14]:
205 |
206 |
207 | # #replace unsuccessful result with ols
208 | # for i in D2:
209 | # if len([j for j in D2[i] if j<0])>0:
210 | # D2[i]=D1[i][1]
211 |
212 |
213 | # In[15]:
214 |
215 |
216 | #create output
217 | output=pd.DataFrame(columns=D2.keys())
218 |
219 | for i in D2:
220 | output[i]=D2[i]
221 |
222 | output=output.T
223 |
224 | output.columns=['constant','gamma','beta','alpha']
225 |
226 | output.index.name='Item'
227 |
228 | output.reset_index(inplace=True)
229 |
230 |
231 | # In[16]:
232 |
233 |
234 | #merge with grand
235 | tres_grand=grand.merge(output,on='Item',how='left')
236 |
237 | tres_grand.to_csv('tres_grand.csv',index=False)
238 |
239 |
240 | # ### backtest
241 |
242 | # In[17]:
243 |
244 |
245 | forecast=pd.read_csv('forecast.csv')
246 |
247 |
248 | # In[18]:
249 |
250 |
251 | #extract historical generic 1st
252 | palm=pd.read_csv('palm.csv')
253 | palm.set_index('Date',inplace=True,)
254 | palm.index=pd.to_datetime(palm.index)
255 | palm.columns=['Palm oil','Rubber1','Rubber2']
256 |
257 |
258 | # In[19]:
259 |
260 |
261 | cme=pd.read_csv('cme.csv')
262 |
263 | #extract cme palm oil futures
264 | palm_futures=cme[cme['date']==cme['date'].iloc[-1]][cme['product_id']==2457]
265 |
266 | palm_futures=palm_futures[['expiration_date','prior_settle']]
267 | palm_futures.columns=['Date','Palm oil']
268 |
269 | palm_futures.set_index('Date',inplace=True)
270 | palm_futures.index=pd.to_datetime(palm_futures.index)
271 |
272 |
273 | # In[20]:
274 |
275 |
276 | #concat, currency convert and get annual avg
277 | temp=palm['Palm oil'][str(beginyear):].apply(lambda x:x*0.23).append(palm_futures['Palm oil'][:str(endyear+5)])
278 |
279 | palmoil=temp.resample('1A').mean()
280 |
281 | #convert datetime index from year end to beginning
282 | palmoil.index=[pd.to_datetime(str(i)[:5]+'01-01') for i in palmoil.index]
283 |
284 |
285 | # In[21]:
286 |
287 |
288 | #create oil palm actual and estimated price
289 | oilpalm_act=grand['price'][grand['Item']=='Oil palm fruit']
290 | oilpalm_act.reset_index(inplace=True,drop=True)
291 |
292 | oilpalm_est=oilpalm_act.iloc[-1:].append(forecast['price'][forecast['Item']=='Oil palm fruit'])
293 | oilpalm_est.reset_index(inplace=True,drop=True)
294 |
295 |
296 | # In[22]:
297 |
298 |
299 | #dual axis plot
300 | fig=plt.figure(figsize=(10,5))
301 | ax=fig.add_subplot(111)
302 |
303 |
304 | ax.set_xlabel('Date')
305 | ax.set_ylabel('Palm Oil Generic 1st',color='#45ADA8')
306 | ax.plot(palmoil.index,palmoil,color='#45ADA8',label='Palm Oil')
307 | ax.tick_params(axis='y',labelcolor='#45ADA8')
308 | ax.yaxis.labelpad=15
309 |
310 | plt.legend(loc=3)
311 | ax2=ax.twinx()
312 |
313 | ax2.set_ylabel('Oil Palm Fruit Price',color='#EC2049',rotation=270)
314 | ax2.plot(palmoil.index[:len(oilpalm_act)],oilpalm_act,
315 | color='#EC2049',label='Oil Palm Act')
316 | ax2.plot(palmoil.index[len(oilpalm_est)-1:],oilpalm_est,
317 | color='#fea5c4',label='Oil Palm Est',linestyle='-.')
318 | ax2.tick_params(axis='y',labelcolor='#EC2049')
319 | ax2.yaxis.labelpad=15
320 |
321 | fig.tight_layout()
322 | plt.legend(loc=4)
323 | plt.grid(False)
324 | plt.title('Palm Oil vs Oil Palm')
325 | plt.show()
326 |
327 |
--------------------------------------------------------------------------------
/Smart Farmers project/preview/cabbage price.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Smart Farmers project/preview/cabbage price.png
--------------------------------------------------------------------------------
/Smart Farmers project/preview/cabbage production.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Smart Farmers project/preview/cabbage production.png
--------------------------------------------------------------------------------
/Smart Farmers project/preview/cabbage regression.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Smart Farmers project/preview/cabbage regression.png
--------------------------------------------------------------------------------
/Smart Farmers project/preview/cocoa price.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Smart Farmers project/preview/cocoa price.png
--------------------------------------------------------------------------------
/Smart Farmers project/preview/cocoa production.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Smart Farmers project/preview/cocoa production.png
--------------------------------------------------------------------------------
/Smart Farmers project/preview/cocoa regression.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Smart Farmers project/preview/cocoa regression.png
--------------------------------------------------------------------------------
/Smart Farmers project/preview/coconut price.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Smart Farmers project/preview/coconut price.png
--------------------------------------------------------------------------------
/Smart Farmers project/preview/coconut production.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Smart Farmers project/preview/coconut production.png
--------------------------------------------------------------------------------
/Smart Farmers project/preview/coconut regression.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Smart Farmers project/preview/coconut regression.png
--------------------------------------------------------------------------------
/Smart Farmers project/preview/demand model.PNG:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Smart Farmers project/preview/demand model.PNG
--------------------------------------------------------------------------------
/Smart Farmers project/preview/mango price.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Smart Farmers project/preview/mango price.png
--------------------------------------------------------------------------------
/Smart Farmers project/preview/mango production.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Smart Farmers project/preview/mango production.png
--------------------------------------------------------------------------------
/Smart Farmers project/preview/mango regression.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Smart Farmers project/preview/mango regression.png
--------------------------------------------------------------------------------
/Smart Farmers project/preview/naive model matrix.PNG:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Smart Farmers project/preview/naive model matrix.PNG
--------------------------------------------------------------------------------
/Smart Farmers project/preview/naive model.PNG:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Smart Farmers project/preview/naive model.PNG
--------------------------------------------------------------------------------
/Smart Farmers project/preview/oil palm price.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Smart Farmers project/preview/oil palm price.png
--------------------------------------------------------------------------------
/Smart Farmers project/preview/oil palm production.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Smart Farmers project/preview/oil palm production.png
--------------------------------------------------------------------------------
/Smart Farmers project/preview/oil palm regression.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Smart Farmers project/preview/oil palm regression.png
--------------------------------------------------------------------------------
/Smart Farmers project/preview/oil palm vs palm oil.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Smart Farmers project/preview/oil palm vs palm oil.png
--------------------------------------------------------------------------------
/Smart Farmers project/preview/overall production.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Smart Farmers project/preview/overall production.png
--------------------------------------------------------------------------------
/Smart Farmers project/preview/price change derivation.PNG:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Smart Farmers project/preview/price change derivation.PNG
--------------------------------------------------------------------------------
/Smart Farmers project/preview/pricing mechanism.PNG:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Smart Farmers project/preview/pricing mechanism.PNG
--------------------------------------------------------------------------------
/Smart Farmers project/preview/ricardian model.PNG:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Smart Farmers project/preview/ricardian model.PNG
--------------------------------------------------------------------------------
/Smart Farmers project/preview/rubber price.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Smart Farmers project/preview/rubber price.png
--------------------------------------------------------------------------------
/Smart Farmers project/preview/rubber production.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Smart Farmers project/preview/rubber production.png
--------------------------------------------------------------------------------
/Smart Farmers project/preview/rubber regression.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Smart Farmers project/preview/rubber regression.png
--------------------------------------------------------------------------------
/Smart Farmers project/preview/workflow.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/Smart Farmers project/preview/workflow.png
--------------------------------------------------------------------------------
/data/bitcoin.csv:
--------------------------------------------------------------------------------
1 | Date,XBTUSD BGN Curncy (R3),SPX Index (L2),GOLDLNPM Index (R2),TLT US Equity (L1),EEM US Equity (R1)
2 | 7/31/2010,0.06,1101.6,1169,100.48,41.4
3 | 8/31/2010,0.06,1049.33,1246,108.56,40.06
4 | 9/30/2010,0.06,1141.2,1307,105.51,44.77
5 | 10/31/2010,0.19,1183.26,1346.75,100.42,46.12
6 | 11/30/2010,0.21,1180.55,1383.5,98.41,44.78
7 | 12/31/2010,0.3,1257.64,1405.5,94.12,47.642
8 | 1/31/2011,0.52,1286.12,1327,91.22,45.81
9 | 2/28/2011,0.86,1327.22,1411,92.4,45.79
10 | 3/31/2011,0.78,1325.83,1439,92.13,48.67
11 | 4/30/2011,2.88,1363.61,1535.5,93.89,50
12 | 5/31/2011,8.74,1345.2,1536.5,96.69,48.53
13 | 6/30/2011,16.1,1320.64,1505.5,94.1,47.6
14 | 7/31/2011,13.5,1292.28,1628.5,97.92,47.11
15 | 8/31/2011,8.2,1218.89,1813.5,107.03,42.75
16 | 9/30/2011,5.14,1131.42,1620,120.8,35.095
17 | 10/31/2011,3.25,1253.3,1722,115.88,40.815
18 | 11/30/2011,2.97,1246.96,1746,117.88,40.01
19 | 12/31/2011,4.25,1257.61,1531,121.25,37.94
20 | 1/31/2012,5.48,1312.41,1744,120.85,42.11
21 | 2/29/2012,4.86,1365.68,1770,117.43,44.33
22 | 3/31/2012,4.86,1408.47,1662.5,112.2,42.945
23 | 4/30/2012,4.95,1397.91,1651.25,117.32,42.215
24 | 5/31/2012,5.18,1310.33,1558,127.6,37.7
25 | 6/30/2012,6.65,1362.16,1598.5,125.2,39.135
26 | 7/31/2012,9.35,1379.32,1622,129.7,39.12
27 | 8/31/2012,10.16,1406.58,1648.5,127.72,39.28
28 | 9/30/2012,12.39,1440.67,1776,124.22,41.325
29 | 10/31/2012,11.2,1412.16,1719,123.36,41.15
30 | 11/30/2012,12.57,1416.18,1726,124.79,41.785
31 | 12/31/2012,13.51,1426.19,1657.5,121.18,44.35
32 | 1/31/2013,20.41,1498.11,1664.75,117.32,44.215
33 | 2/28/2013,33.38,1514.68,1588.5,118.51,43.205
34 | 3/31/2013,90.5,1569.19,1598.25,117.76,42.77
35 | 4/30/2013,133.75,1597.57,1469,123.01,43.29
36 | 5/31/2013,127.2,1630.74,1394.5,114.45,41.195
37 | 6/30/2013,96.99,1606.28,1192,110.44,38.5
38 | 7/31/2013,97.25,1685.73,1314.5,107.7,39.01
39 | 8/31/2013,126.02,1632.97,1394.75,105.99,38.02
40 | 9/30/2013,127.09,1681.55,1326.5,106.4,40.755
41 | 10/31/2013,206.34,1756.54,1324,107.64,42.455
42 | 11/30/2013,1137,1805.81,1253,104.45,42.35
43 | 12/31/2013,746.89,1848.36,1204.5,101.86,41.795
44 | 1/31/2014,806.69,1782.59,1251,108.28,38.19
45 | 2/28/2014,566.91,1859.45,1326.5,108.57,39.48
46 | 3/31/2014,465.75,1872.34,1291.75,109.1,41.01
47 | 4/30/2014,448.23,1883.95,1288.5,111.1,41.33
48 | 5/31/2014,613.92,1923.57,1250.5,114.1,42.55
49 | 6/30/2014,646.97,1960.23,1315,113.52,43.23
50 | 7/31/2014,586.36,1930.67,1285.25,113.98,43.82
51 | 8/31/2014,479.7,2003.37,1285.75,119.05,45.06
52 | 9/30/2014,389.68,1972.29,1216.5,116.27,41.56
53 | 10/31/2014,342.38,2018.05,1164.25,119.25,42.15
54 | 11/30/2014,375.76,2067.56,1182.75,122.49,41.5
55 | 12/31/2014,317.36,2058.9,1206,125.92,39.29
56 | 1/31/2015,229.67,1994.99,1260.25,138.28,39.02
57 | 2/28/2015,254.87,2104.5,1214,129.53,40.74
58 | 3/31/2015,244.14,2067.89,1187,130.69,40.13
59 | 4/30/2015,236.58,2085.51,1180.25,125.95,42.88
60 | 5/31/2015,231.78,2107.39,1191.4,122.71,41.12
61 | 6/30/2015,260.97,2063.11,1171,117.46,39.62
62 | 7/31/2015,284.69,2103.84,1098.4,122.53,37.12
63 | 8/31/2015,230.41,1972.18,1135,121.42,33.84
64 | 9/30/2015,236.99,1920.03,1114,123.54,32.78
65 | 10/31/2015,323.23,2079.36,1142.35,122.78,34.87
66 | 11/30/2015,376.82,2080.41,1061.9,121.45,33.99
67 | 12/31/2015,432.12,2043.94,1060,120.58,32.19
68 | 1/31/2016,377.29,1940.24,1111.8,127.3,30.57
69 | 2/29/2016,433.91,1932.23,1234.9,130.98,30.32
70 | 3/31/2016,416.85,2059.74,1237,130.61,34.25
71 | 4/30/2016,455.15,2065.3,1285.65,129.38,34.39
72 | 5/31/2016,521.74,2096.96,1212.1,130.16,33.12
73 | 6/30/2016,665.27,2098.86,1320.75,138.9,34.36
74 | 7/31/2016,632,2173.6,1342,141.56,36.205
75 | 8/31/2016,572.41,2170.95,1309.25,139.87,36.53
76 | 9/30/2016,605.99,2168.27,1322.5,137.51,37.45
77 | 10/31/2016,702.17,2126.15,1272,131.25,37.14
78 | 11/30/2016,743.26,2198.81,1178.1,120.24,35.5
79 | 12/31/2016,952.01,2238.83,1145.9,119.13,35.01
80 | 1/31/2017,961.69,2278.87,1212.8,120.1,37.34
81 | 2/28/2017,1192.7,2363.64,1255.6,121.74,37.99
82 | 3/31/2017,1074.69,2362.72,1244.85,120.71,39.39
83 | 4/30/2017,1356.03,2384.2,1266.45,122.35,40.06
84 | 5/31/2017,2286.1,2411.8,1266.2,124.4,41.2
85 | 6/30/2017,2502.5,2423.41,1242.25,125.12,41.39
86 | 7/31/2017,2886.71,2470.3,1267.55,124.04,43.8
87 | 8/31/2017,4741.39,2471.65,1311.75,127.99,44.83
88 | 9/30/2017,4171.25,2519.36,1283.1,124.76,44.81
89 | 10/31/2017,6377.45,2575.26,1270.15,124.46,46.28
90 | 11/30/2017,8134.25,2647.58,1280.2,125.12,46.1
91 | 12/31/2017,14043.06,2673.61,1291,126.86,47.12
92 | 1/31/2018,9962.31,2823.81,1345.05,122.73,51.03
93 | 2/28/2018,10538.02,2713.83,1317.85,118.75,48.02
94 | 3/31/2018,6852.51,2640.87,1323.85,121.9,48.28
95 | 4/30/2018,9272.7,2648.05,1313.2,119.1,46.92
96 | 5/31/2018,7538.28,2705.27,1305.35,121.22,45.69
97 | 6/30/2018,5899.64,2718.37,1250.45,121.72,43.33
98 | 7/31/2018,7689.43,2816.29,1220.95,119.7,44.86
99 | 8/31/2018,7044.6,2901.52,1202.45,120.995,43.17
100 | 9/30/2018,6561.3,2913.98,1187.25,117.27,42.92
101 | 10/31/2018,6302.4,2711.74,1214.95,113.58,39.16
102 | 11/30/2018,3930.38,2760.17,1217.55,115.33,41.08
103 | 12/31/2018,3674.18,2506.85,1279,121.51,39.06
104 | 1/31/2019,3417.04,2704.1,1323.25,121.97,43.1
105 | 2/28/2019,3802.53,2784.49,1319.15,120.02,42.44
106 | 3/31/2019,4092.97,2834.4,1295.4,126.44,42.92
107 | 4/30/2019,5237.87,2945.83,1282.3,123.65,43.93
108 | 5/31/2019,8503.38,2752.06,1295.55,131.83,40.71
109 | 6/30/2019,11392.98,2941.76,1409,132.81,42.91
110 | 7/31/2019,10026.61,2980.38,1427.55,132.89,41.77
111 | 8/31/2019,9626.18,2926.46,1528.4,147.28,40.19
112 | 9/30/2019,8240.9,2976.74,1485.3,143.08,40.87
113 | 10/31/2019,9189.42,3037.56,1510.95,141.24,42.58
114 | 11/30/2019,7710.34,3140.98,1460.15,140.42,42.54
115 | 12/31/2019,7158.27,3230.78,1514.75,135.48,44.87
116 | 1/31/2020,9354.29,3225.52,1584.2,145.9,42.11
117 | 2/29/2020,8633.36,2954.22,1609.85,155.31,40.52
118 | 3/31/2020,6481.37,2584.59,1608.95,164.97,34.13
119 | 4/30/2020,8826.56,2912.43,1702.75,166.74,36.64
120 | 5/31/2020,9505.38,3044.31,1728.7,163.59,37.73
121 | 6/30/2020,9147.09,3100.29,1768.1,163.93,39.99
122 | 7/31/2020,11345.63,3271.12,1964.9,171,43.29
123 | 8/31/2020,11678.6,3500.31,1957.35,162.19,44.54
124 | 9/30/2020,10706.95,3363,1886.9,163.26,44.09
125 | 10/31/2020,13850.1,3269.96,1881.85,157.57,44.71
126 | 11/30/2020,19378.61,3621.63,1762.55,160.02,48.73
127 | 12/31/2020,28996.28,3756.07,1887.6,157.73,51.67
128 | 1/31/2021,32601.26,3714.24,1863.8,152,53.31
129 | 2/28/2021,45248.24,3811.15,1742.85,143.12,53.73
130 | 3/31/2021,58960.2,3972.89,1691.05,135.45,53.34
131 | 4/30/2021,56814.44,4181.17,1767.65,138.64,53.98
132 | 5/31/2021,36690.89,4204.11,1899.95,138.44,54.87
133 | 6/30/2021,34585,4297.5,1763.15,144.35,55.15
134 | 7/31/2021,41540.22,4395.26,1825.75,149.52,51.6
135 | 8/27/2021,47129.53,4470,1786.6,148.45,50.96
136 |
--------------------------------------------------------------------------------
/data/cme holidays.csv:
--------------------------------------------------------------------------------
1 | ,DAY,DATE,HOLIDAY
2 | 0,Wed,2020-01-01,New Year's Day
3 | 1,Mon,2020-01-20,M L King Day
4 | 2,Mon,2020-02-17,Presidents' Day
5 | 3,Fri,2020-04-10,Good Friday
6 | 4,Mon,2020-05-25,Memorial Day
7 | 5,Thu,2020-11-26,Thanksgiving Day
8 | 6,Fri,2020-12-25,Christmas
9 | 7,Tue,2019-12-31,New Year's Day
10 | 8,Thu,2020-01-02,New Year's Day
11 | 9,Fri,2020-01-17,M L King Day
12 | 10,Tue,2020-01-21,M L King Day
13 | 11,Fri,2020-02-14,Presidents' Day
14 | 12,Tue,2020-02-18,Presidents' Day
15 | 13,Thu,2020-04-09,Good Friday
16 | 14,Mon,2020-04-13,Good Friday
17 | 15,Fri,2020-05-22,Memorial Day
18 | 16,Tue,2020-05-26,Memorial Day
19 | 17,Wed,2020-11-25,Thanksgiving Day
20 | 18,Fri,2020-11-27,Thanksgiving Day
21 | 19,Thu,2020-12-24,Christmas
22 | 20,Mon,2020-12-28,Christmas
23 | 21,Fri,2021-01-01,New Year's Day
24 | 22,Mon,2021-01-18,M L King Day
25 | 23,Mon,2021-02-15,Presidents' Day
26 | 24,Fri,2021-04-02,Good Friday
27 | 25,Mon,2021-05-31,Memorial Day
28 | 26,Thu,2021-11-25,Thanksgiving Day
29 | 27,Sat,2021-12-25,Christmas
30 | 28,Thu,2020-12-31,New Year's Day
31 | 29,Mon,2021-01-04,New Year's Day
32 | 30,Fri,2021-01-15,M L King Day
33 | 31,Tue,2021-01-19,M L King Day
34 | 32,Fri,2021-02-12,Presidents' Day
35 | 33,Tue,2021-02-16,Presidents' Day
36 | 34,Thu,2021-04-01,Good Friday
37 | 35,Mon,2021-04-05,Good Friday
38 | 36,Fri,2021-05-28,Memorial Day
39 | 37,Tue,2021-06-01,Memorial Day
40 | 38,Wed,2021-11-24,Thanksgiving Day
41 | 39,Fri,2021-11-26,Thanksgiving Day
42 | 40,Fri,2021-12-24,Christmas
43 | 41,Mon,2021-12-27,Christmas
44 | 42,Sat,2022-01-01,New Year's Day
45 | 43,Mon,2022-01-17,M L King Day
46 | 44,Mon,2022-02-21,Presidents' Day
47 | 45,Fri,2022-04-15,Good Friday
48 | 46,Mon,2022-05-30,Memorial Day
49 | 47,Thu,2022-11-24,Thanksgiving Day
50 | 48,Sun,2022-12-25,Christmas
51 | 49,Fri,2021-12-31,New Year's Day
52 | 50,Mon,2022-01-03,New Year's Day
53 | 51,Fri,2022-01-14,M L King Day
54 | 52,Tue,2022-01-18,M L King Day
55 | 53,Fri,2022-02-18,Presidents' Day
56 | 54,Tue,2022-02-22,Presidents' Day
57 | 55,Thu,2022-04-14,Good Friday
58 | 56,Mon,2022-04-18,Good Friday
59 | 57,Fri,2022-05-27,Memorial Day
60 | 58,Tue,2022-05-31,Memorial Day
61 | 59,Wed,2022-11-23,Thanksgiving Day
62 | 60,Fri,2022-11-25,Thanksgiving Day
63 | 61,Fri,2022-12-23,Christmas
64 | 62,Mon,2022-12-26,Christmas
65 |
--------------------------------------------------------------------------------
/data/stoxx50.xlsx:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/data/stoxx50.xlsx
--------------------------------------------------------------------------------
/preview/awesome asset.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/preview/awesome asset.png
--------------------------------------------------------------------------------
/preview/awesome ma.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/preview/awesome ma.png
--------------------------------------------------------------------------------
/preview/awesome oscillator.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/preview/awesome oscillator.png
--------------------------------------------------------------------------------
/preview/awesome positions.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/preview/awesome positions.png
--------------------------------------------------------------------------------
/preview/bollinger bands bottom w pattern.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/preview/bollinger bands bottom w pattern.png
--------------------------------------------------------------------------------
/preview/bollinger bands positions.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/preview/bollinger bands positions.png
--------------------------------------------------------------------------------
/preview/dual thrust positions.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/preview/dual thrust positions.png
--------------------------------------------------------------------------------
/preview/heikin-ashi asset value.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/preview/heikin-ashi asset value.png
--------------------------------------------------------------------------------
/preview/heikin-ashi positions.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/preview/heikin-ashi positions.png
--------------------------------------------------------------------------------
/preview/london breakout positions.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/preview/london breakout positions.png
--------------------------------------------------------------------------------
/preview/london breakout thresholds.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/preview/london breakout thresholds.png
--------------------------------------------------------------------------------
/preview/macd oscillator.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/preview/macd oscillator.png
--------------------------------------------------------------------------------
/preview/macd positions.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/preview/macd positions.png
--------------------------------------------------------------------------------
/preview/options straddle payoff diagram.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/preview/options straddle payoff diagram.png
--------------------------------------------------------------------------------
/preview/pair trading asset.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/preview/pair trading asset.png
--------------------------------------------------------------------------------
/preview/pair trading eg two step.PNG:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/preview/pair trading eg two step.PNG
--------------------------------------------------------------------------------
/preview/pair trading positions.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/preview/pair trading positions.png
--------------------------------------------------------------------------------
/preview/parabolic sar positions.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/preview/parabolic sar positions.png
--------------------------------------------------------------------------------
/preview/quantitative-trading-profile.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/preview/quantitative-trading-profile.png
--------------------------------------------------------------------------------
/preview/rsi head-shoulder pattern.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/preview/rsi head-shoulder pattern.png
--------------------------------------------------------------------------------
/preview/rsi oscillator.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/preview/rsi oscillator.png
--------------------------------------------------------------------------------
/preview/rsi pattern oscillator.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/preview/rsi pattern oscillator.png
--------------------------------------------------------------------------------
/preview/rsi pattern positions.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/preview/rsi pattern positions.png
--------------------------------------------------------------------------------
/preview/rsi positions.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/preview/rsi positions.png
--------------------------------------------------------------------------------
/preview/shooting star positions.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/preview/shooting star positions.png
--------------------------------------------------------------------------------
/preview/vix calculator.PNG:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/je-suis-tm/quant-trading/611b73f2c3f577ac5b28aaa19ac8c43d3236c7a5/preview/vix calculator.PNG
--------------------------------------------------------------------------------