├── Brain.py
├── Calculator.py
├── DataManager.py
├── Downloader.py
├── Market.py
├── Monitor.py
├── Portfolio.py
├── README.md
├── Simulator.py
├── Trader.py
├── folio.py
├── stocks-and-bonds
├── stocks-and-bonds-timing
├── stocks-only
├── upro-only
└── utils.py


/Brain.py:
--------------------------------------------------------------------------------
  1 | import operator as op
  2 | 
  3 | 
  4 | class Brain(object):
  5 | 
  6 |     """A Brain for a Trader. Decisions regarding Portfolio allocations
  7 |     are made here.
  8 | 
  9 |     In other words, a Trader will 'use' their Brain to decide how much
 10 |     of each asset they should have in their Portfolio
 11 | 
 12 |     Attributes:
 13 |     """
 14 | 
 15 |     def __init__(self):
 16 |         """Initializes an empty Brain."""
 17 |         self._market = None
 18 |         self._portfolio = None
 19 |         self.assets_of_interest = set({})
 20 |         self.positions = []
 21 |         self.assets_to_trade = set({})
 22 |         self.rebalancing_period = None
 23 |         self.strategy_type = None
 24 |         self.desired_ratios = {}
 25 |         self.desired_shares = {}
 26 | 
 27 |     def use_market(self, market):
 28 |         """Sets the Market this Brain should use to make decisions."""
 29 |         self._market = market
 30 | 
 31 |     def use_portfolio(self, portfolio):
 32 |         """Sets the Portfolio this Brain should use to make decisions."""
 33 |         self._portfolio = portfolio
 34 | 
 35 |     def set_strategy(self, strategy):
 36 |         """NEEDS DESC WHEN MORE IMPLEMENTED
 37 | 
 38 |         Args:
 39 |             strategy: A keyword specifying which strategy to use."""
 40 |         self.positions = strategy
 41 | 
 42 |     def set_rebalancing_period(self, period):
 43 |         """Sets the rebalancing frequency. The Brain will adjust the
 44 |         shares of all assets at each period to match ratios.
 45 | 
 46 |         Args:
 47 |             period: A value representing the frequency of rebalancing
 48 |                 e.g. 'm' for monthly
 49 |         """
 50 |         self.rebalancing_period = period
 51 | 
 52 |     def decide_needed_shares(self):
 53 |         """Calculates the amount of shares needed of each desired
 54 |         asset."""
 55 |         self.decide_asset_ratios()
 56 |         for asset in list(self.assets_to_trade):
 57 |             self.desired_shares[asset] = int(self._portfolio.value()
 58 |                                              * self.desired_ratios[asset]
 59 |                                              / self._market.query_stock(asset))
 60 |             self.assets_to_trade.remove(asset)
 61 | 
 62 |     def decide_asset_ratios(self):
 63 |         """Calculates the ratios for each desired asset.
 64 | 
 65 |         Specifically, checks buy and sell signals and updates which
 66 |         assets should be traded. Also handles trading all assets if a
 67 |         rebalance is needed."""
 68 |         for position in self.positions:
 69 |             if (position['is_holding']
 70 |                     and self._check_signal(position['sell_signal'])):
 71 |                 position['is_holding'] = False
 72 |                 self.assets_to_trade.add(position['ticker'])
 73 |                 try:
 74 |                     self.desired_ratios[position['ticker']] \
 75 |                         -= position['ratio']
 76 |                 except KeyError:
 77 |                     print("ERR: reducing ratio for {}".format(
 78 |                         position['ticker']))
 79 |                     self.desired_ratios[position['ticker']] = 0
 80 |             elif (not position['is_holding']
 81 |                   and self._check_signal(position['buy_signal'])):
 82 |                 position['is_holding'] = True
 83 |                 self.assets_to_trade.add(position['ticker'])
 84 |                 try:
 85 |                     self.desired_ratios[position['ticker']] \
 86 |                         += position['ratio']
 87 |                 except KeyError:
 88 |                     self.desired_ratios[position['ticker']] = position['ratio']
 89 |             elif (self.rebalancing_period
 90 |                   and self._market.new_period[self.rebalancing_period]):
 91 |                 self.assets_to_trade.add(position['ticker'])
 92 | 
 93 |     def _check_signal(self, signal_code):
 94 |         """Given a (buy or sell) signal code, interprets it and checks
 95 |         whether or not the signal is satisfied by the market.
 96 | 
 97 |         Currently, buy and sell signals come in 'VALUE COMPARE VALUE'
 98 |         format, where COMPARE is an operator value and VALUE is a code
 99 |         for a value in the market (e.g. 'UPRO~SMA_200' for UPRO's
100 |         Standard Moving Average 200 value).
101 | 
102 |         Args:
103 |             signal_code: A code for the buy or sell signal
104 | 
105 |         Returns:
106 |             A value for whether or not the signal is satisfied
107 |         """
108 |         if signal_code == 'ALWAYS':
109 |             return True
110 |         if signal_code == 'NEVER':
111 |             return False
112 |         compare_using = {
113 |             '>': op.gt,
114 |             '<': op.lt,
115 |             '=': op.eq
116 |         }
117 |         (value_a_code, operator, value_b_code) = signal_code.split(' ')
118 |         return compare_using[operator](self._decode_and_get_value(value_a_code),
119 |                                        self._decode_and_get_value(value_b_code))
120 | 
121 |     def _decode_and_get_value(self, value_code):
122 |         """Decodes a value code and returns the appropriate value.
123 | 
124 |         Args:
125 |             value_code: A code for a value in the Market
126 | 
127 |         Returns:
128 |             A value corresponding to the coding
129 |         """
130 |         (ticker, indicator_code) = value_code.split('~')
131 |         if indicator_code == 'PRICE':
132 |             return self._market.query_stock(ticker)
133 |         return self._market.query_stock_indicator(ticker, indicator_code)
134 | 


--------------------------------------------------------------------------------
/Calculator.py:
--------------------------------------------------------------------------------
  1 | import time
  2 | 
  3 | from utils import SteppedAvgLookup
  4 | from DataManager import DataManager
  5 | 
  6 | 
  7 | class Calculator(object):
  8 | 
  9 |     """A Calculator specifically designed to do stock market related
 10 |     calculations on stock data.
 11 | 
 12 |     For example, indicator data series can be calculated here for
 13 |     charting purposes.
 14 | 
 15 |     NOTE: To add an indicator, write two getter methods, one for a
 16 |         dictionary and another for a series, named
 17 |         'get_<indicator-name>' and 'get_<indicator-name>_series'. Then,
 18 |         in the get_indicator method, add the two methods to the correct
 19 |         mapping. Currently indicator getter functions need at least one
 20 |         argument, even if a None will be passed.
 21 | 
 22 |     Currently supports:
 23 |         - Standard Moving Average for a given period
 24 |         - Exponential Moving Average for a given period
 25 |         - Moving Average Convergence/Divergence for a given set of
 26 |             periods
 27 |         - generating theoretical ETF data
 28 |         - Previous High (i.e. the highest price the stock has been,
 29 |             including the current day)
 30 |     """
 31 | 
 32 |     def __init__(self):
 33 |         """Initializes a Calculator."""
 34 | 
 35 |     def get_indicator(self, indicator_code, price_lut, series=False):
 36 |         """A mapping function for indicator functions. Primarily used
 37 |         for cases where indicators are dynamic and hardcoding functions
 38 |         is impractical.
 39 | 
 40 |         Args:
 41 |             indicator_code: A string coding the indicator and period
 42 |             price_lut: A price lookup table for the data on which the
 43 |                 indicator should be applied
 44 |             series: A value for whether or not to map to a series
 45 |                 indicator function
 46 | 
 47 |         Returns:
 48 |             A dictionary mapping dates to indicator values
 49 |         """
 50 |         # decode
 51 |         code_parts = indicator_code.split('_')
 52 |         indicator = code_parts[0]
 53 |         if len(code_parts) == 1:
 54 |             period = None
 55 |         else:
 56 |             period = code_parts[1].split('-')
 57 |             if len(period) == 1:
 58 |                 period = period[0]
 59 |         # create mapping to methods
 60 |         if series:
 61 |             mapping = {
 62 |                 'SMA': self.get_sma_series,
 63 |                 'EMA': self.get_ema_series,
 64 |                 'MACD': self.get_macd_series,
 65 |                 'MACDSIGNAL': self.get_macd_series,
 66 |                 'PREVHIGH': self.get_prev_high_series
 67 |             }
 68 |         else:
 69 |             mapping = {
 70 |                 'SMA': self.get_sma,
 71 |                 'EMA': self.get_ema,
 72 |                 'MACD': self.get_macd,
 73 |                 'MACDSIGNAL': self.get_macd_signal,
 74 |                 'PREVHIGH': self.get_prev_high
 75 |             }
 76 |         # call correct method
 77 |         return mapping[indicator](period, price_lut)
 78 | 
 79 |     def get_sma(self, period, price_lut):
 80 |         """Calculates the Standard Moving Average for a given period
 81 |         and returns a dictionary of SMA values.
 82 | 
 83 |         Args:
 84 |             period: A value representing a number of days
 85 |             price_lut: A price LUT, i.e. a dictionary mapping dates to
 86 |                 prices
 87 | 
 88 |         Returns:
 89 |             A dictionary with dates mapping to SMA values
 90 |         """
 91 |         sma = {}
 92 |         period = int(period)
 93 |         dates = sorted(price_lut.keys())
 94 |         prices = []  # keep track of all prices as we go, for performance
 95 |         for i, date in enumerate(dates):
 96 |             prices.append(price_lut[date])
 97 |             if i < period:
 98 |                 sma[date] = sum(prices) / len(prices)
 99 |             else:
100 |                 sma[date] = sum(prices[-period:]) / period
101 |         return sma
102 | 
103 |     def get_sma_series(self, period, price_lut):
104 |         """Calculates the Standard Moving Average for a given period
105 |         and returns a list of SMA values.
106 | 
107 |         Args:
108 |             period: A value representing a number of days
109 |             price_lut: A price LUT, i.e. a dictionary mapping dates to
110 |                 prices
111 | 
112 |         Returns:
113 |             A list with SMA values corresponding to ordered dates in
114 |             the provided price LUT
115 |         """
116 |         sma = []
117 |         period = int(period)
118 |         dates = sorted(price_lut.keys())
119 |         prices = []  # keep track of all prices as we go, for performance
120 |         for i, date in enumerate(dates):
121 |             prices.append(price_lut[date])
122 |             if i < period:
123 |                 sma.append(sum(prices) / len(prices))
124 |             else:
125 |                 sma.append(sum(prices[-period:]) / period)
126 |         return sma
127 | 
128 |     def get_ema(self, period, price_lut):
129 |         """Calculates the Exponential Moving Average for a given
130 |         period and returns a dictionary of EMA values.
131 | 
132 |         Args:
133 |             period: A value representing a number of days
134 |             price_lut: A price LUT, i.e. a dictionary mapping dates to
135 |                 prices
136 | 
137 |         Returns:
138 |             A dictionary with dates mapping to EMA values
139 |         """
140 |         ema = {}
141 |         period = int(period)
142 |         dates = sorted(price_lut.keys())
143 |         ema = self.get_sma(period, {d: price_lut[d] for d in dates[0:period]})
144 |         multiplier = 2 / (period + 1)  # used in EMA calulations
145 |         for i, date in enumerate(dates[period:]):
146 |             ema[date] = (float(price_lut[date]) * multiplier
147 |                          + ema[dates[period + i - 1]] * (1 - multiplier))
148 |         return ema
149 | 
150 |     def get_ema_series(self, period, price_lut):
151 |         """Calculates the Exponential Moving Average for a given
152 |         period and returns a list of EMA values.
153 | 
154 |         Args:
155 |             period: A value representing a number of days
156 |             price_lut: A price LUT, i.e. a dictionary mapping dates to
157 |                 prices
158 | 
159 |         Returns:
160 |             A list with EMA values corresponding to the ordered dates
161 |             in the provided price LUT
162 |         """
163 |         ema = []
164 |         period = int(period)
165 |         dates = sorted(price_lut.keys())
166 |         ema = self.get_sma_series(period,
167 |                                   {d: price_lut[d] for d in dates[0:period]})
168 |         multiplier = 2 / (period + 1)  # used in EMA calulations
169 |         for i, date in enumerate(dates[period:]):
170 |             ema.append(float(price_lut[date]) * multiplier
171 |                        + ema[-1] * (1 - multiplier))
172 |         return ema
173 | 
174 |     def get_macd(self, periods, price_lut):
175 |         """Calculates the Moving Average Convergence/Divergence for a
176 |         given period and returns a dictionary mapping dates to MACD.
177 | 
178 |         Args:
179 |             period: A set of values representing the days for each
180 |                 MACD period, i.e. [short, long, exponential/signal]
181 |             price_lut: A set of values on which to perform the MACD
182 |                 calculations
183 | 
184 |         Returns:
185 |             A dictionary mapping dates to MACD values
186 |         """
187 |         # ret = {}
188 |         macd = {}
189 |         # signal = {}
190 |         # histogram = {}
191 |         dates = sorted(price_lut.keys())
192 |         macd_short = self.get_ema(periods[0], price_lut)
193 |         macd_long = self.get_ema(periods[1], price_lut)
194 |         # calculate MACD first - needed for signal and histogram
195 |         for date in dates:
196 |             macd[date] = macd_short[date] - macd_long[date]
197 |         return macd
198 |         # calculate signal - needed for histogram
199 |         # signal = self.get_ema(periods[2], macd)
200 |         # # calculate histogram
201 |         # for date in dates:
202 |         #     histogram[date] = macd[date] - signal[date]
203 |         # # now convert everything to return format
204 |         # for date in dates:
205 |         #     ret[date] = [macd[date], signal[date], histogram[date]]
206 |         # return ret
207 | 
208 |     def get_macd_signal(self, periods, price_lut):
209 |         """Calculates the signal line for the Moving Average
210 |         Convergence/Divergence for a given set of periods and returns a
211 |         dictionary mapping dates to signal line values.
212 | 
213 |         Args:
214 |             period: A set of values representing the days for each
215 |                 MACD period, i.e. [short, long, exponential/signal]
216 |             price_lut: A set of values on which to perform the MACD
217 |                 calculations
218 | 
219 |         Returns:
220 |             A dictionary mapping dates to MACD signal values
221 |         """
222 |         macd = {}
223 |         dates = sorted(price_lut.keys())
224 |         macd_short = self.get_ema(periods[0], price_lut)
225 |         macd_long = self.get_ema(periods[1], price_lut)
226 |         # calculate MACD first - needed for signal
227 |         for date in dates:
228 |             macd[date] = macd_short[date] - macd_long[date]
229 |         return self.get_ema(periods[2], macd)
230 | 
231 |     def get_macd_series(self, periods, price_lut):
232 |         """Calculates the Moving Average Convergence/Divergence for a
233 |         given period and returns lists for MACD, signal, and histogram.
234 | 
235 |         Args:
236 |             period: A set of values representing the days for each
237 |                 MACD period
238 |             price_lut: A set of values on which to perform the MACD
239 |                 calculations
240 | 
241 |         Returns:
242 |             A set of sets of values for the MACD, signal line, and MACD
243 |             histogram at each point for the given values, i.e. a set in
244 |             the form [[MACD], [signal line], [MACD histogram]]
245 |         """
246 |         macd = {}
247 |         signal = []
248 |         histogram = []
249 |         dates = sorted(price_lut.keys())
250 |         macd_short = self.get_ema(periods[0], price_lut)
251 |         macd_long = self.get_ema(periods[1], price_lut)
252 |         # calculate MACD first - needed for signal and histogram
253 |         for date in dates:
254 |             macd[date] = macd_short[date] - macd_long[date]
255 |         # calculate signal - needed for histogram
256 |         signal = self.get_ema_series(periods[2], macd)
257 |         # calculate histogram
258 |         for i, date in enumerate(dates):
259 |             histogram.append(macd[date] - signal[i])
260 |         return [[macd[d] for d in dates], signal, histogram]
261 | 
262 |     def get_prev_high(self, period, price_lut):
263 |         """Calculates the previous high value for every point in the
264 |         given LUT.
265 | 
266 |         Args:
267 |             period: A placeholder, just pass None for now
268 |             price_lut: A set f values on which to perform the previous
269 |                 high calculation
270 |         Returns:
271 |             A dictionary of dates mapping to values
272 |         """
273 |         prev_high = {}
274 |         dates = sorted(price_lut.keys())
275 |         prev_high[dates[0]] = price_lut[dates[0]]
276 |         for i, date in enumerate(dates[1:]):
277 |             prev_high[date] = max(price_lut[date], prev_high[dates[i]])
278 |         return prev_high
279 | 
280 |     def get_prev_high_series(self, period, price_lut):
281 |         """Calculates the previous high value for every point in the
282 |         given LUT.
283 | 
284 |         Args:
285 |             period: A placeholder, just pass None for now
286 |             price_lut: A set f values on which to perform the previous
287 |                 high calculation
288 |         Returns:
289 |             A dictionary of dates mapping to values
290 |         """
291 |         prev_high = []
292 |         dates = sorted(price_lut.keys())
293 |         prev_high.append(price_lut[dates[0]])
294 |         for date in dates[1:]:
295 |             prev_high.append(max(price_lut[date], prev_high[-1]))
296 |         return prev_high
297 | 
298 |     def generate_theoretical_data(self, ticker_tgt, ticker_src,
299 |                                   step=0.00005, pos_adj=None, neg_adj=None):
300 |         """Generates theoretical data for a stock based on another
301 |         stock.
302 | 
303 |         Given two tickers, a granularity/precision step, and manual
304 |         offset/adjustments, generates more data for the first stock
305 |         (gen) to match the length of data in the second stock (src).
306 |         The generation is based on averages in existing real data and
307 |         assumes an existing correlation between two stocks (e.g. UPRO
308 |         and SPY supposedly have a correlation, or leverage factor of 3)
309 | 
310 |         Args:
311 |             ticker_tgt: A ticker of the stock for which data should be
312 |                 generated, i.e. the target for the generation
313 |             ticker_src: A ticker of the stock to be used as the data
314 |                 source to aid in data generation.
315 |                 NOTE: This implies the source data should be longer
316 |                 than the data for the stock for which the generation
317 |                 occurs
318 |             step: A value corresponding to a level of precision, or the
319 |                 number of averages calculated and then used to generate
320 |                 the data. NOTE: precision != accuracy and a default
321 |                 value of 0.00005 is used if one is not given, based on
322 |                 testing done on different values
323 |             pos_adj: A value to be used when adjusting movements in the
324 |                 positive direction, i.e. a higher value will lead to
325 |                 more pronounced positive moves (default: None, if None
326 |                 a hardcoded default value will be used depending on
327 |                 the ticker, typically 0)
328 |             neg_adj: A value to be used when adjusting movements in the
329 |                 negative direction, i.e. a higher value will lead to
330 |                 more pronounced negative moves (default: None, if None
331 |                 a hardcoded default value will be used depending on
332 |                 the ticker, typically 0)
333 | 
334 |         Returns:
335 |             A tuple of price LUTs, one LUT containing real data
336 |             appended to a part of the generated data, the other
337 |             containing a full set of generated data. The former is
338 |             intended to be used in backtesting strategies, while the
339 |             latter is intended to be used for verifying generation
340 |             accuracy against existing real data.
341 |         """
342 |         db = DataManager()
343 |         # get prices for tickers
344 |         price_lut_tgt = db.build_price_lut(ticker_tgt)
345 |         price_lut_src = db.build_price_lut(ticker_src)
346 |         # before doing any calculations, check if all data is on disk already
347 |         # NOTE: feature disabled for now, as it didnt respond to changes
348 |         # price_lut_gen_part = db.build_price_lut(ticker_tgt + '--GEN-PART')
349 |         # price_lut_gen_full = db.build_price_lut(ticker_tgt + '--GEN-FULL')
350 |         # if (len(price_lut_gen_part) == len(price_lut_src)
351 |         #         and len(price_lut_gen_full) == len(price_lut_src)):
352 |         #     return (price_lut_gen_part, price_lut_gen_full)
353 |         # sorted dates needed later
354 |         src_dates = sorted(price_lut_src.keys())
355 |         gen_dates = sorted(price_lut_tgt.keys())
356 |         # part of data will be real data
357 |         price_lut_gen_part = price_lut_tgt.copy()
358 |         # fully generated data needs a real point as an anchor
359 |         price_lut_gen_full = {gen_dates[0]: price_lut_tgt[gen_dates[0]]}
360 |         # a set of adjustments to use if not otherwise specified
361 |         adjustments = {
362 |             'UPRO': (0, 0),
363 |             'TMF': (0.01, 0.05),
364 |             'TQQQ': (0.025, 0),
365 |             'UDOW': (0, 0.01)
366 |         }
367 |         if step == 0.00005 and pos_adj is None and neg_adj is None:
368 |             try:
369 |                 pos_adj = adjustments[ticker_tgt.upper()][0]
370 |                 neg_adj = adjustments[ticker_tgt.upper()][1]
371 |             except KeyError:
372 |                 pos_adj = 0
373 |                 neg_adj = 0
374 |         # calculate % movements and leverage ratio, to use for the SA-LUT
375 |         moves = {}
376 |         ratios = {}
377 |         for i in range(len(gen_dates) - 1):
378 |             change_src = (price_lut_src[gen_dates[i + 1]]
379 |                           / price_lut_src[gen_dates[i]]
380 |                           - 1)
381 |             change_gen = (price_lut_tgt[gen_dates[i + 1]]
382 |                           / price_lut_tgt[gen_dates[i]]
383 |                           - 1)
384 |             moves[gen_dates[i + 1]] = change_src
385 |             if change_src == 0:
386 |                 ratios[gen_dates[i + 1]] = 0.0
387 |             else:
388 |                 ratios[gen_dates[i + 1]] = change_gen / change_src
389 |         sa_lut = SteppedAvgLookup(step,
390 |                                   [moves[d] for d in gen_dates[1:]],
391 |                                   [ratios[d] for d in gen_dates[1:]])
392 |         # generate data going forward from gen data's anchor point
393 |         for i in range(len(gen_dates) - 1):
394 |             move = moves[gen_dates[i + 1]]
395 |             if move >= 0:
396 |                 adj = pos_adj
397 |             else:
398 |                 adj = neg_adj
399 |             price_lut_gen_full[gen_dates[i + 1]] = \
400 |                 (price_lut_gen_full[gen_dates[i]]
401 |                  * (move * (sa_lut.get(move) + adj) + 1))
402 |         # generate data going backwards from gen data's anchor point
403 |         for i in range(len(src_dates) - len(gen_dates) - 1, -1, -1):
404 |             move = (price_lut_src[src_dates[i + 1]]
405 |                     / price_lut_src[src_dates[i]]
406 |                     - 1)
407 |             if move >= 0:
408 |                 adj = pos_adj
409 |             else:
410 |                 adj = neg_adj
411 |             gen_price = (price_lut_gen_full[src_dates[i + 1]]
412 |                          / (move * (sa_lut.get(move) + adj) + 1))
413 |             price_lut_gen_full[src_dates[i]] = gen_price
414 |             price_lut_gen_part[src_dates[i]] = gen_price
415 |         # save data to disk for faster retrieval next time
416 |         db.write_stock_data(ticker_tgt + '--GEN-FULL',
417 |                             [[date,
418 |                               '-',
419 |                               '-',
420 |                               '-',
421 |                               str(price_lut_gen_full[date]),
422 |                               '-'] for date in src_dates],
423 |                             False)
424 |         db.write_stock_data(ticker_tgt + '--GEN-PART',
425 |                             [[date,
426 |                               '-',
427 |                               '-',
428 |                               '-',
429 |                               str(price_lut_gen_part[date]),
430 |                               '-'] for date in src_dates],
431 |                             False)
432 |         return (price_lut_gen_part, price_lut_gen_full)
433 | 


--------------------------------------------------------------------------------
/DataManager.py:
--------------------------------------------------------------------------------
  1 | import errno
  2 | import os
  3 | import os.path
  4 | import datetime
  5 | 
  6 | 
  7 | class DataManager(object):
  8 | 
  9 |     DATE_FORMAT = '%Y-%m-%d'
 10 | 
 11 |     """A DataManager is responsible for managing (i.e. storing and
 12 |     retrieving) data on disk.
 13 | 
 14 |     In other words, this class provides an interface by which stock
 15 |     data can be queried from or stored onto disk. With regards to
 16 |     functions found in this class, 'public' functions house high level
 17 |     logic for publicly used actions, while 'private' functions act as
 18 |     wrappers for low level or commonly used and simple, but ugly to
 19 |     write actions.
 20 | 
 21 |     Attributes:
 22 |         data_location: A string indicating where the stock data is
 23 |             stored on disk
 24 | 
 25 |     Todo:
 26 |         - [code improvement, low priority] create independent market
 27 |             open reference dates
 28 |         - [code improvement, low priority] implement reading csv rows
 29 |             to return map
 30 |         - [code improvement, low priority] implement reading csv
 31 |             columns to return map
 32 |     """
 33 | 
 34 |     def __init__(self, data_location='data/'):
 35 |         """Inits DataManager with a data location.
 36 | 
 37 |         Args:
 38 |             data_location: (optional) A string representing where the
 39 |                 data dir will be on disk, default: ./data/
 40 |         """
 41 |         self.data_location = data_location
 42 |         os.makedirs(self.data_location, exist_ok=True)
 43 | 
 44 |     def write_stock_data(self, ticker, data, append):
 45 |         """Writes an array of data to a file on disk.
 46 | 
 47 |         Args:
 48 |             ticker: A string representing the ticker of a stock
 49 |             data: An array in [[date,open,high,low,close,volume],...]
 50 |                 format
 51 |             append: A boolean representing whether or not to append to
 52 |                 existing data
 53 |         """
 54 |         data_to_write = []
 55 |         if append:
 56 |             mode = 'a'
 57 |             existing_data = self.read_stock_data(ticker, 'row')
 58 |             if len(existing_data) == 0:
 59 |                 data_to_write = data
 60 |             elif (existing_data[-1][0] < data[-1][0] and
 61 |                   existing_data[-1][0] > data[0][0]):
 62 |                 index_of_last = next(i for i, _ in enumerate(data)
 63 |                                      if _[0] == existing_data[-1][0])
 64 |                 data_to_write = data[index_of_last + 1:]
 65 |         else:
 66 |             mode = 'w'
 67 |             data_to_write = data
 68 |             if self._has_file_for(ticker):
 69 |                 self._remove_file_for(ticker)
 70 |         self._write_data_to_csv_file(
 71 |             self._filename_for(ticker), data_to_write, mode)
 72 | 
 73 |     def read_stock_data(self, ticker, format):
 74 |         """Retrieves stock data for a given ticker in a given format
 75 |         from disk.
 76 | 
 77 |         Args:
 78 |             ticker: A string representing the ticker of a stock
 79 |             format: A string representing whether the data should be in
 80 |                 'column' or 'row' format
 81 | 
 82 |         Returns:
 83 |             An array in either row or column format contaning the data
 84 |                 for a given stock
 85 |         """
 86 |         if format == 'column':
 87 |             return self._read_csv_file_columns_for(ticker)
 88 |         if format == 'row':
 89 |             return self._read_csv_file_rows_for(ticker)
 90 |         return []
 91 | 
 92 |     def build_price_lut(self, ticker, fill=True):
 93 |         """Builds a price look up table for a given ticker.
 94 | 
 95 |         Args:
 96 |             ticker: A string representing the ticker of a stock
 97 |             fill: Whether or not to fill holidays/weekends with
 98 |                 previous data
 99 |                 NOTE: experimental feature which made some slightly
100 |                 unexpected numbers come up - turned off for now
101 | 
102 |         Returns:
103 |             A dictionary with dates as keys and prices as values
104 |         """
105 |         price_lookup = {}
106 |         file_content = self._readlines_for(ticker)
107 |         # handle corner cases with empty or single-line files
108 |         if len(file_content) == 0:
109 |             return price_lookup
110 |         if len(file_content) == 1:
111 |             line_data = file_content[i].split(',')
112 |             return {line_data[0]: float(line_data[4])}
113 |         # handle multi-line files & fill in holes with previous data
114 |         for i in range(0, len(file_content) - 1):
115 |             curr_line_data = file_content[i].split(',')
116 |             next_line_data = file_content[i + 1].split(',')
117 |             curr_date = datetime.datetime.strptime(
118 |                 curr_line_data[0], DataManager.DATE_FORMAT)
119 |             next_date = datetime.datetime.strptime(
120 |                 next_line_data[0], DataManager.DATE_FORMAT)
121 |             while curr_date < next_date:
122 |                 price_lookup[curr_date.strftime(DataManager.DATE_FORMAT)] \
123 |                     = float(curr_line_data[4])
124 |                 if fill:
125 |                     curr_date = curr_date + datetime.timedelta(1)
126 |                 else:
127 |                     curr_date = next_date
128 |         # handle last line in file separately
129 |         price_lookup[next_date.strftime(
130 |             DataManager.DATE_FORMAT)] = float(next_line_data[4])
131 |         return price_lookup
132 | 
133 |     def build_strategy(self, strategy_name, strategy_dir='./'):
134 |         """Given a strategy name (the name of the file within which
135 |         the strategy is coded) and builds the data structure for Brain
136 |         to use, then returns the structure along with all assets and
137 |         indicators needed in the Market.
138 | 
139 |         Args:
140 |             strategy_name: A name for the strategy to use - corresponds
141 |                 to a file in the strategies dir
142 |             strategy_dir: An optional value containing a custom
143 |                 location for strategies (default: ./)
144 | 
145 |         Returns:
146 |             A tuple containing the strategy structure, a set of assets
147 |             to add to the Market, and a set of indicators to add to the
148 |             Market
149 |         """
150 |         lines = self._readlines(strategy_dir + strategy_name)
151 |         stocks_needed = set({})
152 |         indicators_needed = set({})
153 |         strategy = {
154 |             'assets': set({}),
155 |             'positions': []
156 |         }
157 |         for line in lines:
158 |             (ratio, ticker, buy_signal, sell_signal) = line.split(',')
159 |             strategy['assets'].add(ticker.upper())
160 |             stocks_needed.add(ticker.upper())
161 |             for signal in [buy_signal, sell_signal]:
162 |                 (tickers, indicators) = self._parse_signal(signal)
163 |                 stocks_needed |= tickers
164 |                 indicators_needed |= indicators
165 |             strategy['positions'].append({
166 |                 'is_holding': False,
167 |                 'ratio': float(ratio),
168 |                 'ticker': ticker.upper(),
169 |                 'buy_signal': buy_signal,
170 |                 'sell_signal': sell_signal
171 |             })
172 |         return (strategy, stocks_needed, indicators_needed)
173 | 
174 |     def _parse_signal(self, signal_code):
175 |         """Parses a buy or sell signal and extracts any tickers and
176 |         indicators from it.
177 | 
178 |         Args:
179 |             signal_code: A code for a buy or sell signal
180 | 
181 |         Returns:
182 |             A tuple containing a set of tickers and a set of indicators
183 |         """
184 |         if signal_code in ['ALWAYS', 'NEVER']:
185 |             return (set({}), set({}))
186 |         tickers = set({})
187 |         indicators = set({})
188 |         (val_a_code, _, val_b_code) = signal_code.split(' ')
189 |         for code in [val_a_code, val_b_code]:
190 |             (ticker, indicator) = code.split('~')
191 |             tickers.add(ticker.upper())
192 |             if indicator not in ['PRICE']:
193 |                 indicators.add(indicator.upper())
194 |         return (tickers, indicators)
195 | 
196 |     def _write_data_to_csv_file(self, filename, data, mode):
197 |         """Writes an array of data to disk in CSV format.
198 | 
199 |         Args:
200 |             filename: A string representing the file to which to write
201 |             data: An array in [[date,open,high,low,close,volume],...]
202 |                 format
203 |         """
204 |         with open(filename, mode) as file:
205 |             for line in data:
206 |                 file.write(','.join(line) + '\n')
207 | 
208 |     def _filename_for(self, ticker):
209 |         """Returns the file name for a ticker, including the path to
210 |         said file.
211 | 
212 |         Args:
213 |             ticker: A string representing the ticker of a stock
214 | 
215 |         Returns:
216 |             A String representing the filename, inluding path, for the
217 |             given ticker
218 |         """
219 |         return self.data_location + ticker.upper() + ".csv"
220 | 
221 |     def _readlines(self, filename):
222 |         """Returns the lines of the file for a given ticker.
223 | 
224 |         Args:
225 |             filename: A string representing the name of a file
226 | 
227 |         Returns:
228 |             An array with each element containing a line of the file
229 |         """
230 |         lines = []
231 |         if self._has_file(filename):
232 |             with open(filename, 'r') as file:
233 |                 lines = [line.strip() for line in file]
234 |         return lines
235 | 
236 |     def _readlines_for(self, ticker):
237 |         """Returns the lines of the file for a given ticker.
238 | 
239 |         Args:
240 |             ticker: A string representing the ticker of a stock
241 | 
242 |         Returns:
243 |             An array with each element containing a line of the file
244 |             for the given ticker
245 |         """
246 |         lines = []
247 |         if self._has_file_for(ticker):
248 |             with open(self._filename_for(ticker), 'r') as file:
249 |                 lines = [line.strip() for line in file]
250 |         return lines
251 | 
252 |     def _has_file(self, filename):
253 |         """Returns whether a file exists.
254 | 
255 |         Args:
256 |             filename: A string representing the filename
257 | 
258 |         Returns:
259 |             A boolean value representing whether or not the file exists
260 |         """
261 |         return os.path.isfile(filename)
262 | 
263 |     def _has_file_for(self, ticker):
264 |         """Returns whether a file for a given ticker exists.
265 | 
266 |         Args:
267 |             ticker: A string representing the ticker of a stock
268 | 
269 |         Returns:
270 |             A boolean value representing whether or not a file exists
271 |             for a given ticker
272 |         """
273 |         return os.path.isfile(self._filename_for(ticker))
274 | 
275 |     def _remove_file_for(self, ticker):
276 |         """Removes the file for the given ticker.
277 | 
278 |         Args:
279 |             ticker: A string representing the ticker of a stock
280 |         """
281 |         os.remove(self._filename_for(ticker))
282 | 
283 |     def _read_csv_file_rows_for(self, ticker):
284 |         """Reads and returns the data in a CSV file for a given ticker
285 |         in row-by-row format.
286 | 
287 |         Args:
288 |             ticker: A string representing the ticker of a stock
289 | 
290 |         Returns:
291 |             An array, where each element is an array containing data
292 |             for a row in a CSV file
293 |         """
294 |         data = []
295 |         file_content = self._readlines_for(ticker)
296 |         for line in file_content:
297 |             data.append([value.strip() for value in line.split(',')])
298 |         return data
299 | 
300 |     def _read_csv_file_columns_for(self, ticker):
301 |         """Reads and returns the data in a CSV file for a given ticker
302 |         in column-by-column format.
303 | 
304 |         Args:
305 |             ticker: A string representing the ticker of a stock
306 | 
307 |         Returns:
308 |             An array, where each element is an array containing data
309 |             for a column in a CSV file
310 |         """
311 |         data = []
312 |         file_content = self._readlines_for(ticker)
313 |         # create arrays for each column
314 |         for i in range(0, 6):
315 |             data.append([])
316 |         # iterate through file
317 |         for line in file_content:
318 |             values = line.split(',')
319 |             for i in range(0, 6):
320 |                 data[i].append(values[i].strip())
321 |         return data
322 | 


--------------------------------------------------------------------------------
/Downloader.py:
--------------------------------------------------------------------------------
  1 | import urllib.request
  2 | import os
  3 | import os.path
  4 | import datetime
  5 | import argparse
  6 | from DataManager import DataManager
  7 | 
  8 | 
  9 | class Downloader(object):
 10 | 
 11 |     """A Downloader that handles downloading stock data and returning
 12 |     it in a consitent format.
 13 | 
 14 |     Currently only supports downloading from Google Finance API.
 15 |     Stores data in CSV format with Date,Open,High,Low,Close,Volume
 16 |     columns.
 17 | 
 18 |     Attributes:
 19 |         sources: mapping of source string to handler method
 20 | 
 21 |     Todo:
 22 |         - [code improvement, low priority] implement yahoo downloading
 23 |             as backup
 24 |         - [code improvement, low priority] implement only downloading
 25 |             missing data, rather than all
 26 |         - [code improvement, low priority] implement way to verify
 27 |             data, and downloading holes from 2nd source
 28 |     """
 29 | 
 30 |     def __init__(self):
 31 |         """Initializes a Downlaoder used for downloading stock data."""
 32 |         self.sources = {
 33 |             'yahoo':  self._download_using_yahoo,
 34 |             'google': self._download_using_google
 35 |         }
 36 | 
 37 |     def download(self, ticker, preferred_source, quiet=True):
 38 |         """Downloads and returns stock data for a ticker from a source.
 39 | 
 40 |         A simple routing method that uses a (source -> download method)
 41 |         mapping.
 42 | 
 43 |         Args:
 44 |             ticker: A string representing the ticker of a stock
 45 |             preferred_source: A string representing a download source
 46 |             quiet: A boolean for whether or not to print progress
 47 |                 feedback, default: True
 48 | 
 49 |         Returns:
 50 |             An array containing the stock data in chronological order
 51 |         """
 52 |         return self.sources[preferred_source](ticker, quiet)
 53 | 
 54 |     def _download_using_google(self, ticker, quiet=True):
 55 |         """Downloads and returns stock data from Google.
 56 | 
 57 |         A method that acts as a wrapper for logic pertaining to
 58 |         downloading from Google specifically. Unless start date is
 59 |         known, Google will only provide one year at a time. So this
 60 |         method downloads one year at a time until no new data is
 61 |         downloaded.
 62 | 
 63 |         Args:
 64 |             ticker: A string representing the ticker of a stock
 65 |             quiet: A boolean for whether or not to print progress
 66 |                 feedback, default: True
 67 | 
 68 |         Returns:
 69 |             An array of stock data in chronological order
 70 |         """
 71 |         data = []
 72 |         last_date = datetime.date.today().strftime("%Y-%m-%d")
 73 |         if not quiet:
 74 |             print('Downloading {} data from Google'.format(ticker),
 75 |                   end='', flush=True)
 76 |         while True:
 77 |             new_data = self._download_google_csv_data(ticker, last_date)
 78 |             if not quiet:
 79 |                 print('.', end='', flush=True)
 80 |             if not len(new_data):
 81 |                 break
 82 |             if new_data[0][0] >= last_date:
 83 |                 break
 84 |             data = new_data + data
 85 |             last_date = (datetime.datetime.strptime(new_data[0][0], "%Y-%m-%d")
 86 |                          - datetime.timedelta(1)).strftime("%Y-%m-%d")
 87 |         if not quiet:
 88 |             print('')
 89 |         return data
 90 | 
 91 |     def _download_using_yahoo(self, ticker):
 92 |         """Downloads and returns stock data from Yahoo Finance.
 93 | 
 94 |         Not implemented, since Google is sufficient for my purposes at
 95 |         the moment. However, it will need to scrape a cookie from a
 96 |         response header and then use it to request data
 97 |         NOTE: some people reported the above method doesnt work anymore
 98 | 
 99 |         Args:
100 |             ticker: A string representing the ticker of a stock
101 | 
102 |         Returns:
103 |             An array of stock data in chronological order
104 |         """
105 |         return []
106 | 
107 |     def _google_url(self, ticker, date, market=None):
108 |         """Simple URL generating function for the Google finance API.
109 | 
110 |         Args:
111 |             ticker: A string representing the ticker to download
112 |             date: A string in 'YYYY-MM-DD' format representing the end
113 |                 date for the data
114 |             market: An optional value to specify the market for the
115 |                 ticker
116 |         """
117 |         # special cases 'hack' to handle weirdness of Google API
118 |         special_cases = {
119 |             'TLT': 'NASDAQ'
120 |         }
121 |         dt = datetime.datetime.strptime(date, '%Y-%m-%d')
122 |         base = 'http://finance.google.com/finance/historical'
123 |         if market:
124 |             query = '{}%3A{}'.format(market, ticker)
125 |         else:
126 |             try:
127 |                 query = '{}%3A{}'.format(special_cases[ticker.upper()], ticker)
128 |             except KeyError:
129 |                 query = '{}'.format(ticker)
130 |         enddate = '{}%20{},%20{}'.format(
131 |             dt.strftime('%b'), dt.strftime('%d'), dt.strftime('%Y'))
132 |         output = 'csv'
133 |         return '{}?q={}&enddate={}&output={}'.format(
134 |             base, query, enddate, output)
135 | 
136 |     def _download_google_csv_data(self, ticker, date):
137 |         """Handler for making a URL request to Google and interpretting
138 |         the data.
139 | 
140 |         Requests data from Google via a URL and converts the data to
141 |         [[date, open, high, low, close, volume], ...] format.
142 | 
143 |         Args:
144 |             ticker: A string representing the ticker of a stock
145 |             date: A string in YYYY-MM-DD format representing the end
146 |                 date for the request
147 | 
148 |         Todo:
149 |             - error handling for urllib request
150 |         """
151 |         data = []
152 |         try:
153 |             csv = urllib.request.urlopen(
154 |                 self._google_url(ticker, date)).readlines()
155 |         except urllib.error.HTTPError:
156 |             csv = urllib.request.urlopen(
157 |                 self._google_url(ticker, date, 'NYSE')).readlines()
158 |         for line in csv[1:]:
159 |             data = [line.decode("ASCII").strip().split(',')] + data
160 |             data[0][0] = datetime.datetime.strptime(
161 |                 data[0][0], "%d-%b-%y").strftime("%Y-%m-%d")
162 |         return data
163 | 
164 | 
165 | def download_and_write(ticker, source):
166 |     """Wrapper for downloading and writing.
167 | 
168 |     Args:
169 |         ticker: A string for the ticker data to download
170 |         source: A string for the source to download from
171 |     """
172 |     data = downloader.download(ticker, source, False)
173 |     if len(data) == 0:
174 |         print("No data downloaded for {}".format(ticker))
175 |         return
176 |     db.write_stock_data(ticker, data, True)
177 | 
178 | 
179 | def main():
180 |     """Wrapper for main logic."""
181 |     args = parser.parse_args()
182 |     # handle downloading from list of files
183 |     if args.download_from:
184 |         for file_with_tickers in args.download_from:
185 |             with open(file_with_tickers, 'r') as file:
186 |                 lines = file.readlines()
187 |             for line in lines:
188 |                 download_and_write(line.strip(), args.using)
189 |         exit()
190 |     # handle downlaoding from list of tickers
191 |     if args.download:
192 |         for ticker in args.download:
193 |             download_and_write(ticker, args.using)
194 |         exit()
195 | 
196 | 
197 | if __name__ == "__main__":
198 |     parser = argparse.ArgumentParser(
199 |         description="Downloader for historical stock data.")
200 |     parser.add_argument('--using', default='google', nargs=1,
201 |                         help=('a source/API from which to get the data, '
202 |                               'default: google'))
203 |     download_group = parser.add_mutually_exclusive_group(required=True)
204 |     download_group.add_argument('--download', nargs='+',
205 |                                 help='the stock ticker(s) to download')
206 |     download_group.add_argument('--download-from', nargs='+',
207 |                                 help=('file(s) containing the stock tickers to'
208 |                                       'download'))
209 | 
210 |     downloader = Downloader()
211 |     db = DataManager()
212 | 
213 |     main()
214 |     print("Did nothing.")
215 |     exit()
216 | 


--------------------------------------------------------------------------------
/Market.py:
--------------------------------------------------------------------------------
  1 | from datetime import datetime as dt
  2 | 
  3 | from utils import date_str
  4 | from utils import date_obj
  5 | 
  6 | from DataManager import DataManager
  7 | 
  8 | 
  9 | class Market(object):
 10 | 
 11 |     """A Market containing stocks and a date.
 12 | 
 13 |     Can be queried for stock prices at the current date of this Market
 14 | 
 15 |     Attributes:
 16 |         stocks: A map of stock tickers to price LUTs
 17 |         new_period: A map of flags for market periods
 18 |         dates: An array of dates for the market
 19 |         date: A tuple containing (curr date index in dates, curr date)
 20 | 
 21 |     Todo:
 22 |     """
 23 | 
 24 |     def __init__(self, tickers=None, dates=None):
 25 |         """Intialize a Market with a set of dates and stock tickers
 26 |         with corresponding price LUTs.
 27 | 
 28 |         Args:
 29 |             tickers: An array of tickers for which to build price LUTs
 30 |             dates: An array of dates
 31 |         """
 32 |         self._db = DataManager()
 33 |         self.new_period = {'m': False, 'q': False, 'y': False}
 34 |         self.commissions = 10
 35 |         self.stocks = {}
 36 |         self.stocks_indicators = {}
 37 |         if tickers != None:
 38 |             self.add_stocks(tickers)
 39 |         self.dates = []
 40 |         self.date = (-1, None)
 41 |         if dates != None:
 42 |             self.dates = dates
 43 |             self.date = (0, self.dates[0])
 44 | 
 45 |     def add_stocks(self, tickers):
 46 |         """Creates price LUTs and adds them to the Market. Also sets up
 47 |         the data structures for indicators.
 48 | 
 49 |         Args:
 50 |             tickers: An array of tickers for which to create LUTs
 51 |         """
 52 |         for ticker in tickers:
 53 |             self.stocks[ticker.upper()] \
 54 |                 = self._db.build_price_lut(ticker.upper())
 55 |             # create empty dict to be populated later by indicators
 56 |             self.stocks_indicators[ticker.upper()] = {}
 57 | 
 58 |     def add_indicator(self, ticker, indicator, indicator_lut):
 59 |         """Adds the indicator data for a ticker to this Market.
 60 | 
 61 |         Args:
 62 |             ticker: A ticker for which to add indicator data
 63 |             indicator: A string for the indicator being added
 64 |             indicator_lut: A lookup table for the indicator data itself
 65 |         """
 66 |         self.stocks_indicators[ticker.upper()][indicator.upper()] = \
 67 |             indicator_lut
 68 | 
 69 |     def inject_stock_data(self, ticker, dates, prices, price_lut=None):
 70 |         """Injects provided stock data into this market.
 71 | 
 72 |         Generally used for generated data, but can be used in any case
 73 |         to bypass the default price LUT creation method.
 74 | 
 75 |         Args:
 76 |             ticker: A ticker for which to inject data
 77 |             dates: An array of dates corresponding to the prices
 78 |             prices: An array of prices corresponding to the dates
 79 |             price_lut: A price lookup table with dates mapping to
 80 |                 prices to be used instead of building one from dates
 81 |                 and prices
 82 |         """
 83 |         ticker = ticker.upper()
 84 |         self.stocks_indicators[ticker] = {}
 85 |         if price_lut:
 86 |             self.stocks[ticker] = price_lut
 87 |             return
 88 |         price_lut = {}
 89 |         for i in range(0, len(dates)):
 90 |             price_lut[dates[i]] = prices[i]
 91 |         self.stocks[ticker] = price_lut
 92 | 
 93 |     def current_date(self):
 94 |         """Returns the current date of this Market.
 95 | 
 96 |         Returns:
 97 |             A string representing the current date in this Market
 98 |         """
 99 |         return date_str(self.date[1])
100 | 
101 |     def query_stock(self, ticker, num_days=0):
102 |         """Query a stock at the current date.
103 | 
104 |         Args:
105 |             ticker: A ticker to query
106 |             num_days: A value representing the number of days of prices
107 |                 going backwards from the current date to return. A
108 |                 value of 0 means only a float for today's value will be
109 |                 returned. A value >0 means an array of that many values
110 |                 will be returned (default: 0)
111 | 
112 |         Returns:
113 |             A float representing the price of the stock
114 |         """
115 |         ticker = ticker.upper()
116 |         if num_days:
117 |             dates = self.dates[
118 |                 max(0, self.date[0] - num_days + 1):self.date[0] + 1]
119 |             return [float(self.stocks[ticker][date]) for date in dates]
120 |         try:
121 |             return float(self.stocks[ticker][self.current_date()])
122 |         except KeyError:
123 |             print("NEEDS FIX: no data for " + ticker + " at " + self.date[1])
124 |             return None
125 | 
126 |     def query_stock_indicator(self, ticker, indicator):
127 |         """Query a stock indicator value or set of values at the
128 |         current date.
129 | 
130 |         Args:
131 |             ticker: A ticker to query
132 |             indicator: An identifying string for the indicator value to
133 |                 return
134 | 
135 |         Returns:
136 |             A float or set of floats representing the indicator
137 |             value(s)
138 |         """
139 |         ticker = ticker.upper()
140 |         indicator = indicator.upper()
141 |         try:
142 |             return float(
143 |                 self.stocks_indicators[ticker][indicator][self.current_date()])
144 |         except KeyError:
145 |             print('NEEDS FIX: no {} value for {} at {}'.format(
146 |                 indicator, ticker, self.current_date()))
147 |             return None
148 | 
149 |     def set_date(self, date):
150 |         """Sets this Market to a given date.
151 | 
152 |         Args:
153 |             date: A date to which to set this Market
154 |         """
155 |         if date < self.dates[0]:
156 |             self.date = (0, self.dates[0])
157 |             return 0
158 |         if date > self.dates[-1]:
159 |             self.date = (len(self.dates) - 1, self.dates[-1])
160 |             return 0
161 |         try:
162 |             self.date = (self.dates.index(date), date)
163 |             return 0
164 |         except ValueError:
165 |             print("NEEDS FIX: date does not exist")
166 |             return 1
167 | 
168 |     def set_default_dates(self):
169 |         """Sets a default range for this Market's dates.
170 | 
171 |         Based on existing stocks in this Market, decides an appropriate
172 |         range in which all stocks have prices.
173 |         """
174 |         date_range = (date_str(dt.fromordinal(1)),
175 |                       date_str(dt.fromordinal(999999)))
176 |         for price_lut in self.stocks.values():
177 |             dates = sorted(price_lut.keys())
178 |             date_range = (max(date_range[0], dates[0]), min(
179 |                 date_range[1], dates[-1]))
180 |             date_idxs = (dates.index(
181 |                 date_range[0]), dates.index(date_range[1]))
182 |             self.dates = dates[date_idxs[0]:date_idxs[1] + 1]
183 |         self.date = (0, self.dates[0])
184 | 
185 |     def advance_day(self):
186 |         """Advances this Market's date by one day."""
187 |         self.date = (self.date[0] + 1, self.dates[self.date[0] + 1])
188 |         self._raise_period_flags()
189 | 
190 |     def _raise_period_flags(self):
191 |         """Internal function to handle setting flags at new periods."""
192 |         last_date = date_obj(self.dates[self.date[0] - 1])
193 |         curr_date = date_obj(self.date[1])
194 |         self.new_period = {'m': False, 'q': False, 'y': False}
195 |         if last_date.year < curr_date.year:
196 |             self.new_period = {'m': True, 'q': True, 'y': True}
197 |         elif last_date.month != curr_date.month:
198 |             self.new_period['m'] = True
199 |             if (curr_date.month - 1) % 3 == 0:
200 |                 self.new_period['q'] = True
201 | 


--------------------------------------------------------------------------------
/Monitor.py:
--------------------------------------------------------------------------------
  1 | from datetime import timedelta
  2 | from math import sqrt
  3 | 
  4 | from utils import date_obj
  5 | from utils import days_between
  6 | 
  7 | 
  8 | class Monitor(object):
  9 | 
 10 |     """A Monitor for market and portfolio data in simulations, used by
 11 |     taking snapshots of a market and portfolio at various stages in a
 12 |     simulation.
 13 | 
 14 |     More specifically, as snapshots are taken - on a daily basis - the
 15 |     data taken from those snapshots is recorded and interpretted to
 16 |     provide a means for retrieving statistics (e.g. value history,
 17 |     indicators, max drawdowns, etc.)
 18 | 
 19 |     Supported portfolio statistics and data series:
 20 |         - Per-day portfolio value history
 21 |         - Per-day asset allocation
 22 |         - Per-day contributions vs growth percent of total value
 23 |         - Per-year annual returns
 24 |         - Max drawdown
 25 |         - CAGR calculation
 26 |         - Adjusted CAGR calculation
 27 | 
 28 |     Supported indicators:
 29 |         - Standard Moving Average (SMA) for a given period
 30 |         - Exponential Moving Average (EMA) for a given period
 31 |         - Moving Average Convergence/Divergence (MACD) for a given
 32 |             set of periods
 33 | 
 34 |     Attributes:
 35 |         portfolio: A Portfolio instance to monitor
 36 |         market: A Market instance to reference during monitoring
 37 |     """
 38 | 
 39 |     def __init__(self, trader, market):
 40 |         """Initializes a Monitior with a Portfolio and Market instance.
 41 | 
 42 |         Args:
 43 |             trader: A Trader instance to monitor
 44 |             market: A Market instance to reference during monitoring
 45 |         """
 46 |         # main attributes
 47 |         self.trader = trader
 48 |         self.portfolio = trader.portfolio
 49 |         self.market = market
 50 |         # getter mappings
 51 |         self._data_series_getter_for = {
 52 |             'portfolio_values': self._get_portfolio_value_data_series,
 53 |             'asset_allocations': self._get_asset_alloc_data_series,
 54 |             'annual_returns': self._get_annual_returns_data_series,
 55 |             'contribution_vs_growth': self._get_contrib_vs_growth_data_series
 56 |         }
 57 |         self._statistic_getter_for = {
 58 |             'max_drawdown': self._get_max_drawdown,
 59 |             'cagr': self._get_cagr,
 60 |             'adjusted_cagr': self._get_adjusted_cagr,
 61 |             'sharpe_ratio': self._get_sharpe_ratio,
 62 |             'sortino_ratio': self._get_sortino_ratio
 63 |         }
 64 | 
 65 |     def init_stats(self):
 66 |         """Runs any necessary setup that needs to happen before stats
 67 |         can be recorded."""
 68 |         # init internal values used in record keeping
 69 |         self._dates = []
 70 |         self._all_assets = {}
 71 |         self._daily_value_history = {}
 72 |         self._monthly_value_history = {}
 73 |         self._annual_value_history = {}
 74 |         self._asset_alloc_history = {}
 75 |         self._contrib_vs_growth_history = {}
 76 |         self._daily_returns = {}
 77 |         self._monthly_returns = {}
 78 |         self._annual_returns = {}
 79 |         self._max_drawdown = {
 80 |             'amount': 0, 'from': None, 'to': None, 'recovered_by': None
 81 |         }
 82 |         self._portfolio_max = 0
 83 |         self._portfolio_min_since_max = 0
 84 |         self._potential_drawdown_start = None
 85 |         # init all assets to be monitored
 86 |         self._all_assets = self.trader.get_assets_of_interest()
 87 | 
 88 |     def take_snapshot(self):
 89 |         """Records a snapshot of all supported stats for the Portfolio
 90 |         at the current date."""
 91 |         self._dates.append(self.market.current_date())
 92 |         self._record_portfolio_value()
 93 |         self._record_asset_allocation()
 94 |         self._record_contribution_vs_growth()
 95 |         self._record_monthly_return()
 96 |         self._record_annual_return()
 97 |         self._update_drawdown()
 98 | 
 99 |     def get_data_series(self, series):
100 |         """Returns a set of data in a format meant for plotting.
101 | 
102 |         Args:
103 |             series: A string representing the data series to get
104 | 
105 |         Returns:
106 |             A set of X and Y series to be used in a plot
107 |         """
108 |         return self._data_series_getter_for[series]()
109 | 
110 |     def get_statistic(self, statistic):
111 |         """Returns a statistic for the monitored Portfolio(s).
112 | 
113 |         Args:
114 |             statistic: A string representing the statistic to get
115 | 
116 |         Returns:
117 |             A value or set of values corresponding to the desired
118 |             statistic
119 |         """
120 |         return self._statistic_getter_for[statistic]()
121 | 
122 |     def get_indicator(self, indicator, ticker):
123 |         """Returns an indicator value or values for the monitored Portfolio(s).
124 | 
125 |         Args:
126 |             indicator: A string representing the statistic to get
127 |             ticker: A string representing the ticker for a stock
128 | 
129 |         Returns:
130 |             A value or set of values corresponding to the desired
131 |             statistic
132 |         """
133 |         return self.market.query_stock_indicator(ticker, indicator)
134 | 
135 |     def _record_portfolio_value(self):
136 |         """Internal method for recording the Portfolio value."""
137 |         (curr_year, curr_month, _) = self.market.current_date().split('-')
138 |         self._daily_value_history[self.market.current_date()] \
139 |             = self.portfolio.value()
140 |         if self.market.new_period['m'] or not len(self._monthly_value_history):
141 |             self._monthly_value_history[curr_year + '-' + curr_month] \
142 |                 = self.portfolio.value()
143 |         if self.market.new_period['y'] or not len(self._annual_value_history):
144 |             self._annual_value_history[curr_year] \
145 |                 = self.portfolio.value()
146 | 
147 |     def _record_asset_allocation(self):
148 |         """Internal method for recording the asset allocation of the
149 |         Portfolio."""
150 |         alloc = {}
151 |         for asset, shares in self.portfolio.holdings.items():
152 |             if self.portfolio.value() == 0:
153 |                 alloc[asset] = 0
154 |             else:
155 |                 alloc[asset] = (self.market.query_stock(asset) * int(shares)
156 |                                 / self.portfolio.value())
157 |         self._asset_alloc_history[self.market.current_date()] = alloc
158 | 
159 |     def _record_contribution_vs_growth(self):
160 |         """Internal method for recording the percentages of the
161 |         Portfolio value which are from growth and contributions."""
162 |         ratio = {'contribution': 1, 'growth': 0}
163 |         if self.portfolio.value() != 0:
164 |             ratio['contrib'] = (self.portfolio.total_contributions
165 |                                 / self.portfolio.value())
166 |             ratio['growth'] = max(0, 1 - ratio['contrib'])
167 |         self._contrib_vs_growth_history[self.market.current_date()] = ratio
168 | 
169 |     def _record_monthly_return(self):
170 |         """Internal method for recording the Portfolio's monthly
171 |         returns."""
172 |         if (not self.market.new_period['m']
173 |                 or len(self._monthly_value_history) <= 1):
174 |             return
175 |         this_dt = date_obj(self.market.current_date())
176 |         last_dt = (date_obj(self.market.current_date()).replace(day=1)
177 |                    - timedelta(1))
178 |         this_month = str(this_dt.year) + '-' + ('0' + str(this_dt.month))[-2:]
179 |         last_month = str(last_dt.year) + '-' + ('0' + str(last_dt.month))[-2:]
180 |         self._monthly_returns[last_month] = \
181 |             (self._monthly_value_history[this_month]
182 |              / self._monthly_value_history[last_month]
183 |              - 1)
184 | 
185 |     def _record_annual_return(self):
186 |         """Internal method for recording the Portfolio's annual
187 |         returns."""
188 |         if (not self.market.new_period['y']
189 |                 or len(self._annual_value_history) <= 1):
190 |             return
191 |         (this_year, _, _) = self.market.current_date().split('-')
192 |         last_year = str(int(this_year) - 1)
193 |         self._annual_returns[last_year] = \
194 |             (self._annual_value_history[this_year]
195 |              / self._annual_value_history[last_year]
196 |              - 1)
197 | 
198 |     def _update_drawdown(self):
199 |         """Updates the maximum drawdown for this Monitor's
200 |         Portfolio."""
201 |         if self.portfolio.value() >= self._portfolio_max:
202 |             self._potential_drawdown_start = None
203 |             self._portfolio_max = self.portfolio.value()
204 |             self._portfolio_min_since_max = self._portfolio_max
205 |             if not self._max_drawdown['recovered_by']:
206 |                 self._max_drawdown['recovered_by'] = self.market.current_date()
207 |             return
208 |         if self.portfolio.value() < self._portfolio_min_since_max:
209 |             if not self._potential_drawdown_start:
210 |                 self._potential_drawdown_start = self.market.current_date()
211 |             self._portfolio_min_since_max = self.portfolio.value()
212 |             drawdown = self._portfolio_min_since_max / self._portfolio_max - 1
213 |             if drawdown < self._max_drawdown['amount']:
214 |                 self._max_drawdown['amount'] = drawdown
215 |                 self._max_drawdown['from'] = self._potential_drawdown_start
216 |                 self._max_drawdown['to'] = self.market.current_date()
217 |                 self._max_drawdown['recovered_by'] = None
218 | 
219 |     def _get_portfolio_value_data_series(self):
220 |         """Internal function which returns a data series for a
221 |         portfoio's value history.
222 | 
223 |         The dates in the data series are returned as datetime objects,
224 |         while the value is stored as floats. e.g.
225 |         ([<datetime_obj>, ...], [10000.00, ...])
226 | 
227 |         Returns:
228 |             A tuple of X and Y values meant to be plotted
229 |         """
230 |         dates = sorted(self._daily_value_history.keys())
231 |         return ([date_obj(date) for date in dates],
232 |                 [self._daily_value_history[date] for date in dates])
233 | 
234 |     def _get_asset_alloc_data_series(self):
235 |         """Internal function which returns a tuple of data series in
236 |         (x, y) format for a portfolio's asset allocation history.
237 | 
238 |         The dates in the data series are returned as datetime objects,
239 |         while the allocation ratios are in sets of floats. e.g.
240 |         ([<datetime_obj>, ...], [[0.4, ...], [0.3, ...], ...])
241 | 
242 |         Returns:
243 |             A set of X and Y values meant to be plotted
244 |         """
245 |         dates = sorted(self._asset_alloc_history.keys())
246 |         allocs = [[] for i in range(len(self._all_assets))]
247 |         for date in dates:
248 |             for index, asset in enumerate(sorted(self._all_assets)):
249 |                 try:
250 |                     alloc = self._asset_alloc_history[date][asset]
251 |                 except KeyError:
252 |                     alloc = 0
253 |                 allocs[index].append(alloc)
254 |         return ([date_obj(date) for date in dates], allocs)
255 | 
256 |     def _get_annual_returns_data_series(self):
257 |         """Internal function which returns a tuple of data series in
258 |         (x, y) format for a portfolio's annual returns.
259 | 
260 |         The dates in the data series are returned as string
261 |         representation of years, while annual returns are in a set of
262 |         floats. e.g. (['2009', ...], [0.45, ...])
263 | 
264 |         Returns:
265 |             A set of X and Y values meant to be plotted
266 |         """
267 |         years = sorted(self._annual_returns.keys())
268 |         return ([str(year) for year in years],
269 |                 [self._annual_returns[year] for year in years])
270 | 
271 |     def _get_contrib_vs_growth_data_series(self):
272 |         """Internal function which returns a tuple of data series in
273 |         (x, y) format for a history of a portfolio's contribution vs
274 |         growth as a percent of the whole portfolio.
275 | 
276 |         The dates in the data series are returned as datetime objects,
277 |         while the contributions and growth are in sets of floats. e.g.
278 |         ([<datetime_obj>, ...], [[0.4, ...] [0.6, ...]])
279 | 
280 |         Returns:
281 |             A set of X and Y values meant to be plotted
282 |         """
283 |         dates = sorted(self._daily_value_history.keys())
284 |         ratios = [[], []]
285 |         for date in dates:
286 |             ratios[0].append(self._contrib_vs_growth_history[date]['contrib'])
287 |             ratios[1].append(self._contrib_vs_growth_history[date]['growth'])
288 |         return ([date_obj(date) for date in dates], ratios)
289 | 
290 |     def _get_max_drawdown(self):
291 |         """Internal function for returning the max drawdown.
292 | 
293 |         Returns:
294 |             A dictionary in the form:
295 |                 {'amount': <val>,
296 |                  'from': <date_str>,
297 |                  'to': <date_str>,
298 |                  'recovered by': <date_str>}
299 |         """
300 |         return self._max_drawdown.copy()
301 | 
302 |     def _get_cagr(self):
303 |         """Internal function for calculating the Cumulative Annual
304 |         Growth Rate.
305 | 
306 |         Returns:
307 |             A value representing the CAGR
308 |         """
309 |         start_val = self.portfolio.starting_cash
310 |         end_val = self.portfolio.value()
311 |         years = days_between(self._dates[0], self._dates[-1]) / 365.25
312 |         return (end_val / start_val) ** (1 / years) - 1
313 | 
314 |     def _get_adjusted_cagr(self):
315 |         """Internal function for calculating the adjusted Cumulative
316 |         Annual Growth Rate.
317 | 
318 |         Returns:
319 |             A value representing the adjusted CAGR
320 |         """
321 |         start_val = self.portfolio.starting_cash
322 |         end_val = self.portfolio.value()
323 |         years = days_between(self._dates[0], self._dates[-1]) / 365.25
324 |         contrib = self.portfolio.total_contributions
325 |         return ((end_val - contrib + start_val) / start_val) ** (1 / years) - 1
326 | 
327 |     def _get_sortino_ratio(self):
328 |         """Internal function for calculating the Sortino ratio of a
329 |         portfolio.
330 | 
331 |         Returns:
332 |             A value representing the Sortino ratio.
333 |         """
334 |         # risk_free_return = 0.01  # yearly 08/2017 1-month T-bill rate
335 |         risk_free_return = 0.00083  # monthly 08/2017 1-month T-bill rate
336 |         # excess_returns
337 |         excess_returns = []
338 |         neg_excess_returns = []
339 |         for ret in self._monthly_returns.values():
340 |             excess_returns.append(ret - risk_free_return)
341 |             if ret - risk_free_return < 0:
342 |                 neg_excess_returns.append(ret - risk_free_return)
343 |         excess_return_mean = sum(excess_returns) / len(excess_returns)
344 |         neg_excess_return_mean = (sum(neg_excess_returns)
345 |                                   / len(neg_excess_returns))
346 |         # standard deviation
347 |         if len(neg_excess_returns) <= 1:
348 |             return 'undef'
349 |         stdev = sqrt(
350 |             sum([(ret - neg_excess_return_mean) ** 2
351 |                  for ret in neg_excess_returns])
352 |             / len(excess_returns))
353 |         return excess_return_mean / stdev
354 | 
355 |     def _get_sharpe_ratio(self):
356 |         """Internal function for calculating the Sharpe ratio of a
357 |         portfolio.
358 | 
359 |         Returns:
360 |             A value representing the Sharpe ratio.
361 |         """
362 |         # risk_free_return = 0.01  # yearly 08/2017 1-month T-bill rate
363 |         risk_free_return = 0.00083  # monthly 08/2017 1-month T-bill rate
364 |         # excess_returns
365 |         excess_returns = []
366 |         for ret in self._monthly_returns.values():
367 |             excess_returns.append(ret - risk_free_return)
368 |         excess_return_mean = sum(excess_returns) / len(excess_returns)
369 |         # standard deviation
370 |         stdev = sqrt(
371 |             sum([(ret - excess_return_mean) ** 2 for ret in excess_returns])
372 |             / len(excess_returns))
373 |         return excess_return_mean / stdev
374 | 


--------------------------------------------------------------------------------
/Portfolio.py:
--------------------------------------------------------------------------------
  1 | class Portfolio(object):
  2 | 
  3 |     """A Portfolio with cash, assets (definition of stocks to be held),
  4 |     and holdings (stocks actually held).
  5 | 
  6 |     This portfolio should not do anything on its own, i.e. should not
  7 |     house any business logic with regards to trading or positioning,
  8 |     but rather act as a container for assets to be acted on by an
  9 |     outside agent.
 10 | 
 11 |     Attributes:
 12 |         cash: A float representing the amount of cash in this Portfolio
 13 |         total_contributions: A counter for contributions
 14 |         holdings: A mapping of holdings to a number of shares
 15 | 
 16 |     Todo:
 17 |         - [code improvement, low priority] portfolio interacts with
 18 |             Market for market prices and commissions, instead of taking
 19 |             those args
 20 |         - [code improvement, low priority] market should be private,
 21 |             since it's not this Portfolio's market technically
 22 |     """
 23 | 
 24 |     def __init__(self, cash=0):
 25 |         """Initializes an empty Portfolio.
 26 | 
 27 |         Args:
 28 |             cash: A cash value for this Portfolio to start at,
 29 |                 default: 0
 30 |         """
 31 |         self._market = None
 32 |         self.cash = float(cash)
 33 |         self.starting_cash = float(cash)
 34 |         self.total_contributions = 0
 35 |         self.holdings = {}
 36 |         self.trades = 0
 37 | 
 38 |     def use_market(self, market):
 39 |         """Sets the market this Portfolio should use for looking up
 40 |         prices.
 41 | 
 42 |         Args:
 43 |             market: A Market instance to use
 44 |         """
 45 |         self._market = market
 46 | 
 47 |     def add_cash(self, amount):
 48 |         """Adds a certain amount of cash to the portfolio.
 49 | 
 50 |         Args:
 51 |             amount: A value representing the amount of cash to add
 52 |         """
 53 |         self.cash += float(amount)
 54 |         self.total_contributions += float(amount)
 55 | 
 56 |     def buy(self, ticker, amount):
 57 |         """Adds a holding to the portfolio in the form of a buy.
 58 | 
 59 |         Args:
 60 |             ticker: A string for the ticker of the holding to add
 61 |             amount: A value for the number of shares to buy
 62 |         """
 63 |         if amount <= 0:
 64 |             return 0
 65 |         self.trades += 1
 66 |         price = float(self._market.query_stock(ticker))
 67 |         if (float(amount) * price) > (self.cash - self._market.commissions):
 68 |             print('ERROR: not enough cash({}) to buy {}x{} at {} on {}'.format(
 69 |                 self.cash, ticker, amount, price, self._market.current_date()))
 70 |             print('APPLYING FIX: buying {}x{} instead'.format(
 71 |                 ticker, int((self.cash - self._market.commissions) / price)))
 72 |             return self.buy(ticker, int((self.cash - self._market.commissions)
 73 |                                         / price))
 74 |         try:
 75 |             self.holdings[ticker.upper()] += int(amount)
 76 |         except KeyError:
 77 |             self.holdings[ticker.upper()] = int(amount)
 78 |         self.cash -= int(amount) * price + self._market.commissions
 79 |         return 0
 80 | 
 81 |     def sell(self, ticker, amount):
 82 |         """Removes a holding from the portfolio in the form of a sell.
 83 | 
 84 |         Args:
 85 |             ticker: A string for the ticker of the holding to remove
 86 |             amount: A value for the number of shares to sell
 87 |         """
 88 |         if amount <= 0:
 89 |             return 0
 90 |         self.trades += 1
 91 |         price = float(self._market.query_stock(ticker))
 92 |         try:
 93 |             self.holdings[ticker.upper()] -= int(amount)
 94 |         except KeyError:
 95 |             self.holdings[ticker.upper()] = -int(amount)
 96 |         self.cash += int(amount) * price - self._market.commissions
 97 |         return 0
 98 | 
 99 |     # def short(self, ticker, amount, price, commission):
100 |     #     """Adds a holding to the portfolio in the form of a short, i.e.
101 |     #     negative shares.
102 |     #
103 |     #     Shorting is significantly more involved and contingent on more
104 |     #     variables, so this feature is experimental and not complete.
105 |     #     Currently not used in main program, but still referenced.
106 |     #
107 |     #     Args:
108 |     #         ticker: A string for the ticker of the holding to short
109 |     #         amount: A value for the number of shares to sell
110 |     #         price: A value corresponding to the price of each share
111 |     #         commission: A value corresponding to the cost of the trade
112 |     #     """
113 |     #     self.sell(ticker, amount, price, commission)
114 |     #
115 |     # def cover(self, ticker, amount, price, commission):
116 |     #     """Removes a holding from the portfolio in the form of a cover
117 |     #     i.e. removes existing negative shares.
118 |     #
119 |     #     Shorting is significantly more involved and contingent on more
120 |     #     variables, so this feature is experimental and not complete.
121 |     #     Currently not used in main program, but still referenced.
122 |     #
123 |     #     Args:
124 |     #         ticker: A string for the ticker of the holding to cover
125 |     #         amount: A value for the number of shares to sell
126 |     #         price: A value corresponding to the price of each share
127 |     #         commission: A value corresponding to the cost of the trade
128 |     #     """
129 |     #     self.buy(ticker, amount, price, commission)
130 | 
131 |     def value(self):
132 |         """Returns the total value of this Portfolio (cash + holdings).
133 | 
134 |         Returns:
135 |             A value correspondingto the total value of this Portfolio
136 |         """
137 |         return self.cash + sum(
138 |             [float(self.holdings[asset] * self._market.query_stock(asset))
139 |              for asset in self.holdings.keys()])
140 | 
141 |     def shares_of(self, ticker):
142 |         """Returns the number of shares this portfolio is holding of a
143 |         given ticker."""
144 |         try:
145 |             shares = self.holdings[ticker.upper()]
146 |         except KeyError:
147 |             shares = 0
148 |         return shares
149 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | # portfolio-backtester
  2 | 
  3 | A command-line script I made to help me with making decisions with regards to choosing stocks in the stock market. To oversimplify it, it's a portfolio backtester; i.e. given a portfolio, it'll tell you how that portfolio would've done in the past.
  4 | 
  5 | > NOTE: I've included definitions and links for some words at the bottom, since stock terminology is used in describing some functionality. Words with definitions at the bottom are in blue and clickable. If you mousover, a short summary should appear, but you can click to navigate to the actual, longer definitions
  6 | 
  7 | Main features:
  8 | - Download day-by-day stock data from Google using Downloader.py
  9 | - Draw a price history chart using downloaded data
 10 | - Calculate and overlay [indicators](https://github.com/DmitryGubanov/portfolio-backtester/tree/v3.0-basic-timing-strategies#indicator "tools used to analyze trends and patterns") on the chart. Implemented indicators: [SMA](https://github.com/DmitryGubanov/portfolio-backtester/tree/v3.0-basic-timing-strategies#sma-simple-moving-average "Simple Moving Average, average stock price for last N days"), [EMA](https://github.com/DmitryGubanov/portfolio-backtester/tree/v3.0-basic-timing-strategies#ema-exponential-moving-average "Exponential Moving Average, like SMA, but the prices used in the average are given exponentially decreasing weightings going backwards"), [MACD](https://github.com/DmitryGubanov/portfolio-backtester/tree/v3.0-basic-timing-strategies#macd-moving-average-convergence-divergence "Moving Average Convergence Divergence, a set of values which seek to quantify momentum")
 11 | - Simulate past performance on a day-by-day basis for a portfolio of stocks
 12 | - Supports periodic [rebalancing](https://github.com/DmitryGubanov/portfolio-backtester/tree/v3.0-basic-timing-strategies#rebalance "Restoring the original weights for the assets in your portfolio") and contributing
 13 | - Specify conditional ratios for assets, which could depend on some relationship between stock price and/or indicators (e.g. buy stock X when it's below SMA_50, sell when it's above SMA_10)
 14 | - Summarize portfolio performance with commonly used statistics. Implemented statistics: final value, number of trades made, [(Adjusted) CAGR](https://github.com/DmitryGubanov/portfolio-backtester/tree/v3.0-basic-timing-strategies#adjusted-cagr-compound-annual-growth-rate "Adjusted Compound Annual Growth Rate, your average yearly returns"), [Sharpe Ratio](https://github.com/DmitryGubanov/portfolio-backtester/tree/v3.0-basic-timing-strategies#sharpe-ratio "A ratio quantifying how many returns you make per unit of risk; higher is better"), [Sortino Ratio](https://github.com/DmitryGubanov/portfolio-backtester/tree/v3.0-basic-timing-strategies#sortino-ratio "Similar to Sharpe, but this ratio only factors in negative volatility"), best year, worst year, maximum [drawdown](https://github.com/DmitryGubanov/portfolio-backtester/tree/v3.0-basic-timing-strategies#drawdown "A percent value representing a change from a peak to a valley, i.e. max drawdown is the maximum loss incurred along the way") and time taken to recover from it.
 15 | - Show portfolio status over time by charting some statistics. Implemented charted statistics: portfolio value history, asset allocation/ratios over time, annual returns, contributions vs growth over time.
 16 | 
 17 | Experimental features:
 18 | - Generating data for one stock based on data of another stock. Example: stock A is correlated to stock B, but stock A only has data back to 2009, while stock B has data going back to 1990. You can use this data generation to generate data for stock A back to 1990 based on stock B. Intended for use on [leveraged ETFs](https://github.com/DmitryGubanov/portfolio-backtester/tree/v3.0-basic-timing-strategies#leveraged-etf "To oversimplify, a stock which seeks to multiply the returns of another stock by a factor").
 19 | 
 20 | # 0. Table of contents
 21 | 
 22 | [1. Prerequisites](https://github.com/DmitryGubanov/portfolio-backtester/tree/v3.0-basic-timing-strategies#1-prerequisites)
 23 | 
 24 | [2. Sample usage](https://github.com/DmitryGubanov/portfolio-backtester/tree/v3.0-basic-timing-strategies#2-sample-usage)
 25 | 
 26 | [3. Advanced usage](https://github.com/DmitryGubanov/portfolio-backtester/tree/v3.0-basic-timing-strategies#3-advanced-usage)
 27 | 
 28 | [4. Current work in progress](https://github.com/DmitryGubanov/portfolio-backtester/tree/v3.0-basic-timing-strategies#4-current-work-in-progress)
 29 | 
 30 | [5. Changelog](https://github.com/DmitryGubanov/portfolio-backtester/tree/v3.0-basic-timing-strategies#5-version-featureschangelog)
 31 | 
 32 | [6. Definitions](https://github.com/DmitryGubanov/portfolio-backtester/tree/v3.0-basic-timing-strategies#6-definitions)
 33 | 
 34 | # 1. Prerequisites
 35 | 
 36 | This program was written and tested in Python 3.5.2 (https://www.python.org/downloads/release/python-352/). Use a different version at your own discretion.
 37 | 
 38 | Graphing requires matplotlib.
 39 | ```
 40 | $ pip install matplotlib
 41 | ```
 42 | > NOTE: ensure that the pip you use is installed under python 3.5 with 'pip -V'
 43 | 
 44 | Finally, you're probably going to want to clone this repo.
 45 | 
 46 | # 2. Sample usage
 47 | 
 48 | This will act as an example of how this program can be used to tweak a common portfolio strategy for more desirable performance. I provided some sample strategy files in the repo which we'll use.
 49 | 
 50 | ### 2.0: Downloading the data
 51 | 
 52 | Download stock data for the stocks/funds with tickers SPY and TLT.
 53 | ```
 54 | $ python3.5 Downloader.py --download SPY TLT
 55 | ```
 56 | > NOTE: For the curious, SPY follows the S&P500 index (the stock market as a whole) while TLT follows the long-term treasury bond index (the apparent value of stable and relatively low risk investments). You invest in the stock market for growth purposes, but when the stock market is doing poorly, the viablility of more stable investments rises since they aren't as exposed to poor market conditions. In short, when the stock market is down, there is a better than random chance the bond index is up. As a result, the two are somewhat inversely correlated which makes bonds a 'natural' hedge (something you use to mitigate losses) for stocks.
 57 | 
 58 | ### 2.1: Testing standard strategy (our benchmark)
 59 | 
 60 | Let's see where simply investing 10,000 in the stock market gets us:
 61 | > NOTE: If you pay attention to the command, you'll notice '--strategy stocks-only'. stocks-only is a sample strategy file I provided - we'll be using three different ones.
 62 | 
 63 | > ANOTHER NOTE: Your outputs will differ from mine, since time has passed and the stocks we're using in this example are real stocks which change price over time
 64 | 
 65 | ```
 66 | $ python3.5 folio.py --portfolio 10000 --strategy stocks-only
 67 | 
 68 | ##################################
 69 | # PERFORMANCE SUMMARY
 70 | ##################################
 71 | initial: $10000.00
 72 | final:   $28673.68
 73 | trades:  1
 74 | ---------------------------
 75 | Sharpe Ratio:  0.13015961697451908
 76 | Sortino Ratio: 0.2555517731631023
 77 | ---------------------------
 78 | CAGR:          7.22%
 79 | Adjusted CAGR: 7.22%
 80 | ---------------------------
 81 | best year:  25.13%
 82 | worst year: -35.71%
 83 | ---------------------------
 84 | max drawdown: -56.26%
 85 |   between 2007-10-10 and 2009-03-09, recovered by 2013-03-14
 86 | 
 87 | ```
 88 | 
 89 | Charts:
 90 | - first chart: portfolio value vs time
 91 | - second chart: asset allocation vs time (in this case, we're 100% in stocks the whole time)
 92 | - third chart: annual returns
 93 | - fourth chart: contributions and growth vs time (we start at 100% contributions and as time goes on, we grow as our assets grow - notice in 2009 we have more contributions than actual portfolio value, i.e. we've lost money overall)
 94 | 
 95 | <img src="http://i.imgur.com/NuzTFZ0.png" alt="chart" />
 96 | 
 97 | So on average we get 7.2% a year, but we would have had to weather a 56% drop during the 2008 recession (yikes).
 98 | 
 99 | ### 2.2: Introduce bonds
100 | 
101 | Let's try to add bonds, a 'natural' hedge to stocks, to try and mitigate some of those losses.
102 | ```
103 | $ python3.5 folio.py --portfolio 10000 --strategy stocks-and-bonds
104 | 
105 | ##################################
106 | # PERFORMANCE SUMMARY
107 | ##################################
108 | initial: $10000.00
109 | final:   $23500.12
110 | trades:  2
111 | ---------------------------
112 | Sharpe Ratio:  0.15580397381790653
113 | Sortino Ratio: 0.29533912934209955
114 | ---------------------------
115 | CAGR:          5.82%
116 | Adjusted CAGR: 5.82%
117 | ---------------------------
118 | best year:  15.67%
119 | worst year: -17.55%
120 | ---------------------------
121 | max drawdown: -35.72%
122 |   between 2007-10-10 and 2009-03-09, recovered by 2012-02-24
123 | ```
124 | <img src="http://i.imgur.com/5zhQrJv.png" alt="chart" />
125 | 
126 | By introducing bonds, we've cut down our risk by ~40% at the cost of ~20% of our gains. As a result, the [Sharpe](https://github.com/DmitryGubanov/portfolio-backtester/tree/v3.0-basic-timing-strategies#sharpe-ratio "A ratio quantifying how many returns you make per unit of risk; higher is better") and [Sortino](https://github.com/DmitryGubanov/portfolio-backtester/tree/v3.0-basic-timing-strategies#sortino-ratio "Similar to Sharpe, but this ratio only factors in negative volatility") ratios are both higher.
127 | 
128 | From the graphs, we can see our asset allocations have veered away from what we set intially (0.6 and 0.4, check the sample files).
129 | 
130 | ### 2.3: Maintain ratios by rebalancing
131 | 
132 | Let's rebalance quarterly to maintain our desired ratios of 60% SPY and 40% TLT, as defined by our strategy file.
133 | 
134 | ```
135 | $ python3.5 folio.py --portfolio 10000 --strategy stocks-and-bonds --rebalance q
136 | 
137 | ##################################
138 | # PERFORMANCE SUMMARY
139 | ##################################
140 | initial: $10000.00
141 | final:   $25402.22
142 | trades:  114
143 | ---------------------------
144 | Sharpe Ratio:  0.1765820993420145
145 | Sortino Ratio: 0.3315811452594389
146 | ---------------------------
147 | CAGR:          6.36%
148 | Adjusted CAGR: 6.36%
149 | ---------------------------
150 | best year:  16.98%
151 | worst year: -14.45%
152 | ---------------------------
153 | max drawdown: -33.09%
154 |   between 2007-10-30 and 2009-03-09, recovered by 2011-05-31
155 | ```
156 | <img src="http://i.imgur.com/IIoIR5E.png" alt="chart" />
157 | 
158 | With our ratios maintained throughout the life of our portfolio, we've regained some of those lost gains and actually lost even more risk. You'll notice the Sharpe and Sortino ratios have once again increased.
159 | 
160 | ### 2.4: Experiment with timing
161 | 
162 | Let's try a timing strategy based on the Simple Moving Average indicator. In this case we'll use the SMA 100, a fairly long term indicator. In short, we'll sell when there's a sharp enough negative movement to break a positive 100-day trend, but buy it back when it recovers above that trend. Theoretically, this is to avoid big negative movements; realistically, we'll see:
163 | 
164 | ```
165 | $ python3.5 folio.py --portfolio 10000 --strategy stocks-and-bonds --rebalance q
166 | 
167 | ##################################
168 | # PERFORMANCE SUMMARY
169 | ##################################
170 | initial: $10000.00
171 | final:   $20147.15
172 | trades:  317
173 | ---------------------------
174 | Sharpe Ratio:  0.16891167389583966
175 | Sortino Ratio: 0.393798491370434
176 | ---------------------------
177 | CAGR:          4.74%
178 | Adjusted CAGR: 4.74%
179 | ---------------------------
180 | best year:  11.59%
181 | worst year: -8.71%
182 | ---------------------------
183 | max drawdown: -12.76%
184 |   between 2007-06-05 and 2009-03-09, recovered by 2009-09-16
185 | ```
186 | <img src="http://i.imgur.com/Aq37jCM.png" alt="chart" />
187 | 
188 | First thing to notice is the asset allocations. The blue one (SPY) is bouncing between 0.6 and 0.2, because our strategy sells 40% below the SMA 100, and buys it back when it comes back above it.
189 | 
190 | Before moving on, it might help to visualize this:
191 | ```
192 | $ python3.5 folio.py --draw SPY --indicators SMA_100
193 | ```
194 | 
195 | From our original, we've lost ~35% of our gains, but we've also lost ~80% of our risk. In fact, this is not immediately obvious, but the Sharpe and Sortinio ratios indicate this strategy sacrifices some upward movement to avoid a lot of downward movement. We're also making ~317 trades over the course of 15 years, which is a lot more than the original of 1 trade, but that comes out to about 20 trades a year, which really isn't that much.
196 | 
197 | ### 2.5 Conclusion
198 | 
199 | I knew these tweaks would have these results ahead of time, so it's entirely possible to get worse results from your tweaks. However, the point is this program makes it fairly easy to play around with various strategies to see how they would perform in the market conditions of the past.
200 | 
201 | # 3. Advanced usage
202 | 
203 | This section is for using some of the more advanced features.
204 | 
205 | ### 3.0 Advanced features
206 | 
207 | > Disclaimer: These are advanced features of a stock program, so they do require a bit more knowledge and a more involved perspective on the stock market to be understood. I'll try my best to convey the motivations, functionality, and results as succinctly as possible.
208 | 
209 | [3.1 Generating data](https://github.com/DmitryGubanov/portfolio-backtester/tree/v3.0-basic-timing-strategies#31-generating-data)
210 | 
211 | [3.2 Adjusting/creating  strategies](https://github.com/DmitryGubanov/portfolio-backtester/tree/v3.0-basic-timing-strategies#32-adjusting-timing-strategies)
212 | 
213 | ### 3.1 Generating data
214 | 
215 | This demonstration will use two tickers, SPY and UPRO. UPRO tries to multiply the returns of SPY by 3, which simply means if SPY moves 1%, UPRO tries to move 3%. This seems attractive:
216 | 
217 | ```
218 | $ python3.5 Downloader.py --download SPY UPRO
219 | $ python3.5 folio.py --draw UPRO
220 | ```
221 | 
222 | <img src="http://i.imgur.com/FmXWZ7L.png" alt="chart" />
223 | 
224 | That's more than a 10x increase in value over the last ~7 years. This is all the data there is on UPRO, so all existing data suggests that this is a good investment. However, UPRO was conveniently started after the recession in 2008, so there exists no data on how it would have performed during that time. Let's find out (theoretically speaking) by generating UPRO based on SPY.
225 | 
226 | This generation is using existing UPRO and SPY data to build a relationship between the two, then using that relationship to generate the part of UPRO that doesn't exist where SPY does exist. Luckily SPY goes back all the way to the 1990s, so we can generate UPRO that far.
227 | 
228 | ```
229 | $ python3.5 folio.py --draw UPRO --use-generated UPRO SPY
230 | ```
231 | 
232 | > NOTE: Notice the added --use-generated argument on the command-line. --use-generated simply bypasses the original data source for any feature, and replaces it with the generated data.
233 | 
234 | <img src="http://i.imgur.com/W2hU8rL.png" alt="chart" />
235 | 
236 | Anyway, we can see that UPRO dropped quite a bit (~95%) during the recession and during the dot-com crash. With this new information, it's unlikely that many would feel comfortable investing in something that lost over 90% of its value on two occasions in the last 20 years.
237 | 
238 | Let's build a portfolio using UPRO the same way we did with SPY in section 2.1.
239 | 
240 | ```
241 | python3.5 folio.py --portfolio 10000 --strategy upro-only --use-generated UPRO SPY
242 | 
243 | ##################################
244 | # PERFORMANCE SUMMARY
245 | ##################################
246 | initial: $10000.00
247 | final:   $503322.53
248 | trades:  1
249 | ---------------------------
250 | Sharpe Ratio:  0.16743168735558275
251 | Sortino Ratio: 0.3561463844157349
252 | ---------------------------
253 | CAGR:          17.26%
254 | Adjusted CAGR: 17.26%
255 | ---------------------------
256 | best year:  146.13%
257 | worst year: -85.67%
258 | ---------------------------
259 | max drawdown: -96.18%
260 |   between 2000-03-27 and 2009-03-09, recovered by 2016-12-07
261 | ```
262 | 
263 | Although the yearly returns look good, starting in 2000 you would have lost money until you lost 96% and would only recover by the end of 2016. To most, this would be a deal-breaker, which is why I consider this feature handy in testing leveraged ETFs in situations to which they've not been exposed.
264 | 
265 | This is an experimental feature, in that there is no way to verify its accuracy. However, I've used this method to generate data that does exist (for verification purposes; in any other case, I wouldn't need to generate data which already exists) and it was pretty accurate.
266 | 
267 | Using the standalone generate functionality, you can compare generated data against real data:
268 | 
269 | ```
270 | $ python3.5 folio.py --generate UPRO SPY
271 | ```
272 | 
273 | <img src="http://i.imgur.com/NNSRDGc.png" alt="chart" />
274 | 
275 | At the top we see the real vs the generated, at the bottom we see the generated and what the generated is generated from.
276 | 
277 | ### 3.2 Adjusting timing strategies
278 | 
279 | In each example so far, there have been strategy files used. They're in CSV format and have four columns: weight, ticker, buy signal, sell signal. Here's 'stocks-only':
280 | 
281 | ```
282 | $ cat stocks-only
283 | 
284 | 1.0,SPY,ALWAYS,NEVER
285 | 0.0,TLT,ALWAYS,NEVER
286 | ```
287 | 
288 | The weight is the portion of the portfolio dedicated to that asset or position. In this case, 1.0 (or 100%) SPY and 0.0 (or 0%) TLT. The 0.0 line isn't necessary, but it's there for consistency between strategies.
289 | 
290 | A signal is like a raised flag, if the buy/sell signal is satisfied, the strategy says to buy/sell that portion of the portfolio. In this case, the buy signals are ALWAYS (always buy this position) and sell signals are NEVER (never sell this position). This just represents a buy-and-hold portfolio.
291 | 
292 | Here's 'stocks-and-bonds-timing':
293 | 
294 | ```
295 | $ cat stocks-and-bonds-timing
296 | 
297 | 0.2,SPY,ALWAYS,NEVER
298 | 0.4,SPY,SPY~PRICE > SPY~SMA_100,SPY~PRICE < SPY~SMA_100
299 | 0.4,TLT,ALWAYS,NEVER
300 | ```
301 | 
302 | The buy and sell signals for the second portion is more involved now, but it's simply saying buy when SPY's price is above SPY's SMA_100 and sell when the opposite happens. Without using any fancy regex, the pattern is basically:
303 | 
304 | ```
305 | <ticker>~<indicator> <relation> <ticker>~<indicator>.
306 | ```
307 | 
308 | Ticker can be any real ticker for which you have data.
309 | 
310 | Indicator currently has to be one of the following, where X, Y, Z are positive integers:
311 | - PRICE
312 | - SMA_X
313 | - EMA_X
314 | - MACD_X-Y-Z
315 | - MACDSIGNAL_X-Y-Z
316 | 
317 | Relation is either < or >
318 | 
319 | 
320 | # 4. Current work in progress
321 | 
322 | ### 4.0 Short-term (v3.0, trades based on indicators):
323 | 
324 | x create shell for Brain class, a class dedicated to making decisions based on strategies  
325 | x hardcode a basic strategy into Brain (assesses market daily, provides shares to Trader)  
326 | x probably need to refactor Trader by moving rebalancing into Brain  
327 | x program Brain to handle strategies based on different indicators and periods  
328 | x implement a way to read strategies from file in DataManager  
329 | x implement Sharpe and Sortino ratios  
330 | x implement previous high as indicator  
331 | o add some sort of tolerance/adjustments to previous high to not make it useless for years after crashes (need to brainstorm)  
332 | x initialize both ratios and shares in Brain to 0 for all assets before anything runs  
333 | o dynamic/adjusted buy and sell signals (keyword -> filled in during simulation)  
334 | o buy and sell signals with ANDs and ORs  
335 | o relative strength index  
336 | o identify peaks and valleys (draw functionality for now)  
337 | o identify support and resistance lines (draw functionality for now)  
338 | o logarithmic charts or daily returns instead of daily prices  
339 | o chart pattern: head and shoulders  
340 | o chart pattern: double top, double bottom  
341 | 
342 | ### 4.1 Long-term:
343 | 
344 | o interface (e.g. web)  
345 | o dynamic portfolio ratios depending on conditions  
346 | o benchmarks  
347 | o reimplement withdrawals   
348 | o gather very short term data (minutely or less) (possibly other program)
349 | 
350 | # 5. Version features/changelog
351 | 
352 | Current version: 3.0  
353 | WIP: 3.0
354 | 
355 | ## Version 1
356 | 
357 | > Goals: get data, store data, project data, show data
358 | 
359 | #### v1.0, basic data
360 | - download stock data given ticker
361 | - download stock(s) data from list of stocks in file
362 | - read CSV file with stock data and convert to arrays
363 | - graph stock data using pyplot
364 | 
365 | #### v1.1, basic indicators
366 | - implement some indicators (sma, ema, macd) with custom date ranges
367 | - display indicators using pyplot
368 | 
369 | #### v1.2, playing around with data
370 | - calculate growth of all stocks in a file
371 | - specify time period for analysis
372 | - implement some utils to make analysis consistent (e.g. date math, nearest date before/after given date)
373 | 
374 | #### v1.3, ETF data generation
375 | - given two tickers, create relationship between the two and extrapolate data for one based on data in other (e.g. UPRO is 3x the S&P500, read S&P before UPRO's inception to calculate what UPRO would have been had it existed before its inception)
376 | - tweak data generation to improve accuracy
377 | - test generation by generating existing data and comparing
378 | 
379 | #### v1.4, cleanup
380 | - move repeated code into functions
381 | - rewrite some functions to be more legible and have clearer logic flow
382 | 
383 | 
384 | ## Version 2
385 | 
386 | > Goals: simulate a basic portfolio, create framework-esque platform
387 | 
388 | #### v2.0, basic portfolio
389 | - create portfolio class, which has cash, holdings, and assets
390 | - create portfolio behaviour (buy, sell, short, cover)
391 | 
392 | #### v2.1, basic market
393 | - create market class, which has a date and stocks
394 | - create market behaviour (query stocks on date, advance date, add stocks, inject data)
395 | 
396 | #### v2.2, basic simulation
397 | - create simulator class, which has portfolio, market, and start/end date(s)
398 | - create simulator simulation behaviour
399 | 
400 | #### v2.3, simulation features
401 | - add contributions and rebalancing of portfolio holdings to simulator
402 | - add optional commission costs
403 | - add portfolio statistics for graphing purposes (portfolio value, asset allocation, annual return, contribution vs growth)
404 | 
405 | #### v2.4, validation, cleanup, and fixes
406 | - separate download logic into own Downloader class
407 | - implement downloading from google, since yahoo stopped their free/easy to use service
408 | - separated all classes into own files and put all util classes/functions into own file
409 | - implement Trader class for trading logic
410 | - implement DataManager class for managing data on disk
411 | - implement Monitor class for statistics and record keeping during simulations
412 | - implement Calculator class for stand-alone calculations outside simulations
413 | - rewrote all files to follow PEP-8 and Google docstrings coding style
414 | 
415 | 
416 | ## Version 3
417 | 
418 | > Goals: more intricate user programmed strategies
419 | 
420 | #### v3.0, basic indicator related strategies
421 | - implement Brain class, where all decision making will happen
422 | - Trader now has a Brain, but otherwise only executes trades based on what Brain has decided (i.e. Brain calculates needed shares, Trader then references needed shares and executes trades so their Portfolio matches said shares)
423 | - implement custom strategies read from file (all needed data is automatically extracted from the strategies file so only the files need to be changed to test a new strategy)
424 | - Sharpe and Sortino ratios implemented (helps compare strategy effectiveness)
425 | - separated MACD into two indicators: MACD and MACDSIGNAL
426 | 
427 | # 6. Definitions
428 | 
429 | > NOTE: Some definitions have been pulled from or influenced by Investopedia. Terminology is also simplified to avoid using undefined terms in definitions.
430 | 
431 | #### Indicator
432 | Indicators are statistics used to measure current conditions as well as to forecast financial or economic trends.
433 | 
434 | http://www.investopedia.com/terms/i/indicator.asp
435 | 
436 | 
437 | #### SMA (Simple Moving Average)
438 | Always has a period (number of days, X) associated with it. The average price for a stock over the last X days. Typically used to quantify trends.
439 | 
440 | http://www.investopedia.com/terms/s/sma.asp
441 | 
442 | 
443 | #### EMA (Exponential Moving Average)
444 | Always has a period (number of days, X) associated with it. Similar to the SMA, but the weight given to each price goes down exponentially as you go backwards in time. Whereas in a SMA, equal weight is given to each day.
445 | 
446 | http://www.investopedia.com/terms/e/ema.asp
447 | 
448 | 
449 | #### MACD (Moving Average Convergence Divergence)
450 | Typically has three periods (number of days, X, Y, Z) associated with it. The standard periods are 12, 26, 9, but these can be changed. The math is too complicated for this definition, but in general, it tries to quantify the momentum of a stock, rather than the trend, by subtracting a long-term trend from a short-term trend (in an attempt to see the 'net' trend).
451 | 
452 | http://www.investopedia.com/terms/m/macd.asp
453 | 
454 | 
455 | #### Rebalance
456 | When you build a portfolio of assets, a standard strategy is to specify weights for each asset (e.g. if you have 4 assets, you might give each a weight of 25% in your portfolio). However, over time asset values change and these weights/ratios might stray from what you originally specified. Rebalancing is simply buying/selling until the original weights/ratios are restored.
457 | 
458 | http://www.investopedia.com/terms/r/rebalancing.asp
459 | 
460 | 
461 | #### [Adjusted] CAGR (Compound Annual Growth Rate)
462 | Simply put, this is the average rate at which your portfolio grew every year. Adjusted CAGR is applicable only when contributions have been made to the portfolio after its inception; it doesn't include these contributions in the growth and tells you the 'net' growth per year.
463 | > NOTE: growth is exponential, so this is not total growth divided by years.
464 | 
465 | http://www.investopedia.com/terms/c/cagr.asp
466 | 
467 | 
468 | #### Sharpe Ratio
469 | A ratio of returns:volatility. In other words, a value meant to quantify how much return you get on per unit of risk you take on. Often times risk is the variable controlled for when managing a portfolio. For example, two portfolios moved up 10% in a year, but the first moved drastically up and down along the way, while another moved in a straight line. The former is very volatile and would have a low ratio, while the latter is not volatile and would have a higher ratio. Typically, higher is better.
470 | 
471 | http://www.investopedia.com/terms/s/sharperatio.asp
472 | 
473 | 
474 | #### Sortino Ratio
475 | A ratio of returns:negative volatility. Similar to Sharpe, but this ignores volatility in the positive direction, since drastic upward moves are considered good.
476 | 
477 | http://www.investopedia.com/terms/s/sortinoratio.asp
478 | 
479 | 
480 | #### Drawdown
481 | A percent change between a peak and a valley on a chart. For our purposes, we care about maximum drawdowns, which is the biggest loss you incur along the way.
482 | 
483 | http://www.investopedia.com/terms/d/drawdown.asp
484 | 
485 | 
486 | #### ETF (Exchange Traded Fund)
487 | For all practical purposes, this is just another stock. The difference is, ETFs aren't based on spefic companies usually, but rather on and index or collections of companies/commodities/etc., usually based on some criteria.
488 | 
489 | http://www.investopedia.com/terms/e/etf.asp
490 | 
491 | 
492 | #### Leveraged ETF
493 | Assume there exists an ETF X. A leveraged ETF based on X would seek to multiply the returns of X by some factor (usually 2 or 3).
494 | > NOTE: returns can be negative, so multiplying returns is typically considered very risky.
495 | 
496 | http://www.investopedia.com/terms/l/leveraged-etf.asp
497 | 


--------------------------------------------------------------------------------
/Simulator.py:
--------------------------------------------------------------------------------
  1 | import datetime
  2 | from datetime import datetime as dt
  3 | 
  4 | from utils import date_str
  5 | from utils import date_obj
  6 | 
  7 | from Calculator import Calculator
  8 | 
  9 | 
 10 | class Simulator(object):
 11 | 
 12 |     """A simulator for how a portfolio would act in a market.
 13 | 
 14 |     Generally speaking, this is a coordinator between the Trader and
 15 |     the Market. A Market has stocks and dates, while a Trader has a
 16 |     Portfolio and strategies; both are indepedent, so this Simulator
 17 |     moves the Market and makes the Trader react to it.
 18 |     Additionally, this Simulator takes stats of the Portfolio at each
 19 |     day.
 20 | 
 21 |     Supported Statistics:
 22 |         - Per-day portfolio value history
 23 |         - Per-day asset allocation
 24 |         - Per-day contributions vs growth percent of total value
 25 |         - Per-year annual returns
 26 | 
 27 |     Attributes:
 28 |         dates_testing: A tuple indicating a range of dates to test
 29 | 
 30 |     Todo:
 31 |         - [new feature] multiple portfolios/traders
 32 |     """
 33 | 
 34 |     def __init__(self):
 35 |         """Initializes an empty simulator."""
 36 |         self._calc = Calculator()
 37 |         self._trader = None
 38 |         self._market = None
 39 |         self._monitor = None
 40 |         self._stocks = set({})
 41 |         self._indicators = set({})
 42 |         self.dates_testing = (None, None)
 43 | 
 44 |     def add_trader(self, trader):
 45 |         """Sets the Trader for this Simulator.
 46 | 
 47 |         Args:
 48 |             trader: A Trader instance
 49 |         """
 50 |         self._trader = trader
 51 | 
 52 |     def use_market(self, market):
 53 |         """Sets the Market for this Simulator to use.
 54 | 
 55 |         Args:
 56 |             market: A Market instance to use
 57 |         """
 58 |         self._market = market
 59 | 
 60 |     def use_monitor(self, monitor):
 61 |         """Sets the Monitor for this Simulator to use.
 62 | 
 63 |         Args:
 64 |             monitor: A Monitor instance to use
 65 |         """
 66 |         self._monitor = monitor
 67 | 
 68 |     def use_stocks(self, tickers):
 69 |         """Adds a set of stocks to the stocks with which to populate
 70 |         the Market.
 71 | 
 72 |         Args:
 73 |             tickers: A set of tickers
 74 |         """
 75 |         self._stocks |= set(tickers)
 76 | 
 77 |     def use_indicators(self, indicators):
 78 |         """Adds a set of indicators to the indicators with which to
 79 |         populate the Market.
 80 | 
 81 |         Args:
 82 |             indicators: A set of indicators
 83 |         """
 84 |         self._indicators |= set(indicators)
 85 | 
 86 |     def set_start_date(self, date):
 87 |         """Sets the start date for this Simulator.
 88 | 
 89 |         Args:
 90 |             date: A start date
 91 |         """
 92 |         self.dates_testing = (date_str(date), self.dates_testing[1])
 93 | 
 94 |     def set_end_date(self, date):
 95 |         """Sets the end date for this Simulator.
 96 | 
 97 |         Args:
 98 |             date: An end date
 99 |         """
100 |         self.dates_testing = (self.dates_testing[0], date_str(date))
101 | 
102 |     def remove_date_limits(self):
103 |         """Removes any date range for this Simulator."""
104 |         self.dates_testing = (None, None)
105 | 
106 |     def simulate(self):
107 |         """Runs this Simulator with the current configuration."""
108 |         self._init_market()
109 |         self._init_dates()
110 |         self._init_trader()
111 |         self._monitor.init_stats()
112 |         while self._market.current_date() < self.dates_testing[1]:
113 |             self._market.advance_day()
114 |             self._trader.adjust_portfolio()
115 |             self._monitor.take_snapshot()
116 | 
117 |     def _init_market(self):
118 |         """Initializes/resets the Market to work with the current
119 |         Simulator setup.
120 | 
121 |         Specifically, adds all stocks to the Market and resets the
122 |         Market's dates. Then, adds all relevant indicators."""
123 |         for asset in self._stocks:
124 |             if asset not in self._market.stocks.keys():
125 |                 self._market.add_stocks([asset])
126 |             for indicator in self._indicators:
127 |                 self._market.add_indicator(
128 |                     asset,
129 |                     indicator,
130 |                     self._calc.get_indicator(indicator,
131 |                                              self._market.stocks[asset]))
132 |         self._market.set_default_dates()    
133 | 
134 |     def _init_dates(self):
135 |         """Initializes/resets the testing dates for this Simulator.
136 | 
137 |         Specifically, aligns the Simulator's dates with the Market's
138 |         dates."""
139 |         if (not self.dates_testing[0]
140 |                 or self.dates_testing[0] < self._market.dates[0]):
141 |             self.dates_testing = (self._market.dates[0],
142 |                                   self.dates_testing[1])
143 |         else:
144 |             self._market.set_date(self.dates_testing[0])
145 |         if (not self.dates_testing[1]
146 |                 or self.dates_testing[1] > self._market.dates[-1]):
147 |             self.dates_testing = (self.dates_testing[0],
148 |                                   self._market.dates[-1])
149 | 
150 |     def _init_trader(self):
151 |         """Initializes/resets the Trader(s) for this Simulator.
152 | 
153 |         Specifically, asigns a Market for the Trader to uses and
154 |         initializes the Trader's Portfolio.
155 |         """
156 |         self._trader.use_market(self._market)
157 |         self._trader.initialize_portfolio()
158 | 


--------------------------------------------------------------------------------
/Trader.py:
--------------------------------------------------------------------------------
  1 | from Brain import Brain
  2 | 
  3 | 
  4 | class Trader(object):
  5 |     """A Trader is meant to emulate the role of a trader in the stock
  6 |     market.
  7 | 
  8 |     A Trader will have a Portfolio and make decisions based on a set of
  9 |     criteria.
 10 | 
 11 |     Supported Portfolio Management:
 12 |         - contributions every month/quarter/year
 13 |         - rebalancing every month/quarter/year
 14 | 
 15 |     Attributes:
 16 |         portfolio: A Portfolio instance to manage
 17 |         assets_of_interest: An array of strings, each of which
 18 |             represent a ticker of interest
 19 | 
 20 |     Todo:
 21 |         - [new feature] reimplement withdrawals
 22 |         - [new feature] dynamic strategies (need to plan specifics)
 23 |     """
 24 | 
 25 |     def __init__(self, starting_cash, portfolio, market):
 26 |         """Initializes a Trader.
 27 | 
 28 |         Args:
 29 |             starting_cash: A value representing the amount of cash this
 30 |                 Trader starts with
 31 |             portfolio: A Portfolio to be managed by this Trader
 32 |             assets_of_interest: An array of strings, each of which
 33 |                 represent a ticker of interest
 34 |         """
 35 |         # general setup
 36 |         self._brain = Brain()
 37 |         self._contributions = None
 38 |         self.use_portfolio(portfolio)
 39 |         self.use_market(market)
 40 |         self.set_starting_cash(starting_cash)
 41 | 
 42 |     def use_portfolio(self, portfolio):
 43 |         """Sets the Portfolio this Trader should use. Propagates the
 44 |         Portfolio to the Brain as well.
 45 | 
 46 |         Args:
 47 |             portfolio: A Portfoio instance to use
 48 |         """
 49 |         self.portfolio = portfolio
 50 |         self._brain.use_portfolio(portfolio)
 51 | 
 52 |     def use_market(self, market):
 53 |         """Sets the market this Trader should use for looking up
 54 |         prices. Propagates the Market to the Brain as well.
 55 | 
 56 |         Args:
 57 |             market: A Market instance to use
 58 |         """
 59 |         self._market = market
 60 |         self._brain.use_market(market)
 61 |         self.portfolio.use_market(market)
 62 | 
 63 |     def set_starting_cash(self, amount):
 64 |         """Sets the starting cash of this trader.
 65 | 
 66 |         Args:
 67 |             amount: A value representing the starting cash
 68 |         """
 69 |         self.starting_cash = float(amount)
 70 |         self.portfolio.starting_cash = float(amount)
 71 | 
 72 |     def set_contributions(self, amount, period):
 73 |         """Sets the amount of cash to contribute per period.
 74 | 
 75 |         Args:
 76 |             amount: A value representing the amount to contribute
 77 |             period: A value representing the frequency of contributions
 78 |                 e.g. 'm' for monthly
 79 |         """
 80 |         self._contributions = (amount, period)
 81 | 
 82 |     def set_rebalancing_period(self, period):
 83 |         """Sets the rebalancing frequency. Propagates the value to the
 84 |         Brain. The Trader will rebalance every new period, regardless
 85 |         of ratios or if the Portfolio was recently rebalanced.
 86 | 
 87 |         Args:
 88 |             period: A value representing the frequency of rebalancing
 89 |                 e.g. 'm' for monthly
 90 |         """
 91 |         self._brain.set_rebalancing_period(period)
 92 | 
 93 |     def set_strategy(self, strategy):
 94 |         """Sets to one of the predefined strategies."""
 95 |         self._brain.set_strategy(strategy)
 96 | 
 97 |     def initialize_portfolio(self):
 98 |         """Sets up the portfolio to the current desired ratios.
 99 |         Intended to run once at start."""
100 |         self.portfolio.add_cash(self.starting_cash)
101 |         self._brain.decide_needed_shares()
102 |         self._execute_trades()
103 | 
104 |     def adjust_portfolio(self):
105 |         """Decides a new portfolio asset allocation, if applicable, and
106 |         adjusts the Portfolio to it."""
107 |         self._contribute()
108 |         self._brain.decide_needed_shares()
109 |         self._execute_trades()
110 | 
111 |     def get_assets_of_interest(self):
112 |         """Returns this Trader's assets of interest.
113 | 
114 |         Returns:
115 |             A list of assets
116 |         """
117 |         return self._brain.assets_of_interest.copy()
118 | 
119 |     def add_asset_of_interest(self, ticker):
120 |         """Adds a ticker for an asset to the assets of interest.
121 | 
122 |         Args:
123 |             ticker: A string representing the ticker of a desired asset
124 |         """
125 |         self._brain.assets_of_interest.add(ticker)
126 |         self._brain.desired_ratios[ticker] = 0
127 |         self._brain.desired_shares[ticker] = 0
128 | 
129 |     def add_assets_of_interest(self, tickers):
130 |         """Adds a set tickers to the assets of interest.
131 | 
132 |         Args:
133 |             ticker: A set of tickers
134 |         """
135 |         self._brain.assets_of_interest |= tickers
136 |         for ticker in tickers:
137 |             self._brain.desired_ratios[ticker] = 0
138 |             self._brain.desired_shares[ticker] = 0
139 | 
140 |     def set_desired_asset_ratio(self, ticker, ratio):
141 |         """Sets an allocation for an asset.
142 | 
143 |         Args:
144 |             ticker: A string representing the ticker of a desired asset
145 |             ratio: An int corresponding to the desired ratio for the
146 |                 given ticker
147 |         """
148 |         self._brain.desired_asset_ratios[ticker] = ratio
149 | 
150 |     def _execute_trades(self):
151 |         """Calculates the trades needed to be made to satisfy the
152 |         desired shares and executes the trades.
153 | 
154 |         NOTE: Sometimes there isn't the right amount of cash for a buy,
155 |             in which case a buy/sell performs the maximum amount it can
156 |             do. As a result, the desired shares need to be updated to
157 |             avoid trying to buy/sell the remaining shares every
158 |             following day."""
159 |         # calculate trades needed
160 |         desired_trades = {'buy': {}, 'sell': {}}
161 |         for asset in self._brain.assets_of_interest:
162 |             change = (self._brain.desired_shares[asset]
163 |                       - self.portfolio.shares_of(asset))
164 |             if change < 0:
165 |                 desired_trades['sell'][asset] = abs(change)
166 |             elif change > 0:
167 |                 desired_trades['buy'][asset] = abs(change)
168 |         # perform sells
169 |         for (ticker, amount) in desired_trades['sell'].items():
170 |             self.portfolio.sell(ticker, amount)
171 |             self._brain.desired_shares[ticker] \
172 |                 = self.portfolio.shares_of(ticker)
173 |         # perform buys
174 |         for (ticker, amount) in desired_trades['buy'].items():
175 |             self.portfolio.buy(ticker, amount)
176 |             self._brain.desired_shares[ticker] \
177 |                 = self.portfolio.shares_of(ticker)
178 | 
179 |     def _contribute(self):
180 |         """Contributes to the portfolio according to the set
181 |         contribution settings."""
182 |         if self._contributions == None:
183 |             return
184 |         if self._market.new_period[self._contributions[1]]:
185 |             self.portfolio.add_cash(float(self._contributions[0]))
186 | 


--------------------------------------------------------------------------------
/folio.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/python
  2 | 
  3 | import argparse
  4 | import urllib
  5 | import os
  6 | import os.path
  7 | import matplotlib.pyplot as pyplot
  8 | import matplotlib.dates as mdates
  9 | import datetime
 10 | from datetime import datetime as dt
 11 | import calendar
 12 | import time
 13 | 
 14 | from Downloader import Downloader
 15 | from DataManager import DataManager
 16 | from Market import Market
 17 | from Portfolio import Portfolio
 18 | from Simulator import Simulator
 19 | from Monitor import Monitor
 20 | from Trader import Trader
 21 | from Calculator import Calculator
 22 | from utils import *
 23 | 
 24 | ##############################################################################
 25 | # MAIN
 26 | ##############################################################################
 27 | 
 28 | 
 29 | def main():
 30 |     args = parser.parse_args()
 31 | 
 32 |     if args.draw:
 33 |         # parse arguments and collect data
 34 |         # price history
 35 |         if args.use_generated:
 36 |             gen = args.use_generated[0]
 37 |             src = args.use_generated[1]
 38 |             (data, _) = calc.generate_theoretical_data(gen, src)
 39 |         else:
 40 |             data = db.build_price_lut(args.draw[0])
 41 |         dates = [date_obj(date) for date in sorted(data.keys())]
 42 |         prices = [data[date_str(date)] for date in dates]
 43 |         # indicators and plot counts
 44 |         if not args.indicators:
 45 |             args.indicators = []
 46 |         plots = 1
 47 |         indicators = {}
 48 |         for indicator_code in args.indicators:
 49 |             if indicator_code[0:4] == 'MACD':
 50 |                 plots = 2
 51 |             indicators[indicator_code] = calc.get_indicator(indicator_code,
 52 |                                                             data, True)
 53 | 
 54 |         # plot main price data
 55 |         pyplot.subplot(plots * 100 + 11)
 56 |         pyplot.plot(dates, prices, label='{} price'.format(args.draw[0]))
 57 |         pyplot.legend(loc='upper left')
 58 | 
 59 |         # plot indicators
 60 |         for (indicator_code, series) in indicators.items():
 61 |             code_parts = indicator_code.split('_')
 62 |             indicator = code_parts[0]
 63 |             if len(code_parts) > 1:
 64 |                 period_code = code_parts[1]
 65 |             if indicator == 'MACD':
 66 |                 pyplot.subplot(plots * 100 + 12)
 67 |                 pyplot.plot(dates, series[0], label=indicator_code)
 68 |                 pyplot.plot(dates, series[1],
 69 |                             label='Signal_{}'.format(period_code))
 70 |                 pyplot.legend(loc='upper left')
 71 |             else:
 72 |                 pyplot.subplot(plots * 100 + 11)
 73 |                 pyplot.plot(dates, series, label=indicator_code)
 74 |                 pyplot.legend(loc='upper left')
 75 | 
 76 |         pyplot.show()
 77 | 
 78 |     if args.generate:
 79 |         (part, full) = calc.generate_theoretical_data(args.generate[0],
 80 |                                                       args.generate[1])
 81 |         tgt_lut = db.build_price_lut(args.generate[0])
 82 |         src_lut = db.build_price_lut(args.generate[1])
 83 |         tgt_dates = [date_obj(d) for d in sorted(tgt_lut.keys())]
 84 |         src_dates = [date_obj(d) for d in sorted(part.keys())]
 85 |         tgt_gen_part_prices = [part[date_str(d)] for d in src_dates]
 86 |         tgt_gen_full_prices = [full[date_str(d)] for d in src_dates]
 87 |         src_prices = [src_lut[date_str(d)] for d in src_dates]
 88 | 
 89 |         pyplot.subplot(211)
 90 |         pyplot.plot([date_obj(d) for d in tgt_dates],
 91 |                     tgt_gen_full_prices[-len(tgt_dates):],
 92 |                     label='{}-generated'.format(args.generate[0]))
 93 |         pyplot.plot([date_obj(d) for d in tgt_dates],
 94 |                     tgt_gen_part_prices[-len(tgt_dates):],
 95 |                     label='{}'.format(args.generate[0]))
 96 |         pyplot.legend(loc='upper left')
 97 | 
 98 |         pyplot.subplot(212)
 99 |         pyplot.plot(src_dates, tgt_gen_part_prices, label='{}-generated'.format(args.generate[0]))
100 |         pyplot.plot(src_dates, src_prices, label='{}'.format(args.generate[1]))
101 |         pyplot.legend(loc='upper left')
102 | 
103 |         pyplot.show()
104 | 
105 |     if args.portfolio:
106 |         # init main objects
107 |         my_market = Market()
108 |         my_portfolio = Portfolio()
109 |         my_trader = Trader(args.portfolio[0], my_portfolio, my_market)
110 | 
111 |         # init simulator
112 |         my_monitor = Monitor(my_trader, my_market)
113 |         my_sim = Simulator()
114 |         my_sim.add_trader(my_trader)
115 |         my_sim.use_market(my_market)
116 |         my_sim.use_monitor(my_monitor)
117 | 
118 |         (strategy, tickers, indicators) = db.build_strategy(args.strategy[0])
119 |         my_trader.add_assets_of_interest(strategy['assets'])
120 |         my_trader.set_strategy(strategy['positions'])
121 |         my_sim.use_stocks(tickers)
122 |         my_sim.use_indicators(indicators)
123 | 
124 |         if args.contribute:
125 |             my_trader.set_contributions(args.contribute[0], args.contribute[1])
126 | 
127 |         if args.rebalance:
128 |             my_trader.set_rebalancing_period(args.rebalance[0])
129 | 
130 |         if args.use_generated:
131 |             for i in range(len(args.use_generated) // 2):
132 |                 gen = args.use_generated[i * 2]
133 |                 src = args.use_generated[i * 2 + 1]
134 |                 (data, _) = calc.generate_theoretical_data(gen, src)
135 |                 my_market.inject_stock_data(gen, None, None, data)
136 | 
137 |         # run simulation
138 |         my_sim.simulate()
139 | 
140 |         # print some stats
141 |         print('##################################')
142 |         print('# PERFORMANCE SUMMARY')
143 |         print('##################################')
144 |         print('initial: $' + currency(my_trader.starting_cash))
145 |         print('final:   $' + currency(my_trader.portfolio.value()))
146 |         print('trades:  {}'.format(my_portfolio.trades))
147 |         print('---------------------------')
148 |         print('Sharpe Ratio:  {}'.format(
149 |             my_monitor.get_statistic('sharpe_ratio')))
150 |         print('Sortino Ratio: {}'.format(
151 |             my_monitor.get_statistic('sortino_ratio')))
152 |         print('---------------------------')
153 |         print('CAGR:          {}%'.format(
154 |             percent(my_monitor.get_statistic('cagr'))))
155 |         print('Adjusted CAGR: {}%'.format(
156 |             percent(my_monitor.get_statistic('adjusted_cagr'))))
157 |         print('---------------------------')
158 |         print('best year:  {}%'.format(
159 |             percent(max(my_monitor.get_data_series('annual_returns')[1]))))
160 |         print('worst year: {}%'.format(
161 |             percent(min(my_monitor.get_data_series('annual_returns')[1]))))
162 |         print('---------------------------')
163 |         drawdown = my_monitor.get_statistic('max_drawdown')
164 |         print('max drawdown: {}%'.format(percent(drawdown['amount'])))
165 |         print('  between {} and {}, recovered by {}'.format(
166 |             drawdown['from'], drawdown['to'], drawdown['recovered_by']))
167 | 
168 |         # show plots
169 |         (x, y) = my_monitor.get_data_series('portfolio_values')
170 |         pyplot.subplot(411)
171 |         pyplot.plot(x, y)
172 |         pyplot.grid(b=False, which='major', color='grey', linestyle='-')
173 | 
174 |         (x, y) = my_monitor.get_data_series('asset_allocations')
175 |         pyplot.subplot(412)
176 |         pyplot.stackplot(x, y, alpha=0.5)
177 |         pyplot.grid(b=True, which='major', color='grey', linestyle='-')
178 |         pyplot.legend(sorted(strategy['assets']), loc='upper left')
179 | 
180 |         (x, y) = my_monitor.get_data_series('annual_returns')
181 |         ax = pyplot.subplot(413)
182 |         pyplot.bar(list(range(len(x))), y, 0.5, color='blue')
183 |         ax.set_xticks(list(range(len(x))))
184 |         ax.set_xticklabels(x)
185 |         pyplot.grid(b=True, which='major', color='grey', linestyle='-')
186 | 
187 |         (x, y) = my_monitor.get_data_series('contribution_vs_growth')
188 |         pyplot.subplot(414)
189 |         pyplot.stackplot(x, y, alpha=0.5)
190 |         pyplot.grid(b=True, which='major', color='grey', linestyle='-')
191 |         pyplot.legend(['Contributions', 'Growth'], loc='upper left')
192 | 
193 |         pyplot.show()
194 | 
195 |     exit()
196 | 
197 | 
198 | if __name__ == "__main__":
199 |     parser = argparse.ArgumentParser(description='Stock backtester (WIP).')
200 |     parser.add_argument('--draw', nargs=1, help='Draw a chart for a ticker')
201 |     parser.add_argument('--indicators', nargs='+',
202 |                         help='Use with --draw. Specify an indicator or set of indicators to show on top of the chart for --draw. Example: SMA_50 SMA_20')
203 |     parser.add_argument('--generate', nargs=2,
204 |                         help='Generate data for first based on second. Standalone.')
205 |     parser.add_argument('--portfolio', nargs=1,
206 |                         help='Specify a portfolio amount.')
207 |     parser.add_argument('--strategy', nargs=1,
208 |                         help='Use with --portfolio. Specify a strategy file to use. Currently this is necessary for your portfolio to do anything interesting.')
209 |     parser.add_argument('--contribute', nargs=2,
210 |                         help='Use with --portfolio. Specify an amount to contribute with a frequency')
211 |     parser.add_argument('--rebalance', nargs=1,
212 |                         help='Use with --portfolio. Specify a frequency at which to rebalance.')
213 |     parser.add_argument('--use-generated', nargs='+',
214 |                         help='Use with --portfolio or --draw. Specify pairs of tickers, wherein the first of the pair will be generated based on the second. This will replace the data used in --draw or --portfolio.')
215 | 
216 |     db = DataManager()
217 |     calc = Calculator()
218 | 
219 |     main()
220 | 


--------------------------------------------------------------------------------
/stocks-and-bonds:
--------------------------------------------------------------------------------
1 | 0.6,SPY,ALWAYS,NEVER
2 | 0.4,TLT,ALWAYS,NEVER
3 | 


--------------------------------------------------------------------------------
/stocks-and-bonds-timing:
--------------------------------------------------------------------------------
1 | 0.2,SPY,ALWAYS,NEVER
2 | 0.4,SPY,SPY~PRICE > SPY~SMA_100,SPY~PRICE < SPY~SMA_100
3 | 0.4,TLT,ALWAYS,NEVER
4 | 


--------------------------------------------------------------------------------
/stocks-only:
--------------------------------------------------------------------------------
1 | 1.0,SPY,ALWAYS,NEVER
2 | 0.0,TLT,ALWAYS,NEVER
3 | 


--------------------------------------------------------------------------------
/upro-only:
--------------------------------------------------------------------------------
1 | 1.0,UPRO,ALWAYS,NEVER
2 | 


--------------------------------------------------------------------------------
/utils.py:
--------------------------------------------------------------------------------
  1 | """A collection of utility classes and methods to be used throughout
  2 | the diferent modules and classes in this project.
  3 | 
  4 | Todo:
  5 |     - [code improvement, low priority] rewrite nearest_index using
  6 |         next() and enumerate()
  7 |     - [for fun, low priority] run benchmark on nearest_date_index
  8 | """
  9 | 
 10 | import datetime
 11 | from datetime import datetime as dt
 12 | import os
 13 | import os.path
 14 | 
 15 | STOCK_DIR = "data/"
 16 | DATE_FORMAT = "%Y-%m-%d"
 17 | 
 18 | ######
 19 | # CLASSES
 20 | #####
 21 | 
 22 | 
 23 | class SteppedAvgLookup(object):
 24 | 
 25 |     """A look up table/mapping of values to averages..
 26 | 
 27 |     Given a set of keys and values and a step range, calculates the
 28 |     average value for all steps based on the keys that lie in those
 29 |     steps.
 30 |     e.g. if step = 1, (keys,values) = (1.1, 10), (2.3, 5), (2.5, 3),
 31 |         then the LUT it will build is:
 32 |             0 to 2   -> 10 (0 to 2 is steps 0 to 1, and 1 to 2,
 33 |                             which has pairs (1.1, 10))
 34 |             2 to inf -> 4  (2 to inf is steps 2 to 3, 3 to 4, ... etc,
 35 |                             which has pairs (2.3, 5), (2.5, 3))
 36 | 
 37 |     Used for calculating the average "near" certain values, given
 38 |     enough data for such a notion to be useful, but not enough data
 39 |     to have an average for every single value.
 40 | 
 41 |     Currently, primary use is estimating an ETF's true leverage factor
 42 |     at its underlying asset's movement, e.g. UPRO and SPY. There is
 43 |     enough data (every day between July 2009 and today) to estimate
 44 |     how UPRO moves relative to SPY (supposedly 3x, but in reality it
 45 |     varies) at a given movement of SPY.
 46 |     """
 47 | 
 48 |     def __init__(self, step, keys, vals):
 49 |         """Initializes an empty lookup and then builds it based on the
 50 |         given values.
 51 | 
 52 |         Args:
 53 |             step: A value specifying the step size, while a higher
 54 |                 value may give more precision, that does not always
 55 |                 imply more accuracy
 56 |             keys: An array of keys which correspond to the values
 57 |             vals: An array of values which correspond to the keys
 58 |         """
 59 |         self._lut = {}
 60 |         self._num_points = {}
 61 |         self._build_lut(step, keys, vals)
 62 | 
 63 |     def get(self, val):
 64 |         """Gets the average at the neatest key greater than the given
 65 |         value.
 66 | 
 67 |         Args:
 68 |             val: A value for which an average is wanted
 69 | 
 70 |         Returns:
 71 |             A value corresponding the the average at the given value
 72 |         """
 73 |         for key in sorted(self._lut.keys()):
 74 |             if val < key:
 75 |                 return self._lut[key]
 76 | 
 77 |     def get_num_points(self, val):
 78 |         """Returns the number of data points at the given step.
 79 | 
 80 |         Used internally for calculating averages.
 81 | 
 82 |         Args:
 83 |             val: A value for which the number of data points is wanted
 84 | 
 85 |         Returns:
 86 |             A value corresponding to the number of data points at the
 87 |             given value
 88 |         """
 89 |         for key in sorted(self._num_points.keys()):
 90 |             if val < key:
 91 |                 return self._num_points[key]
 92 | 
 93 |     def _build_lut(self, step, keys, vals):
 94 |         """Internal function for building the LUT.
 95 | 
 96 |         Args:
 97 |             step: A value specifying the step size, while a higher
 98 |                 value may give more precision, that does not always
 99 |                 imply more accuracy
100 |             keys: An array of keys which correspond to the values
101 |             vals: An array of values which correspond to the keys
102 |         """
103 |         for i in range(int(min(keys) // step), int(max(keys) // step)):
104 |             self._lut[i * step] = 0
105 |             self._num_points[i * step] = 0
106 |         self._lut[float("inf")] = 0
107 |         self._num_points[float("inf")] = 0
108 |         steps = sorted(self._lut.keys())
109 |         for i in range(0, len(keys)):
110 |             for j in range(0, len(steps)):
111 |                 if keys[i] < steps[j]:
112 |                     self._lut[steps[j]] = \
113 |                         ((self._lut[steps[j]]
114 |                           * self._num_points[steps[j]] + vals[i])
115 |                          / (self._num_points[steps[j]] + 1))
116 |                     break
117 |         for key in sorted(self._lut.keys()):
118 |             if self._lut[key] == 0:
119 |                 del self._lut[key]
120 | 
121 | 
122 | ######
123 | # FUNCTIONS
124 | #####
125 | 
126 | def currency(number):
127 |     """Nicer looking wrapper for converting to currency format.
128 | 
129 |     Args:
130 |         number: A number value to covert to currency format
131 | 
132 |     Returns:
133 |         A number in currency format
134 |     """
135 |     return "{0:.2f}".format(float(number))
136 | 
137 | 
138 | def percent(number):
139 |     """Nicer looking wrapper for converting to percent format.
140 | 
141 |     Args:
142 |         number: A number value to covert to percent format
143 | 
144 |     Returns:
145 |         A number in percent format
146 |     """
147 |     return "{0:.2f}".format(float(number * 100))
148 | 
149 | 
150 | def date_obj(date):
151 |     """Returns the equivalent datetime object for the given date or
152 |     date object.
153 | 
154 |     Args:
155 |         number: A date string or date/datetime object
156 | 
157 |     Returns:
158 |         A datetime object representing the given date
159 |     """
160 |     if type(date) is dt:
161 |         return date
162 |     if type(date) is datetime.date:
163 |         return dt(date.year, date.month, date.day)
164 |     return dt.strptime(date, DATE_FORMAT)
165 | 
166 | 
167 | def date_str(date):
168 |     """Returns the equivalent date string for the given date or
169 |     date object.
170 | 
171 |     Args:
172 |         number: A date string or date/datetime object
173 | 
174 |     Returns:
175 |         A date string representing the given date
176 |     """
177 |     if type(date) is str:
178 |         return date
179 |     return date.strftime(DATE_FORMAT)
180 | 
181 | def days_between(date_a, date_b):
182 |     """Returns the number of days between two dates.
183 | 
184 |     Args:
185 |         date_a: A date string or object representing the earlier date
186 |         date_b: A date string or object representing the later date
187 | 
188 |     Returns:
189 |         A value representing a number of days
190 |     """
191 |     return (date_obj(date_b) - date_obj(date_a)).days
192 | 
193 | def write_list_to_file(list, filename, overwrite):
194 |     """Writes a list to a newline separated file.
195 | 
196 |     Args:
197 |         list: An array/list to write to file
198 |         filename: A filename of a file to which to write
199 |         overwrite: A boolean for whether or not to overwrite an
200 |             existing file
201 | 
202 |     Returns:
203 |         Number of lines written
204 |     """
205 |     if overwrite and os.path.isfile(filename):
206 |         os.remove(filename)
207 |     written = 0
208 |     with open(filename, 'a') as file:
209 |         for item in list:
210 |             written += 1
211 |             file.write(item + '\n')
212 |     return written
213 | 
214 | 
215 | def list_from_csv(filename, col, s_char, r_chars):
216 |     """Extracts a specific column from a CSV file, given a split char.
217 |     Also removes all given chars from the values.
218 | 
219 |     Args:
220 |         filename: A filename of a CSV file
221 |         col: A value for a column in a CSV file
222 |         s_char: A character representing a delimiter
223 |         r_chars: An array of characters to remove
224 | 
225 |     Returns:
226 |         An array of values corresponding to the stripped column
227 |     """
228 |     lines = readlines(filename)
229 |     with open(filename, 'r') as file:
230 |         lines = [line.strip() for line in file]
231 |     column_lines = []
232 |     for i in range(1, len(lines)):
233 |         column_lines.append(lines[i].split(sfilname_char)[col].strip())
234 |         for r in r_chars:
235 |             column_lines[-1] = column_lines[-1].replace(r, '')
236 |     return column_lines
237 | 
238 | 
239 | def subtract_date(period, unit, date):
240 |     """Subtracts the period from the given date, returns date in same
241 |     type as input.
242 | 
243 |     Args:
244 |         period: A period value, e.g. 3 (for 3 days/months/years)
245 |         unit: A unit for the period, e.g. 'm' for month
246 |         date: A date from which to subtract
247 | 
248 |     Returns:
249 |         A new date value in the same type as input
250 |     """
251 |     diffs = {'y': 0, 'm': 0, 'd': 0}
252 |     diffs[unit.lower()] = int(period)
253 |     new = {}
254 |     new['y'] = date_obj(date).year - diffs['y'] - diffs['m'] // 12
255 |     new['m'] = date_obj(date).month - diffs['m'] % 12
256 |     if new['m'] < 1:
257 |         new['y'] = new['y'] + (new['m'] - 1) // 12
258 |         new['m'] = new['m'] - ((new['m'] - 1) // 12) * 12
259 |     new['d'] = min(calendar.monthrange(
260 |         new['y'], new['m'])[1], date_obj(date).day)
261 |     new_date = dt(new['y'], new['m'], new['d']) - \
262 |         datetime.timedelta(diffs['d'])
263 |     if type(date) is str:
264 |         return date_str(new_date)
265 |     return new_date
266 | 
267 | 
268 | def nearest_index(val, vals, direction, val_type=None):
269 |     """Given a value, finds the index of the nearest value before/after
270 |     said value in an array of values.
271 | 
272 |     Using val_type uses an optimization. Currently only supports
273 |     'date' as a val_type, since dates are relatively predictable in
274 |     their distribution.
275 | 
276 |     Args:
277 |         val: A value for which to find the nearest value in values
278 |         vals: An array of values to look through
279 |         direction: A 'direction' (-1 or +1) for looking, i.e. to look
280 |             for a nearest lower value or nearest higher value
281 |         val_type: A type of value - used for optimizations
282 | 
283 |     Returns:
284 |         An index for the nearest value, -1 otherwise
285 |     """
286 |     if val_type == 'date':
287 |         return nearest_date_index(val, vals, direction)
288 |     if (len(vals) == 0
289 |         or (vals[-1] < val and direction > 0)
290 |         or (vals[0] > val and direction < 0)):
291 |         return -1
292 |     if direction > 0 and vals[0] > val:
293 |         return 0
294 |     if direction < 0 and vals[-1] < val:
295 |         return len(vals) - 1
296 |     for i in range(0, (len(vals) - 1)):
297 |         if (val > vals[i] and val <= vals[i + 1] and direction > 0):
298 |             return i + 1
299 |         if (val <= vals[i] and val > vals[i + 1] and direction < 0):
300 |             return i
301 |     return -1
302 | 
303 | 
304 | def nearest_date_index(date, dates, direction):
305 |     """Optimization for nearest index for date types.
306 | 
307 |     Approximates where the date would be based on starting and ending
308 |     dates in list and starts search there. In practise, only takes a
309 |     few steps.
310 | 
311 |     Args:
312 |         date: A date for which to find the nearest date in dates
313 |         dates: An array of dates to look through
314 |         direction: A 'direction' (-1 or +1) for looking, i.e. to look
315 |             for a nearest lower value or nearest higher value
316 | 
317 |     Returns:
318 |         An index for the nearest date
319 |     """
320 |     if len(dates) == 0 or date_str(dates[-1]) < date_str(date):
321 |         return -1
322 |     if date_str(dates[0]) >= date_str(date):
323 |         return 0
324 |     last_date = date_obj(dates[-1])
325 |     first_date = date_obj(dates[0])
326 |     target_date = date_obj(date)
327 |     approx_factor = len(dates) / (last_date - first_date).days
328 |     i = int((target_date - first_date).days * approx_factor)
329 |     if i > 0:
330 |         i -= 1
331 |     if date_str(dates[i]) == date_str(date):
332 |         return i
333 |     if date_str(dates[i]) < date_str(date):
334 |         while date_str(dates[i]) < date_str(date):
335 |             i += 1
336 |     else:
337 |         while date_str(dates[i - 1]) >= date_str(date):
338 |             i -= 1
339 |     if direction == 0:
340 |         return min([i, i - 1],
341 |                    key=lambda x: abs((date_obj(dates[x])
342 |                                       - date_obj(date)).days))
343 |     if direction < 0:
344 |         return i - 1
345 |     if direction > 0:
346 |         return i
347 | 


--------------------------------------------------------------------------------