├── smallaccountsize ├── __init__.py ├── instrument_list │ ├── __init__.py │ └── instrument_list.py ├── diversification_plots.py ├── generatesystems.py ├── smallaccount.yaml └── roundingeffects.py ├── dockertest ├── dockerscript ├── Dockerfile ├── dockertest.py └── dockertestresults.py ├── README.md ├── breakout ├── temp.py ├── breakoutfuturesestimateconfig.yaml └── breakout.py ├── riskenvelope └── riskenvelope.py ├── riskmanagement ├── parameteruncertainty.py └── measurement.py ├── somemoretradingrules ├── relativecarry.py ├── cross_sectional.py ├── shortvol.py ├── momentum.py └── allrules.py ├── fitting ├── get_raw_data.py └── bruteforce.py ├── forecastscaling └── forecastscaling.py ├── mythbusting ├── ewmacperformance.py ├── wholesystem.py ├── fitforecastweights.py ├── timevariationreturns.py └── overextend.py ├── variablecapital └── variablecapital.py ├── forecasting ├── correlatedassets.py ├── crossassets.py └── forecasting.py ├── regressionrule ├── tradingrule.py ├── realdata.py └── randomdata.py ├── optimisation ├── optimisation.py ├── optimisationwithcosts.py └── uncertainty.py ├── rateconditiong.py └── vix ├── vix.py └── US_monthly_returns.csv /smallaccountsize/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /smallaccountsize/instrument_list/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dockertest/dockerscript: -------------------------------------------------------------------------------- 1 | sudo docker run -t -i -v /home/rob/results:/results robcarver17/pysystemtrade 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # pysystemtrade_examples 2 | Examples using pysystemtrade for [my blog](qoppac.blogspot.com) 3 | 4 | requires: [pysystemtrade](https://github.com/robcarver17/pysystemtrade) -------------------------------------------------------------------------------- /dockertest/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python 2 | MAINTAINER Rob Carver 3 | RUN pip3 install pandas 4 | RUN pip3 install pyyaml 5 | RUN pip3 install scipy 6 | RUN pip3 install matplotlib 7 | COPY pysystemtrade/ /pysystemtrade/ 8 | ENV PYTHONPATH /pysystemtrade:$PYTHONPATH 9 | CMD [ "python3", "/pysystemtrade/examples/dockertest/dockertest.py" ] -------------------------------------------------------------------------------- /dockertest/dockertest.py: -------------------------------------------------------------------------------- 1 | """ 2 | Let's get the chapter 15 system 3 | """ 4 | 5 | from systems.provided.futures_chapter15.basesystem import futures_system 6 | from matplotlib.pyplot import show 7 | 8 | resultsdir = "/results" 9 | system = futures_system(log_level="on") 10 | print(system.accounts.portfolio().sharpe()) 11 | system.pickle_cache("", resultsdir + "/dockertest.pck") 12 | -------------------------------------------------------------------------------- /dockertest/dockertestresults.py: -------------------------------------------------------------------------------- 1 | from systems.provided.futures_chapter15.basesystem import futures_system 2 | from matplotlib.pyplot import show 3 | 4 | resultsdir = "/home/rob/results" 5 | 6 | system = futures_system(log_level="on") 7 | system.unpickle_cache("", resultsdir + "/dockertest.pck") 8 | # this will run much faster and reuse previous calculations 9 | print(system.accounts.portfolio().sharpe()) 10 | -------------------------------------------------------------------------------- /breakout/temp.py: -------------------------------------------------------------------------------- 1 | from matplotlib.pyplot import show, title 2 | from systems.provided.futures_chapter15.estimatedsystem import futures_system 3 | 4 | system = futures_system() 5 | system.config.forecast_weight_estimate["pool_instruments"] = True 6 | system.config.forecast_weight_estimate["method"] = "bootstrap" 7 | system.config.forecast_weight_estimate["equalise_means"] = False 8 | system.config.forecast_weight_estimate["monte_runs"] = 200 9 | system.config.forecast_weight_estimate["bootstrap_length"] = 104 10 | 11 | system = futures_system(config=system.config) 12 | 13 | system.combForecast.get_raw_forecast_weights("CORN").plot() 14 | title("CORN") 15 | show() 16 | -------------------------------------------------------------------------------- /riskenvelope/riskenvelope.py: -------------------------------------------------------------------------------- 1 | from systems.provided.futures_chapter15.basesystem import futures_system 2 | from matplotlib.pyplot import show, hist 3 | import numpy as np 4 | 5 | system = futures_system(log_level="on") 6 | 7 | system.config.capital_multiplier['func'] = 'syscore.capital.half_compounding' 8 | 9 | system.accounts.portfolio().percent().cumsum().plot() 10 | show() 11 | 12 | drawdowns = system.accounts.portfolio().percent().drawdown() 13 | drawdowns.plot() 14 | show() 15 | 16 | drawdowns = system.accounts.portfolio_with_multiplier().percent().plot() 17 | show() 18 | 19 | drawdowns = system.accounts.portfolio_with_multiplier().percent().drawdown() 20 | drawdowns.plot() 21 | show() 22 | 23 | distr_drawdowns = list(drawdowns.values) 24 | distr_drawdowns = [x for x in distr_drawdowns if not np.isnan(x)] 25 | 26 | hist(distr_drawdowns) 27 | show() 28 | -------------------------------------------------------------------------------- /riskmanagement/parameteruncertainty.py: -------------------------------------------------------------------------------- 1 | ## make a gaussian distribution, size N 2 | from scipy.stats import norm 3 | from random import gauss 4 | import numpy as np 5 | from matplotlib.pyplot import show, hist, gca 6 | 7 | stdev=3.0 8 | Nlength=500 9 | monte_length=10000 10 | var_point=5 11 | std_correction = norm.ppf(1-var_point/100.0) 12 | 13 | assert (var_point*monte_length/100.0)>=2.0 14 | 15 | all_std = [] 16 | all_var = [] 17 | all_es = [] 18 | all_sigma=[] 19 | 20 | for unused in range(monte_length): 21 | 22 | data=[gauss(0.0, stdev) for Unused in range(Nlength)] 23 | sigma_est = np.std(data) 24 | stdev_est = np.mean(data) - std_correction*np.std(data) 25 | var_est = np.percentile(data, var_point) 26 | small_data = [x for x in data if x <= var_est] 27 | es_est = np.mean(small_data) 28 | 29 | all_std.append(stdev_est) 30 | all_var.append(var_est) 31 | all_es.append(es_est) 32 | all_sigma.append(sigma_est) 33 | 34 | hist(all_std, bins=50, range=[-6.1, -3.8]) 35 | #np.std(all_std) 36 | 37 | hist(all_var, bins=50, range=[-6.1, -3.8]) 38 | np.std(all_var) 39 | 40 | hist(all_es, bins=50) -------------------------------------------------------------------------------- /somemoretradingrules/relativecarry.py: -------------------------------------------------------------------------------- 1 | from systems.provided.futures_chapter15.basesystem import futures_system 2 | 3 | sys=futures_system() 4 | 5 | save_absolute_carry_rule = sys.config.trading_rules['carry'] 6 | 7 | # equal weighting across instruments 8 | del(sys.config.instrument_weights) 9 | 10 | # but estimated IDM 11 | sys.config.use_instrument_div_mult_estimates=True 12 | 13 | sys.config.trading_rules=dict(carry=save_absolute_carry_rule, relativecarry=dict(function='systems.provided.moretradingrules.morerules.relative_carry', 14 | data=['rawdata.smoothed_carry','rawdata.median_carry_for_asset_class'])) 15 | 16 | # going to estimate forecast weights 17 | 18 | sys.config.use_forecast_scale_estimates=True 19 | sys.config.use_forecast_weight_estimates=True 20 | sys.config.use_forecast_div_mult_estimates=True 21 | 22 | del(sys.config.forecast_weights) 23 | 24 | ans = sys.accounts.pandl_for_all_trading_rules() 25 | ans2 = sys.accounts.pandl_for_all_trading_rules_unweighted() 26 | 27 | ans2.to_frame().cumsum().plot() 28 | ans2.get_stats("sharpe",freq="monthly") 29 | 30 | ans.monthly.stats() 31 | -------------------------------------------------------------------------------- /somemoretradingrules/cross_sectional.py: -------------------------------------------------------------------------------- 1 | from systems.provided.futures_chapter15.basesystem import futures_system 2 | 3 | sys=futures_system() 4 | 5 | # equal weighting across instruments 6 | del(sys.config.instrument_weights) 7 | 8 | # but use estimated IDM 9 | sys.config.use_instrument_div_mult_estimates=True 10 | 11 | sys.config.trading_rules=dict(mr=dict(function='systems.provided.moretradingrules.morerules.cross_sectional_mean_reversion', 12 | data=['rawdata.cumulative_norm_return', 13 | 'rawdata.normalised_price_for_asset_class'],other_args=dict(horizon=125))) 14 | 15 | # going to estimate forecast weights 16 | 17 | sys.config.use_forecast_scale_estimates=True 18 | sys.config.use_forecast_weight_estimates=True 19 | sys.config.use_forecast_div_mult_estimates=True 20 | 21 | del(sys.config.forecast_weights) 22 | 23 | mkt="US10" 24 | sys.rawdata.cumulative_norm_return(mkt).plot() 25 | sys.rawdata.normalised_price_for_asset_class(mkt).plot() 26 | 27 | x=sys.rawdata.cumulative_norm_return(mkt)- sys.rawdata.normalised_price_for_asset_class(mkt) 28 | x.plot() 29 | 30 | sys.rules.get_raw_forecast(mkt, "mr").plot() 31 | 32 | sys.accounts.pandl_for_instrument_forecast(mkt, "mr").cumsum().plot() 33 | 34 | sys.accounts.portfolio().cumsum().plot() 35 | -------------------------------------------------------------------------------- /somemoretradingrules/shortvol.py: -------------------------------------------------------------------------------- 1 | from systems.provided.futures_chapter15.basesystem import * 2 | 3 | sys=futures_system() 4 | sys.config.instrument_weights = dict(VIX=0.5, V2X = 0.5) 5 | sys.config.trading_rules=dict(shortbias=dict(function='systems.provided.moretradingrules.morerules.short_bias')) 6 | sys.config.use_forecast_scale_estimates=True 7 | del(sys.config.forecast_weights) 8 | 9 | ans=sys.accounts.portfolio() 10 | 11 | ans.plot() 12 | 13 | ans.monthly.stats() 14 | 15 | # measure skew on equities 16 | sys.cache.clear() 17 | sys.config.instrument_weights = dict(SP500 = 1.0) 18 | sys.accounts.portfolio().monthly.stats() 19 | 20 | # what about if we throw carry and trend following back into the mix? 21 | sys=futures_system() 22 | sys.config.instrument_weights = dict(VIX=0.5, V2X = 0.5) 23 | sys.config.use_forecast_scale_estimates=True 24 | sys.config.use_forecast_div_mult_estimates=True 25 | sys.config.use_forecast_weight_estimates = True 26 | ans2 = sys.accounts.portfolio() 27 | 28 | # now with short bias 29 | sys=futures_system() 30 | sys.config.instrument_weights = dict(VIX=0.5, V2X = 0.5) 31 | sys.config.use_forecast_scale_estimates=True 32 | sys.config.use_forecast_div_mult_estimates=True 33 | sys.config.use_forecast_weight_estimates = True 34 | sys.config.trading_rules['shortbias']=dict(function='systems.provided.moretradingrules.morerules.short_bias') 35 | ans3 = sys.accounts.portfolio() 36 | 37 | 38 | -------------------------------------------------------------------------------- /fitting/get_raw_data.py: -------------------------------------------------------------------------------- 1 | from systems.provided.futures_chapter15.basesystem import futures_system 2 | from copy import copy 3 | from pickle import dump, load 4 | 5 | base_system=futures_system() 6 | base_config = base_system.config 7 | ## run all possible combinations of TF to get base performance 8 | 9 | instruments = base_system.get_instrument_list() 10 | 11 | results = dict() 12 | wlist = [1,2,3,4,5,6,7,8,9,10,15,20,25,30,35,40,45,50,60,70,80,90,100,125,150,175,200,250] 13 | #wlist = [1,250] 14 | 15 | instrument_list = base_system.get_instrument_list() 16 | 17 | from syscore.genutils import progressBar 18 | thing=progressBar(len(wlist)*len(wlist)*len(instrument_list)) 19 | 20 | for Aspeed in wlist: 21 | for Bspeed in wlist: 22 | 23 | if Aspeed==Bspeed: 24 | continue 25 | 26 | config=copy(base_config) 27 | trading_rules = dict(rule=dict(function='systems.provided.futures_chapter15.rules.ewmac', data=['rawdata.get_daily_prices', 'rawdata.daily_returns_volatility' 28 | ], other_args=dict(Lfast=Aspeed, Lslow=Bspeed))) 29 | 30 | config.trading_rules = trading_rules 31 | config.use_forecast_scale_estimates = True 32 | 33 | new_system=futures_system(config=config) 34 | 35 | for instrument in instrument_list: 36 | results_key = (Aspeed, Bspeed, instrument) 37 | acc=new_system.accounts.pandl_for_instrument_forecast(instrument, "rule").gross.as_ts() 38 | 39 | results[results_key]=acc 40 | thing.iterate() 41 | 42 | f = open('/home/rob/results.pck', "wb") 43 | dump(results, f) 44 | f.close() 45 | 46 | -------------------------------------------------------------------------------- /smallaccountsize/diversification_plots.py: -------------------------------------------------------------------------------- 1 | from pickle import load, dump 2 | from syscore.pdutils import align_to_joint 3 | 4 | ans = load(open("/home/rob/data.pck", "rb")) 5 | 6 | [roll_acc, idm, acc_curve, mkt_counters] = ans 7 | 8 | # plot IDM against market count, scattered 9 | 10 | mkt_count_for_scatter = [] 11 | idm_for_scatter = [] 12 | roll_acc_for_scatter = [] 13 | risk_for_scatter = [] 14 | 15 | for (roll_acc_item, idm_item, mkt_count_item) in zip(roll_acc, idm, 16 | mkt_counters): 17 | 18 | (roll_acc_item, mkt_count_item) = align_to_joint( 19 | roll_acc_item, mkt_count_item, ffill=(True, True)) 20 | (idm_item, mkt_count_item) = align_to_joint( 21 | idm_item, mkt_count_item, ffill=(True, True)) 22 | norm_risk = .2 / (roll_acc_item.resample("A").mean( 23 | ).iloc[:, 0] / idm_item.resample("A").mean().iloc[:, 0]) 24 | 25 | roll_acc_item_rs = list( 26 | roll_acc_item.resample("A").mean().iloc[:, 0].values) 27 | idm_item_rs = list(idm_item.resample("A").mean().iloc[:, 0].values) 28 | mktcount_item_rs = list(mkt_count_item.resample("A").mean().values) 29 | 30 | mkt_count_for_scatter = mkt_count_for_scatter + mktcount_item_rs 31 | roll_acc_for_scatter = roll_acc_for_scatter + roll_acc_item_rs 32 | idm_for_scatter = idm_for_scatter + idm_item_rs 33 | risk_for_scatter = risk_for_scatter + list(norm_risk.values) 34 | 35 | from matplotlib.pyplot import plot, show, scatter 36 | 37 | scatter(mkt_count_for_scatter, idm_for_scatter) 38 | show() 39 | 40 | scatter(mkt_count_for_scatter, risk_for_scatter) 41 | show() 42 | -------------------------------------------------------------------------------- /forecastscaling/forecastscaling.py: -------------------------------------------------------------------------------- 1 | from copy import copy 2 | from matplotlib.pyplot import show, bar 3 | from systems.provided.futures_chapter15.estimatedsystem import futures_system 4 | """ 5 | cross sectional 6 | """ 7 | 8 | system = futures_system() 9 | 10 | # don't pool 11 | system.config.forecast_scalar_estimate['pool_instruments'] = False 12 | 13 | instrument_list = system.get_instrument_list() 14 | print(instrument_list) 15 | 16 | results = [] 17 | for instrument_code in instrument_list: 18 | results.append( 19 | round( 20 | float( 21 | system.forecastScaleCap.get_forecast_scalar( 22 | instrument_code, "ewmac2_8").tail(1).values), 2)) 23 | print(results) 24 | 25 | results = [] 26 | for instrument_code in instrument_list: 27 | results.append( 28 | round( 29 | float( 30 | system.forecastScaleCap.get_forecast_scalar( 31 | instrument_code, "carry").tail(1).values), 2)) 32 | print(results) 33 | """ 34 | Use an expanding window 35 | """ 36 | 37 | system = futures_system() 38 | 39 | # Let's use a one year rolling window instead 40 | system.config.forecast_scalar_estimate['window'] = 250 41 | system.config.forecast_scalar_estimate['min_periods'] = 250 42 | 43 | system.forecastScaleCap.get_forecast_scalar("EDOLLAR", "ewmac64_256").plot() 44 | show() 45 | """ 46 | Goldilocks amount of minimum data - not too much, not too little 47 | """ 48 | 49 | system = futures_system() 50 | 51 | # stupidly small number of min periods 52 | system.config.forecast_scalar_estimate['min_periods'] = 50 53 | 54 | # don't pool 55 | system.config.forecast_scalar_estimate['pool_instruments'] = False 56 | 57 | system.forecastScaleCap.get_forecast_scalar("EDOLLAR", "ewmac64_256").plot() 58 | show() 59 | 60 | system.rules.get_raw_forecast("EDOLLAR", "ewmac64_256").plot() 61 | show() 62 | """ 63 | 64 | """ 65 | -------------------------------------------------------------------------------- /mythbusting/ewmacperformance.py: -------------------------------------------------------------------------------- 1 | from systems.provided.futures_chapter15.basesystem import * 2 | import pandas as pd 3 | from matplotlib.pyplot import show, plot 4 | 5 | system = futures_system() 6 | system.set_logging_level("on") 7 | 8 | trading_rules = system.rules.trading_rules() 9 | 10 | # so we use all the markets we have, equal weighted 11 | del (system.config.instrument_weights) 12 | 13 | instrument_list = system.get_instrument_list() 14 | system.config.instrument_weights = dict([(code, 1.0 / len(instrument_list)) 15 | for code in instrument_list]) 16 | 17 | # not used anyway; so we have all trading rules 18 | del (system.config.forecast_weights) 19 | 20 | # trading_rules=system.combForecast.get_trading_rule_list("US10") 21 | 22 | for rule_name in trading_rules: 23 | 24 | print(rule_name) 25 | # system.accounts.pandl_for_trading_rule(rule_name).to_ncg_frame().cumsum().plot() 26 | # show() 27 | 28 | print(system.accounts.pandl_for_trading_rule(rule_name).t_test()) 29 | print(system.accounts.pandl_for_trading_rule(rule_name).sharpe()) 30 | print(rule_name) 31 | 32 | print("***********************") 33 | 34 | for rule_name in ["ewmac16_64", "carry"]: 35 | print(rule_name) 36 | print("*****************") 37 | for instr_code in instrument_list: 38 | data = system.accounts.pandl_for_instrument_forecast( 39 | instr_code, rule_name) 40 | print("%s %.4f %.3f" % (instr_code, data.t_test()[1], data.sharpe())) 41 | 42 | bigresults = [] 43 | for rule_name in trading_rules: 44 | results = [] 45 | for instr_code in instrument_list: 46 | data = system.accounts.pandl_for_instrument_forecast( 47 | instr_code, rule_name) 48 | results.append(data.sharpe()) 49 | 50 | results.sort() 51 | print(rule_name) 52 | print(results) 53 | bigresults = bigresults + results 54 | -------------------------------------------------------------------------------- /riskmanagement/measurement.py: -------------------------------------------------------------------------------- 1 | from systems.provided.futures_chapter15.basesystem import futures_system 2 | from systems.provided.moretradingrules.morerules import long_bias, short_bias 3 | from copy import copy 4 | import numpy as np 5 | import pandas as pd 6 | 7 | base_system=futures_system() 8 | 9 | config = copy(base_system.config) 10 | 11 | config.percentage_vol_target=16.0 12 | 13 | ## long lived vol estimate. comment out to use standard one 14 | #config.volatility_calculation['days']=999999999 15 | config.volatility_calculation['days']=35 16 | config.trading_rules=dict(long_bias=long_bias, short_bias=short_bias) 17 | config.instrument_weights=dict(US10=0.5, US5=0.5) 18 | #config.instrument_weights=dict(US10=0.5, SP500=0.5) 19 | # 20 | #config.forecast_weights=dict(US10=dict(long_bias=1.0), SP500=dict(long_bias=1.0)) 21 | #config.forecast_weights=dict(US10=dict(long_bias=1.0), US5=dict(long_bias=1.0)) 22 | # or for relative value 23 | config.forecast_weights=dict(US10=dict(long_bias=1.0), US5=dict(short_bias=1.0)) 24 | 25 | config.forecast_div_multiplier = 1.0 26 | 27 | system=futures_system(config=config) 28 | pandl=system.accounts.portfolio().percent() 29 | pandl=pandl[pd.datetime(1990,1,1):] 30 | #pandl=pandl[pd.datetime(1998,1,1):] 31 | pandl[pandl==0]=np.nan 32 | pandl[abs(pandl)>10.0]=np.nan 33 | 34 | config.instrument_div_multiplier=config.instrument_div_multiplier/pandl.std() 35 | 36 | system=futures_system(config=config) 37 | pandl=system.accounts.portfolio().percent() 38 | pandl[pandl==0]=np.nan 39 | pandl[abs(pandl)>5.0]=np.nan 40 | pandl=pandl[pd.datetime(1990,1,1):] 41 | #pandl=pandl[pd.datetime(1998,1,1):] 42 | 43 | import statsmodels.api as sm 44 | import pylab 45 | 46 | 47 | sm.qqplot(pandl.values, line='45') 48 | pylab.show() 49 | 50 | pandl.ffill().rolling(100).std().plot() 51 | 52 | x=system.accounts.pandl_across_subsystems() 53 | y=x.to_frame() 54 | z=y.rolling(100).corr() 55 | values = [-z.iloc[rowid,:][1] for rowid in range(int(len(z)/2))] 56 | values = pd.DataFrame(list(values), x.index) -------------------------------------------------------------------------------- /variablecapital/variablecapital.py: -------------------------------------------------------------------------------- 1 | from matplotlib.pyplot import show 2 | from systems.provided.futures_chapter15.basesystem import futures_system 3 | 4 | system = futures_system(log_level="on") 5 | system.config.instrument_weights = dict(EDOLLAR=1.0) 6 | 7 | system.config.capital_multiplier['func'] = 'syscore.capital.fixed_capital' 8 | """ 9 | system.accounts.portfolio().curve().plot() 10 | show() 11 | 12 | 13 | 14 | system.accounts.portfolio().percent().curve().plot() 15 | show() 16 | 17 | system.accounts.portfolio().cumulative().curve().plot() 18 | show() 19 | """ 20 | pandl_fixed = system.accounts.portfolio() 21 | 22 | print(system.accounts.portfolio().capital) 23 | """ 24 | system = futures_system(log_level="on") 25 | system.config.instrument_weights=dict(EDOLLAR=1.0) 26 | system.config.capital_multiplier['func']='syscore.capital.full_compounding' 27 | 28 | system.accounts.capital_multiplier().plot() 29 | show() 30 | 31 | system.accounts.portfolio_with_multiplier().capital.plot() 32 | show() 33 | 34 | system.accounts.portfolio_with_multiplier().curve().plot() 35 | show() 36 | 37 | 38 | 39 | system.accounts.get_buffered_position_with_multiplier("EDOLLAR", False).plot() 40 | system.accounts.get_buffered_position("EDOLLAR", False).plot() 41 | show() 42 | 43 | 44 | """ 45 | 46 | system = futures_system(log_level="on") 47 | system.config.instrument_weights = dict(EDOLLAR=1.0) 48 | system.config.capital_multiplier['func'] = 'syscore.capital.half_compounding' 49 | """ 50 | system.accounts.capital_multiplier().plot() 51 | show() 52 | 53 | system.accounts.portfolio_with_multiplier().capital.plot() 54 | show() 55 | 56 | system.accounts.portfolio_with_multiplier().curve().plot() 57 | show() 58 | 59 | 60 | 61 | system.accounts.get_buffered_position_with_multiplier("EDOLLAR", False).plot() 62 | system.accounts.get_buffered_position("EDOLLAR", False).plot() 63 | show() 64 | """ 65 | 66 | pandl_variable = system.accounts.portfolio_with_multiplier() 67 | pandl_fixed.curve().plot() 68 | pandl_variable.curve().plot() 69 | show() 70 | -------------------------------------------------------------------------------- /mythbusting/wholesystem.py: -------------------------------------------------------------------------------- 1 | from systems.provided.futures_chapter15.basesystem import * 2 | import pandas as pd 3 | import numpy as np 4 | from matplotlib.pyplot import show, plot, scatter, gca 5 | from syscore.pdutils import align_to_joint, uniquets, divide_df_single_column 6 | from syscore.dateutils import generate_fitting_dates 7 | from syscore.algos import robust_vol_calc 8 | 9 | from systems.portfolio import Portfolios 10 | config = Config("systems.provided.futures_chapter15.futuresconfig.yaml") 11 | 12 | # so we use all the markets we have, equal weighted 13 | del (config.instrument_weights) 14 | config.notional_trading_capital = 10000000 15 | config.use_instrument_weight_estimates = True 16 | config.use_forecast_weight_estimates = True 17 | 18 | system = System([ 19 | Account(), Portfolios(), PositionSizing(), FuturesRawData(), 20 | ForecastCombine(), ForecastScaleCap(), Rules() 21 | ], csvFuturesData(), config) 22 | system.set_logging_level("on") 23 | 24 | # avgs 25 | instrument_list = system.get_instrument_list() 26 | trading_rules = system.rules.trading_rules().keys() 27 | 28 | ans = dict() 29 | for instrument_code in instrument_list: 30 | ans[instrument_code] = dict() 31 | for rule in trading_rules: 32 | ans[instrument_code][ 33 | rule] = system.accounts.pandl_for_instrument_forecast( 34 | instrument_code, rule).sharpe() 35 | 36 | # average rule / instrument 37 | ans = [] 38 | for instrument_code in instrument_list: 39 | for rule in trading_rules: 40 | ans.append( 41 | system.accounts.pandl_for_instrument_forecast( 42 | instrument_code, rule).sharpe()) 43 | 44 | np.mean(ans) 45 | 46 | # average Rule 47 | ans = [] 48 | for rule in trading_rules: 49 | ans.append(system.accounts.pandl_for_trading_rule(rule).sharpe()) 50 | 51 | print(ans) 52 | 53 | # average instrument 54 | ans = [] 55 | for instrument_code in instrument_list: 56 | ans.append(system.accounts.pandl_for_subsystem(instrument_code).sharpe()) 57 | 58 | print(ans) 59 | 60 | # portfolio 61 | system.accounts.portfolio().sharpe() 62 | -------------------------------------------------------------------------------- /mythbusting/fitforecastweights.py: -------------------------------------------------------------------------------- 1 | from systems.provided.futures_chapter15.basesystem import * 2 | import pandas as pd 3 | import numpy as np 4 | from matplotlib.pyplot import show, plot, scatter, gca 5 | from syscore.pdutils import align_to_joint, uniquets, divide_df_single_column 6 | from syscore.dateutils import generate_fitting_dates 7 | from syscore.algos import robust_vol_calc 8 | 9 | from systems.portfolio import Portfolios 10 | config = Config("systems.provided.futures_chapter15.futuresconfig.yaml") 11 | 12 | # so we use all the markets we have, equal weighted 13 | del (config.instrument_weights) 14 | config.notional_trading_capital = 10000000 15 | config.use_instrument_weight_estimates = True 16 | config.use_forecast_weight_estimates = True 17 | config.forecast_weight_estimate = dict( 18 | pool_instruments=False, method="one_period") 19 | 20 | system = System([ 21 | Account(), Portfolios(), PositionSizing(), FuturesRawData(), 22 | ForecastCombine(), ForecastScaleCap(), Rules() 23 | ], csvFuturesData(), config) 24 | system.set_logging_level("on") 25 | a1 = system.accounts.portfolio() 26 | 27 | config = Config("systems.provided.futures_chapter15.futuresconfig.yaml") 28 | 29 | # so we use all the markets we have, equal weighted 30 | del (config.instrument_weights) 31 | config.notional_trading_capital = 10000000 32 | config.use_instrument_weight_estimates = True 33 | config.use_forecast_weight_estimates = True 34 | config.forecast_weight_estimate = dict( 35 | pool_instruments=True, method="one_period") 36 | 37 | system = System([ 38 | Account(), Portfolios(), PositionSizing(), FuturesRawData(), 39 | ForecastCombine(), ForecastScaleCap(), Rules() 40 | ], csvFuturesData(), config) 41 | system.set_logging_level("on") 42 | a2 = system.accounts.portfolio() 43 | 44 | from syscore.accounting import account_test 45 | 46 | print("Fit by instrument out of sample:") 47 | print(a1.stats()) 48 | print("") 49 | print("Fit across instruments out of sample") 50 | print(a2.stats()) 51 | 52 | print("Test ") 53 | print(account_test(a1, a2)) 54 | 55 | a3 = pd.concat([a1.curve(), a2.curve()], axis=1) 56 | a3.columns = ["byinstr", "pooled"] 57 | a3.plot() 58 | show() 59 | -------------------------------------------------------------------------------- /smallaccountsize/generatesystems.py: -------------------------------------------------------------------------------- 1 | instrument_list = [ 2 | 'KR3', 'V2X', 'EDOLLAR', 'MXP', 'CORN', 'EUROSTX', 'GAS_US', 'PLAT', 'US2', 3 | 'LEANHOG', 'GBP', 'VIX', 'CAC', 'COPPER', 'CRUDE_W', 'BOBL', 'WHEAT', 4 | 'JPY', 'NASDAQ', 'GOLD', 'US5', 'SOYBEAN', 'AUD', 'SP500', 'PALLAD', 5 | 'KR10', 'LIVECOW', 'NZD', 'KOSPI', 'US10', 'SMI', 'EUR', 'OAT', 'AEX', 6 | 'BUND', 'BTP', 'US20' 7 | ] 8 | 9 | instrument_sets = [] 10 | for idx in range(9)[1:]: 11 | instrument_sets.append(instrument_list[:idx]) 12 | 13 | for idx in [15, 20, 25, 38]: 14 | instrument_sets.append(instrument_list[:idx]) 15 | 16 | from systems.portfolio import PortfoliosEstimated 17 | from systems.provided.futures_chapter15.basesystem import * 18 | from syscore.correlations import get_avg_corr 19 | from copy import copy 20 | import numpy as np 21 | from pickle import dump, load 22 | 23 | config = Config("examples.smallaccountsize.smallaccount.yaml") 24 | 25 | mkt_counters = [] 26 | 27 | idm = [] 28 | acc_curve = [] 29 | roll_acc = [] 30 | 31 | for (idx, instr_set) in enumerate(instrument_sets): 32 | config.instruments = instr_set 33 | system = System([ 34 | Account(), PortfoliosEstimated(), PositionSizing(), FuturesRawData(), 35 | ForecastCombineFixed(), ForecastScaleCapFixed(), Rules() 36 | ], csvFuturesData(), config) 37 | 38 | system.config.instrument_div_mult_estimate['dm_max'] = 100.0 39 | system.set_logging_level("on") 40 | 41 | idm.append(system.portfolio.get_instrument_diversification_multiplier()) 42 | 43 | acc = system.accounts.portfolio(roundpositions=False) 44 | acc_curve.append(acc) 45 | roll_acc.append(acc.weekly.rolling_ann_std(22)) 46 | mktcount = acc.to_frame().shape[1] - np.isnan(acc.to_frame()).sum(axis=1) 47 | mkt_counters.append(mktcount) 48 | 49 | import pandas as pd 50 | roll_acc = [] 51 | mkt_counters = [] 52 | for acc in acc_curve: 53 | y = pd.rolling_std( 54 | acc.weekly.as_df(), 20, min_periods=4, 55 | center=True) * acc.weekly._vol_scalar 56 | 57 | roll_acc.append(y) 58 | 59 | mktcount = acc.to_frame().shape[1] - np.isnan(acc.to_frame()).sum(axis=1) 60 | mkt_counters.append(mktcount) 61 | 62 | ans = [roll_acc, idm, acc_curve, mkt_counters] 63 | with open("/home/rob/data.pck", "wb") as f: 64 | dump(ans, f) 65 | -------------------------------------------------------------------------------- /forecasting/correlatedassets.py: -------------------------------------------------------------------------------- 1 | from matplotlib.pyplot import plot, scatter 2 | import numpy as np 3 | from scipy.stats import linregress 4 | import statsmodels.formula.api as sm 5 | 6 | from systems.provided.futures_chapter15.basesystem import futures_system 7 | import pandas as pd 8 | from systems.forecasting import TradingRule 9 | from systems.provided.moretradingrules.morerules import long_bias 10 | from copy import copy 11 | 12 | system=futures_system() 13 | carry_acc=system.accounts.pandl_for_instrument_forecast("US10", "carry") 14 | ewmac8_acc=system.accounts.pandl_for_instrument_forecast("US10", "ewmac8_32") 15 | ewmac16_acc=system.accounts.pandl_for_instrument_forecast("US10", "ewmac16_64") 16 | 17 | all_rets = pd.concat([carry_acc, ewmac8_acc, ewmac16_acc], axis=1) 18 | 19 | # benchmarking 20 | all_curve = system.accounts.pandl_for_instrument("MXP").percent() 21 | all_curve=1.5*all_curve 22 | 23 | new_rule = TradingRule(long_bias) 24 | config = copy(system.config) 25 | config.trading_rules['long_bias']=new_rule 26 | 27 | ## If you're using fixed weights and scalars 28 | 29 | config.forecast_scalar=1.0 30 | config.forecast_weights=dict(long_bias=1.0) ## all existing forecast weights will need to be updated 31 | config.forecast_div_multiplier=1.0 32 | system2 = futures_system(config=config) 33 | 34 | long_acc=system2.accounts.pandl_for_instrument_forecast("MXP", "long_bias") 35 | 36 | both = pd.concat([long_acc, all_curve], axis=1) 37 | both[pd.datetime(1982,9,15):pd.datetime(1982,9,21)]=np.nan 38 | 39 | both.columns = ['Long_only', 'Strategy'] 40 | 41 | 42 | longonly_values=[] 43 | strategyvalues=[] 44 | for i in range(len(both.index)): 45 | long_value=both.Long_only[i] 46 | strategy_value = both.Strategy[i] 47 | if not(np.isnan(long_value) or np.isnan(strategy_value)): 48 | longonly_values.append(long_value) 49 | strategyvalues.append(strategy_value) 50 | 51 | scatter(longonly_values, strategyvalues) 52 | 53 | result = sm.ols(formula="Strategy ~ Long_only", data=both).fit() 54 | 55 | 56 | 57 | ewmac8_acc=system.accounts.pandl_for_instrument_forecast("US10", "ewmac64_256") 58 | ewmac16_acc=system.accounts.pandl_for_instrument_forecast("US10", "ewmac32_128") 59 | 60 | both = pd.concat([ewmac8_acc, ewmac16_acc], axis=1) 61 | 62 | both.columns = ['Existing', 'New'] 63 | 64 | existingvalues=[] 65 | strategyvalues=[] 66 | for i in range(len(both.index)): 67 | existing_value=both.Existing[i] 68 | strategy_value = both.New[i] 69 | if not(np.isnan(existing_value) or np.isnan(strategy_value)): 70 | existingvalues.append(existing_value) 71 | strategyvalues.append(strategy_value) 72 | 73 | scatter(existingvalues, strategyvalues) 74 | 75 | result = sm.ols(formula="New ~ Existing", data=both).fit() 76 | -------------------------------------------------------------------------------- /somemoretradingrules/momentum.py: -------------------------------------------------------------------------------- 1 | from systems.provided.futures_chapter15.basesystem import futures_system 2 | from systems.forecasting import create_variations, TradingRule 3 | 4 | dict_of_speeds= [dict(Lfast=4, Lslow=16), dict(Lfast=8, Lslow=32), 5 | dict(Lfast=16, Lslow=64), 6 | dict(Lfast=32, Lslow=128), dict(Lfast=64, Lslow=256)] 7 | 8 | normmom_base_rule= TradingRule('systems.provided.futures_chapter15.rules.ewmac', data=['rawdata.cumulative_norm_return', 9 | 'rawdata.daily_returns_volatility']) 10 | normmom_variations = create_variations(normmom_base_rule, dict_of_speeds, 11 | key_argname='Lfast', nameformat="normmom_%s:%s") 12 | 13 | aggmom_base_rule = TradingRule('systems.provided.futures_chapter15.rules.ewmac', data=['rawdata.normalised_price_for_asset_class', 14 | 'rawdata.daily_returns_volatility']) 15 | aggmom_variations = create_variations(aggmom_base_rule, dict_of_speeds, 16 | key_argname='Lfast', nameformat="aggmom_%s:%s") 17 | 18 | standardmom_base_rule = TradingRule('systems.provided.futures_chapter15.rules.ewmac', data= ['rawdata.get_daily_prices', 'rawdata.daily_returns_volatility']) 19 | standardmom_variations = create_variations(standardmom_base_rule, dict_of_speeds, 20 | key_argname='Lfast', nameformat="standardmom_%s:%s") 21 | 22 | 23 | # I f***** love python 3 24 | variations = {**normmom_variations, **aggmom_variations, **standardmom_variations} 25 | 26 | sys=futures_system(trading_rules=variations) 27 | 28 | # equal weighting across instruments 29 | del(sys.config.instrument_weights) 30 | 31 | # but estimated IDM 32 | sys.config.use_instrument_div_mult_estimates=True 33 | 34 | # going to estimate forecast weights 35 | 36 | sys.config.use_forecast_scale_estimates=True 37 | sys.config.use_forecast_weight_estimates=True 38 | sys.config.use_forecast_div_mult_estimates=True 39 | 40 | del(sys.config.forecast_weights) 41 | 42 | allpandl=sys.accounts.pandl_for_all_trading_rules().to_frame() 43 | standardpandl=allpandl[list(standardmom_variations.keys())] 44 | aggmompandl=allpandl[list(aggmom_variations.keys())]*2.08 45 | normmomepandl=allpandl[list(normmom_variations.keys())]*2.17 46 | 47 | import pandas as pd 48 | thing=pd.concat([standardpandl.sum(axis=1).cumsum(), normmomepandl.sum(axis=1).cumsum()],axis=1) 49 | thing.columns=['Standard', 'Normalised'] 50 | thing.plot() 51 | 52 | thing=pd.concat([standardpandl.sum(axis=1).cumsum(),aggmompandl.sum(axis=1).cumsum(), normmomepandl.sum(axis=1).cumsum()],axis=1) 53 | thing.columns=['Standard', 'Aggregate', 'Normalised'] -------------------------------------------------------------------------------- /smallaccountsize/smallaccount.yaml: -------------------------------------------------------------------------------- 1 | #YAML 2 | # 3 | # Trading rules 4 | # 5 | trading_rules: 6 | ewmac2_8: 7 | function: systems.provided.futures_chapter15.rules.ewmac 8 | data: 9 | - "rawdata.get_daily_prices" 10 | - "rawdata.daily_returns_volatility" 11 | other_args: 12 | Lfast: 2 13 | Lslow: 8 14 | forecast_scalar: 10.6 15 | 16 | ewmac4_16: 17 | function: systems.provided.futures_chapter15.rules.ewmac 18 | data: 19 | - "rawdata.get_daily_prices" 20 | - "rawdata.daily_returns_volatility" 21 | other_args: 22 | Lfast: 4 23 | Lslow: 16 24 | forecast_scalar: 7.5 25 | 26 | ewmac8_32: 27 | function: systems.provided.futures_chapter15.rules.ewmac 28 | data: 29 | - "rawdata.get_daily_prices" 30 | - "rawdata.daily_returns_volatility" 31 | other_args: 32 | Lfast: 8 33 | Lslow: 32 34 | forecast_scalar: 5.3 35 | 36 | ewmac16_64: 37 | function: systems.provided.futures_chapter15.rules.ewmac 38 | data: 39 | - "rawdata.get_daily_prices" 40 | - "rawdata.daily_returns_volatility" 41 | other_args: 42 | Lfast: 16 43 | Lslow: 64 44 | forecast_scalar: 3.75 45 | 46 | ewmac32_128: 47 | function: systems.provided.futures_chapter15.rules.ewmac 48 | data: 49 | - "rawdata.get_daily_prices" 50 | - "rawdata.daily_returns_volatility" 51 | other_args: 52 | Lfast: 32 53 | Lslow: 128 54 | forecast_scalar: 2.65 55 | 56 | ewmac64_256: 57 | function: systems.provided.futures_chapter15.rules.ewmac 58 | data: 59 | - "rawdata.get_daily_prices" 60 | - "rawdata.daily_returns_volatility" 61 | other_args: 62 | Lfast: 64 63 | Lslow: 256 64 | forecast_scalar: 1.87 65 | carry: 66 | function: systems.provided.futures_chapter15.rules.carry 67 | data: 68 | - "rawdata.daily_annualised_roll" 69 | - "rawdata.daily_returns_volatility" 70 | other_args: 71 | smooth_days: 90 72 | forecast_scalar: 30 73 | # 74 | # forecast combination 75 | # 76 | forecast_weights: 77 | ewmac16_64: 0.21 78 | ewmac32_128: 0.08 79 | ewmac64_256: 0.21 80 | carry: 0.50 81 | forecast_div_multiplier: 1.31 82 | # 83 | # Capital correction 84 | # 85 | percentage_vol_target: 20.0 86 | notional_trading_capital: 10000 87 | base_currency: "USD" 88 | # 89 | # Portfolio creation 90 | # 91 | instruments: ['CORN', 'LEANHOG', 'LIVECOW', 'SOYBEAN', 'WHEAT', 'KR10', 'KR3', 'BOBL', 'BTP', 'BUND', 'OAT', 'SHATZ', 'US10', 'US2', 'US20', 'US5', 'V2X','VIX', 'KOSPI', 'AEX', 'CAC', 'SMI', 'NASDAQ', 'SP500', 'AUD', 'EUR', 'GBP', 'JPY', 'MXP', 'NZD', 'COPPER', 'GOLD', 'PALLAD', 'PLAT', 'CRUDE_W','GAS_US', 'EDOLLAR', 'EUROSTX'] 92 | 93 | 94 | -------------------------------------------------------------------------------- /regressionrule/tradingrule.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | import numpy as np 3 | from syscore.dateutils import UNIXTIME_IN_YEAR 4 | 5 | 6 | def my_regr(df, idx=None, timewindow=None, min_periods=10): 7 | """ 8 | Runs a single regression at point idx, using window length window, returning gradient / beta 9 | 10 | :param df: stuff to regress over 11 | :type df: pd.Series with columns ['y','x'] 12 | 13 | :param idx: where we are in time - uses all previous data 14 | :type idx: int or None (in which does entire data frame) 15 | 16 | :param timewindow: Rolling window to run regression over 17 | :type timewindow: int 18 | 19 | :param min_periods: minimum amount of data to use 20 | :type min_periods: int 21 | 22 | :returns: float 23 | 24 | """ 25 | 26 | ## default to regressing from last point in data 27 | if idx is None: 28 | idx = len(df.index) - 1 29 | 30 | ## default to using entire data frame for regression 31 | if timewindow is None: 32 | timewindow = len(df.index) 33 | 34 | data_start = max(idx - timewindow + 1, 0) 35 | 36 | df_subset = df[data_start:idx] 37 | 38 | ## remove nans in both x andd y 39 | clean_x = [ 40 | xvalue 41 | for (xvalue, 42 | yvalue) in zip(df_subset["x"].values, df_subset["y"].values) 43 | if not (np.isnan(xvalue) or np.isnan(yvalue)) 44 | ] 45 | clean_y = [ 46 | yvalue 47 | for (xvalue, 48 | yvalue) in zip(df_subset["x"].values, df_subset["y"].values) 49 | if not (np.isnan(xvalue) or np.isnan(yvalue)) 50 | ] 51 | 52 | ## enforce minimum amount of data 53 | if len(clean_x) < min_periods: 54 | return np.nan 55 | 56 | ## do the regression 57 | gradient, intercept = np.polyfit(clean_x, clean_y, 1) 58 | 59 | return gradient 60 | 61 | 62 | def regression_rule(price, volatility, timewindow=256, min_periods=10): 63 | """ 64 | Runs the regression rule to detect trends 65 | 66 | :param price: price to check, assumed to be week day frequency 67 | :type price: pd.Series 68 | 69 | :param volatility: vol to standardise price change by 70 | :type volatility: pd.Series (same dimensions and index as price) 71 | 72 | :param timewindow: Rolling window to run regression over, week day frequency 73 | :type timewindow: int 74 | 75 | :returns: pd.Series (same dimensions and index as price) 76 | 77 | """ 78 | 79 | assert type(timewindow) is int 80 | 81 | ## Create a time index where 1.0 is one year 82 | ## internal pandas implementation of date time is unix time 83 | 84 | rawx = list(price.index.astype(np.int64)) 85 | x = [(xvalue - rawx[0]) / UNIXTIME_IN_YEAR for xvalue in rawx] 86 | x = pd.Series(x, price.index) 87 | 88 | ## For regression y=mx + b, where y is price, m is gradient or forecast value 89 | df = pd.concat([price, x], axis=1) 90 | df.columns = ["y", "x"] 91 | 92 | ## Rolling regression, returns gradient which is also forecast 93 | ols_ans = [ 94 | my_regr( 95 | df, idx=idx_start, timewindow=timewindow, min_periods=min_periods) 96 | for idx_start in range(len(df.index)) 97 | ] 98 | 99 | ols_ans = pd.Series(ols_ans, index=price.index) 100 | 101 | ols_ans = ols_ans / volatility 102 | 103 | return (ols_ans) 104 | -------------------------------------------------------------------------------- /regressionrule/realdata.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from matplotlib.pyplot import show, legend 3 | 4 | from sysdata.csvdata import csvFuturesData 5 | from sysdata.configdata import Config 6 | 7 | from systems.forecasting import Rules 8 | from systems.forecasting import TradingRule 9 | 10 | from systems.basesystem import System 11 | 12 | from systems.rawdata import RawData 13 | from systems.forecast_combine import ForecastCombineFixed, ForecastCombineEstimated 14 | from systems.forecast_scale_cap import ForecastScaleCapFixed, ForecastScaleCapEstimated 15 | from systems.positionsizing import PositionSizing 16 | from systems.portfolio import PortfoliosFixed 17 | from systems.account import Account 18 | 19 | from examples.regressionrule.tradingrule import regression_rule 20 | 21 | ## roughly one, two, three weeks; one... twelve months in business days 22 | 23 | #WINDOWS_TO_USE=[42,256] 24 | WINDOWS_TO_USE = [10, 14, 20, 28, 40, 57, 80, 113, 160] 25 | 26 | 27 | def create_rules_for_random_system(): 28 | ## create a series of regression trading rules different lookbacks 29 | 30 | rules_dict = dict() 31 | for timewindow in WINDOWS_TO_USE: 32 | rule_name = "regression%d" % timewindow 33 | 34 | min_periods = max(2, int(np.ceil(timewindow / 4.0))) 35 | 36 | new_rule = TradingRule((regression_rule, [ 37 | "rawdata.get_daily_prices", "rawdata.daily_returns_volatility" 38 | ], dict(timewindow=timewindow, min_periods=min_periods))) 39 | 40 | rules_dict[rule_name] = new_rule 41 | 42 | my_rules = Rules(rules_dict) 43 | 44 | return my_rules 45 | 46 | 47 | def random_system_for_regression(config, rules, log_level="on"): 48 | 49 | my_system = System([ 50 | Account(), PortfoliosFixed(), PositionSizing(), 51 | ForecastCombineEstimated(), ForecastScaleCapEstimated(), rules, 52 | RawData() 53 | ], csvFuturesData(), config) 54 | 55 | my_system.set_logging_level(log_level) 56 | 57 | return my_system 58 | 59 | 60 | rules = create_rules_for_random_system() 61 | config = Config( 62 | dict( 63 | use_forecast_scale_estimates=True, 64 | use_forecast_weight_estimates=True, 65 | forecast_scalar_estimate=dict(pool_instruments=True), 66 | instrument_weights=dict(EDOLLAR=.25, US10=.25, CORN=.25, SP500=.25), 67 | instrument_div_multiplier=1.5)) 68 | 69 | system = random_system_for_regression(config, rules) 70 | 71 | system.accounts.portfolio().cumsum().plot() 72 | show() 73 | 74 | ans = system.accounts.pandl_for_all_trading_rules_unweighted() 75 | 76 | ans.gross.to_frame().cumsum().plot() 77 | legend() 78 | show() 79 | 80 | ans.costs.to_frame().cumsum().plot() 81 | legend() 82 | show() 83 | 84 | ans.net.to_frame().cumsum().plot() 85 | legend() 86 | show() 87 | 88 | instrument_code = "SP500" 89 | ans = system.combForecast.get_forecast_correlation_matrices( 90 | instrument_code).corr_list 91 | print(ans[-1]) 92 | 93 | for instrument_code in system.get_instrument_list(): 94 | for timewindow in WINDOWS_TO_USE: 95 | rule_name = "regression%d" % timewindow 96 | 97 | scaling = system.forecastScaleCap.get_forecast_scalar( 98 | instrument_code, rule_name).values[-1] 99 | turnover = system.accounts.forecast_turnover(instrument_code, 100 | rule_name) 101 | cost_sr = system.accounts.get_SR_cost_for_instrument_forecast( 102 | instrument_code, rule_name) 103 | 104 | print("*** %s Window length %d scalar %.3f turnover %.3f costs %.6f" % 105 | (instrument_code, timewindow, scaling, turnover, cost_sr)) 106 | -------------------------------------------------------------------------------- /optimisation/optimisation.py: -------------------------------------------------------------------------------- 1 | from matplotlib.pyplot import show, title 2 | 3 | from systems.provided.futures_chapter15.estimatedsystem import futures_system 4 | 5 | system = futures_system() 6 | system.set_logging_level("on") 7 | """ 8 | system.forecastScaleCap.get_scaled_forecast("EDOLLAR", "carry").plot() 9 | system.forecastScaleCap.get_scaled_forecast("V2X", "ewmac64_256").plot() 10 | system.forecastScaleCap.get_scaled_forecast("CORN", "ewmac64_256").plot() 11 | 12 | show() 13 | 14 | 15 | system.combForecast.pandl_for_instrument_rules("US10").cumsum().plot() 16 | show() 17 | 18 | 19 | system.config.forecast_weight_estimate["pool_instruments"]=False 20 | system.config.forecast_weight_estimate["method"]="bootstrap" ## speed things up 21 | system.config.forecast_weight_estimate["equalise_means"]=False 22 | system.config.forecast_weight_estimate["monte_runs"]=200 23 | system.config.forecast_weight_estimate["bootstrap_length"]=104 24 | 25 | 26 | 27 | 28 | system=futures_system(config=system.config) 29 | 30 | system.combForecast.get_forecast_weights("CORN").plot() 31 | title("CORN") 32 | show() 33 | 34 | system.combForecast.get_forecast_weights("EDOLLAR").plot() 35 | title("EDOLLAR") 36 | show() 37 | """ 38 | 39 | # reset the config 40 | system = futures_system() 41 | system.config.forecast_weight_estimate["pool_instruments"] = True 42 | system.config.forecast_weight_estimate["method"] = "bootstrap" 43 | system.config.forecast_weight_estimate["equalise_means"] = False 44 | system.config.forecast_weight_estimate["monte_runs"] = 200 45 | system.config.forecast_weight_estimate["bootstrap_length"] = 104 46 | """ 47 | system=futures_system(config=system.config) 48 | 49 | system.combForecast.get_raw_forecast_weights("CORN").plot() 50 | title("CORN") 51 | show() 52 | 53 | ## check same weights 54 | system.combForecast.get_raw_forecast_weights("US10").plot() 55 | title("US10") 56 | show() 57 | 58 | system.combForecast.get_forecast_weights("CORN").plot() 59 | title("CORN") 60 | show() 61 | 62 | 63 | 64 | system.combForecast.get_forecast_diversification_multiplier("EDOLLAR").plot() 65 | show() 66 | 67 | system.combForecast.get_forecast_diversification_multiplier("V2X").plot() 68 | show() 69 | 70 | system.combForecast.get_combined_forecast("EUROSTX").plot() 71 | show() 72 | 73 | 74 | system.positionSize.get_price_volatility("EUROSTX").plot() 75 | show() 76 | 77 | system.positionSize.get_block_value("EUROSTX").plot() 78 | show() 79 | 80 | system.positionSize.get_instrument_currency_vol("EUROSTX").plot() 81 | show() 82 | 83 | system.positionSize.get_instrument_value_vol("EUROSTX").plot() 84 | show() 85 | 86 | system.positionSize.get_volatility_scalar("EUROSTX").plot() 87 | show() 88 | 89 | system.positionSize.get_subsystem_position("EUROSTX").plot() 90 | show() 91 | 92 | instrument_codes=system.get_instrument_list() 93 | 94 | import pandas as pd 95 | 96 | pandl_subsystems=[system.accounts.pandl_for_subsystem(code, percentage=True) 97 | for code in instrument_codes] 98 | 99 | pandl=pd.concat(pandl_subsystems, axis=1) 100 | pandl.columns=instrument_codes 101 | 102 | pandl=pandl.cumsum().plot() 103 | show() 104 | 105 | """ 106 | 107 | system.config.instrument_weight_estimate[ 108 | "method"] = "bootstrap" # speed things up 109 | system.config.instrument_weight_estimate["equalise_means"] = False 110 | system.config.instrument_weight_estimate["monte_runs"] = 200 111 | system.config.instrument_weight_estimate["bootstrap_length"] = 104 112 | 113 | system.portfolio.get_instrument_weights().plot() 114 | show() 115 | 116 | system.portfolio.get_instrument_diversification_multiplier().plot() 117 | show() 118 | 119 | print(system.portfolio.get_instrument_correlation_matrix().corr_list[16]) 120 | print(system.portfolio.get_instrument_correlation_matrix().corr_list[25]) 121 | 122 | system.portfolio.get_notional_position("EUROSTX").plot() 123 | show() 124 | 125 | print(system.accounts.portfolio().stats()) 126 | 127 | system.accounts.portfolio().cumsum().plot() 128 | 129 | show() 130 | -------------------------------------------------------------------------------- /breakout/breakoutfuturesestimateconfig.yaml: -------------------------------------------------------------------------------- 1 | #YAML 2 | # 3 | # Raw data 4 | # 5 | # Trading rules 6 | # 7 | trading_rules: 8 | ewmac2_8: 9 | function: systems.provided.futures_chapter15.rules.ewmac 10 | data: 11 | - "rawdata.get_daily_prices" 12 | - "rawdata.daily_returns_volatility" 13 | other_args: 14 | Lfast: 2 15 | Lslow: 8 16 | ewmac4_16: 17 | function: systems.provided.futures_chapter15.rules.ewmac 18 | data: 19 | - "rawdata.get_daily_prices" 20 | - "rawdata.daily_returns_volatility" 21 | other_args: 22 | Lfast: 4 23 | Lslow: 16 24 | ewmac8_32: 25 | function: systems.provided.futures_chapter15.rules.ewmac 26 | data: 27 | - "rawdata.get_daily_prices" 28 | - "rawdata.daily_returns_volatility" 29 | other_args: 30 | Lfast: 8 31 | Lslow: 32 32 | ewmac16_64: 33 | function: systems.provided.futures_chapter15.rules.ewmac 34 | data: 35 | - "rawdata.get_daily_prices" 36 | - "rawdata.daily_returns_volatility" 37 | other_args: 38 | Lfast: 16 39 | Lslow: 64 40 | ewmac32_128: 41 | function: systems.provided.futures_chapter15.rules.ewmac 42 | data: 43 | - "rawdata.get_daily_prices" 44 | - "rawdata.daily_returns_volatility" 45 | other_args: 46 | Lfast: 32 47 | Lslow: 128 48 | ewmac64_256: 49 | function: systems.provided.futures_chapter15.rules.ewmac 50 | data: 51 | - "rawdata.get_daily_prices" 52 | - "rawdata.daily_returns_volatility" 53 | other_args: 54 | Lfast: 64 55 | Lslow: 256 56 | carry: 57 | function: systems.provided.futures_chapter15.rules.carry 58 | data: 59 | - "rawdata.daily_annualised_roll" 60 | - "rawdata.daily_returns_volatility" 61 | other_args: 62 | smooth_days: 90 63 | breakout10: 64 | function: systems.provided.moretradingrules.morerules.breakout 65 | data: 66 | - "rawdata.get_daily_prices" 67 | other_args: 68 | lookback: 10 69 | breakout20: 70 | function: systems.provided.moretradingrules.morerules.breakout 71 | data: 72 | - "rawdata.get_daily_prices" 73 | other_args: 74 | lookback: 20 75 | breakout40: 76 | function: systems.provided.moretradingrules.morerules.breakout 77 | data: 78 | - "rawdata.get_daily_prices" 79 | other_args: 80 | lookback: 40 81 | breakout80: 82 | function: systems.provided.moretradingrules.morerules.breakout 83 | data: 84 | - "rawdata.get_daily_prices" 85 | other_args: 86 | lookback: 80 87 | breakout160: 88 | function: systems.provided.moretradingrules.morerules.breakout 89 | data: 90 | - "rawdata.get_daily_prices" 91 | other_args: 92 | lookback: 160 93 | breakout320: 94 | function: systems.provided.moretradingrules.morerules.breakout 95 | data: 96 | - "rawdata.get_daily_prices" 97 | other_args: 98 | lookback: 320 99 | # 100 | use_forecast_scale_estimates: True 101 | # 102 | rule_variations: ['ewmac8_32', 'carry', 'breakout40', 'breakout10', 'ewmac2_8', 'breakout320', 'breakout80', 'ewmac16_64', 'breakout160', 'ewmac32_128', 'breakout20', 'ewmac4_16', 'ewmac64_256'] 103 | # 104 | forecast_scalar_estimate: 105 | pool_instruments: True 106 | # 107 | # forecast combination 108 | # 109 | use_forecast_weight_estimates: True 110 | # 111 | forecast_weight_estimate: 112 | method: shrinkage 113 | # 114 | # 115 | # 116 | # 117 | forecast_correlation_estimate: 118 | pool_instruments: True 119 | # 120 | use_instrument_weight_estimates: True 121 | # 122 | instrument_weight_estimate: 123 | method: "shrinkage" 124 | # 125 | # 126 | # Portfolio creation 127 | # 128 | # commenting out the next line means we default to using all instruments 129 | instruments: ['EDOLLAR','US10','EUROSTX', 'MXP', 'CORN', 'V2X'] 130 | 131 | -------------------------------------------------------------------------------- /regressionrule/randomdata.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from matplotlib.pyplot import show 3 | 4 | from syscore.dateutils import BUSINESS_DAYS_IN_YEAR 5 | 6 | from sysdata.randomdata import RandomData 7 | from sysdata.configdata import Config 8 | 9 | from systems.forecasting import Rules 10 | from systems.forecasting import TradingRule 11 | 12 | from systems.basesystem import System 13 | 14 | from systems.rawdata import RawData 15 | from systems.forecast_combine import ForecastCombineFixed, ForecastCombineEstimated 16 | from systems.forecast_scale_cap import ForecastScaleCapFixed, ForecastScaleCapEstimated 17 | from systems.positionsizing import PositionSizing 18 | from systems.portfolio import PortfoliosFixed 19 | from systems.account import Account 20 | 21 | from examples.regressionrule.tradingrule import regression_rule 22 | 23 | VOLS_TO_USE = [0.05] 24 | 25 | ## roughly one, three, six, nine, twelve months in business days 26 | LENGTHS_TO_USE = [64] 27 | ## roughly one, two, three weeks; one... twelve months in business days 28 | WINDOWS_TO_USE = [10, 14, 20, 28, 40, 57, 80, 113, 160, 226] 29 | 30 | 31 | def create_data_for_random_system(): 32 | data = RandomData() 33 | 34 | ## 20 year periods 35 | Nlength = int(BUSINESS_DAYS_IN_YEAR * 20) 36 | 37 | ## arbitrary to make scaling nice 38 | Xamplitude = 50.0 39 | 40 | print("creating random data") 41 | for Volscale in VOLS_TO_USE: 42 | for Tlength in LENGTHS_TO_USE: 43 | 44 | instrument_code = "fake_T%.2fV_%d" % (Volscale, Tlength) 45 | print(instrument_code) 46 | data.generate_random_data(instrument_code, Nlength, Tlength, 47 | Xamplitude, Volscale) 48 | 49 | return data 50 | 51 | 52 | def create_rules_for_random_system(): 53 | ## create a series of regression trading rules different lookbacks 54 | 55 | rules_dict = dict() 56 | for timewindow in WINDOWS_TO_USE: 57 | rule_name = "regression%d" % timewindow 58 | 59 | min_periods = max(2, int(np.ceil(timewindow / 4.0))) 60 | 61 | new_rule = TradingRule((regression_rule, [ 62 | "rawdata.get_daily_prices", "rawdata.daily_returns_volatility" 63 | ], dict(timewindow=timewindow, min_periods=min_periods))) 64 | 65 | rules_dict[rule_name] = new_rule 66 | 67 | my_rules = Rules(rules_dict) 68 | 69 | return my_rules 70 | 71 | 72 | def random_system_for_regression(data, config, rules, log_level="on"): 73 | 74 | my_system = System([ 75 | Account(), PortfoliosFixed(), PositionSizing(), 76 | ForecastCombineEstimated(), ForecastScaleCapEstimated(), rules, 77 | RawData() 78 | ], data, config) 79 | 80 | my_system.set_logging_level(log_level) 81 | 82 | return my_system 83 | 84 | 85 | data = create_data_for_random_system() 86 | rules = create_rules_for_random_system() 87 | config = Config(dict(use_forecast_scale_estimates=True)) 88 | 89 | system = random_system_for_regression(data, config, rules) 90 | 91 | Volscale = 0.05 92 | Tlength = 64 93 | print("Correlation for Vol scale %.2f Trend length %d" % (Volscale, Tlength)) 94 | 95 | instrument_code = "fake_T%.2fV_%d" % (Volscale, Tlength) 96 | 97 | ans = system.combForecast.get_forecast_correlation_matrices( 98 | instrument_code).corr_list 99 | print(ans[-1]) 100 | 101 | for Volscale in VOLS_TO_USE: 102 | for Tlength in LENGTHS_TO_USE: 103 | instrument_code = "fake_T%.2fV_%d" % (Volscale, Tlength) 104 | 105 | for timewindow in WINDOWS_TO_USE: 106 | rule_name = "regression%d" % timewindow 107 | 108 | sr = system.accounts.pandl_for_instrument_forecast( 109 | instrument_code, rule_name).sharpe() 110 | turnover = system.accounts.forecast_turnover( 111 | instrument_code, rule_name) 112 | 113 | print( 114 | "*** Vol scale %.2f Trend length %d Window length %d SR %.3f turnover %.3f" 115 | % (Volscale, Tlength, timewindow, sr, turnover)) 116 | -------------------------------------------------------------------------------- /somemoretradingrules/allrules.py: -------------------------------------------------------------------------------- 1 | from systems.provided.futures_chapter15.basesystem import futures_system 2 | from systems.forecasting import create_variations, TradingRule 3 | from copy import copy 4 | 5 | 6 | sys = futures_system() 7 | del(sys.config.instrument_weights) 8 | 9 | save_absolute_carry_rule = sys.config.trading_rules['carry'] 10 | 11 | # Trading rules 12 | short_vol=dict(shortbias=dict(function='systems.provided.moretradingrules.morerules.short_bias')) 13 | 14 | mean_reversion=dict(mr=dict(function='systems.provided.moretradingrules.morerules.cross_sectional_mean_reversion', 15 | data=['rawdata.cumulative_norm_return', 16 | 'rawdata.normalised_price_for_asset_class'],other_args=dict(horizon=250))) 17 | 18 | relative_carry_rule = dict(relativecarry=dict(function='systems.provided.moretradingrules.morerules.relative_carry', 19 | data=['rawdata.smoothed_carry','rawdata.median_carry_for_asset_class'])) 20 | 21 | absolute_carry_rule =dict(carry=save_absolute_carry_rule) 22 | 23 | dict_of_speeds= [dict(Lfast=4, Lslow=16), dict(Lfast=8, Lslow=32), 24 | dict(Lfast=16, Lslow=64), 25 | dict(Lfast=32, Lslow=128), dict(Lfast=64, Lslow=256)] 26 | 27 | normmom_base_rule= TradingRule('systems.provided.futures_chapter15.rules.ewmac', data=['rawdata.cumulative_norm_return', 28 | 'rawdata.daily_returns_volatility']) 29 | normmom_variations = create_variations(normmom_base_rule, dict_of_speeds, 30 | key_argname='Lfast', nameformat="normmom_%s:%s") 31 | 32 | aggmom_base_rule = TradingRule('systems.provided.futures_chapter15.rules.ewmac', data=['rawdata.normalised_price_for_asset_class', 33 | 'rawdata.daily_returns_volatility']) 34 | aggmom_variations = create_variations(aggmom_base_rule, dict_of_speeds, 35 | key_argname='Lfast', nameformat="aggmom_%s:%s") 36 | 37 | standardmom_base_rule = TradingRule('systems.provided.futures_chapter15.rules.ewmac', data= ['rawdata.get_daily_prices', 'rawdata.daily_returns_volatility']) 38 | standardmom_variations = create_variations(standardmom_base_rule, dict_of_speeds, 39 | key_argname='Lfast', nameformat="standardmom_%s:%s") 40 | 41 | 42 | # I f***** love python 3 43 | 44 | new_trading_rules = {**normmom_variations, **aggmom_variations, **standardmom_variations, **short_vol, **mean_reversion, 45 | **relative_carry_rule, **absolute_carry_rule} 46 | 47 | original_trading_rules = {**standardmom_variations, **absolute_carry_rule} 48 | 49 | trading_rule_names=list(new_trading_rules.keys()) 50 | trading_rule_names_without_shortbias= copy(trading_rule_names) 51 | trading_rule_names_without_shortbias.remove("shortbias") 52 | 53 | sys_old=futures_system(trading_rules=original_trading_rules) 54 | sys_new=futures_system(trading_rules=new_trading_rules) 55 | 56 | # equal weighting across instruments 57 | del(sys_new.config.instrument_weights) 58 | del(sys_old.config.instrument_weights) 59 | 60 | sys_new.config.rule_variations=dict([(instrument_code, trading_rule_names_without_shortbias) for instrument_code in sys_new.get_instrument_list()]) 61 | sys_new.config.rule_variations["VIX"]=trading_rule_names 62 | sys_new.config.rule_variations["V2X"]=trading_rule_names 63 | 64 | 65 | # but use estimated IDM 66 | sys_new.config.use_instrument_div_mult_estimates=True 67 | sys_old.config.use_instrument_div_mult_estimates=True 68 | 69 | # going to estimate forecast weights 70 | 71 | sys_old.config.use_forecast_scale_estimates=True 72 | sys_old.config.use_forecast_weight_estimates=True 73 | sys_old.config.use_forecast_div_mult_estimates=True 74 | del(sys_old.config.forecast_weights) 75 | 76 | sys_new.config.use_forecast_scale_estimates=True 77 | sys_new.config.use_forecast_weight_estimates=True 78 | sys_new.config.use_forecast_div_mult_estimates=True 79 | del(sys_new.config.forecast_weights) 80 | 81 | sys_old.accounts.portfolio().cumsum().plot() 82 | sys_new.accounts.portfolio().cumsum().plot() 83 | 84 | sys_old.cache.pickle("examples.somemoretradingrules.oldsystem.pck") 85 | sys_old.cache.pickle("examples.somemoretradingrules.newsystem.pck") 86 | 87 | 88 | 89 | sys_old.cache.unpickle("examples.somemoretradingrules.oldsystem.pck") 90 | sys_new.cache.unpickle("examples.somemoretradingrules.newsystem.pck") -------------------------------------------------------------------------------- /mythbusting/timevariationreturns.py: -------------------------------------------------------------------------------- 1 | from systems.provided.futures_chapter15.basesystem import * 2 | import pandas as pd 3 | import numpy as np 4 | from matplotlib.pyplot import show, plot, scatter, gca 5 | from syscore.pdutils import align_to_joint, uniquets, divide_df_single_column 6 | from syscore.dateutils import generate_fitting_dates 7 | from syscore.algos import robust_vol_calc 8 | 9 | from systems.portfolio import Portfolios 10 | config = Config("systems.provided.futures_chapter15.futuresconfig.yaml") 11 | 12 | rulename = ["ewmac64_256"] 13 | rule_name = rulename[0] 14 | 15 | # so we use all the markets we have, equal weighted 16 | del (config.instrument_weights) 17 | config.notional_trading_capital = 10000000 18 | config.forecast_weights = dict([(rule, 1.0) for rule in rulename]) 19 | config.use_instrument_weight_estimates = True 20 | config.notional_trading_capital = 10000000 21 | 22 | system = System([ 23 | Account(), Portfolios(), PositionSizing(), FuturesRawData(), 24 | ForecastCombine(), ForecastScaleCap(), Rules() 25 | ], csvFuturesData(), config) 26 | system.set_logging_level("on") 27 | 28 | a2 = system.accounts.portfolio() 29 | 30 | # autocorrelation 31 | pd.concat([a2.weekly.as_df(), a2.weekly.as_df().shift(1)], axis=1).corr() 32 | pd.concat([a2.monthly.as_df(), a2.monthly.as_df().shift(1)], axis=1).corr() 33 | pd.concat([a2.annual.as_df(), a2.annual.as_df().shift(1)], axis=1).corr() 34 | 35 | # recent volatility (market by market...) 36 | instrument_code = "EDOLLAR" 37 | return_period = int((250 / 7.5)) 38 | days = 256 39 | 40 | 41 | def get_scatter_data_for_code_vol(system, 42 | instrument_code, 43 | rule_name, 44 | return_period=5, 45 | days=64): 46 | 47 | denom_price = system.rawdata.daily_denominator_price(instrument_code) 48 | x = system.rawdata.daily_returns(instrument_code) 49 | vol = robust_vol_calc(x, days) 50 | perc_vol = 100.0 * divide_df_single_column(vol, denom_price.shift(1)) 51 | 52 | volavg = pd.rolling_median(perc_vol, 1250, min_periods=10) 53 | vol_qq = (perc_vol - volavg) / volavg 54 | 55 | # work out return for the N days after the forecast 56 | 57 | norm_data = system.accounts.pandl_for_instrument_forecast( 58 | instrument_code, rule_name) 59 | 60 | (vol_qq, norm_data) = align_to_joint( 61 | vol_qq, norm_data, ffill=(True, False)) 62 | 63 | period_returns = pd.rolling_sum(norm_data, return_period, min_periods=1) 64 | 65 | ex_post_returns = period_returns.shift(-return_period) 66 | lagged_vol = vol_qq.shift(1) 67 | 68 | return (list(ex_post_returns.iloc[:, 0].values), 69 | list(lagged_vol.iloc[:, 0].values)) 70 | 71 | 72 | def clean_data(x, y, maxstd=6.0): 73 | 74 | xcap = np.nanstd(x) * maxstd 75 | ycap = np.nanstd(y) * maxstd 76 | 77 | def _cap(xitem, cap): 78 | if np.isnan(xitem): 79 | return xitem 80 | if xitem > cap: 81 | return cap 82 | if xitem < -cap: 83 | return -cap 84 | return xitem 85 | 86 | x = [_cap(xitem, xcap) for xitem in x] 87 | y = [_cap(yitem, ycap) for yitem in y] 88 | 89 | return (x, y) 90 | 91 | 92 | def bin_fit(x, y, buckets=3): 93 | 94 | assert buckets in [3, 25] 95 | 96 | xstd = np.nanstd(x) 97 | 98 | if buckets == 3: 99 | binlimits = [np.nanmin(x), -xstd / 2.0, xstd / 2.0, np.nanmax(x)] 100 | elif buckets == 25: 101 | 102 | steps = xstd / 4.0 103 | binlimits = np.arange(-xstd * 3.0, xstd * 3.0, steps) 104 | 105 | binlimits = [np.nanmin(x)] + list(binlimits) + [np.nanmax(x)] 106 | 107 | fit_y = [] 108 | err_y = [] 109 | x_values_to_plot = [] 110 | for binidx in range(len(binlimits))[1:]: 111 | lower_bin_x = binlimits[binidx - 1] 112 | upper_bin_x = binlimits[binidx] 113 | 114 | x_values_to_plot.append(np.mean([lower_bin_x, upper_bin_x])) 115 | 116 | y_in_bin = [ 117 | y[idx] for idx in range(len(y)) 118 | if x[idx] >= lower_bin_x and x[idx] < upper_bin_x 119 | ] 120 | 121 | fit_y.append(np.nanmedian(y_in_bin)) 122 | err_y.append(np.nanstd(y_in_bin)) 123 | 124 | # no zeros 125 | 126 | return (binlimits, x_values_to_plot, fit_y, err_y) 127 | 128 | 129 | instrument_list = system.get_instrument_list() 130 | 131 | all_scatter = dict(returns=[], vol=[]) 132 | 133 | for instrument_code in instrument_list: 134 | this_instrument_data = get_scatter_data_for_code_vol( 135 | system, instrument_code, rule_name, return_period, days) 136 | all_scatter['returns'] = all_scatter['returns'] + this_instrument_data[0] 137 | all_scatter['vol'] = all_scatter['vol'] + this_instrument_data[1] 138 | 139 | (returns, forecast) = clean_data(all_scatter['returns'], all_scatter['vol']) 140 | 141 | (binlimits, x_values_to_plot, fit_y, err_y) = bin_fit(forecast, returns) 142 | 143 | # this time we multiply the forecast by the fitted Value 144 | -------------------------------------------------------------------------------- /fitting/bruteforce.py: -------------------------------------------------------------------------------- 1 | # assumes get raw data has been run 2 | 3 | from pickle import load 4 | import numpy as np 5 | import matplotlib.pyplot as plt 6 | import pandas as pd 7 | 8 | f = open('/home/rob/results.pck', "rb") 9 | results = load(f) 10 | f.close() 11 | 12 | irange = [xkey[0] for xkey in results.keys()] 13 | jrange = [xkey[1] for xkey in results.keys()] 14 | instruments = [xkey[2] for xkey in results.keys()] 15 | 16 | irange = list(set(irange)) 17 | jrange = list(set(jrange)) 18 | irange.sort() 19 | jrange.sort() 20 | 21 | instruments = list(set(instruments)) 22 | 23 | 24 | instrument="V2X" 25 | 26 | def get_results(results, i, j, instrument): 27 | if i==j: 28 | return np.nan 29 | return results[(i, j, instrument)].mean() * 250 30 | 31 | plot_results = np.array([[get_results(results,i,j, instrument) for i in irange] for j in jrange]) 32 | 33 | plt.imshow(plot_results) 34 | plt.colorbar() 35 | 36 | ax = plt.gca() # grab the current axis 37 | ax.set_xticks(range(len(irange))[1::2][:-1]) 38 | ax.set_xticklabels(irange[1::2][:-1]) 39 | ax.set_yticks(range(len(irange))[1::2][:-1]) 40 | ax.set_yticklabels(jrange[1::2][:-1]) 41 | 42 | ax.set_xlabel("A") 43 | ax.set_ylabel("B") 44 | 45 | plt.show() 46 | 47 | plot_results[np.isnan(plot_results)]=-1000000 48 | max_values=np.unravel_index(plot_results.argmax(), plot_results.shape) 49 | 50 | print("A max %d; B max %d" % (irange[max_values[1]], jrange[max_values[0]])) 51 | 52 | ## Run a series of t-tests versus the maximum 53 | def t_test(acc1, acc2): 54 | ## acc2 needs to be higher, or will get negative numbesr 55 | jacc = pd.concat([acc1, acc2], axis=1) 56 | corr = jacc.corr()[0][1] 57 | adj_factor = acc1.std() / acc2.std() 58 | 59 | diff = acc1.mean() - (acc2.mean()*adj_factor) 60 | 61 | omega_1 = acc1.std() / (len(acc1.index)**.5) 62 | 63 | var_diff = 2 * (omega_1**2) * (1-corr) 64 | t_stat = diff / (var_diff**.5) 65 | 66 | return t_stat 67 | 68 | 69 | def get_ttest_results(results, i, j, instrument, max_acc): 70 | if i==j: 71 | return np.nan 72 | return t_test(max_acc, results[(i, j, instrument)]) 73 | 74 | 75 | def get_corr_results(results, i, j, instrument, max_acc): 76 | if i==j: 77 | return np.nan 78 | 79 | jacc = pd.concat([max_acc, results[(i, j, instrument)]],axis=1) 80 | corr = jacc.corr()[0][1] 81 | 82 | return corr 83 | 84 | 85 | #max_acc = results[(irange[max_values[1]], jrange[max_values[0]], instrument)] 86 | max_acc = results[(7, 70, instrument)] 87 | 88 | plot_results = np.array([[get_ttest_results(results,i,j, instrument, max_acc) for i in irange] for j in jrange]) 89 | #plot_results = np.array([[get_corr_results(results,i,j, instrument, max_acc) for i in irange] for j in jrange]) 90 | 91 | 92 | # delete if want to show all 93 | #plot_results[plot_results<2]=np.nan 94 | #plot_results[plot_results>0.99]=np.nan 95 | 96 | plt.imshow(plot_results) 97 | plt.colorbar() 98 | 99 | ax = plt.gca() # grab the current axis 100 | ax.set_xticks(range(len(irange))[1::2][:-1]) 101 | ax.set_xticklabels(irange[1::2][:-1]) 102 | ax.set_yticks(range(len(irange))[1::2][:-1]) 103 | ax.set_yticklabels(jrange[1::2][:-1]) 104 | 105 | ax.set_xlabel("A") 106 | ax.set_ylabel("B") 107 | 108 | ax.set_title(instrument) 109 | plt.show() 110 | 111 | # and now replot 112 | # correlations 113 | 114 | ## pool the lot of em 115 | 116 | 117 | def pooled_results(results, i,j): 118 | all_results = pd.concat([results[(i, j, instrument)] for instrument in instruments], axis=0) 119 | return all_results 120 | 121 | def get_results_all_instruments(results, i, j): 122 | if i==j: 123 | return np.nan 124 | 125 | return pooled_results(results, i, j).mean() * 250 126 | 127 | plot_results = np.array([[get_results_all_instruments(results,i,j) for i in irange] for j in jrange]) 128 | 129 | plt.imshow(plot_results) 130 | plt.colorbar() 131 | 132 | ax = plt.gca() # grab the current axis 133 | ax.set_xticks(range(len(irange))[1::2][:-1]) 134 | ax.set_xticklabels(irange[1::2][:-1]) 135 | ax.set_yticks(range(len(irange))[1::2][:-1]) 136 | ax.set_yticklabels(jrange[1::2][:-1]) 137 | 138 | ax.set_xlabel("A") 139 | ax.set_ylabel("B") 140 | 141 | plt.show() 142 | 143 | def get_ttest_results_pooled(results, i, j, max_acc): 144 | if i==j: 145 | return np.nan 146 | return t_test(max_acc, pooled_results(results, i, j)) 147 | 148 | plot_results[np.isnan(plot_results)]=-1000000 149 | max_values=np.unravel_index(plot_results.argmax(), plot_results.shape) 150 | 151 | print("A max %d; B max %d" % (irange[max_values[1]], jrange[max_values[0]])) 152 | 153 | max_acc = pooled_results(results,irange[max_values[1]], jrange[max_values[0]]) 154 | 155 | plot_results = np.array([[get_ttest_results_pooled(results,i,j, max_acc) for i in irange] for j in jrange]) 156 | 157 | plot_results[plot_results<2]=np.nan 158 | 159 | plt.imshow(plot_results) 160 | plt.colorbar() 161 | 162 | ax = plt.gca() # grab the current axis 163 | ax.set_xticks(range(len(irange))[1::2][:-1]) 164 | ax.set_xticklabels(irange[1::2][:-1]) 165 | ax.set_yticks(range(len(irange))[1::2][:-1]) 166 | ax.set_yticklabels(jrange[1::2][:-1]) 167 | 168 | ax.set_xlabel("A") 169 | ax.set_ylabel("B") 170 | 171 | plt.show() 172 | -------------------------------------------------------------------------------- /smallaccountsize/instrument_list/instrument_list.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on 4 Mar 2016 3 | 4 | @author: rob 5 | ''' 6 | 7 | from systems.provided.futures_chapter15.estimatedsystem import PortfoliosEstimated 8 | from systems.provided.futures_chapter15.basesystem import * 9 | from syscore.correlations import get_avg_corr 10 | from copy import copy 11 | import numpy as np 12 | 13 | config = Config("examples.smallaccountsize.smallaccount.yaml") 14 | 15 | system = System([ 16 | Account(), PortfoliosEstimated(), PositionSizing(), FuturesRawData(), 17 | ForecastCombineFixed(), ForecastScaleCapFixed(), Rules() 18 | ], csvFuturesData(), config) 19 | 20 | system.set_logging_level("on") 21 | 22 | assetclasses = system.data.get_instrument_asset_classes() 23 | all_assets = list(set(assetclasses.values)) 24 | 25 | # corrmat=system.portfolio.get_instrument_correlation_matrix().corr_list[-1] 26 | instruments = system.get_instrument_list() 27 | max_positions = dict([(instrument_code, 2 * float( 28 | system.positionSize.get_volatility_scalar(instrument_code)[-250:].mean())) 29 | for instrument_code in instruments]) 30 | 31 | 32 | def instr_asset_class(instrument_code, assetclasses): 33 | return str(assetclasses[instrument_code]) 34 | 35 | 36 | def asset_class_count(portfolio, all_assets, assetclasess): 37 | # for a particular, return how many of each asset class 38 | assets_in_portfolio = [ 39 | instr_asset_class(code, assetclasses) for code in portfolio 40 | ] 41 | ans = dict([(asset_class, assets_in_portfolio.count(asset_class)) 42 | for asset_class in all_assets]) 43 | 44 | return ans 45 | 46 | 47 | def which_asset_classes_next(my_portfolio, suitable_instruments, all_assets, 48 | assetclasses): 49 | # returns a list of asset classes we wish to stock up on 50 | # will be those for which my_portfolio is short and suitable_instruments 51 | # have to spare 52 | if len(my_portfolio) == 0: 53 | return all_assets 54 | 55 | assets_in_my_portfolio = asset_class_count(my_portfolio, all_assets, 56 | assetclasses) 57 | suitable_assets = asset_class_count(suitable_instruments, all_assets, 58 | assetclasses) 59 | 60 | available = [ 61 | asset_class for asset_class in all_assets 62 | if suitable_assets[asset_class] > 0 63 | ] 64 | 65 | largest_asset_class_size_in_portfolio = max( 66 | [assets_in_my_portfolio[asset_class] for asset_class in available]) 67 | 68 | if all([ 69 | assets_in_my_portfolio[asset_class] == 70 | largest_asset_class_size_in_portfolio for asset_class in available 71 | ]): 72 | return available 73 | 74 | underweight = [ 75 | asset_class for asset_class in all_assets 76 | if assets_in_my_portfolio[asset_class] < 77 | largest_asset_class_size_in_portfolio and 78 | suitable_assets[asset_class] > 0 79 | ] 80 | 81 | return underweight 82 | 83 | 84 | def rank_within_asset_class(asset_class, suitable_instruments, assetclasses, 85 | max_positions): 86 | # returns the largest of a list of instrument codes, ordered by maximum 87 | # position size 88 | instruments_to_check = [ 89 | code for code in suitable_instruments 90 | if instr_asset_class(code, assetclasses) == asset_class 91 | ] 92 | 93 | instrument_max_positions = [ 94 | max_positions[code] for code in instruments_to_check 95 | ] 96 | 97 | order = np.array(instrument_max_positions).argsort() 98 | 99 | highest_max_idx = order[-1] 100 | 101 | return instruments_to_check[highest_max_idx] 102 | 103 | 104 | def choose_best_instrument(suitable_asset_classes, suitable_instruments, 105 | assetclasses, max_positions): 106 | # select the instrument with the best max position across asset classes 107 | best_max_by_class = [ 108 | rank_within_asset_class(asset_class, suitable_instruments, 109 | assetclasses, max_positions) 110 | for asset_class in suitable_asset_classes 111 | ] 112 | 113 | instrument_max_positions = [ 114 | max_positions[code] for code in best_max_by_class 115 | ] 116 | order = np.array(instrument_max_positions).argsort() 117 | highest_max_idx = order[-1] 118 | 119 | return best_max_by_class[highest_max_idx] 120 | 121 | 122 | def average_correlation(instrument_code, 123 | corrmat, 124 | max_positions, 125 | instruments, 126 | portfolio=[]): 127 | # returns the avg correlation of instrument_code with portfolio 128 | 129 | if len(portfolio) == 0: 130 | # compare to everything 131 | portfolio = copy(instruments) 132 | portfolio.pop(portfolio.index(instrument_code)) 133 | 134 | portfolio_index = [portfolio.index(code) for code in portfolio] 135 | sub_corrmat = corrmat[:, [portfolio_index]][instruments.index( 136 | instrument_code), :] 137 | avg_corr = np.mean(sub_corrmat) 138 | 139 | return avg_corr 140 | 141 | 142 | def return_lowest_five(avg_correlation_list, suitable_instruments): 143 | """ 144 | Returns names of instruments with lowest 5 correlations 145 | """ 146 | 147 | order = np.array(avg_correlation_list).argsort() 148 | top_five = order[:5] 149 | shortlist = [suitable_instruments[xidx] for xidx in top_five] 150 | 151 | return shortlist 152 | 153 | 154 | def return_highest_max_position(shortlist, max_positions): 155 | 156 | maxpos_shortlist = [max_positions[code] for code in shortlist] 157 | order = np.array(maxpos_shortlist).argsort() 158 | return shortlist[order[-1]] 159 | 160 | 161 | my_portfolio = [] 162 | 163 | suitable_instruments = copy(instruments) 164 | suitable_instruments.pop(suitable_instruments.index("SHATZ")) 165 | 166 | while len(suitable_instruments) > 0: 167 | suitable_asset_classes = which_asset_classes_next( 168 | my_portfolio, suitable_instruments, all_assets, assetclasses) 169 | best = choose_best_instrument(suitable_asset_classes, suitable_instruments, 170 | assetclasses, max_positions) 171 | print('{0: <12} '.format(best), max_positions[best]) 172 | 173 | my_portfolio.append(best) 174 | suitable_instruments.pop(suitable_instruments.index(best)) 175 | -------------------------------------------------------------------------------- /rateconditiong.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | from pysystemtrade.systems.provided.futures_chapter15.basesystem import futures_system 4 | from pysystemtrade.syscore.pdutils import pd_readcsv 5 | 6 | from matplotlib import pyplot 7 | import numpy as np 8 | import pandas as pd 9 | from pandas.tseries.offsets import Day 10 | from random import uniform 11 | from scipy.stats import ttest_ind 12 | 13 | rate_to_use = pd_readcsv("/home/rob/workspace3/pysystemtrade_examples/DFF.csv", date_index_name="DATE") 14 | us_5_year = pd_readcsv("/home/rob/workspace3/pysystemtrade_examples/FRED-DGS5.csv", date_index_name="Date") 15 | 16 | rate_to_use = us_5_year.Value 17 | rate_to_use = rate_to_use.sort_index() 18 | rate_to_use = rate_to_use.asfreq(Day()).ffill() 19 | 20 | DAYSINYEAR = 365 21 | rate_average = rate_to_use.rolling(int(DAYSINYEAR * 15), min_periods=DAYSINYEAR).mean() 22 | rate_adjusted = rate_to_use / rate_average # mean 0.79 current level 0.68 23 | rate_raw = rate_to_use 24 | 25 | rate_change = rate_to_use.ffill().diff().rolling(window=DAYSINYEAR).sum() 26 | rate_change_adjusted = rate_change / rate_average 27 | 28 | system = futures_system() 29 | system.config.instrument_weights = dict(EDOLLAR = .25, US5 = .25, US10 = .25, US20 = .25) 30 | 31 | # use this line for sp500 32 | #system.config.instrument_weights = dict(SP500=1.0) 33 | 34 | system.config.use_instrument_div_mult_estimates=True 35 | system.config.forecast_weights = dict(ewmac16_64 = 0.2, ewmac32_128 = 0.2, ewmac64_256 = 0.2, carry=0.4) 36 | 37 | # partition 38 | def partition(pandlcurve, conditioning_variable, pbins=5): 39 | 40 | # match 41 | conditioning_variable_match = conditioning_variable.reindex(pandlcurve.index, method="ffill") 42 | 43 | # avoid look ahead 44 | conditioning_variable_shift = conditioning_variable_match.shift(1) 45 | 46 | cmin = np.nanmin(conditioning_variable_shift) 47 | cmax = np.nanmax(conditioning_variable_shift) 48 | crange = cmax - cmin 49 | 50 | bounds = [np.nanpercentile(conditioning_variable_shift, xpoint) for xpoint in np.arange(0, 100.001, step=100.0/(pbins))] 51 | 52 | pandl_list = [] 53 | bound_names = [] 54 | for (lower_bound, upper_bound) in zip(bounds[:-1], bounds[1:]): 55 | pandl = pandlcurve[(conditioning_variable_shift>lower_bound) & (conditioning_variable_shift<=upper_bound)] 56 | 57 | pandl_list.append(pandl) 58 | 59 | bound_name = "%.2f to %.2f (SR:%.2f)" % (lower_bound, upper_bound, pandl.mean()*16/pandl.std()) 60 | bound_names.append(bound_name) 61 | 62 | return (pandl_list, bound_names) 63 | 64 | 65 | data = [rate_raw, rate_adjusted, rate_change, rate_change_adjusted] 66 | 67 | 68 | def partitionit(system, data, predictor_name , instrument_name, cond_name , pbins=2): 69 | 70 | rate_raw, rate_adjusted, rate_change, rate_change_adjusted = data 71 | 72 | if predictor_name =="ALL": 73 | if instrument_name == "ALL": 74 | # all predictors, all instruments 75 | pandl_curve = system.accounts.portfolio().percent() 76 | else: 77 | # all predictors, one instrument 78 | pandl_curve = system.accounts.pandl_for_instrument(instrument_name).percent() 79 | 80 | elif predictor_name == "LONG_ONLY": 81 | if instrument_name =="ALL": 82 | pandl_curves = [system.rawdata.daily_returns(instrument_code) / system.rawdata.daily_denominator_price( 83 | instrument_code) for instrument_code in system.get_instrument_list()] 84 | 85 | # normalise individual vol 86 | pandl_curves = [pandl / pandl.std() for pandl in pandl_curves] 87 | 88 | # combine 89 | pandl_curve = pd.concat(pandl_curves, axis=1) 90 | pandl_curve = pandl_curve.sum(axis=1) 91 | 92 | else: 93 | # single instrument, long only 94 | pandl_curve = 100*system.rawdata.daily_returns(instrument_name)/system.rawdata.daily_denominator_price(instrument_name) 95 | 96 | else: 97 | if instrument_name == "ALL": 98 | # individual predictors, all instruments 99 | 100 | pandl_curve = system.accounts.pandl_for_trading_rule_weighted(predictor_name).percent() 101 | else: 102 | # individual predictor, one instrument 103 | pandl_curve = system.accounts.pandl_for_instrument_forecast(instrument_name, predictor_name).percent() 104 | 105 | pandl_curve = pandl_curve[abs(pandl_curve) < 10] 106 | 107 | if cond_name == "Raw": 108 | conditioning_variable = rate_raw 109 | elif cond_name == "Adjusted": 110 | conditioning_variable = rate_adjusted 111 | elif cond_name =="Change": 112 | conditioning_variable = rate_change 113 | elif cond_name =="Adjusted Change": 114 | conditioning_variable = rate_change_adjusted 115 | else: 116 | raise Exception("%s not valid" % cond_name) 117 | 118 | pandl_list, bounds = partition(pandl_curve, conditioning_variable, pbins) 119 | 120 | return pandl_curve, pandl_list, bounds, conditioning_variable 121 | 122 | 123 | def box_whisker_plot(system, data, predictor_name , instrument_name, cond_name , pbins=2): 124 | pandl_curve, pandl_list, bounds, conditioning_variable = partitionit(system, data, predictor_name , instrument_name, cond_name , pbins) 125 | 126 | pyplot.boxplot(pandl_list) 127 | locs, labels = pyplot.xticks() 128 | pyplot.xticks(locs, bounds) 129 | pyplot.title("%s %s, Interest Rate %s, Current value %.2f" % (predictor_name, instrument_name, cond_name, conditioning_variable.values[-1])) 130 | 131 | # bootstrap sampling distributions 132 | 133 | def SR_sample(sample_pandl): 134 | # business days 135 | return (DAYSINYEAR**.5)*sample_pandl.mean()/sample_pandl.std() 136 | 137 | def generate_random_index(boot_length): 138 | random_index = [int(uniform(0, boot_length)) for notUsed in range(boot_length)] 139 | return random_index 140 | 141 | def sampling_distribution_SR(single_pandl, boot_count = 100): 142 | 143 | boot_length = len(single_pandl.index) 144 | list_of_indices = [generate_random_index(boot_length) for notUsed in range(boot_count)] 145 | list_of_random_sample_periods = [single_pandl[single_index] for single_index in list_of_indices] 146 | 147 | SR_samples = [SR_sample(sample_period) for sample_period in list_of_random_sample_periods] 148 | 149 | return SR_samples 150 | 151 | 152 | def t_tests(pandl_list): 153 | SR_list = [SR_sample(sample_pandl) for sample_pandl in pandl_list] 154 | 155 | lowest_SR = SR_list.index(min(SR_list)) 156 | 157 | target_pandl = pandl_list[lowest_SR] 158 | t_list = [] 159 | for compare_pandl in pandl_list: 160 | t_list.append(ttest_ind(target_pandl, compare_pandl)) 161 | 162 | return t_list 163 | 164 | def multiple_hist_plot(pandl_list, bounds, boot_count = 100): 165 | 166 | SR_distributions = [sampling_distribution_SR(single_pandl, boot_count) for single_pandl in pandl_list] 167 | 168 | bins = min(int(boot_count/50), 10) 169 | 170 | for SR_dist in SR_distributions: 171 | pyplot.hist(SR_dist, bins=bins) 172 | 173 | t_results = t_tests(pandl_list) 174 | plot_labels = ["%s p: %.3f" % (bound_label, test_result.pvalue) 175 | for (bound_label, test_result) in zip(bounds, t_results)] 176 | 177 | pyplot.legend(plot_labels) 178 | 179 | pandl_curve, pandl_list, bounds, conditioning_variable = partitionit(system, data, "ALL", "EDOLLAR", "Adjusted Change", pbins=2) 180 | 181 | 182 | multiple_hist_plot(pandl_list, bounds, boot_count = 2000) -------------------------------------------------------------------------------- /forecasting/crossassets.py: -------------------------------------------------------------------------------- 1 | from matplotlib.pyplot import plot, scatter 2 | from sysdata.csvdata import csvFuturesData 3 | import pandas as pd 4 | import numpy as np 5 | from scipy.stats import linregress 6 | from syscore.algos import robust_vol_calc 7 | 8 | from syscore.dateutils import fit_dates_object 9 | from syscore.genutils import progressBar 10 | 11 | codes=["US10", "US5", "SP500"] 12 | 13 | 14 | data_object=csvFuturesData() 15 | prices=[data_object[code] for code in codes] 16 | prices=pd.concat(prices, axis=1) 17 | prices.columns = codes 18 | prices = prices[pd.datetime(1998,9,10):] 19 | 20 | underlying = [data_object.get_instrument_raw_carry_data(code)['PRICE'] for code in codes] 21 | underlying=pd.concat(underlying, axis=1) 22 | underlying.columns = codes 23 | underlying = underlying[pd.datetime(1998,9,10):] 24 | 25 | perc=(prices - prices.shift(1))/underlying.shift(1) 26 | perc['US10'][abs(perc['US10'])>0.03]=np.nan 27 | 28 | def get_expost_data(perc): 29 | fitting_dates = generate_fitting_dates(perc, "rolling") ## only using annual dates rolling doesn't matter 30 | expost_data = [perc[fit_date.period_start:fit_date.period_end] for fit_date in fitting_dates[1:]] 31 | 32 | return expost_data 33 | 34 | 35 | def generate_fitting_dates(data, date_method, period="12M",rollperiods=20): 36 | """ 37 | generate a list 4 tuples, one element for each period in the data 38 | each tuple contains [fit_start, fit_end, period_start, period_end] datetime objects 39 | the last period will be a 'stub' if we haven't got an exact number of years 40 | 41 | date_method can be one of 'in_sample', 'expanding', 'rolling' 42 | 43 | if 'rolling' then use rollperiods variable 44 | """ 45 | 46 | if date_method not in ["in_sample", "rolling", "expanding"]: 47 | raise Exception( 48 | "don't recognise date_method %s should be one of in_sample, expanding, rolling" 49 | % date_method) 50 | 51 | if isinstance(data, list): 52 | start_date = min([dataitem.index[0] for dataitem in data]) 53 | end_date = max([dataitem.index[-1] for dataitem in data]) 54 | else: 55 | start_date = data.index[0] 56 | end_date = data.index[-1] 57 | 58 | # now generate the dates we use to fit 59 | if date_method == "in_sample": 60 | # single period 61 | return [fit_dates_object(start_date, end_date, start_date, end_date)] 62 | 63 | # generate list of dates, one period apart, including the final date 64 | period_starts = list(pd.date_range(start_date, end_date, freq=period)) + [ 65 | end_date 66 | ] 67 | 68 | # loop through each period 69 | periods = [] 70 | for tidx in range(len(period_starts))[1:-1]: 71 | # these are the dates we test in 72 | period_start = period_starts[tidx] 73 | period_end = period_starts[tidx + 1] 74 | 75 | # now generate the dates we use to fit 76 | if date_method == "expanding": 77 | fit_start = start_date 78 | elif date_method == "rolling": 79 | yearidx_to_use = max(0, tidx - rollperiods) 80 | fit_start = period_starts[yearidx_to_use] 81 | else: 82 | raise Exception( 83 | "don't recognise date_method %s should be one of in_sample, expanding, rolling" 84 | % date_method) 85 | 86 | if date_method in ['rolling', 'expanding']: 87 | fit_end = period_start 88 | else: 89 | raise Exception("don't recognise date_method %s " % date_method) 90 | 91 | periods.append( 92 | fit_dates_object(fit_start, fit_end, period_start, period_end)) 93 | 94 | if date_method in ['rolling', 'expanding']: 95 | # add on a dummy date for the first year, when we have no data 96 | periods = [ 97 | fit_dates_object( 98 | start_date, 99 | start_date, 100 | start_date, 101 | period_starts[1], 102 | no_data=True) 103 | ] + periods 104 | 105 | return periods 106 | 107 | 108 | 109 | def calc_historic_confidence(perc, function_to_use, rollperiods=250): 110 | fitting_dates=generate_fitting_dates(perc, "rolling", rollperiods=rollperiods) 111 | 112 | list_of_confidence=[] 113 | thing=progressBar(len(fitting_dates)-1) 114 | for fit_date in fitting_dates[1:]: 115 | list_of_confidence.append(function_to_use(perc, fit_date)) 116 | thing.iterate() 117 | thing.finished() 118 | 119 | 120 | list_of_confidence = pd.DataFrame(list_of_confidence, index=[fit_date.fit_end for fit_date in fitting_dates[1:]]) 121 | list_of_confidence.columns = ["lower", "upper"] 122 | 123 | return list_of_confidence 124 | 125 | def single_bootstrap_from_data(data): 126 | n = len(data) 127 | bootstraps = [int(np.random.uniform(high=n)) for not_used in range(n)] 128 | bs_data=[data.iloc[bsnumber] for bsnumber in bootstraps] 129 | return bs_data 130 | 131 | def gen_bootstraps_from_data(data, monte_carlo=100): 132 | 133 | all_bs = [single_bootstrap_from_data(data) for not_used in range(monte_carlo)] 134 | 135 | return all_bs 136 | 137 | 138 | def calc_historic_corr_distr_annual__us10_us5(perc, fit_date): 139 | data=perc[fit_date.fit_start:fit_date.fit_end] 140 | 141 | ## bootstrap 142 | all_bs = gen_bootstraps_from_data(data) 143 | 144 | all_bs_corr = [pd.concat(bs_item, axis=1).transpose().corr() for bs_item in all_bs] 145 | all_bs_corr = [bs_corr_item.values[0][1] for bs_corr_item in all_bs_corr] 146 | 147 | conf_interval = [np.percentile(all_bs_corr,2.5), np.percentile(all_bs_corr, 97.5)] 148 | 149 | return list(conf_interval) 150 | 151 | def calc_historic_corr_distr_annual__us10_sp500(perc, fit_date=None): 152 | 153 | data=perc[fit_date.fit_start:fit_date.fit_end] 154 | 155 | ## bootstrap 156 | all_bs = gen_bootstraps_from_data(data) 157 | 158 | all_bs_corr = [pd.concat(bs_item, axis=1).transpose().corr() for bs_item in all_bs] 159 | all_bs_corr = [bs_corr_item.values[0][2] for bs_corr_item in all_bs_corr] 160 | 161 | conf_interval = [np.percentile(all_bs_corr,2.5), np.percentile(all_bs_corr, 97.5)] 162 | 163 | return list(conf_interval) 164 | 165 | expost_data = get_expost_data(perc) 166 | 167 | 168 | list_of_confidence = calc_historic_confidence(perc, function_to_use=calc_historic_corr_distr_annual__us10_us5) 169 | expost_dates = list_of_confidence.index 170 | expost_corr = [period_data.corr().values[0][1] for period_data in expost_data] 171 | expost_corr = pd.DataFrame(expost_corr, expost_dates) 172 | datatoplot = pd.concat([list_of_confidence, expost_corr], axis=1) 173 | datatoplot.columns=["lower","upper","actual"] 174 | 175 | list_of_confidence2 = calc_historic_confidence(perc, function_to_use=calc_historic_corr_distr_annual__us10_sp500) 176 | expost_corr = [period_data.corr().values[0][2] for period_data in expost_data] 177 | expost_corr = pd.DataFrame(expost_corr, expost_dates) 178 | datatoplot = pd.concat([list_of_confidence2, expost_corr], axis=1) 179 | datatoplot.columns=["lower","upper","actual"] 180 | 181 | def corr1(period_data): 182 | return period_data.corr().values[0][1] 183 | 184 | def corr2(period_data): 185 | return period_data.corr().values[0][2] 186 | 187 | 188 | data=perc 189 | period="5D" 190 | rollperiods=1 ## set to very large number if using exp std 191 | expost_function_to_use=corr2 192 | exante_function_to_use=corr2 193 | fitting_dates=generate_fitting_dates(perc, "rolling", period=period, rollperiods=rollperiods) 194 | ex_ante_value_list=[] 195 | ex_post_value_list=[] 196 | 197 | for fit_date in fitting_dates[1:]: 198 | ex_ante_data=data[fit_date.fit_start:fit_date.fit_end] 199 | ex_ante_value = exante_function_to_use(ex_ante_data) 200 | 201 | ex_post_data=data[fit_date.period_start:fit_date.period_end] 202 | ex_post_value = expost_function_to_use(ex_post_data) 203 | 204 | if not (np.isnan(ex_ante_value) or np.isnan(ex_post_value)): 205 | 206 | ex_post_value_list.append(ex_post_value) 207 | ex_ante_value_list.append(ex_ante_value) 208 | 209 | scatter(ex_ante_value_list, ex_post_value_list) 210 | 211 | print(linregress(ex_ante_value_list, ex_post_value_list)) 212 | -------------------------------------------------------------------------------- /breakout/breakout.py: -------------------------------------------------------------------------------- 1 | from syscore.accounting import account_test 2 | 3 | from syscore.pdutils import turnover 4 | from sysdata.configdata import Config 5 | 6 | from systems.provided.futures_chapter15.estimatedsystem import futures_system 7 | from systems.provided.moretradingrules.morerules import breakout 8 | 9 | import pandas as pd 10 | import numpy as np 11 | from matplotlib.pyplot import show, legend, matshow 12 | 13 | bvariations = ["breakout" + str(ws) for ws in [10, 20, 40, 80, 160, 320]] 14 | evariations = [ 15 | "ewmac%d_%d" % (fast, fast * 4) for fast in [2, 4, 8, 16, 32, 64] 16 | ] 17 | """ 18 | my_config = Config("examples.breakout.breakoutfuturesestimateconfig.yaml") 19 | 20 | system = futures_system(config=my_config, log_level="on") 21 | 22 | price=system.data.daily_prices("CRUDE_W") 23 | 24 | price.plot() 25 | show() 26 | 27 | lookback=250 28 | 29 | roll_max = price.rolling(lookback, min_periods=min(len(price), np.ceil(lookback/2.0))).max() 30 | roll_min = price.rolling(lookback, min_periods=min(len(price), np.ceil(lookback/2.0))).min() 31 | 32 | 33 | all=pd.concat([price, roll_max, roll_min], axis=1) 34 | all.columns=["price", "max", "min"] 35 | all.plot() 36 | legend(loc="top left") 37 | show() 38 | 39 | roll_mean = (roll_max+roll_min)/2.0 40 | 41 | all=pd.concat([price, roll_max, roll_mean,roll_min], axis=1) 42 | all.columns=["price", "max", "mean", "min"] 43 | all.plot() 44 | legend(loc="top left") 45 | show() 46 | 47 | 48 | ## gives a nice natural scaling 49 | output = 40.0*((price - roll_mean) / (roll_max - roll_min)) 50 | 51 | output.plot() 52 | show() 53 | 54 | 55 | print(turnover(output, 10.0)) 56 | 57 | smooth=int(250/4.0) 58 | smoothed_output = output.ewm(span=smooth, min_periods=np.ceil(smooth/2.0)).mean() 59 | print(turnover(smoothed_output, 10.0)) 60 | 61 | smoothed_output.plot() 62 | show() 63 | 64 | ## check window size correlation and also turnover properties 65 | outputall=[] 66 | 67 | wslist=[4,5,6,7,8,9,10,15,20,25,30,35,40, 50, 60, 70, 80, 90, 100, 120, 140, 160, 68 | 180, 200, 240, 280, 320, 360, 500] 69 | 70 | for ws in wslist: 71 | smoothed_output = breakout(price, ws) 72 | 73 | ## 74 | avg_forecast=float(smoothed_output.abs().mean()) 75 | print("WS %d turnover %.2f" % (ws, turnover(smoothed_output, avg_forecast))) 76 | outputall.append(smoothed_output) 77 | 78 | outputall=pd.concat(outputall, axis=1) 79 | outputall.columns=wslist 80 | print(outputall.corr()) 81 | 82 | print() 83 | 84 | outputall.iloc[:,[6, 8, 12, 16, 21, 26]].round(2) 85 | 86 | matshow(outputall.iloc[:,[6, 8, 12, 16, 21, 26]].corr()) 87 | show() 88 | 89 | system.rules.get_raw_forecast("CRUDE_W", "breakout160").plot() 90 | show() 91 | 92 | 93 | 94 | my_config = Config("examples.breakout.breakoutfuturesestimateconfig.yaml") 95 | my_config.forecast_scalar_estimate['pool_instruments']=False 96 | del(my_config.instruments) 97 | 98 | ## logging off as printing results 99 | system = futures_system(config=my_config, log_level="off") 100 | 101 | 102 | instr_list=system.get_instrument_list() 103 | variations=["breakout"+str(ws) for ws in [10, 20, 40, 80, 160, 320]] 104 | 105 | for rule_name in variations: 106 | all_scalars=[] 107 | for instrument in instr_list: 108 | scalar=float(system.forecastScaleCap.get_forecast_scalar(instrument, rule_name).tail(1).values) 109 | all_scalars.append((instrument, scalar)) 110 | all_scalars=sorted(all_scalars, key=lambda x: x[1]) 111 | print("%s: %s %s" % (rule_name, str(all_scalars[0]), str(all_scalars[-1]))) 112 | all_scalar_values=[x[1] for x in all_scalars] 113 | print("mean %.3f std %.3f min %.3f max %.3f" % (np.nanmean(all_scalar_values), np.nanstd(all_scalar_values), 114 | np.nanmin(all_scalar_values), np.nanmax(all_scalar_values))) 115 | 116 | ## reload config so we get scalars estimated with pooling behaviour 117 | my_config = Config("examples.breakout.breakoutfuturesestimateconfig.yaml") 118 | 119 | ## logging off as printing results 120 | system = futures_system(config=my_config, log_level="off" ) 121 | for rule_name in variations: 122 | instrument="CRUDE_W" ## doesn't matter 123 | scalar=float(system.forecastScaleCap.get_forecast_scalar(instrument, rule_name).tail(1).values) 124 | print("%s: %.3f" % (rule_name, scalar)) 125 | 126 | ## now turnovers 127 | for rule_name in variations: 128 | all_turnovers=[] 129 | for instrument in instr_list: 130 | turnover_value=system.accounts.forecast_turnover(instrument, rule_name) 131 | all_turnovers.append((instrument, turnover_value)) 132 | all_turnovers=sorted(all_turnovers, key=lambda x: x[1]) 133 | print("%s: %s %s" % (rule_name, str(all_turnovers[0]), str(all_turnovers[-1]))) 134 | all_turnover_values=[x[1] for x in all_turnovers] 135 | print("mean %.3f std %.3f" % (np.nanmean(all_turnover_values), np.nanstd(all_turnover_values))) 136 | 137 | 138 | ## limit the rules to just breakout for now 139 | my_config = Config("examples.breakout.breakoutfuturesestimateconfig.yaml") 140 | my_config.trading_rules = dict([(rule_name, my_config.trading_rules[rule_name]) for rule_name in variations]) 141 | 142 | system = futures_system(config=my_config, log_level="on") 143 | 144 | print(system.combForecast.get_forecast_weights("EUROSTX").irow(-1)) 145 | print(system.combForecast.get_forecast_weights("V2X").irow(-1)) 146 | 147 | ## now include other rules 148 | 149 | 150 | my_config = Config("examples.breakout.breakoutfuturesestimateconfig.yaml") 151 | #my_config.forecast_weight_estimate["method"]="bootstrap" 152 | 153 | system = futures_system(config=my_config, log_level="on") 154 | 155 | #cProfile.run("system.accounts.pandl_for_all_trading_rules_unweighted().to_frame()","restats") 156 | system.accounts.pandl_for_all_trading_rules_unweighted().to_frame().loc[:, bvariations].cumsum().plot() 157 | show() 158 | 159 | 160 | variations=bvariations+evariation+["carry"] 161 | 162 | corr_result=system.combForecast.get_forecast_correlation_matrices("US10") 163 | matrix=corr_result.corr_list[-1] 164 | matrix=pd.DataFrame(matrix, columns=corr_result.columns) 165 | matrix=matrix.round(2) 166 | matrix.index=corr_result.columns 167 | matrix=matrix.loc[variations][variations] 168 | short_names=["brk"+str(ws) for ws in [10, 20, 40, 80, 160, 320]]+[ 169 | "ewm%d" % fast for fast in [2,4,8,16,32, 64]]+["carry"] 170 | matrix.index=matrix.columns=short_names 171 | 172 | matrix.to_csv("correlations.csv") 173 | 174 | 175 | system.combForecast.get_forecast_weights("V2X").iloc[-1,][variations].plot(kind="barh") 176 | show() 177 | 178 | 179 | allpandl=[] 180 | for rule_variation_name in variations: 181 | 182 | allpandl.append(system.accounts.pandl_for_trading_rule(rule_variation_name).as_df()) 183 | 184 | allpandl=pd.concat(allpandl, axis=1) 185 | allpandl.columns=variations 186 | 187 | allpandl.cumsum().plot() 188 | show() 189 | 190 | 191 | allpandl=[] 192 | for rule_variation_name in bvariations: 193 | 194 | allpandl.append(system.accounts.pandl_for_trading_rule_unweighted(rule_variation_name).as_df()) 195 | 196 | allpandl=pd.concat(allpandl, axis=1) 197 | allpandl.columns=bvariations 198 | 199 | allpandl.cumsum().plot() 200 | show() 201 | 202 | allpandl=[] 203 | for rule_variation_name in bvariations: 204 | 205 | allpandl.append(system.accounts.pandl_for_trading_rule(rule_variation_name).as_df()) 206 | 207 | allpandl=pd.concat(allpandl, axis=1) 208 | 209 | allpandl.cumsum().sum(axis=1).plot() 210 | show() 211 | 212 | ### show grouped courves 213 | my_config = Config("examples.breakout.breakoutfuturesestimateconfig.yaml") 214 | my_config.forecast_weight_estimate["method"]="equal_weights" 215 | system=futures_system(config=my_config, log_level="on") 216 | 217 | 218 | allrulespandl=system.accounts.pandl_for_all_trading_rules() 219 | 220 | ## 221 | ewmac_all=allrulespandl.to_frame().loc[:,evariations].sum(axis=1) 222 | break_all=allrulespandl.to_frame().loc[:,bvariations].sum(axis=1) 223 | 224 | both_plot=pd.concat([ewmac_all, break_all], axis=1) 225 | print(both_plot.corr()) 226 | both_plot.plot() 227 | show() 228 | 229 | """ 230 | # full backtest compare 231 | 232 | my_config = Config("examples.breakout.breakoutfuturesestimateconfig.yaml") 233 | # will do all instruments we have data for 234 | del (my_config.instruments) 235 | 236 | # temporarily remove breakout rules 237 | my_config.rule_variations = evariations 238 | my_config.forecast_weight_estimate["method"] = "equal_weights" 239 | system_old = futures_system(config=my_config, log_level="on") 240 | 241 | # new system has all trading rules 242 | new_config = Config("examples.breakout.breakoutfuturesestimateconfig.yaml") 243 | new_config.rule_variations = bvariations 244 | new_config.forecast_weight_estimate["method"] = "equal_weights" 245 | del (new_config.instruments) 246 | 247 | system_new = futures_system(config=new_config, log_level="on") 248 | 249 | curve1 = system_old.accounts.portfolio() 250 | curve2 = system_new.accounts.portfolio() 251 | 252 | print(curve1.stats()) 253 | print(curve2.stats()) 254 | 255 | print(account_test(curve2, curve1)) 256 | 257 | curves_to_plot = pd.concat([curve1.as_df(), curve2.as_df()], axis=1) 258 | curves_to_plot.columns = ["ewmac", "breakout"] 259 | 260 | print(curves_to_plot.corr()) 261 | curves_to_plot.cumsum().plot() 262 | show() 263 | -------------------------------------------------------------------------------- /vix/vix.py: -------------------------------------------------------------------------------- 1 | from syscore.pdutils import pd_readcsv_frompackage 2 | from syscore.genutils import progressBar 3 | from copy import copy 4 | from scipy.stats import norm 5 | import matplotlib.mlab as mlab 6 | import matplotlib.cm as cm 7 | 8 | import pandas as pd 9 | import numpy as np 10 | from sysdata.csvdata import csvFuturesData 11 | import matplotlib.pyplot as plt 12 | 13 | def process_row(x): 14 | try: 15 | return float(x[0]) 16 | except: 17 | return np.nan 18 | 19 | 20 | def get_vix_data(): 21 | vix_spot=pd_readcsv_frompackage("examples.vix.VIX.csv") 22 | 23 | return pd.Series([process_row(x) for x in vix_spot.values], vix_spot.index).ffill() 24 | 25 | def get_us_long_data(): 26 | us_long_data = pd_readcsv_frompackage("examples.vix.US_monthly_returns.csv") 27 | return us_long_data 28 | 29 | # functions to do conditional binning 30 | def half_split(conditioner): 31 | binrange = [[conditioner.min(), conditioner.median()], [conditioner.median(), conditioner.max()]] 32 | return binrange 33 | 34 | def deciles(conditioner): 35 | return [[conditioner.min(), np.percentile(conditioner.values, 10)], [np.percentile(conditioner.values, 90), conditioner.max()]] 36 | 37 | def deciles_and_mid(conditioner): 38 | return [[conditioner.min(), np.percentile(conditioner.values, 10)], 39 | [np.percentile(conditioner.values, 10), np.percentile(conditioner.values, 90)], 40 | [np.percentile(conditioner.values, 90), conditioner.max()]] 41 | 42 | def no_binning(conditioner): 43 | return [[conditioner.min(), conditioner.max()]] 44 | 45 | def below_10(conditioner): 46 | return [[conditioner.min(), 10.0], [10.0, conditioner.max()]] 47 | 48 | # actually do the analysis 49 | 50 | def produce_conditioned_distributions(conditioner, bin_func, response_change): 51 | conditioner_cleaned = conditioner.dropna() 52 | my_bins=bin_func(conditioner_cleaned) 53 | 54 | binned_response = [response_change[(conditioner>bin[0]) & (conditioner<=bin[1])].values for bin in my_bins] 55 | 56 | binned_response = [[x for x in bin_data if not np.isnan(x)] for bin_data in binned_response] 57 | 58 | my_bin_labels = ["%.2f to %.2f" % (bin[0], bin[1]) for bin in my_bins] 59 | 60 | return (my_bin_labels, binned_response) 61 | 62 | 63 | 64 | def do_a_plot(response_variable, conditioner, lag_response=True, frequency="D", time_horizon=30, bin_func=no_binning, 65 | calculate_sharpe_ratios=False, 66 | index_to_use = None, drawfittedlines=False, target_in_bin=20, 67 | calculate_returns = True): 68 | 69 | response_variable = copy(response_variable) 70 | conditioner = copy(conditioner) 71 | 72 | if lag_response: 73 | conditioner = conditioner.shift(1) 74 | 75 | if index_to_use is None: 76 | index_to_use = response_variable.index 77 | 78 | conditioner = conditioner.reindex(index_to_use) 79 | response_variable = response_variable.reindex(index_to_use) 80 | 81 | 82 | conditioner=conditioner.resample(frequency).last().ffill() 83 | response_variable=response_variable.resample(frequency).last().ffill() 84 | 85 | if calculate_returns: 86 | response_variable = (response_variable.shift(-time_horizon) - response_variable) /response_variable 87 | else: 88 | response_variable = response_variable.shift(-time_horizon) 89 | 90 | if calculate_sharpe_ratios: 91 | annualisation = (time_horizon / 250)**.5 92 | vol = pd.rolling_std(response_variable, time_horizon) 93 | response_variable = response_variable / vol.shift(-time_horizon) 94 | response_variable = response_variable * annualisation 95 | 96 | 97 | (my_bin_labels, bin_data) = produce_conditioned_distributions(conditioner, bin_func, response_variable) 98 | 99 | colors = ["red", "blue", "green"] 100 | all_bin_range = np.max(np.max(np.array(bin_data))) - np.min(np.min(np.array(bin_data))) 101 | desired_bin_count = sum([len(x) for x in bin_data]) / target_in_bin 102 | desired_bin_width = all_bin_range / desired_bin_count 103 | 104 | for my_color, one_bin in zip(colors, bin_data): 105 | nrange = np.max(one_bin) - np.min(one_bin) 106 | bin_count = int(nrange / desired_bin_width) 107 | n, bins, patches = plt.hist(one_bin, bin_count, facecolor=my_color, alpha=0.5) 108 | 109 | if drawfittedlines: 110 | (mu, sigma) = norm.fit(one_bin) 111 | y = mlab.normpdf( bins, mu, sigma) 112 | l = plt.plot(bins, y, color=my_color, linewidth=2) 113 | 114 | 115 | plt.legend(my_bin_labels) 116 | plt.show() 117 | meanstring=["%.5f (%s) " % (np.mean(one_bin), bin_label) for one_bin, bin_label in zip(bin_data, my_bin_labels)] 118 | print("Means %s" % "".join(meanstring)) 119 | 120 | return bin_data 121 | 122 | def _sample_this(thingtosample, sample_length): 123 | indices= [int(np.random.uniform() * len(thingtosample)) for notused in range(sample_length)] 124 | return [thingtosample[idx] for idx in indices] 125 | 126 | def paired_test(bin_data, number_of_runs=200): 127 | 128 | sample_length=np.min([len(onebin) for onebin in bin_data]) 129 | 130 | diffs = [] 131 | pb=progressBar(number_of_runs) 132 | for notUsed in range(number_of_runs): 133 | first_sample = _sample_this(bin_data[0], sample_length) 134 | second_sample = _sample_this(bin_data[1], sample_length) 135 | sample_diff = [x-y for x,y in zip(first_sample, second_sample)] 136 | 137 | this_difference = np.mean(sample_diff) 138 | diffs.append(this_difference) 139 | pb.iterate() 140 | 141 | return diffs 142 | 143 | def plot_paired_test(bin_data, number_of_runs=200): 144 | diff_dist = paired_test(bin_data, number_of_runs=number_of_runs) 145 | bin_count = int(number_of_runs/50) 146 | plt.hist(diff_dist, bin_count) 147 | legend_string = "Mean difference %.5f" % np.mean(diff_dist) 148 | plt.title(legend_string) 149 | plt.show() 150 | 151 | count_positive = len([x for x in diff_dist if x>0.0]) 152 | count_fraction = float(count_positive) / len(diff_dist) 153 | 154 | print("Proportion positive %.5f" % count_fraction) 155 | 156 | # get data 157 | vix_spot=get_vix_data() 158 | 159 | my_data_handler=csvFuturesData() 160 | 161 | sp500future = my_data_handler.get_raw_price("SP500") 162 | vixfuture = my_data_handler.get_raw_price("VIX") 163 | 164 | # we don't use this, but can check the results with it 165 | # note this is monthly data 166 | # You'd need to adjust 'frequency' and time_horizon appropriately 167 | usequitylong = get_us_long_data().USA_TR 168 | 169 | 170 | 171 | vix_spot.plot() 172 | plt.show() 173 | 174 | # start with unconditional plot 175 | 176 | do_a_plot(sp500future, vix_spot, lag_response=False, frequency="D", time_horizon=20, bin_func=no_binning, 177 | index_to_use = None, drawfittedlines=False) 178 | 179 | # add conditioning, half and half 180 | bin_data=do_a_plot(sp500future, vix_spot, lag_response=True, frequency="D", time_horizon=20, bin_func=half_split, 181 | index_to_use = None, drawfittedlines=False) 182 | 183 | plot_paired_test(bin_data, number_of_runs=5000) 184 | 185 | # add conditioning, half and half, with SR 186 | bin_data=do_a_plot(sp500future, vix_spot, lag_response=True, frequency="D", time_horizon=20, bin_func=half_split, 187 | index_to_use = None, drawfittedlines=False, calculate_sharpe_ratios=True) 188 | 189 | plot_paired_test(bin_data, number_of_runs=5000) 190 | 191 | # below 10 192 | bin_data=do_a_plot(sp500future, vix_spot, lag_response=True, frequency="D", time_horizon=20, bin_func=below_10, 193 | index_to_use = None, drawfittedlines=False, calculate_sharpe_ratios=True) 194 | 195 | plot_paired_test(bin_data, number_of_runs=5000) 196 | 197 | 198 | # deciles and middle 199 | # this is for the plot 200 | bin_data=do_a_plot(sp500future, vix_spot, lag_response=True, frequency="D", time_horizon=20, bin_func=deciles_and_mid, 201 | index_to_use = None, drawfittedlines=False, calculate_sharpe_ratios=True) 202 | 203 | plot_paired_test([bin_data[0], bin_data[1]], number_of_runs=5000) 204 | plot_paired_test([bin_data[1], bin_data[2]], number_of_runs=5000) 205 | plot_paired_test([bin_data[0], bin_data[2]], number_of_runs=5000) 206 | 207 | ### 208 | one_month_vol = pd.rolling_std(sp500future.diff(), 20) 209 | 210 | bin_data=do_a_plot(one_month_vol, vix_spot, lag_response=True, frequency="D", time_horizon=20, bin_func=deciles_and_mid, 211 | calculate_sharpe_ratios=False, calculate_returns=False) 212 | 213 | plot_paired_test([bin_data[0], bin_data[1]], number_of_runs=5000) 214 | plot_paired_test([bin_data[1], bin_data[2]], number_of_runs=5000) 215 | plot_paired_test([bin_data[0], bin_data[2]], number_of_runs=5000) 216 | 217 | 218 | one_month_vol = pd.rolling_std(sp500future.diff(), 20) 219 | 220 | bin_data=do_a_plot(one_month_vol, vix_spot, lag_response=True, frequency="D", time_horizon=20, bin_func=deciles_and_mid, 221 | calculate_sharpe_ratios=False, calculate_returns=False) 222 | 223 | plot_paired_test([bin_data[0], bin_data[1]], number_of_runs=5000) 224 | plot_paired_test([bin_data[1], bin_data[2]], number_of_runs=5000) 225 | plot_paired_test([bin_data[0], bin_data[2]], number_of_runs=5000) 226 | 227 | one_month_vol = pd.rolling_std(sp500future.diff(), 20) 228 | 229 | 230 | 231 | 232 | bin_data=do_a_plot(vix_spot, vix_spot, lag_response=True, frequency="D", time_horizon=500, bin_func=deciles_and_mid, 233 | calculate_sharpe_ratios=False, calculate_returns=False) 234 | 235 | plot_paired_test([bin_data[0], bin_data[1]], number_of_runs=5000) 236 | plot_paired_test([bin_data[1], bin_data[2]], number_of_runs=5000) 237 | plot_paired_test([bin_data[0], bin_data[2]], number_of_runs=5000) 238 | -------------------------------------------------------------------------------- /forecasting/forecasting.py: -------------------------------------------------------------------------------- 1 | from matplotlib.pyplot import plot, scatter 2 | from sysdata.csvdata import csvFuturesData 3 | import pandas as pd 4 | import numpy as np 5 | from scipy.stats import linregress 6 | from syscore.algos import robust_vol_calc 7 | 8 | from syscore.dateutils import fit_dates_object 9 | from syscore.genutils import progressBar 10 | 11 | code="US10" 12 | 13 | 14 | data_object=csvFuturesData() 15 | prices=data_object[code] 16 | perc=(prices - prices.shift(1))/data_object.get_instrument_raw_carry_data(code)['PRICE'] 17 | perc[abs(perc)>0.03]=np.nan 18 | 19 | def get_expost_data(perc): 20 | fitting_dates = generate_fitting_dates(perc, "rolling") ## only using annual dates rolling doesn't matter 21 | expost_data = [perc[fit_date.period_start:fit_date.period_end] for fit_date in fitting_dates[1:]] 22 | 23 | return expost_data 24 | 25 | 26 | 27 | def calc_historic_confidence(perc, function_to_use, rollperiods=250): 28 | fitting_dates=generate_fitting_dates(perc, "rolling", rollperiods=rollperiods) 29 | 30 | list_of_confidence=[] 31 | thing=progressBar(len(fitting_dates)-1) 32 | for fit_date in fitting_dates[1:]: 33 | list_of_confidence.append(function_to_use(perc, fit_date)) 34 | thing.iterate() 35 | thing.finished() 36 | 37 | 38 | list_of_confidence = pd.DataFrame(list_of_confidence, index=[fit_date.fit_end for fit_date in fitting_dates[1:]]) 39 | list_of_confidence.columns = ["lower", "upper"] 40 | 41 | return list_of_confidence 42 | 43 | def calc_historic_mean_distr_annual(perc, fit_date): 44 | data=perc[fit_date.fit_start:fit_date.fit_end] 45 | avg = data.mean() 46 | std = data.std() 47 | 48 | conf_interval_mult = (std**2 / len(data))**.5 49 | conf_interval_days = [avg - 1.96 * conf_interval_mult, avg + 1.96 * conf_interval_mult] 50 | conf_interval = np.array(conf_interval_days)*250 51 | 52 | return list(conf_interval) 53 | 54 | 55 | 56 | def single_bootstrap_from_data(data): 57 | n = len(data) 58 | bootstraps = [int(np.random.uniform(high=n)) for not_used in range(n)] 59 | bs_data=[data[bsnumber] for bsnumber in bootstraps] 60 | return bs_data 61 | 62 | def gen_bootstraps_from_data(data, monte_carlo=500): 63 | 64 | all_bs = [single_bootstrap_from_data(data) for not_used in range(monte_carlo)] 65 | 66 | return all_bs 67 | 68 | 69 | def calc_historic_SR_distr_annual(perc, fit_date): 70 | data=perc[fit_date.fit_start:fit_date.fit_end] 71 | SR = data.mean() / data.std() 72 | SR_std = ((1+.5*SR**2)/len(data))**.5 73 | 74 | conf_interval = (16*(SR - 2*SR_std), 16*(SR + 2*SR_std)) 75 | 76 | return conf_interval 77 | 78 | 79 | def calc_historic_std_distr_annual(perc, fit_date): 80 | data=perc[fit_date.fit_start:fit_date.fit_end] 81 | 82 | ## bootstrap 83 | all_bs = gen_bootstraps_from_data(data) 84 | all_bs_std = [np.nanstd(bs_data)*16 for bs_data in all_bs] 85 | conf_interval = [np.percentile(all_bs_std,2.5), np.percentile(all_bs_std, 97.5)] 86 | 87 | return list(conf_interval) 88 | 89 | def calc_historic_distr_annual(perc, fit_date): 90 | data = perc[fit_date.fit_start:fit_date.fit_end] 91 | 92 | data_mean = data.mean() 93 | data_std = data.std() 94 | 95 | conf_interval = (data_mean - 1.96*data_std, data_mean + 1.96*data_std) 96 | 97 | return conf_interval 98 | 99 | 100 | def ewmav(x, span=35): 101 | vol = x.ewm(adjust=True, span=span, min_periods=5).std() 102 | return vol[-1] 103 | 104 | def generate_fitting_dates(data, date_method, period="12M",rollperiods=20): 105 | """ 106 | generate a list 4 tuples, one element for each period in the data 107 | each tuple contains [fit_start, fit_end, period_start, period_end] datetime objects 108 | the last period will be a 'stub' if we haven't got an exact number of years 109 | 110 | date_method can be one of 'in_sample', 'expanding', 'rolling' 111 | 112 | if 'rolling' then use rollperiods variable 113 | """ 114 | 115 | if date_method not in ["in_sample", "rolling", "expanding"]: 116 | raise Exception( 117 | "don't recognise date_method %s should be one of in_sample, expanding, rolling" 118 | % date_method) 119 | 120 | if isinstance(data, list): 121 | start_date = min([dataitem.index[0] for dataitem in data]) 122 | end_date = max([dataitem.index[-1] for dataitem in data]) 123 | else: 124 | start_date = data.index[0] 125 | end_date = data.index[-1] 126 | 127 | # now generate the dates we use to fit 128 | if date_method == "in_sample": 129 | # single period 130 | return [fit_dates_object(start_date, end_date, start_date, end_date)] 131 | 132 | # generate list of dates, one period apart, including the final date 133 | period_starts = list(pd.date_range(start_date, end_date, freq=period)) + [ 134 | end_date 135 | ] 136 | 137 | # loop through each period 138 | periods = [] 139 | for tidx in range(len(period_starts))[1:-1]: 140 | # these are the dates we test in 141 | period_start = period_starts[tidx] 142 | period_end = period_starts[tidx + 1] 143 | 144 | # now generate the dates we use to fit 145 | if date_method == "expanding": 146 | fit_start = start_date 147 | elif date_method == "rolling": 148 | yearidx_to_use = max(0, tidx - rollperiods) 149 | fit_start = period_starts[yearidx_to_use] 150 | else: 151 | raise Exception( 152 | "don't recognise date_method %s should be one of in_sample, expanding, rolling" 153 | % date_method) 154 | 155 | if date_method in ['rolling', 'expanding']: 156 | fit_end = period_start 157 | else: 158 | raise Exception("don't recognise date_method %s " % date_method) 159 | 160 | periods.append( 161 | fit_dates_object(fit_start, fit_end, period_start, period_end)) 162 | 163 | if date_method in ['rolling', 'expanding']: 164 | # add on a dummy date for the first year, when we have no data 165 | periods = [ 166 | fit_dates_object( 167 | start_date, 168 | start_date, 169 | start_date, 170 | period_starts[1], 171 | no_data=True) 172 | ] + periods 173 | 174 | return periods 175 | 176 | 177 | 178 | expost_data = get_expost_data(perc) 179 | 180 | 181 | list_of_confidence = calc_historic_confidence(perc, function_to_use=calc_historic_mean_distr_annual) 182 | expost_dates = list_of_confidence.index 183 | 184 | expost_means = [period_data.mean()*250 for period_data in expost_data] 185 | expost_means = pd.DataFrame(expost_means, expost_dates) 186 | 187 | 188 | datatoplot = pd.concat([list_of_confidence, expost_means], axis=1) 189 | datatoplot.columns=["lower","upper","actual"] 190 | 191 | 192 | list_of_confidence = calc_historic_confidence(perc, function_to_use=calc_historic_std_distr_annual) 193 | expost_std = [period_data.std()*16 for period_data in expost_data] 194 | expost_std = pd.DataFrame(expost_std, expost_dates) 195 | datatoplot = pd.concat([list_of_confidence, expost_std], axis=1) 196 | datatoplot.columns=["lower","upper","actual"] 197 | 198 | list_of_confidence = calc_historic_confidence(perc, function_to_use=calc_historic_SR_distr_annual) 199 | expost_ssr = [16*period_data.mean()/period_data.std() for period_data in expost_data] 200 | expost_ssr = pd.DataFrame(expost_ssr, expost_dates) 201 | datatoplot = pd.concat([list_of_confidence, expost_ssr], axis=1) 202 | datatoplot.columns=["lower","upper","actual"] 203 | 204 | entire_distribution = calc_historic_confidence(perc, function_to_use=calc_historic_distr_annual) 205 | entire_distribution.plot() 206 | perc.plot() 207 | 208 | def sr_func(data): 209 | return data.mean()/data.std() 210 | 211 | data=perc 212 | period="10B" 213 | rollperiods=1 ## set to very large number if using exp std 214 | expost_function_to_use=np.nanstd 215 | exante_function_to_use=np.nanstd 216 | multiplier_to_apply=16.0 217 | fitting_dates=generate_fitting_dates(perc, "rolling", period=period, rollperiods=rollperiods) 218 | ex_ante_value_list=[] 219 | ex_post_value_list=[] 220 | 221 | for fit_date in fitting_dates[1:]: 222 | ex_ante_data=data[fit_date.fit_start:fit_date.fit_end] 223 | ex_ante_value = exante_function_to_use(ex_ante_data)*multiplier_to_apply 224 | 225 | ex_post_data=data[fit_date.period_start:fit_date.period_end] 226 | ex_post_value = expost_function_to_use(ex_post_data)*multiplier_to_apply 227 | 228 | if not (np.isnan(ex_ante_value) or np.isnan(ex_post_value)): 229 | 230 | ex_post_value_list.append(ex_post_value) 231 | ex_ante_value_list.append(ex_ante_value) 232 | 233 | scatter(ex_ante_value_list, ex_post_value_list) 234 | 235 | print(linregress(ex_ante_value_list, ex_post_value_list)) 236 | 237 | def calc_ewmac_forecast(price, Lfast=64, Lslow=256): 238 | """ 239 | Calculate the ewmac trading fule forecast, given a price and EWMA speeds 240 | Lfast, Lslow and vol_lookback 241 | 242 | """ 243 | # price: This is the stitched price series 244 | # We can't use the price of the contract we're trading, or the volatility 245 | # will be jumpy 246 | # And we'll miss out on the rolldown. See 247 | # http://qoppac.blogspot.co.uk/2015/05/systems-building-futures-rolling.html 248 | 249 | price = price.resample("1B").last() 250 | 251 | if Lslow is None: 252 | Lslow = 4 * Lfast 253 | 254 | # We don't need to calculate the decay parameter, just use the span 255 | # directly 256 | fast_ewma = price.ewm(span=Lfast).mean() 257 | slow_ewma = price.ewm(span=Lslow).mean() 258 | raw_ewmac = fast_ewma - slow_ewma 259 | 260 | vol = robust_vol_calc(price.diff()) 261 | return raw_ewmac / vol 262 | 263 | price = data_object.daily_prices(code) 264 | ewmac = calc_ewmac_forecast(price, 16, 64) 265 | 266 | 267 | 268 | data=perc 269 | period="1M" 270 | rollperiods=1 ## set to very large number if using exp std 271 | exante_function_to_use=sr_func 272 | multiplier_to_apply=16.0 273 | fitting_dates=generate_fitting_dates(perc, "rolling", period=period, rollperiods=rollperiods) 274 | ex_ante_value_list=[] 275 | ex_post_value_list=[] 276 | 277 | 278 | for fit_date in fitting_dates[1:]: 279 | ex_ante_value = ewmac[:fit_date.fit_end][-1] 280 | 281 | ex_post_data=data[fit_date.period_start:fit_date.period_end] 282 | ex_post_value = expost_function_to_use(ex_post_data)*multiplier_to_apply 283 | 284 | if not (np.isnan(ex_ante_value) or np.isnan(ex_post_value)): 285 | 286 | ex_post_value_list.append(ex_post_value) 287 | ex_ante_value_list.append(ex_ante_value) 288 | 289 | scatter(ex_ante_value_list, ex_post_value_list) 290 | 291 | print(linregress(ex_ante_value_list, ex_post_value_list)) 292 | 293 | from systems.provided.futures_chapter15.basesystem import futures_system -------------------------------------------------------------------------------- /smallaccountsize/roundingeffects.py: -------------------------------------------------------------------------------- 1 | """ 2 | rounding effects 3 | """ 4 | 5 | # Need a binary / thresholder 6 | 7 | from systems.portfolio import PortfoliosFixed 8 | from systems.forecast_combine import ForecastCombineFixed, apply_cap, multiply_df 9 | import pandas as pd 10 | from sysdata.csvdata import csvFuturesData 11 | 12 | 13 | class ForecastWithBinary(ForecastCombineFixed): 14 | def get_combined_forecast(self, instrument_code): 15 | def _get_combined_forecast(system, instrument_code, this_stage): 16 | this_stage.log.msg( 17 | "Calculating combined forecast for %s" % (instrument_code), 18 | instrument_code=instrument_code) 19 | 20 | forecast_weights = this_stage.get_forecast_weights(instrument_code) 21 | rule_variation_list = list(forecast_weights.columns) 22 | 23 | forecasts = this_stage.get_all_forecasts(instrument_code, 24 | rule_variation_list) 25 | forecast_div_multiplier = this_stage.get_forecast_diversification_multiplier( 26 | instrument_code) 27 | forecast_cap = this_stage.get_forecast_cap() 28 | 29 | # multiply weights by forecasts 30 | combined_forecast = multiply_df(forecast_weights, forecasts) 31 | 32 | # sum 33 | combined_forecast = combined_forecast.sum( 34 | axis=1).to_frame("comb_forecast") 35 | 36 | # apply fdm 37 | # (note in this simple version we aren't adjusting FDM if forecast_weights change) 38 | forecast_div_multiplier = forecast_div_multiplier.reindex( 39 | forecasts.index, method="ffill") 40 | raw_combined_forecast = multiply_df(combined_forecast, 41 | forecast_div_multiplier) 42 | 43 | combined_forecast = apply_cap(raw_combined_forecast, forecast_cap) 44 | 45 | combined_forecast[combined_forecast > 0.0] = 10.0 46 | combined_forecast[combined_forecast < 0.0] = 10.0 47 | 48 | return combined_forecast 49 | 50 | combined_forecast = self.parent.calc_or_cache( 51 | 'get_combined_forecast', instrument_code, _get_combined_forecast, 52 | self) 53 | return combined_forecast 54 | 55 | 56 | class ForecastWithThreshold(ForecastCombineFixed): 57 | def get_combined_forecast(self, instrument_code): 58 | def _get_combined_forecast(system, instrument_code, this_stage): 59 | this_stage.log.msg( 60 | "Calculating combined forecast for %s" % (instrument_code), 61 | instrument_code=instrument_code) 62 | 63 | forecast_weights = this_stage.get_forecast_weights(instrument_code) 64 | rule_variation_list = list(forecast_weights.columns) 65 | 66 | forecasts = this_stage.get_all_forecasts(instrument_code, 67 | rule_variation_list) 68 | forecast_div_multiplier = this_stage.get_forecast_diversification_multiplier( 69 | instrument_code) 70 | forecast_cap = this_stage.get_forecast_cap() 71 | 72 | # multiply weights by forecasts 73 | combined_forecast = multiply_df(forecast_weights, forecasts) 74 | 75 | # sum 76 | combined_forecast = combined_forecast.sum( 77 | axis=1).to_frame("comb_forecast") 78 | 79 | # apply fdm 80 | # (note in this simple version we aren't adjusting FDM if forecast_weights change) 81 | forecast_div_multiplier = forecast_div_multiplier.reindex( 82 | forecasts.index, method="ffill") 83 | raw_combined_forecast = multiply_df(combined_forecast, 84 | forecast_div_multiplier) 85 | 86 | def map_forecast_value(x): 87 | x = float(x) 88 | if x < -20.0: 89 | return -30.0 90 | if x >= -20.0 and x < -10.0: 91 | return -(abs(x) - 10.0) * 3 92 | if x >= -10.0 and x <= 10.0: 93 | return 0.0 94 | if x > 10.0 and x <= 20.0: 95 | return (abs(x) - 10.0) * 3 96 | return 30.0 97 | 98 | combined_forecast = pd.DataFrame( 99 | [map_forecast_value(x) 100 | for x in combined_forecast.values], combined_forecast.index) 101 | 102 | return combined_forecast 103 | 104 | combined_forecast = self.parent.calc_or_cache( 105 | 'get_combined_forecast', instrument_code, _get_combined_forecast, 106 | self) 107 | return combined_forecast 108 | 109 | 110 | ''' 111 | Created on 4 Mar 2016 112 | 113 | @author: rob 114 | ''' 115 | 116 | from systems.provided.futures_chapter15.estimatedsystem import PortfoliosEstimated 117 | from systems.provided.futures_chapter15.basesystem import * 118 | from syscore.correlations import get_avg_corr 119 | from copy import copy 120 | import numpy as np 121 | 122 | data = csvFuturesData() 123 | all_instruments = data.keys() 124 | 125 | config = Config("examples.smallaccountsize.smallaccount.yaml") 126 | 127 | all_accounts = [] 128 | for instrument_code in all_instruments: 129 | 130 | config.instruments = [instrument_code] 131 | 132 | system1 = System([ 133 | Account(), PortfoliosEstimated(), PositionSizing(), FuturesRawData(), 134 | ForecastCombineFixed(), ForecastScaleCapFixed(), Rules() 135 | ], csvFuturesData(), config) 136 | 137 | system1.set_logging_level("on") 138 | 139 | max_position = float( 140 | system1.positionSize.get_volatility_scalar(instrument_code).mean() * 141 | 2.0) 142 | 143 | original_capital = system1.config.notional_trading_capital 144 | 145 | accounts_this_instr = [] 146 | for target_max in [1.0, 2.0, 3.0, 4.0]: 147 | 148 | # reset trading capital 149 | config = system1.config 150 | config.use_SR_costs = False 151 | config.notional_trading_capital = original_capital * target_max / max_position 152 | 153 | system1 = System([ 154 | Account(), PortfoliosFixed(), PositionSizing(), FuturesRawData(), 155 | ForecastCombineFixed(), ForecastScaleCapFixed(), Rules() 156 | ], csvFuturesData(), config) 157 | 158 | system1.set_logging_level("on") 159 | 160 | system1_rounded = system1.accounts.portfolio(roundpositions=True) 161 | system1_unrounded = system1.accounts.portfolio(roundpositions=False) 162 | 163 | system2 = System([ 164 | Account(), PortfoliosFixed(), PositionSizing(), FuturesRawData(), 165 | ForecastWithBinary(), ForecastScaleCapFixed(), Rules() 166 | ], csvFuturesData(), config) 167 | 168 | system2.set_logging_level("on") 169 | 170 | system2_rounded = system2.accounts.portfolio(roundpositions=True) 171 | system2_unrounded = system2.accounts.portfolio(roundpositions=False) 172 | 173 | system3 = System([ 174 | Account(), PortfoliosFixed(), PositionSizing(), FuturesRawData(), 175 | ForecastWithThreshold(), ForecastScaleCapFixed(), Rules() 176 | ], csvFuturesData(), config) 177 | 178 | system3.set_logging_level("on") 179 | 180 | system3_rounded = system3.accounts.portfolio(roundpositions=True) 181 | system3_unrounded = system3.accounts.portfolio(roundpositions=False) 182 | 183 | accounts_this_instr.append([ 184 | system1_rounded, system1_unrounded, system2_rounded, 185 | system2_unrounded, system3_rounded, system3_unrounded 186 | ]) 187 | 188 | all_accounts.append(accounts_this_instr) 189 | 190 | targetref = 2 # 0 is 1.0 target_max etc 191 | 192 | acc_names = [ 193 | "Arbitrary, unrounded", "Binary, unrounded", "Threshold, unrounded", 194 | "Arbitrary, round", "Binary, round", "Threshold, round" 195 | ] 196 | 197 | allaccresults = dict(gross=[], net=[], costs=[], vol=[], SR=[], maxstd=[]) 198 | allstackresults = dict(gross=[], net=[], costs=[], vol=[], SR=[]) 199 | 200 | for accref in [1, 3, 5, 0, 2, 4]: # 0 is system1_rounded and so on 201 | 202 | allresults_gross = [] 203 | allresults_net = [] 204 | allresults_costs = [] 205 | allresults_vol = [] 206 | allresults_sr = [] 207 | allresults_max_std = [] 208 | 209 | allstacknet = [] 210 | allstackcosts = [] 211 | allstackgross = [] 212 | 213 | for (instridx, instrument_code) in enumerate(all_instruments): 214 | allresults_gross.append( 215 | all_accounts[instridx][targetref][accref].gross.weekly.ann_mean()) 216 | allresults_net.append( 217 | all_accounts[instridx][targetref][accref].net.weekly.ann_mean()) 218 | allresults_costs.append( 219 | all_accounts[instridx][targetref][accref].costs.weekly.ann_mean()) 220 | allresults_vol.append( 221 | all_accounts[instridx][targetref][accref].net.weekly.ann_std()) 222 | allresults_max_std.append( 223 | float( 224 | pd.rolling_std( 225 | all_accounts[instridx][targetref][accref] 226 | .net.weekly.as_df(), 227 | 26, 228 | min_periods=4, 229 | center=True).max()) * (52**.5)) 230 | 231 | try: 232 | sharpe = all_accounts[instridx][targetref][ 233 | accref].net.weekly.ann_mean() / all_accounts[instridx][ 234 | targetref][accref].net.weekly.ann_std() 235 | 236 | except ZeroDivisionError: 237 | sharpe = np.nan 238 | 239 | allresults_sr.append(sharpe) 240 | 241 | allstacknet.append( 242 | list(all_accounts[instridx][targetref][accref] 243 | .net.weekly.iloc[:, 0].values)) 244 | allstackcosts.append( 245 | list(all_accounts[instridx][targetref][accref] 246 | .costs.weekly.iloc[:, 0].values)) 247 | allstackgross.append( 248 | list(all_accounts[instridx][targetref][accref] 249 | .gross.weekly.iloc[:, 0].values)) 250 | 251 | allstacknet = sum(allstacknet, []) 252 | allstackgross = sum(allstackgross, []) 253 | allstackcosts = sum(allstackcosts, []) 254 | 255 | allaccresults['gross'].append(np.mean(allresults_gross)) 256 | allaccresults['net'].append(np.mean(allresults_net)) 257 | allaccresults['costs'].append(np.mean(allresults_costs)) 258 | allaccresults['vol'].append(np.mean(allresults_vol)) 259 | allaccresults['SR'].append(np.nanmean(allresults_sr)) 260 | allaccresults['maxstd'].append(np.nanmean(allresults_max_std)) 261 | 262 | #allstacknet=[x for x in allstacknet if not x==0.0] 263 | #allstackgross=[x for x in allstacknet if not x==0.0] 264 | #allstackcosts=[x for x in allstackcosts if not x==0.0] 265 | 266 | allstackresults['gross'].append(np.nanmean(allstackgross) * 52) 267 | allstackresults['net'].append(np.nanmean(allstacknet) * 52) 268 | allstackresults['costs'].append(np.nanmean(allstackcosts) * 52) 269 | allstackresults['vol'].append(np.nanstd(allstacknet) * (52**.5)) 270 | allstackresults['SR'].append( 271 | np.nanmean(allstacknet) * (52**.5) / np.nanstd(allstacknet)) 272 | -------------------------------------------------------------------------------- /vix/US_monthly_returns.csv: -------------------------------------------------------------------------------- 1 | DATETIME,USA_TR 2 | 31/12/69,100 3 | 30/01/70,92.886 4 | 27/02/70,98.13 5 | 31/03/70,98.729 6 | 30/04/70,90.2 7 | 29/05/70,85.125 8 | 30/06/70,81.667 9 | 31/07/70,87.823 10 | 31/08/70,91.445 11 | 30/09/70,94.574 12 | 30/10/70,93.91 13 | 30/11/70,99.08 14 | 31/12/70,104.83 15 | 29/01/71,109.712 16 | 26/02/71,110.885 17 | 31/03/71,115.446 18 | 30/04/71,119.987 19 | 31/05/71,115.381 20 | 30/06/71,116.229 21 | 30/07/71,112.147 22 | 31/08/71,116.828 23 | 30/09/71,115.434 24 | 29/10/71,111.371 25 | 30/11/71,111.897 26 | 31/12/71,118.976 27 | 31/01/72,121.455 28 | 29/02/72,124.948 29 | 31/03/72,126.991 30 | 28/04/72,127.288 31 | 31/05/72,127.239 32 | 30/06/72,125.054 33 | 31/07/72,124.441 34 | 31/08/72,129.073 35 | 29/09/72,129.375 36 | 31/10/72,131.162 37 | 30/11/72,136.315 38 | 29/12/72,138.685 39 | 31/01/73,135.496 40 | 28/02/73,129.271 41 | 30/03/73,127.862 42 | 30/04/73,123.065 43 | 31/05/73,121.224 44 | 29/06/73,121.493 45 | 31/07/73,126.387 46 | 31/08/73,122.291 47 | 28/09/73,126.776 48 | 31/10/73,127.435 49 | 30/11/73,114.036 50 | 31/12/73,116.157 51 | 31/01/74,114.282 52 | 28/02/74,114.27 53 | 29/03/74,111.59 54 | 30/04/74,107.411 55 | 31/05/74,104.2 56 | 28/06/74,102.561 57 | 31/07/74,94.53 58 | 30/08/74,86.081 59 | 30/09/74,76.191 60 | 31/10/74,89.746 61 | 29/11/74,85.702 62 | 31/12/74,84.012 63 | 31/01/75,94.945 64 | 28/02/75,100.942 65 | 31/03/75,103.442 66 | 30/04/75,108.144 67 | 30/05/75,113.121 68 | 30/06/75,118.595 69 | 31/07/75,110.687 70 | 29/08/75,108.301 71 | 30/09/75,104.554 72 | 31/10/75,111.787 73 | 28/11/75,115.207 74 | 31/12/75,114.104 75 | 30/01/76,127.461 76 | 27/02/76,126.193 77 | 31/03/76,130.651 78 | 30/04/76,129.331 79 | 31/05/76,127.993 80 | 30/06/76,133.733 81 | 30/07/76,133.06 82 | 31/08/76,132.995 83 | 30/09/76,136.431 84 | 29/10/76,133.951 85 | 30/11/76,133.468 86 | 31/12/76,140.63 87 | 31/01/77,133.864 88 | 28/02/77,131.223 89 | 31/03/77,129.807 90 | 29/04/77,130.303 91 | 31/05/77,127.953 92 | 30/06/77,134.267 93 | 29/07/77,132.697 94 | 31/08/77,130.304 95 | 30/09/77,130.472 96 | 31/10/77,125.16 97 | 30/11/77,128.508 98 | 30/12/77,129.353 99 | 31/01/78,121.914 100 | 28/02/78,119.291 101 | 31/03/78,122.298 102 | 28/04/78,133.96 103 | 31/05/78,134.608 104 | 30/06/78,132.576 105 | 31/07/78,140.702 106 | 31/08/78,144.507 107 | 29/09/78,143.978 108 | 31/10/78,131.816 109 | 30/11/78,134.563 110 | 29/12/78,137.066 111 | 31/01/79,143.397 112 | 28/02/79,138.508 113 | 30/03/79,146.444 114 | 30/04/79,146.503 115 | 31/05/79,143.722 116 | 29/06/79,149.302 117 | 31/07/79,151.002 118 | 31/08/79,158.52 119 | 28/09/79,158.296 120 | 31/10/79,148.176 121 | 30/11/79,154.339 122 | 31/12/79,156.871 123 | 31/01/80,166.343 124 | 29/02/80,164.679 125 | 31/03/80,150.127 126 | 30/04/80,156.657 127 | 30/05/80,164.213 128 | 30/06/80,169.553 129 | 31/07/80,180.816 130 | 29/08/80,182.07 131 | 30/09/80,186.663 132 | 31/10/80,191.383 133 | 28/11/80,211.35 134 | 31/12/80,204.002 135 | 30/01/81,196.434 136 | 27/02/81,200.605 137 | 31/03/81,207.305 138 | 30/04/81,203.476 139 | 29/05/81,202.915 140 | 30/06/81,202.518 141 | 31/07/81,204.017 142 | 31/08/81,192.382 143 | 30/09/81,183.347 144 | 30/10/81,192.458 145 | 30/11/81,200.892 146 | 31/12/81,195.58 147 | 29/01/82,193.369 148 | 26/02/82,183.389 149 | 31/03/82,182.399 150 | 30/04/82,190.406 151 | 31/05/82,184.098 152 | 30/06/82,181.792 153 | 30/07/82,178.554 154 | 31/08/82,200.536 155 | 30/09/82,203.196 156 | 29/10/82,225.851 157 | 30/11/82,234.64 158 | 31/12/82,238.888 159 | 31/01/83,248.474 160 | 28/02/83,253.785 161 | 31/03/83,262.619 162 | 29/04/83,283.293 163 | 31/05/83,278.964 164 | 30/06/83,289.822 165 | 29/07/83,280.481 166 | 31/08/83,286.262 167 | 30/09/83,290.035 168 | 31/10/83,286.821 169 | 30/11/83,293.048 170 | 30/12/83,291.501 171 | 31/01/84,288.315 172 | 29/02/84,276.421 173 | 30/03/84,281.595 174 | 30/04/84,284.581 175 | 31/05/84,268.699 176 | 29/06/84,274.478 177 | 31/07/84,272.413 178 | 31/08/84,302.067 179 | 28/09/84,302.595 180 | 31/10/84,304.195 181 | 30/11/84,301.281 182 | 31/12/84,308.921 183 | 31/01/85,333.786 184 | 28/02/85,337.785 185 | 29/03/85,338.745 186 | 30/04/85,338.042 187 | 31/05/85,359.024 188 | 28/06/85,364.338 189 | 31/07/85,363.296 190 | 30/08/85,360.048 191 | 30/09/85,348.707 192 | 31/10/85,364.583 193 | 29/11/85,389.965 194 | 31/12/85,410.105 195 | 31/01/86,412.709 196 | 28/02/86,443.997 197 | 31/03/86,468.801 198 | 30/04/86,462.301 199 | 30/05/86,485.082 200 | 30/06/86,491.419 201 | 31/07/86,465.208 202 | 29/08/86,499.603 203 | 30/09/86,457.133 204 | 31/10/86,483.078 205 | 28/11/86,494.433 206 | 31/12/86,481.976 207 | 30/01/87,545.985 208 | 27/02/87,565.593 209 | 31/03/87,580.841 210 | 30/04/87,575.593 211 | 29/05/87,578.843 212 | 30/06/87,608.817 213 | 31/07/87,637.762 214 | 31/08/87,662.893 215 | 30/09/87,648.76 216 | 30/10/87,511.075 217 | 30/11/87,467.875 218 | 31/12/87,500.821 219 | 29/01/88,522.273 220 | 29/02/88,544.14 221 | 31/03/88,525.935 222 | 29/04/88,530.765 223 | 31/05/88,535.331 224 | 30/06/88,560.25 225 | 29/07/88,558.518 226 | 31/08/88,539.784 227 | 30/09/88,562.85 228 | 31/10/88,580.103 229 | 30/11/88,571.162 230 | 30/12/88,580.518 231 | 31/01/89,624.69 232 | 28/02/89,605.958 233 | 31/03/89,619.615 234 | 28/04/89,653.282 235 | 31/05/89,678.503 236 | 30/06/89,675.979 237 | 31/07/89,737.954 238 | 31/08/89,749.128 239 | 29/09/89,747.547 240 | 31/10/89,730.628 241 | 30/11/89,745.749 242 | 29/12/89,762.584 243 | 31/01/90,713.133 244 | 28/02/90,723.758 245 | 30/03/90,740.92 246 | 30/04/90,725.864 247 | 31/05/90,793.009 248 | 29/06/90,790.122 249 | 31/07/90,788.972 250 | 31/08/90,716.853 251 | 28/09/90,683.662 252 | 31/10/90,681.623 253 | 30/11/90,726.455 254 | 31/12/90,746.708 255 | 31/01/91,782.388 256 | 28/02/91,836.52 257 | 29/03/91,856.964 258 | 30/04/91,860.373 259 | 31/05/91,896.179 260 | 28/06/91,855.168 261 | 31/07/91,895.261 262 | 30/08/91,917.013 263 | 30/09/91,901.496 264 | 31/10/91,917.089 265 | 29/11/91,880.074 266 | 31/12/91,980.63 267 | 31/01/92,964.318 268 | 28/02/92,975.39 269 | 31/03/92,957.436 270 | 30/04/92,985.481 271 | 29/05/92,989.539 272 | 30/06/92,974.462 273 | 31/07/92,1015.136 274 | 31/08/92,990.832 275 | 30/09/92,1001.632 276 | 30/10/92,1007.776 277 | 30/11/92,1041.304 278 | 31/12/92,1052.826 279 | 29/01/93,1062.116 280 | 26/02/93,1076.114 281 | 31/03/93,1098.308 282 | 30/04/93,1074.114 283 | 31/05/93,1101.624 284 | 30/06/93,1105.089 285 | 30/07/93,1103.94 286 | 31/08/93,1143.856 287 | 30/09/93,1132.651 288 | 29/10/93,1152.626 289 | 30/11/93,1144.907 290 | 31/12/93,1158.852 291 | 31/01/94,1201.082 292 | 28/02/94,1167.437 293 | 31/03/94,1117.015 294 | 29/04/94,1133.025 295 | 31/05/94,1152.004 296 | 30/06/94,1119.778 297 | 29/07/94,1157.778 298 | 31/08/94,1203.548 299 | 30/09/94,1178.293 300 | 31/10/94,1205.588 301 | 30/11/94,1164.663 302 | 30/12/94,1182.011 303 | 31/01/95,1214.883 304 | 28/02/95,1263.074 305 | 31/03/95,1297.222 306 | 28/04/95,1338.256 307 | 31/05/95,1390.833 308 | 30/06/95,1425.372 309 | 31/07/95,1474.114 310 | 31/08/95,1474.553 311 | 29/09/95,1541.034 312 | 31/10/95,1540.647 313 | 30/11/95,1608.724 314 | 29/12/95,1633.386 315 | 31/01/96,1692.017 316 | 29/02/96,1709.998 317 | 29/03/96,1727.687 318 | 30/04/96,1753.61 319 | 31/05/96,1800.809 320 | 28/06/96,1811.811 321 | 31/07/96,1732.479 322 | 30/08/96,1771.655 323 | 30/09/96,1871.283 324 | 31/10/96,1919.512 325 | 29/11/96,2064.995 326 | 31/12/96,2026.293 327 | 31/01/97,2165.497 328 | 28/02/97,2181.224 329 | 31/03/97,2084.009 330 | 30/04/97,2221.712 331 | 30/05/97,2347.995 332 | 30/06/97,2455.442 333 | 31/07/97,2650.06 334 | 29/08/97,2493.214 335 | 30/09/97,2623.869 336 | 31/10/97,2552.82 337 | 28/11/97,2676.92 338 | 31/12/97,2716.982 339 | 30/01/98,2752.48 340 | 27/02/98,2946.083 341 | 31/03/98,3099.36 342 | 30/04/98,3136.193 343 | 29/05/98,3073.75 344 | 30/06/98,3206.355 345 | 31/07/98,3175.11 346 | 31/08/98,2733.618 347 | 30/09/98,2913.732 348 | 30/10/98,3139.696 349 | 30/11/98,3353.374 350 | 31/12/98,3551.723 351 | 29/01/99,3705.311 352 | 26/02/99,3601.942 353 | 31/03/99,3751.828 354 | 30/04/99,3887.652 355 | 31/05/99,3797.28 356 | 30/06/99,4001.631 357 | 30/07/99,3872.025 358 | 31/08/99,3847.674 359 | 30/09/99,3733.527 360 | 29/10/99,3977.809 361 | 30/11/99,4063.205 362 | 31/12/99,4346.659 363 | 31/01/00,4113.942 364 | 29/02/00,4016.239 365 | 31/03/00,4417.244 366 | 28/04/00,4274.448 367 | 31/05/00,4161.501 368 | 30/06/00,4261.432 369 | 31/07/00,4185.351 370 | 31/08/00,4403.062 371 | 29/09/00,4163.637 372 | 31/10/00,4134.069 373 | 30/11/00,3807.643 374 | 29/12/00,3801.778 375 | 31/01/01,3940.799 376 | 28/02/01,3588.647 377 | 30/03/01,3359.872 378 | 30/04/01,3621.65 379 | 31/05/01,3643.445 380 | 29/06/01,3560.065 381 | 31/07/01,3526.865 382 | 31/08/01,3292.648 383 | 28/09/01,3040.927 384 | 31/10/01,3085.453 385 | 30/11/01,3323.252 386 | 31/12/01,3344.37 387 | 31/01/02,3297.208 388 | 28/02/02,3232.827 389 | 29/03/02,3353.375 390 | 30/04/02,3135.884 391 | 31/05/02,3112.559 392 | 28/06/02,2875.354 393 | 31/07/02,2668.65 394 | 30/08/02,2681.354 395 | 30/09/02,2378.786 396 | 31/10/02,2594.869 397 | 29/11/02,2751.065 398 | 31/12/02,2584.929 399 | 31/01/03,2522.505 400 | 28/02/03,2484.112 401 | 31/03/03,2507.246 402 | 30/04/03,2716.722 403 | 30/05/03,2862.49 404 | 30/06/03,2897.97 405 | 31/07/03,2951.784 406 | 29/08/03,3007.171 407 | 30/09/03,2972.457 408 | 31/10/03,3142.133 409 | 28/11/03,3172.555 410 | 31/12/03,3337.41 411 | 30/01/04,3397.874 412 | 27/02/04,3439.102 413 | 31/03/04,3384.481 414 | 30/04/04,3332.517 415 | 31/05/04,3376.846 416 | 30/06/04,3440.639 417 | 30/07/04,3323.49 418 | 31/08/04,3340.507 419 | 30/09/04,3377.187 420 | 29/10/04,3428.833 421 | 30/11/04,3569.833 422 | 31/12/04,3694.97 423 | 31/01/05,3603.494 424 | 28/02/05,3678.646 425 | 31/03/05,3619.894 426 | 29/04/05,3554.49 427 | 31/05/05,3670.69 428 | 30/06/05,3679.889 429 | 29/07/05,3818.157 430 | 31/08/05,3782.703 431 | 30/09/05,3815.117 432 | 31/10/05,3752.529 433 | 30/11/05,3904.158 434 | 30/12/05,3906.351 435 | 31/01/06,4012.973 436 | 28/02/06,4017.045 437 | 31/03/06,4068.566 438 | 28/04/06,4122.201 439 | 31/05/06,4000.222 440 | 30/06/06,4004.709 441 | 31/07/06,4018.081 442 | 31/08/06,4115.145 443 | 29/09/06,4219.909 444 | 31/10/06,4364.118 445 | 30/11/06,4450.719 446 | 29/12/06,4504.774 447 | 31/01/07,4586.413 448 | 28/02/07,4502.528 449 | 30/03/07,4551.226 450 | 30/04/07,4746.318 451 | 31/05/07,4912.532 452 | 29/06/07,4830.393 453 | 31/07/07,4680.877 454 | 31/08/07,4751.956 455 | 28/09/07,4932.33 456 | 31/10/07,5015.452 457 | 30/11/07,4803.743 458 | 31/12/07,4776.315 459 | 31/01/08,4486.439 460 | 29/02/08,4346.96 461 | 31/03/08,4330.946 462 | 30/04/08,4545.319 463 | 30/05/08,4617.845 464 | 30/06/08,4241.463 465 | 31/07/08,4192.954 466 | 29/08/08,4250.383 467 | 30/09/08,3860.514 468 | 31/10/08,3200.304 469 | 28/11/08,2964.17 470 | 31/12/08,3002.604 471 | 30/01/09,2757.992 472 | 27/02/09,2475.326 473 | 31/03/09,2687.434 474 | 30/04/09,2945.475 475 | 29/05/09,3107.532 476 | 30/06/09,3114.565 477 | 31/07/09,3349.331 478 | 31/08/09,3465.82 479 | 30/09/09,3600.257 480 | 30/10/09,3530.52 481 | 30/11/09,3740.563 482 | 31/12/09,3817.617 483 | 29/01/10,3683.695 484 | 26/02/10,3797.381 485 | 31/03/10,4024.789 486 | 30/04/10,4089.279 487 | 31/05/10,3759.572 488 | 30/06/10,3559.653 489 | 30/07/10,3808.436 490 | 31/08/10,3639.268 491 | 30/09/10,3970.138 492 | 29/10/10,4126.484 493 | 30/11/10,4131.084 494 | 31/12/10,4407.359 495 | 31/01/11,4513.031 496 | 28/02/11,4664.337 497 | 31/03/11,4669.889 498 | 29/04/11,4812.786 499 | 31/05/11,4760.617 500 | 30/06/11,4679.897 501 | 29/07/11,4588.744 502 | 31/08/11,4334.932 503 | 30/09/11,4023.147 504 | 31/10/11,4465.154 505 | 30/11/11,4452.91 506 | 30/12/11,4495.07 507 | 31/01/12,4706.759 508 | 29/02/12,4914.136 509 | 30/03/12,5071.5 510 | 30/04/12,5040.928 511 | 31/05/12,4731.22 512 | 29/06/12,4918.302 513 | 31/07/12,4984.978 514 | 31/08/12,5101.984 515 | 28/09/12,5232.081 516 | 31/10/12,5137.66 517 | 30/11/12,5171.124 518 | 31/12/12,5220.28 519 | 31/01/13,5496.521 520 | 28/02/13,5567.152 521 | 29/03/13,5776.561 522 | 30/04/13,5890.806 523 | 31/05/13,6014.938 524 | 28/06/13,5934.743 525 | 31/07/13,6246.996 526 | 30/08/13,6074.659 527 | 30/09/13,6275.865 528 | 31/10/13,6554.499 529 | 29/11/13,6741.446 530 | 31/12/13,6922.497 531 | 31/01/14,6687.485 532 | 28/02/14,7001.931 533 | 31/03/14,7050.019 534 | 30/04/14,7092.383 535 | 30/05/14,7261.646 536 | 30/06/14,7417.225 537 | 31/07/14,7312.88 538 | 29/08/14,7606.391 539 | 30/09/14,7487.563 540 | 31/10/14,7668.259 541 | 28/11/14,7871.617 542 | 31/12/14,7847.602 543 | 30/01/15,7626.069 544 | 27/02/15,8072.21 545 | 31/03/15,7955.684 546 | 30/04/15,8028.432 547 | 29/05/15,8135.42 548 | 30/06/15,7981.335 549 | 31/07/15,8140.837 550 | 31/08/15,7646.982 551 | 30/09/15,7444.467 552 | 30/10/15,8058.738 553 | 30/11/15,8086.848 554 | 31/12/15,7950.974 555 | 29/01/16,7528.367 556 | 29/02/16,7512.496 557 | 31/03/16,8027.008 558 | 29/04/16,8066.671 559 | -------------------------------------------------------------------------------- /mythbusting/overextend.py: -------------------------------------------------------------------------------- 1 | from systems.provided.futures_chapter15.basesystem import * 2 | import pandas as pd 3 | import numpy as np 4 | from matplotlib.pyplot import show, plot, scatter, gca 5 | from syscore.pdutils import align_to_joint, uniquets 6 | from syscore.dateutils import generate_fitting_dates 7 | 8 | 9 | def clean_data(x, y, maxstd=6.0): 10 | 11 | xcap = np.nanstd(x) * maxstd 12 | ycap = np.nanstd(y) * maxstd 13 | 14 | def _cap(xitem, cap): 15 | if np.isnan(xitem): 16 | return xitem 17 | if xitem > cap: 18 | return cap 19 | if xitem < -cap: 20 | return -cap 21 | return xitem 22 | 23 | x = [_cap(xitem, xcap) for xitem in x] 24 | y = [_cap(yitem, ycap) for yitem in y] 25 | 26 | return (x, y) 27 | 28 | 29 | def bin_fit(x, y, buckets=3): 30 | 31 | assert buckets in [3, 25] 32 | 33 | xstd = np.nanstd(x) 34 | 35 | if buckets == 3: 36 | binlimits = [np.nanmin(x), -xstd / 2.0, xstd / 2.0, np.nanmax(x)] 37 | elif buckets == 25: 38 | 39 | steps = xstd / 4.0 40 | binlimits = np.arange(-xstd * 3.0, xstd * 3.0, steps) 41 | 42 | binlimits = [np.nanmin(x)] + list(binlimits) + [np.nanmax(x)] 43 | 44 | fit_y = [] 45 | err_y = [] 46 | x_values_to_plot = [] 47 | for binidx in range(len(binlimits))[1:]: 48 | lower_bin_x = binlimits[binidx - 1] 49 | upper_bin_x = binlimits[binidx] 50 | 51 | x_values_to_plot.append(np.mean([lower_bin_x, upper_bin_x])) 52 | 53 | y_in_bin = [ 54 | y[idx] for idx in range(len(y)) 55 | if x[idx] >= lower_bin_x and x[idx] < upper_bin_x 56 | ] 57 | 58 | fit_y.append(np.nanmedian(y_in_bin)) 59 | err_y.append(np.nanstd(y_in_bin)) 60 | 61 | # Adjust for intercept error 62 | y_shift = float(np.interp(0.0, x_values_to_plot, fit_y)) 63 | 64 | fit_y = [y_value - y_shift for y_value in fit_y] 65 | 66 | return (binlimits, x_values_to_plot, fit_y, err_y) 67 | 68 | 69 | def get_scatter_data_for_code_forecast(system, 70 | instrument_code, 71 | rule_name, 72 | startdate=None, 73 | enddate=None, 74 | return_period=5): 75 | 76 | norm_data = system.rawdata.norm_returns(instrument_code) 77 | forecast = system.rules.get_raw_forecast(instrument_code, rule_name) 78 | 79 | if startdate is None: 80 | startdate = forecast.index[0] 81 | if enddate is None: 82 | enddate = forecast.index[-1] 83 | 84 | (forecast, norm_data) = align_to_joint( 85 | forecast[startdate:enddate], 86 | norm_data[startdate:enddate], 87 | ffill=(True, False)) 88 | 89 | # work out return for the N days after the forecast 90 | period_returns = pd.rolling_sum(norm_data, return_period, min_periods=1) 91 | 92 | ex_post_returns = period_returns.shift(-return_period) 93 | lagged_forecast = forecast.shift(1) 94 | 95 | return (list(ex_post_returns.iloc[:, 0].values), 96 | list(lagged_forecast.iloc[:, 0].values)) 97 | 98 | 99 | def do_a_little_plot(system): 100 | 101 | rule_name = "carry" 102 | return_period = 20 103 | instrument_list = system.get_instrument_list() 104 | 105 | all_scatter = dict(returns=[], forecast=[]) 106 | for instrument_code in instrument_list: 107 | this_instrument_data = get_scatter_data_for_code_forecast( 108 | system, instrument_code, rule_name, return_period) 109 | all_scatter[ 110 | 'returns'] = all_scatter['returns'] + this_instrument_data[0] 111 | all_scatter[ 112 | 'forecast'] = all_scatter['forecast'] + this_instrument_data[1] 113 | 114 | (returns, forecast) = clean_data(all_scatter['returns'], 115 | all_scatter['forecast']) 116 | 117 | (binlimits, x_values_to_plot, fit_y, err_y) = bin_fit(forecast, returns) 118 | 119 | scatter( 120 | all_scatter['forecast'], 121 | all_scatter['returns'], 122 | alpha=0.05, 123 | color="black") 124 | ax = gca() 125 | ax.errorbar(x_values_to_plot, fit_y, yerr=err_y) 126 | show() 127 | 128 | 129 | def fit_a_filter_datewise(system, 130 | rule_name, 131 | instrument_code=None, 132 | return_period=5, 133 | date_method="expanding", 134 | rollyears=999, 135 | buckets=3): 136 | """ 137 | 138 | if instrument_code is None, fits across all instruments 139 | """ 140 | 141 | if instrument_code is None: 142 | instrument_list = system.get_instrument_list() 143 | else: 144 | instrument_list = [instrument_code] 145 | 146 | fit_dates = generate_fitting_dates([ 147 | system.rules.get_raw_forecast(instrument_code, rule_name) 148 | for instrument_code in instrument_list 149 | ], date_method, rollyears) 150 | 151 | filter_data = [] 152 | for fit_period in fit_dates: 153 | system.log.msg("Estimating fitting from %s to %s" % 154 | (fit_period.period_start, fit_period.period_end)) 155 | 156 | if fit_period.no_data: 157 | data = [None, None] 158 | else: 159 | 160 | data = fit_a_filter(system, rule_name, instrument_list, 161 | fit_period.fit_start, fit_period.fit_end, 162 | return_period, buckets) 163 | 164 | filter_data.append(data) 165 | 166 | return (fit_dates, filter_data) 167 | 168 | 169 | def fit_a_filter(system, 170 | rule_name, 171 | instrument_list, 172 | start_date=None, 173 | end_date=None, 174 | return_period=5, 175 | buckets=3): 176 | 177 | all_scatter = dict(returns=[], forecast=[]) 178 | for instrument_code in instrument_list: 179 | this_instrument_data = get_scatter_data_for_code_forecast( 180 | system, instrument_code, rule_name, start_date, end_date, 181 | return_period) 182 | all_scatter[ 183 | 'returns'] = all_scatter['returns'] + this_instrument_data[0] 184 | all_scatter[ 185 | 'forecast'] = all_scatter['forecast'] + this_instrument_data[1] 186 | 187 | (returns, forecast) = clean_data(all_scatter['returns'], 188 | all_scatter['forecast']) 189 | 190 | (binlimits, x_values_to_plot, fit_y, err_y) = bin_fit( 191 | forecast, returns, buckets) 192 | 193 | return (x_values_to_plot, fit_y) 194 | 195 | 196 | def filtering_function(raw_forecast, 197 | x_bins=None, 198 | fit_y=None, 199 | startdate=None, 200 | enddate=None): 201 | """ 202 | This is an additional stage that sits between raw forecast and forecast scaling 203 | 204 | x_bins: defines upper and lower limits of ranges 205 | y_points: defines y values 206 | 207 | We then interpolate to get the appropriate forecast value 208 | 209 | Note that this will screw up any forecast scaling - this needs to be re-done 210 | 211 | """ 212 | if x_bins is None: 213 | x_bins = [-100.0, 100.0] 214 | 215 | if fit_y is None: 216 | fit_y = [-100.0, 100.0] 217 | 218 | if startdate is None: 219 | startdate = raw_forecast.index[0] 220 | if enddate is None: 221 | enddate = raw_forecast.index[-1] 222 | 223 | sub_forecast = raw_forecast[startdate:enddate] 224 | 225 | new_values = np.interp(sub_forecast.iloc[:, 0].values, x_bins, fit_y) 226 | 227 | return pd.DataFrame(new_values, index=sub_forecast.index) 228 | 229 | 230 | from systems.forecast_scale_cap import ForecastScaleCapEstimated, ALL_KEYNAME, str2Bool 231 | 232 | from copy import copy 233 | 234 | 235 | class newfsc(ForecastScaleCapEstimated): 236 | def get_raw_forecast(self, instrument_code, rule_name): 237 | """ 238 | override method to filter 239 | """ 240 | return self.get_filtered_forecast(instrument_code, rule_name) 241 | 242 | def get_filtered_forecast(self, instrument_code, rule_variation_name): 243 | """ 244 | Filter the forecast 245 | """ 246 | 247 | def _get_filtered_forecast(system, instrument_code, 248 | rule_variation_name, this_stage): 249 | 250 | raw_forecast = this_stage.get_actual_raw_forecast( 251 | instrument_code, rule_variation_name) 252 | 253 | (fit_dates, filter_data) = this_stage.get_fitted_values( 254 | instrument_code, rule_variation_name) 255 | 256 | filtered_list = [] 257 | for (fit_period, data_this_period) in zip(fit_dates, filter_data): 258 | (x_bins, fit_y) = data_this_period 259 | 260 | filtered_list.append( 261 | filtering_function( 262 | raw_forecast, 263 | startdate=fit_period.period_start, 264 | enddate=fit_period.period_end, 265 | x_bins=x_bins, 266 | fit_y=fit_y)) 267 | 268 | filtered_forecast = pd.concat(filtered_list, axis=0) 269 | 270 | return uniquets(filtered_forecast) 271 | 272 | filtered_forecast = self.parent.calc_or_cache_nested( 273 | "get_filtered_forecast", instrument_code, rule_variation_name, 274 | _get_filtered_forecast, self) 275 | 276 | return filtered_forecast 277 | 278 | def get_fitted_values(self, instrument_code, rule_variation_name): 279 | def _get_fitted_values(system, instrument_code, rule_variation_name, 280 | this_stage, **kwargs): 281 | this_stage.log.terse("Fitting mapping for %s %s " % 282 | (instrument_code, rule_variation_name)) 283 | if instrument_code == ALL_KEYNAME: 284 | instrument_code = None 285 | 286 | (fit_dates, filter_data) = fit_a_filter_datewise( 287 | system, rule_variation_name, instrument_code, **kwargs) 288 | 289 | return (fit_dates, filter_data) 290 | 291 | instrument_fit_config = copy(system.config.instrument_fit) 292 | pool_instruments = str2Bool( 293 | instrument_fit_config.pop("pool_instruments")) 294 | 295 | if pool_instruments: 296 | # pooled, same for all instruments 297 | instrument_code_key = ALL_KEYNAME 298 | 299 | else: 300 | ## not pooled 301 | instrument_code_key = instrument_code 302 | 303 | fitted_values = self.parent.calc_or_cache_nested( 304 | "get_fitted_values", instrument_code_key, rule_variation_name, 305 | _get_fitted_values, self, **instrument_fit_config) 306 | 307 | return fitted_values 308 | 309 | def get_actual_raw_forecast(self, instrument_code, rule_variation_name): 310 | """ 311 | Old method for raw forecast, keep so we can pipe this in 312 | """ 313 | raw_forecast = self.parent.rules.get_raw_forecast( 314 | instrument_code, rule_variation_name) 315 | 316 | return raw_forecast 317 | 318 | 319 | rulename = "ewmac64_256" 320 | return_period = 30 321 | 322 | from systems.portfolio import Portfolios 323 | config = Config("systems.provided.futures_chapter15.futuresconfig.yaml") 324 | 325 | config.use_forecast_scale_estimates = True 326 | config.instrument_fit = dict( 327 | pool_instruments=True, 328 | return_period=return_period, 329 | buckets=3, 330 | date_method="in_sample") 331 | config.forecast_weights = dict([(rule, 1.0) for rule in [rulename]]) 332 | # so we use all the markets we have, equal weighted 333 | del (config.instrument_weights) 334 | config.notional_trading_capital = 10000000 335 | config.forecast_cap = 40.0 336 | 337 | system = System([ 338 | Account(), Portfolios(), PositionSizing(), FuturesRawData(), 339 | ForecastCombine(), newfsc(), Rules() 340 | ], csvFuturesData(), config) 341 | 342 | system.set_logging_level("on") 343 | 344 | a1 = system.accounts.portfolio() 345 | 346 | from systems.portfolio import Portfolios 347 | config = Config("systems.provided.futures_chapter15.futuresconfig.yaml") 348 | 349 | config.forecast_weights = dict([(rule, 1.0) for rule in [rulename]]) 350 | # so we use all the markets we have, equal weighted 351 | del (config.instrument_weights) 352 | config.notional_trading_capital = 10000000 353 | system.config.forecast_weights = dict([(rule, 1.0) for rule in rulename]) 354 | config.use_forecast_scale_estimates = True 355 | 356 | system.config.notional_trading_capital = 10000000 357 | 358 | system = System([ 359 | Account(), Portfolios(), PositionSizing(), FuturesRawData(), 360 | ForecastCombine(), ForecastScaleCap(), Rules() 361 | ], csvFuturesData(), config) 362 | system.set_logging_level("on") 363 | 364 | a2 = system.accounts.portfolio() 365 | 366 | from syscore.accounting import account_test 367 | 368 | print("Filtered:") 369 | print(a1.stats()) 370 | print("") 371 | print("No filter") 372 | print(a2.stats()) 373 | 374 | print("Test") 375 | print(account_test(a1, a2)) 376 | 377 | a3 = pd.concat([a1.curve(), a2.curve()], axis=1) 378 | a3.columns = ["filter", "nofilter"] 379 | a3.plot() 380 | show() 381 | -------------------------------------------------------------------------------- /optimisation/optimisationwithcosts.py: -------------------------------------------------------------------------------- 1 | from matplotlib.pyplot import show, title 2 | 3 | from systems.provided.futures_chapter15.estimatedsystem import futures_system 4 | 5 | rule_variations = [ 6 | 'carry', 'ewmac2_8', 'ewmac4_16', 'ewmac8_32', 'ewmac16_64', 'ewmac32_128', 7 | 'ewmac64_256' 8 | ] 9 | """ 10 | ## pool everything, no costs 11 | system=futures_system() 12 | system.set_logging_level("on") 13 | del(system.config.rule_variations) 14 | 15 | system=futures_system() 16 | system.set_logging_level("on") 17 | 18 | system.config.rule_variations=rule_variations 19 | system.config.forecast_weight_estimate['apply_cost_weight']=False 20 | system.config.forecast_cost_estimates['use_pooled_costs']=True 21 | system.config.forecast_weight_estimate['pool_gross_returns']=True 22 | system.config.forecast_weight_estimate['cost_multiplier']=0.0 23 | system.config.forecast_weight_estimate['ceiling_cost_SR']=999.0 24 | system.config.forecast_weight_estimate['method']="bootstrap" 25 | system.config.forecast_weight_estimate['equalise_gross']=False 26 | 27 | 28 | print(system.combForecast.get_forecast_weights("EUROSTX").tail(1)) ## cheap market 29 | system.combForecast.get_forecast_weights("EUROSTX").iloc[-1,:].loc[rule_variations].plot(kind="barh") 30 | show() 31 | 32 | print(system.combForecast.get_forecast_weights("V2X").tail(1)) ## expensive market 33 | 34 | 35 | ## pool everything with costs 36 | system=futures_system() 37 | system.set_logging_level("on") 38 | del(system.config.rule_variations) 39 | 40 | system=futures_system() 41 | system.set_logging_level("on") 42 | 43 | system.config.rule_variations=rule_variations 44 | system.config.forecast_weight_estimate['apply_cost_weight']=False 45 | system.config.forecast_cost_estimates['use_pooled_costs']=True 46 | system.config.forecast_weight_estimate['pool_gross_returns']=True 47 | system.config.forecast_weight_estimate['cost_multiplier']=1.0 48 | system.config.forecast_weight_estimate['ceiling_cost_SR']=999.0 49 | system.config.forecast_weight_estimate['method']="bootstrap" 50 | system.config.forecast_weight_estimate['equalise_gross']=False 51 | 52 | 53 | print(system.combForecast.get_forecast_weights("EUROSTX").tail(1)) ## cheap market 54 | system.combForecast.get_forecast_weights("EUROSTX").iloc[-1,:].loc[rule_variations].plot(kind="barh") 55 | show() 56 | 57 | 58 | print(system.combForecast.get_forecast_weights("V2X").tail(1)) ## expensive market 59 | system.combForecast.get_forecast_weights("V2X").iloc[-1,:].loc[rule_variations].plot(kind="barh") 60 | show() 61 | 62 | ## individual market estimation 63 | system=futures_system() 64 | system.set_logging_level("on") 65 | del(system.config.rule_variations) 66 | 67 | system=futures_system() 68 | system.set_logging_level("on") 69 | 70 | system.config.rule_variations=rule_variations 71 | 72 | system.config.forecast_weight_estimate['apply_cost_weight']=False 73 | system.config.forecast_cost_estimates['use_pooled_costs']=False 74 | system.config.forecast_weight_estimate['pool_gross_returns']=False 75 | system.config.forecast_weight_estimate['cost_multiplier']=1.0 76 | system.config.forecast_weight_estimate['ceiling_cost_SR']=999.0 77 | system.config.forecast_weight_estimate['method']="bootstrap" 78 | system.config.forecast_weight_estimate['equalise_gross']=False 79 | 80 | 81 | print(system.combForecast.get_forecast_weights("EUROSTX").tail(1)) ## cheap market 82 | system.combForecast.get_forecast_weights("EUROSTX").iloc[-1,:].loc[rule_variations].plot(kind="barh") 83 | show() 84 | 85 | print(system.combForecast.get_forecast_weights("V2X").tail(1)) ## expensive market 86 | system.combForecast.get_forecast_weights("V2X").iloc[-1,:].loc[rule_variations].plot(kind="barh") 87 | show() 88 | 89 | ## dont pool costs 90 | system=futures_system() 91 | system.set_logging_level("on") 92 | del(system.config.rule_variations) 93 | 94 | system=futures_system() 95 | system.set_logging_level("on") 96 | 97 | system.config.rule_variations=rule_variations 98 | system.config.forecast_weight_estimate['apply_cost_weight']=False 99 | system.config.forecast_cost_estimates['use_pooled_costs']=False 100 | system.config.forecast_weight_estimate['pool_gross_returns']=True 101 | system.config.forecast_weight_estimate['cost_multiplier']=1.0 102 | system.config.forecast_weight_estimate['ceiling_cost_SR']=999.0 103 | system.config.forecast_weight_estimate['method']="bootstrap" 104 | system.config.forecast_weight_estimate['equalise_gross']=False 105 | 106 | 107 | print(system.combForecast.get_forecast_weights("EUROSTX").tail(1)) ## cheap market 108 | system.combForecast.get_forecast_weights("EUROSTX").iloc[-1,:].loc[rule_variations].plot(kind="barh") 109 | show() 110 | 111 | print(system.combForecast.get_forecast_weights("V2X").tail(1)) ## expensive market 112 | system.combForecast.get_forecast_weights("V2X").iloc[-1,:].loc[rule_variations].plot(kind="barh") 113 | show() 114 | 115 | 116 | ## equalise gross, only use costs 117 | system=futures_system() 118 | system.set_logging_level("on") 119 | del(system.config.rule_variations) 120 | 121 | system=futures_system() 122 | system.set_logging_level("on") 123 | 124 | system.config.rule_variations=rule_variations 125 | system.config.forecast_weight_estimate['apply_cost_weight']=False 126 | system.config.forecast_cost_estimates['use_pooled_costs']=False 127 | system.config.forecast_weight_estimate['pool_gross_returns']=True 128 | system.config.forecast_weight_estimate['cost_multiplier']=1.0 129 | system.config.forecast_weight_estimate['ceiling_cost_SR']=999.0 130 | system.config.forecast_weight_estimate['method']="bootstrap" 131 | system.config.forecast_weight_estimate['equalise_gross']=True 132 | 133 | 134 | print(system.combForecast.get_forecast_weights("EUROSTX").tail(1)) ## cheap market 135 | system.combForecast.get_forecast_weights("EUROSTX").iloc[-1,:].loc[rule_variations].plot(kind="barh") 136 | show() 137 | 138 | print(system.combForecast.get_forecast_weights("V2X").tail(1)) ## expensive market 139 | system.combForecast.get_forecast_weights("V2X").iloc[-1,:].loc[rule_variations].plot(kind="barh") 140 | show() 141 | 142 | 143 | 144 | ## subtract costs from gross return multiply by a factor 145 | system=futures_system() 146 | system.set_logging_level("on") 147 | del(system.config.rule_variations) 148 | 149 | system=futures_system() 150 | system.set_logging_level("on") 151 | 152 | system.config.rule_variations=rule_variations 153 | system.config.forecast_weight_estimate['apply_cost_weight']=False 154 | system.config.forecast_cost_estimates['use_pooled_costs']=False 155 | system.config.forecast_weight_estimate['pool_gross_returns']=True 156 | system.config.forecast_weight_estimate['cost_multiplier']=3.0 157 | system.config.forecast_weight_estimate['ceiling_cost_SR']=999.0 158 | system.config.forecast_weight_estimate['method']="bootstrap" 159 | system.config.forecast_weight_estimate['equalise_gross']=False 160 | 161 | 162 | print(system.combForecast.get_forecast_weights("EUROSTX").tail(1)) ## cheap market 163 | system.combForecast.get_forecast_weights("EUROSTX").iloc[-1,:].loc[rule_variations].plot(kind="barh") 164 | show() 165 | 166 | print(system.combForecast.get_forecast_weights("V2X").tail(1)) ## expensive market 167 | system.combForecast.get_forecast_weights("V2X").iloc[-1,:].loc[rule_variations].plot(kind="barh") 168 | show() 169 | 170 | ## cost weighting 171 | system=futures_system() 172 | system.set_logging_level("on") 173 | del(system.config.rule_variations) 174 | 175 | system=futures_system() 176 | system.set_logging_level("on") 177 | 178 | system.config.rule_variations=rule_variations 179 | system.config.forecast_weight_estimate['apply_cost_weight']=True 180 | system.config.forecast_cost_estimates['use_pooled_costs']=False 181 | system.config.forecast_weight_estimate['pool_gross_returns']=True 182 | system.config.forecast_weight_estimate['cost_multiplier']=0.0 183 | system.config.forecast_weight_estimate['ceiling_cost_SR']=999.0 184 | system.config.forecast_weight_estimate['method']="bootstrap" 185 | system.config.forecast_weight_estimate['equalise_gross']=False 186 | 187 | 188 | print(system.combForecast.get_forecast_weights("EUROSTX").tail(1)) ## cheap market 189 | system.combForecast.get_forecast_weights("EUROSTX").iloc[-1,:].loc[rule_variations].plot(kind="barh") 190 | show() 191 | 192 | print(system.combForecast.get_forecast_weights("V2X").tail(1)) ## expensive market 193 | system.combForecast.get_forecast_weights("V2X").iloc[-1,:].loc[rule_variations].plot(kind="barh") 194 | show() 195 | 196 | ## cost threshold 197 | system=futures_system() 198 | system.set_logging_level("on") 199 | del(system.config.rule_variations) 200 | 201 | system=futures_system() 202 | system.set_logging_level("on") 203 | 204 | system.config.rule_variations=rule_variations 205 | system.config.forecast_weight_estimate['apply_cost_weight']=False 206 | system.config.forecast_cost_estimates['use_pooled_costs']=False 207 | system.config.forecast_weight_estimate['pool_gross_returns']=True 208 | system.config.forecast_weight_estimate['cost_multiplier']=1.0 209 | system.config.forecast_weight_estimate['ceiling_cost_SR']=.13 210 | system.config.forecast_weight_estimate['method']="bootstrap" 211 | system.config.forecast_weight_estimate['equalise_gross']=False 212 | 213 | 214 | print(system.combForecast.get_forecast_weights("EUROSTX").tail(1)) ## cheap market 215 | system.combForecast.get_forecast_weights("EUROSTX").iloc[-1,:].loc[rule_variations].plot(kind="barh") 216 | show() 217 | 218 | print(system.combForecast.get_forecast_weights("V2X").tail(1)) ## expensive market 219 | system.combForecast.get_forecast_weights("V2X").iloc[-1,:].loc[rule_variations].plot(kind="barh") 220 | show() 221 | 222 | 223 | ## favourite 224 | system=futures_system() 225 | system.set_logging_level("on") 226 | del(system.config.rule_variations) 227 | 228 | system=futures_system() 229 | system.set_logging_level("on") 230 | 231 | system.config.rule_variations=rule_variations 232 | system.config.forecast_weight_estimate['apply_cost_weight']=True 233 | system.config.forecast_cost_estimates['use_pooled_costs']=False 234 | system.config.forecast_weight_estimate['pool_gross_returns']=True 235 | system.config.forecast_weight_estimate['cost_multiplier']=0.0 236 | system.config.forecast_weight_estimate['ceiling_cost_SR']=0.13 237 | system.config.forecast_weight_estimate['method']="bootstrap" 238 | system.config.forecast_weight_estimate['equalise_gross']=False 239 | 240 | 241 | print(system.combForecast.get_forecast_weights("EUROSTX").tail(1)) ## cheap market 242 | system.combForecast.get_forecast_weights("EUROSTX").iloc[-1,:].loc[rule_variations].plot(kind="barh") 243 | show() 244 | 245 | print(system.combForecast.get_forecast_weights("V2X").tail(1)) ## expensive market 246 | system.combForecast.get_forecast_weights("V2X").iloc[-1,:].loc[rule_variations].plot(kind="barh") 247 | show() 248 | 249 | 250 | ## shrinkage 251 | system=futures_system() 252 | system.set_logging_level("on") 253 | del(system.config.rule_variations) 254 | 255 | system=futures_system() 256 | system.set_logging_level("on") 257 | 258 | system.config.rule_variations=rule_variations 259 | system.config.forecast_weight_estimate['method']="shrinkage" 260 | system.config.forecast_weight_estimate['equalise_gross']=False 261 | 262 | 263 | print(system.combForecast.get_forecast_weights("EUROSTX").tail(1)) ## cheap market 264 | system.combForecast.get_forecast_weights("EUROSTX").iloc[-1,:].loc[rule_variations].plot(kind="barh") 265 | show() 266 | 267 | print(system.combForecast.get_forecast_weights("V2X").tail(1)) ## expensive market 268 | system.combForecast.get_forecast_weights("V2X").iloc[-1,:].loc[rule_variations].plot(kind="barh") 269 | show() 270 | 271 | ## equal weights 272 | system=futures_system() 273 | system.set_logging_level("on") 274 | del(system.config.rule_variations) 275 | 276 | system=futures_system() 277 | system.set_logging_level("on") 278 | 279 | system.config.rule_variations=rule_variations 280 | system.config.forecast_weight_estimate['method']="equal_weights" 281 | system.config.forecast_weight_estimate['apply_cost_weight']=False 282 | 283 | print(system.combForecast.get_forecast_weights("EUROSTX").tail(1)) ## cheap market 284 | system.combForecast.get_forecast_weights("EUROSTX").iloc[-1,:].loc[rule_variations].plot(kind="barh") 285 | show() 286 | 287 | print(system.combForecast.get_forecast_weights("V2X").tail(1)) ## expensive market 288 | system.combForecast.get_forecast_weights("V2X").iloc[-1,:].loc[rule_variations].plot(kind="barh") 289 | show() 290 | 291 | 292 | 293 | ## instruments 294 | system=futures_system() 295 | system.set_logging_level("on") 296 | del(system.config.rule_variations) 297 | 298 | system=futures_system() 299 | system.set_logging_level("on") 300 | 301 | system.config.rule_variations=rule_variations 302 | system.config.forecast_weight_estimate['method']="shrinkage" 303 | 304 | system.config.instrument_weight_estimate['method']="bootstrap" 305 | system.config.instrument_weight_estimate['apply_cost_weight']=True 306 | system.config.instrument_weight_estimate['cost_multiplier']=0.0 307 | system.config.instrument_weight_estimate['ceiling_cost_SR']=0.13 308 | system.config.instrument_weight_estimate['equalise_gross']=False 309 | 310 | 311 | print(system.portfolio.get_instrument_weights()) 312 | system.portfolio.get_instrument_weights().iloc[-1,:].plot(kind="barh") 313 | show() 314 | """ 315 | 316 | # instruments - equal weights 317 | system = futures_system() 318 | system.set_logging_level("on") 319 | del (system.config.rule_variations) 320 | 321 | system = futures_system() 322 | system.set_logging_level("on") 323 | 324 | system.config.rule_variations = rule_variations 325 | system.config.forecast_weight_estimate['method'] = "equal_weights" 326 | 327 | system.config.instrument_weight_estimate['method'] = "equal_weights" 328 | system.config.instrument_weight_estimate['apply_cost_weight'] = True 329 | system.config.instrument_weight_estimate['cost_multiplier'] = 0.0 330 | system.config.instrument_weight_estimate['ceiling_cost_SR'] = 0.13 331 | system.config.instrument_weight_estimate['equalise_gross'] = False 332 | 333 | print(system.portfolio.get_instrument_weights()) 334 | system.portfolio.get_instrument_weights().iloc[-1, :].plot(kind="barh") 335 | show() 336 | -------------------------------------------------------------------------------- /optimisation/uncertainty.py: -------------------------------------------------------------------------------- 1 | from syscore.genutils import progressBar 2 | from sysdata.csvdata import csvFuturesData 3 | import pandas as pd 4 | import random 5 | from syscore.optimisation import optimise, sigma_from_corr_and_std 6 | import numpy as np 7 | from matplotlib.pyplot import show, legend 8 | from syscore.correlations import boring_corr_matrix, get_avg_corr 9 | 10 | # get some returns to play with 11 | # 3 asset problem 12 | 13 | data=csvFuturesData() 14 | 15 | def calc_weekly_return(instrument_code, start_date=pd.datetime(1998,1,1)): 16 | price = data[instrument_code] 17 | price=price[start_date:] 18 | weekly_price = price.resample("W").last() 19 | denom_price = data.get_instrument_raw_carry_data(instrument_code).PRICE 20 | denom_weekly_price = denom_price.reindex(weekly_price.index, method="ffill") 21 | 22 | weekly_returns = (weekly_price - weekly_price.shift(1))/denom_weekly_price 23 | 24 | return weekly_returns[1:] 25 | 26 | code_list = ["SP500", "US10", "US5"] 27 | corr_pairs = ["SP500/US5", "SP500/US10", "US5/US10"] 28 | 29 | returns = dict([(instrument_code, calc_weekly_return(instrument_code)) for instrument_code in code_list]) 30 | returns = pd.DataFrame(returns) 31 | 32 | 33 | def some_random_bootstrapped_returns(returns, horizon=None): 34 | if horizon is None: 35 | horizon = len(returns.index) 36 | 37 | draws = [int(random.uniform(0, len(returns))) for notused in range(horizon)] 38 | 39 | return returns.iloc[draws,] 40 | 41 | def statistic_from_bootstrap(returns, stat_function, horizon=None): 42 | 43 | subset_returns = some_random_bootstrapped_returns(returns, horizon=horizon) 44 | 45 | return list(stat_function(subset_returns)) 46 | 47 | def distribution_of_statistic(returns, stat_function, monte_length=1000, horizon=None, colnames=code_list): 48 | list_of_bs_stats = [] 49 | thing=progressBar(monte_length) 50 | for notUsed in range(monte_length): 51 | list_of_bs_stats.append(statistic_from_bootstrap(returns, stat_function, horizon=horizon)) 52 | thing.iterate() 53 | 54 | ans=pd.DataFrame(np.array(list_of_bs_stats)) 55 | ans.columns = colnames 56 | 57 | return ans 58 | 59 | def sharpe_ratio(weekly_returns): 60 | return annualised_mean(weekly_returns) / annualised_std(weekly_returns) 61 | 62 | def annualised_mean(weekly_returns): 63 | return (weekly_returns.mean()*52.0) 64 | 65 | def annualised_std(weekly_returns): 66 | return (weekly_returns.std()*(52**.5)) 67 | 68 | def corr_as_vector(weekly_returns): 69 | corr = weekly_returns.corr() 70 | return [corr[code_list[0]][code_list[2]], corr[code_list[0]][code_list[1]], corr[code_list[1]][code_list[2]]] 71 | 72 | def optimisation(weekly_returns, equalisecorr=False, equaliseSR=False, riskweights=False): 73 | if equalisecorr: 74 | corrmatrix = np.diag([1.0]*len(returns.columns)) 75 | else: 76 | corrmatrix = weekly_returns.corr().values 77 | mean_list = list(annualised_mean(weekly_returns).values) 78 | stdev_list = list(annualised_std(weekly_returns).values) 79 | 80 | if equaliseSR: 81 | sr_list = [each_return / each_std for each_return, each_std in zip(mean_list, stdev_list)] 82 | 83 | avg_sr = np.nanmean(sr_list) 84 | mean_list = [each_std * avg_sr for each_std in stdev_list] 85 | 86 | if riskweights: 87 | avg_std = np.nanmean(stdev_list) 88 | mean_list = [this_mean * avg_std / this_std for this_mean, this_std in zip(mean_list, stdev_list)] 89 | stdev_list = [avg_std]*len(stdev_list) 90 | 91 | return optimisation_with_data(corrmatrix, mean_list, stdev_list) 92 | 93 | def optimisation_with_data(corrmatrix, mean_list, stdev_list): 94 | sigma = sigma_from_corr_and_std(stdev_list, corrmatrix) 95 | 96 | weights = optimise(sigma, mean_list) 97 | 98 | return weights 99 | 100 | 101 | cset=["red","blue","green","black", "yellow"] 102 | 103 | annualised_mean(returns) 104 | annualised_std(returns) 105 | sharpe_ratio(returns) 106 | corr_as_vector(returns) 107 | optimisation(returns) 108 | 109 | """ 110 | Equalisation 111 | """ 112 | 113 | optimisation(returns, equalisecorr=False, equaliseSR=True) 114 | optimisation(returns, equalisecorr=True, equaliseSR=False) 115 | optimisation(returns, riskweights=True) 116 | optimisation(returns, equalisecorr=False, equaliseSR=True, riskweights=True) 117 | optimisation(returns, equalisecorr=True, equaliseSR=False, riskweights=True) 118 | optimisation(returns, equalisecorr=True, equaliseSR=True, riskweights=True) 119 | 120 | ans_mean=distribution_of_statistic(returns, annualised_mean, colnames=code_list, monte_length=10000) 121 | print(ans_mean.quantile(0.1)) 122 | print(ans_mean.quantile(0.9)) 123 | for color, cname in zip(cset,code_list): 124 | ans_mean[cname].hist(bins=50, color=color, histtype='step', alpha=.5) 125 | legend(code_list) 126 | show() 127 | 128 | ans_std=distribution_of_statistic(returns, annualised_std, colnames=code_list,monte_length=10000) 129 | print(ans_std.quantile(0.1)) 130 | print(ans_std.quantile(0.9)) 131 | for color, cname in zip(cset,code_list): 132 | ans_std[cname].hist(bins=50, color=color, histtype='step', alpha=.5) 133 | legend(code_list) 134 | show() 135 | 136 | ans_sr=distribution_of_statistic(returns, sharpe_ratio, colnames=code_list, monte_length=10000) 137 | print(ans_sr.quantile(0.1)) 138 | print(ans_sr.quantile(0.9)) 139 | for color, cname in zip(cset,code_list): 140 | ans_sr[cname].hist(bins=50, color=color, histtype='step', alpha=.5) 141 | 142 | legend(code_list) 143 | show() 144 | 145 | ans_corr=distribution_of_statistic(returns, corr_as_vector, colnames=corr_pairs, monte_length=10000) 146 | for color, cname in zip(cset,corr_pairs): 147 | ans_corr[cname].hist(bins=50, color=color, histtype="step", alpha=.5) 148 | print(ans_corr.quantile(0.1)) 149 | print(ans_corr.quantile(0.9)) 150 | 151 | legend(corr_pairs) 152 | show() 153 | 154 | # get rolling estimates over time 155 | def get_rolling_estimate(returns, func_name, value_index=0, monte_length=1000): 156 | slice_end=pd.date_range(returns.index[1], returns.index[-1], freq="12M") 157 | thing=progressBar(len(slice_end)) 158 | 159 | lower_points=[] 160 | upper_points=[] 161 | for end_point in slice_end: 162 | subset_returns = returns[:end_point] 163 | subset_distribution = distribution_of_statistic(subset_returns, func_name, monte_length=monte_length) 164 | lower_points.append(subset_distribution.quantile(0.1)[value_index]) 165 | upper_points.append(subset_distribution.quantile(0.9)[value_index]) 166 | thing.iterate() 167 | 168 | output = pd.DataFrame(dict(upper=upper_points, lower=lower_points), index=slice_end) 169 | 170 | return output 171 | 172 | sr_5y= get_rolling_estimate(returns, sharpe_ratio, monte_length=1000, value_index=2) 173 | std_5y=get_rolling_estimate(returns, annualised_std, monte_length=1000, value_index=2) 174 | corr_5y_sp=get_rolling_estimate(returns, corr_as_vector, monte_length=1000, value_index=0) 175 | 176 | # check upper and lower boundaries 177 | corrmatrix = returns.corr().values 178 | mean_list = list(annualised_mean(returns).values) 179 | stdev_list = list(annualised_std(returns).values) 180 | 181 | # tweak these to get extreme results 182 | corrmatrix[0][2] = corrmatrix[2][0] = ans_corr.quantile(0.9)[0] 183 | mean_list[0]=ans_mean.quantile(0.9)[0] 184 | mean_list[2]=ans_mean.quantile(0.1)[2] 185 | 186 | stdev_list[0]=ans_std.quantile(0.9)[0] 187 | 188 | print(optimisation_with_data(corrmatrix, mean_list, stdev_list)) 189 | 190 | # Generate some fake data to illustrate bootstrapping 191 | def threeassetportfolio(yearsdata=30, SRlist=[0.5, 0.5, 0.5], annual_vol=[.15,.15,.15], clist=[.0, .0, .0], 192 | index_start=pd.datetime(2000, 1, 1)): 193 | 194 | plength = yearsdata * 52 195 | (c1, c2, c3) = clist 196 | dindex = pd.date_range(index_start, periods=plength, freq="W") 197 | 198 | daily_vol = [vol_item / 16.0 for vol_item in annual_vol] 199 | means = [SR_item * vol_item / 250.0 for SR_item, vol_item in zip(SRlist, annual_vol)] 200 | stds = np.diagflat(daily_vol) 201 | corr = np.array([[1.0, c1, c2], [c1, 1.0, c3], [c2, c3, 1.0]]) 202 | 203 | covs = np.dot(stds, np.dot(corr, stds)) 204 | plength = len(dindex) 205 | 206 | m = np.random.multivariate_normal(means, covs, plength).T 207 | 208 | portreturns = pd.DataFrame(dict(one=m[0], two=m[1], three=m[2]), dindex) 209 | portreturns = portreturns[['one', 'two', 'three']] 210 | 211 | # adjust targets for mean 212 | avgs = list(portreturns.mean()) 213 | differential = [avg_item - mean_item for avg_item, mean_item in zip(avgs, means)] 214 | differential = pd.DataFrame([differential]*len(portreturns.index), portreturns.index, columns=portreturns.columns) 215 | 216 | portreturns = portreturns - differential 217 | 218 | return portreturns 219 | 220 | fake_data1=threeassetportfolio() 221 | fake_data2=threeassetportfolio(SRlist=[0.5, 0.5, 0.0], annual_vol=[.1,.1,.2], clist=[.9, .0, .0]) 222 | 223 | def rolling_optimisation(weekly_returns, opt_function=optimisation, **kwargs): 224 | slice_ends = pd.date_range(returns.index[1], returns.index[-1], freq="12M") 225 | weights=[] 226 | thing=progressBar(len(slice_ends)) 227 | 228 | for end_point in slice_ends: 229 | subset_data = weekly_returns[:end_point] 230 | period_weights = opt_function(subset_data, **kwargs) 231 | weights.append(period_weights) 232 | thing.iterate() 233 | 234 | weights=pd.DataFrame(weights, index=slice_ends, columns=weekly_returns.columns) 235 | 236 | return weights 237 | 238 | def bootstrapped_optimisation(weekly_returns, monte_carlo=1000, **kwargs): 239 | weights=[] 240 | for notUsed in range(monte_carlo): 241 | sampled_data=some_random_bootstrapped_returns(weekly_returns) 242 | sample_weights = optimisation(sampled_data, **kwargs) 243 | weights.append(sample_weights) 244 | 245 | weights=np.array(weights) 246 | avg_weights = weights.mean(axis=0) 247 | avg_weights = list(avg_weights) 248 | 249 | return avg_weights 250 | 251 | rolling_optimisation(fake_data1).plot() 252 | rolling_optimisation(fake_data2).plot() 253 | 254 | rolling_optimisation(fake_data1, bootstrapped_optimisation).plot() 255 | rolling_optimisation(fake_data2, bootstrapped_optimisation).plot() 256 | 257 | # real data 258 | rolling_optimisation(returns, bootstrapped_optimisation).plot() 259 | 260 | # conditional 261 | 262 | def measure_12_month_momentum(weekly_returns): 263 | return_last_year = 52.0*weekly_returns.rolling(center=False, window=52, min_periods=1).mean() 264 | std_last_year = (52.0**.5)*weekly_returns.rolling(center=False, window=52, min_periods=1).std() 265 | sr_last_year = return_last_year / std_last_year 266 | 267 | return sr_last_year 268 | 269 | def measure_abs_momentum(weekly_returns): 270 | code_list = weekly_returns.columns 271 | abs_mom={} 272 | for code in code_list: 273 | abs_mom[code] = measure_12_month_momentum(weekly_returns[code]) 274 | 275 | abs_mom_all = pd.DataFrame(abs_mom) 276 | 277 | return abs_mom_all 278 | 279 | # we don't use this function, but it could be useful 280 | def measure_relative_momentum(weekly_returns): 281 | 282 | abs_mom_all = measure_abs_momentum(weekly_returns) 283 | abs_mom_avg = abs_mom_all.mean(axis=1) 284 | 285 | rel_mom=dict([(code, abs_mom_all[code]-abs_mom_avg) for code in code_list]) 286 | rel_mom = pd.DataFrame(rel_mom) 287 | 288 | return rel_mom 289 | 290 | 291 | def subset_data_by_momentum_range(weekly_returns, mrange=[-999, -2.0]): 292 | 293 | mom_estimates = measure_abs_momentum(weekly_returns) 294 | mom_estimates = mom_estimates.shift(1) 295 | 296 | subset_data_list=[] 297 | for code in weekly_returns.columns: 298 | mom_estimates_instrument = mom_estimates[code] 299 | subset_data_for_instrument=weekly_returns[code] 300 | subset_data = subset_data_for_instrument[mom_estimates_instrument>mrange[0]] 301 | subset_data = subset_data[mom_estimates_instrument<=mrange[1]] 302 | 303 | subset_data_list.append(subset_data) 304 | 305 | subset_data_as_pd=pd.concat(subset_data_list, axis=0) 306 | 307 | return subset_data 308 | 309 | def distribution_by_momentum_range(weekly_returns, stat_function, mrange=[-999, -2.0]): 310 | subset_returns = pd.DataFrame(subset_data_by_momentum_range(weekly_returns, mrange=mrange)) 311 | distr = distribution_of_statistic(subset_returns, stat_function, colnames=["%2.f:%.2f" % (mrange[0], mrange[1])]) 312 | 313 | return distr 314 | 315 | 316 | 317 | all = distribution_by_momentum_range(returns, sharpe_ratio, mrange=[-999, 999]).iloc[:,0].values 318 | 319 | # points taken off distribution of abs_mom: 10,25,75,90 320 | #sr_dist_very_low = distribution_by_momentum_range(returns, sharpe_ratio, mrange=[-999, -0.75]).iloc[:,0].values 321 | #sr_dist_low = distribution_by_momentum_range(returns, sharpe_ratio, mrange=[-0.75, -0.05]).iloc[:,0].values 322 | #sr_dist_middle = distribution_by_momentum_range(returns, sharpe_ratio, mrange=[-0.05, 1.25]).iloc[:,0].values 323 | #sr_dist_high = distribution_by_momentum_range(returns, sharpe_ratio, mrange=[1.25, 1.85]).iloc[:,0].values 324 | #sr_dist_very_high = distribution_by_momentum_range(returns, sharpe_ratio, mrange=[1.85, 999]).iloc[:,0].values 325 | 326 | sr_dist_low = distribution_by_momentum_range(returns, sharpe_ratio, mrange=[-0.75, -0.05]).iloc[:,0].values 327 | sr_dist_high = distribution_by_momentum_range(returns, sharpe_ratio, mrange=[1.25, 1.85]).iloc[:,0].values 328 | 329 | #sr_conditional = dict(Very_low = sr_dist_very_low, Low=sr_dist_low, Middle=sr_dist_middle, 330 | # High=sr_dist_high, Very_high=sr_dist_very_high) 331 | 332 | sr_conditional = dict(low = sr_dist_low, high=sr_dist_high) 333 | 334 | cond_names = list(sr_conditional.keys()) 335 | 336 | sr_conditional = pd.DataFrame(sr_conditional, index=range(len(sr_conditional[cond_names[0]]))) 337 | 338 | for color, cond_name in zip(cset,cond_names): 339 | ans=sr_conditional[cond_name] 340 | ans.hist(bins=50, color=color, histtype='step', alpha=.5) 341 | 342 | legend(cond_names) 343 | show() 344 | 345 | # used to calculate conditional bootstrap estimates 346 | 347 | def distances(weekly_returns): 348 | mom_estimates = measure_abs_momentum(weekly_returns) 349 | mom_estimates = mom_estimates.shift(1) 350 | all_distances={} 351 | for code in mom_estimates.keys(): 352 | this_code_distance=[] 353 | for date_index_value, value in zip(mom_estimates.index, mom_estimates[code]): 354 | this_value_distance=get_distance_versus_value(value, mom_estimates, date_index_value) 355 | this_code_distance.append(this_value_distance) 356 | all_distances[code]=this_code_distance 357 | 358 | return all_distances 359 | 360 | def get_distance_versus_value(value, mom_estimates, date_index_value): 361 | pass 362 | 363 | def single_value_calc(value, other_value): 364 | return 1 / abs(1+value - other_value) 365 | 366 | ### risk weighting plus bayesian 367 | 368 | def optimisation_bayesian_riskweights(weekly_returns, shrinkcorr=0.0, shrinkSR=0.0, priorcorroffdiag=0.5, 369 | priorSR=0.25): 370 | 371 | prior_corrmatrix = boring_corr_matrix(len(weekly_returns.columns), priorcorroffdiag) 372 | 373 | est_corrmatrix = weekly_returns.corr().values 374 | 375 | shrink_corrmatrix = (prior_corrmatrix*shrinkcorr)+(est_corrmatrix*(1-shrinkcorr)) 376 | 377 | est_mean_list = list(annualised_mean(weekly_returns).values) 378 | est_stdev_list = list(annualised_std(weekly_returns).values) 379 | 380 | est_sr_list = [each_return / each_std for each_return, each_std in zip(est_mean_list, est_stdev_list)] 381 | prior_sr_list = [priorSR] * len(est_sr_list) 382 | 383 | shrink_sr_list = [(prior*shrinkSR)+(est*(1-shrinkSR)) for prior, est in zip(prior_sr_list, est_sr_list)] 384 | 385 | shrunk_mean_list = [each_std * shrunk_sr for each_std, shrunk_sr in zip(est_stdev_list, shrink_sr_list)] 386 | 387 | ## apply risk weights 388 | avg_std = np.nanmean(est_stdev_list) 389 | riskwt_mean_list = [this_mean * avg_std / this_std for this_mean, this_std in zip(shrunk_mean_list, est_stdev_list)] 390 | norm_stdev_list = [avg_std]*len(est_stdev_list) 391 | 392 | return optimisation_with_data(shrink_corrmatrix, riskwt_mean_list, norm_stdev_list) 393 | 394 | 395 | for shrinkcorr in [0.0,.25,.5,.75,1.0]: 396 | for shrinkSR in [0.0,.25,.5,.75,1.0]: 397 | print("\nSR %f corr %f" % (shrinkSR, shrinkcorr)) 398 | print("weights:") 399 | print(optimisation_bayesian_riskweights(returns, shrinkcorr=shrinkcorr, shrinkSR=shrinkSR)) 400 | 401 | ## bootstrapping over time with risk weights 402 | ans=rolling_optimisation(returns, bootstrapped_optimisation, riskweights=True) 403 | ans2=rolling_optimisation(returns, bootstrapped_optimisation, riskweights=False) 404 | ans3=rolling_optimisation(returns, bootstrapped_optimisation, riskweights=True, equaliseSR=True) 405 | 406 | ## bayesian 407 | ans4= rolling_optimisation(returns, optimisation_bayesian_riskweights, shrinkcorr=0.5, shrinkSR=0.95) --------------------------------------------------------------------------------