├── .idea ├── .gitignore ├── misc.xml ├── encodings.xml ├── inspectionProfiles │ └── profiles_settings.xml ├── modules.xml ├── vcs.xml └── option_future_research.iml ├── research ├── daily_models │ ├── P_12_0.2_0.2.jpg │ ├── P_26_0.2_0.2.jpg │ ├── P_5_0.2_0.2.jpg │ ├── rb_5_0.2_0.2.jpg │ ├── rb_12_0.2_0.2.jpg │ ├── rb_26_0.2_0.2.jpg │ ├── test.py │ ├── __init__.py │ ├── back_test_result.csv │ ├── com_analysis.py │ ├── basis_index.py │ └── com_daily.py ├── __init__.py ├── ml_models │ ├── __init__.py │ ├── ts_transformer.py │ ├── xgb_demo.py │ ├── reg_demos.py │ └── select_features.py ├── tick_models │ ├── __init__.py │ ├── sta_reference.py │ └── FactorProcess.py └── data_process │ ├── __init__.py │ └── DataProcess.py ├── __init__.py ├── backtester ├── Event.py ├── __init__.py ├── Account.py ├── Position.py ├── Factor.py ├── backtest.py └── BackTester.py ├── conf ├── __init__.py ├── test.editorconfig └── strategy.ini ├── utils ├── __init__.py ├── utils.py └── define.py ├── strategy ├── __init__.py ├── Signal.py ├── T0Signal.py └── RegSignal.py ├── corr_2021-07-09.csv ├── corr_2021-07-12.csv ├── corr_2021-07-02.csv ├── corr_2021-07-05.csv ├── corr_2021-07-06.csv ├── corr_2021-07-08.csv ├── corr_2021-07-07.csv ├── README.md ├── .gitignore ├── main.py └── corr.csv /.idea/.gitignore: -------------------------------------------------------------------------------- 1 | # Default ignored files 2 | /shelf/ 3 | /workspace.xml 4 | -------------------------------------------------------------------------------- /research/daily_models/P_12_0.2_0.2.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiaoqiyu/option_future_research/HEAD/research/daily_models/P_12_0.2_0.2.jpg -------------------------------------------------------------------------------- /research/daily_models/P_26_0.2_0.2.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiaoqiyu/option_future_research/HEAD/research/daily_models/P_26_0.2_0.2.jpg -------------------------------------------------------------------------------- /research/daily_models/P_5_0.2_0.2.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiaoqiyu/option_future_research/HEAD/research/daily_models/P_5_0.2_0.2.jpg -------------------------------------------------------------------------------- /research/daily_models/rb_5_0.2_0.2.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiaoqiyu/option_future_research/HEAD/research/daily_models/rb_5_0.2_0.2.jpg -------------------------------------------------------------------------------- /research/daily_models/rb_12_0.2_0.2.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiaoqiyu/option_future_research/HEAD/research/daily_models/rb_12_0.2_0.2.jpg -------------------------------------------------------------------------------- /research/daily_models/rb_26_0.2_0.2.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiaoqiyu/option_future_research/HEAD/research/daily_models/rb_26_0.2_0.2.jpg -------------------------------------------------------------------------------- /__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2021/9/3 9:53 4 | # @Author : rpyxqi@gmail.com 5 | # @Site : 6 | # @File : __init__.py.py 7 | -------------------------------------------------------------------------------- /backtester/Event.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2021/10/25 13:11 4 | # @Author : rpyxqi@gmail.com 5 | # @Site : 6 | # @File : Event.py 7 | -------------------------------------------------------------------------------- /conf/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2021/9/28 18:10 4 | # @Author : rpyxqi@gmail.com 5 | # @Site : 6 | # @File : __init__.py.py 7 | -------------------------------------------------------------------------------- /utils/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2021/9/28 18:12 4 | # @Author : rpyxqi@gmail.com 5 | # @Site : 6 | # @File : __init__.py.py 7 | -------------------------------------------------------------------------------- /backtester/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2021/9/28 18:10 4 | # @Author : rpyxqi@gmail.com 5 | # @Site : 6 | # @File : __init__.py.py 7 | -------------------------------------------------------------------------------- /research/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2021/9/3 9:53 4 | # @Author : rpyxqi@gmail.com 5 | # @Site : 6 | # @File : __init__.py.py 7 | -------------------------------------------------------------------------------- /research/daily_models/test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2022/4/26 22:47 4 | # @Author : rpyxqi@gmail.com 5 | # @Site : 6 | # @File : test.py 7 | -------------------------------------------------------------------------------- /strategy/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2021/9/28 18:10 4 | # @Author : rpyxqi@gmail.com 5 | # @Site : 6 | # @File : __init__.py.py 7 | -------------------------------------------------------------------------------- /research/ml_models/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2021/9/28 18:09 4 | # @Author : rpyxqi@gmail.com 5 | # @Site : 6 | # @File : __init__.py.py 7 | -------------------------------------------------------------------------------- /research/tick_models/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2021/9/28 18:09 4 | # @Author : rpyxqi@gmail.com 5 | # @Site : 6 | # @File : __init__.py.py 7 | -------------------------------------------------------------------------------- /research/daily_models/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2021/9/28 18:08 4 | # @Author : rpyxqi@gmail.com 5 | # @Site : 6 | # @File : __init__.py.py 7 | -------------------------------------------------------------------------------- /research/data_process/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2021/9/28 18:13 4 | # @Author : rpyxqi@gmail.com 5 | # @Site : 6 | # @File : __init__.py.py 7 | -------------------------------------------------------------------------------- /.idea/misc.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /corr_2021-07-09.csv: -------------------------------------------------------------------------------- 1 | ,log_return,cos,label_1 2 | log_return,1.0,0.6220641113253699,0.34446323257182804 3 | cos,0.6220641113253699,1.0,0.12010698745745302 4 | label_1,0.34446323257182804,0.12010698745745302,1.0 5 | -------------------------------------------------------------------------------- /corr_2021-07-12.csv: -------------------------------------------------------------------------------- 1 | ,log_return,cos,label_1 2 | log_return,1.0,0.595690342982082,0.33677370179085736 3 | cos,0.595690342982082,1.0,0.11572490496155427 4 | label_1,0.33677370179085736,0.11572490496155427,1.0 5 | -------------------------------------------------------------------------------- /research/ml_models/ts_transformer.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2022/6/13 13:24 4 | # @Author : rpyxqi@gmail.com 5 | # @Site : 6 | # @File : ts_transformer.py 7 | -------------------------------------------------------------------------------- /research/tick_models/sta_reference.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2022/5/27 14:19 4 | # @Author : rpyxqi@gmail.com 5 | # @Site : 6 | # @File : sta_reference.py 7 | -------------------------------------------------------------------------------- /.idea/encodings.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /.idea/inspectionProfiles/profiles_settings.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 6 | -------------------------------------------------------------------------------- /.idea/modules.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /.idea/vcs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /corr_2021-07-02.csv: -------------------------------------------------------------------------------- 1 | ,log_return,log_return_0,wap_log_return,label_1 2 | log_return,1.0,0.4313170766562893,-0.6303306281863491,0.3650985121997103 3 | log_return_0,0.4313170766562893,1.0,-0.5175661326767479,0.08960881877394036 4 | wap_log_return,-0.6303306281863491,-0.5175661326767479,1.0,0.015029392587053821 5 | label_1,0.3650985121997103,0.08960881877394036,0.015029392587053821,1.0 6 | -------------------------------------------------------------------------------- /corr_2021-07-05.csv: -------------------------------------------------------------------------------- 1 | ,log_return,log_return_0,wap_log_return,label_1 2 | log_return,1.0,0.3953102674119525,-0.5778133642341495,0.31712850967467987 3 | log_return_0,0.3953102674119525,1.0,-0.4319660868247206,0.06694354783989923 4 | wap_log_return,-0.5778133642341495,-0.4319660868247206,1.0,0.06141961438248771 5 | label_1,0.31712850967467987,0.06694354783989923,0.06141961438248771,1.0 6 | -------------------------------------------------------------------------------- /corr_2021-07-06.csv: -------------------------------------------------------------------------------- 1 | ,log_return,log_return_0,wap_log_return,label_1 2 | log_return,1.0,0.44155529940037,-0.6398386290268137,0.3648210115950664 3 | log_return_0,0.44155529940037,1.0,-0.5472088304586968,0.06895602055364117 4 | wap_log_return,-0.6398386290268137,-0.5472088304586968,1.0,0.029864960338698857 5 | label_1,0.3648210115950664,0.06895602055364117,0.029864960338698857,1.0 6 | -------------------------------------------------------------------------------- /corr_2021-07-08.csv: -------------------------------------------------------------------------------- 1 | ,log_return,log_return_0,wap_log_return,label_1 2 | log_return,1.0,0.43150704887559765,-0.620525720744645,0.35911705943478667 3 | log_return_0,0.43150704887559765,1.0,-0.4997037944820132,0.0575499468906213 4 | wap_log_return,-0.620525720744645,-0.4997037944820132,1.0,0.03517371116977774 5 | label_1,0.35911705943478667,0.0575499468906213,0.03517371116977774,1.0 6 | -------------------------------------------------------------------------------- /corr_2021-07-07.csv: -------------------------------------------------------------------------------- 1 | ,log_return,log_return_0,wap_log_return,label_1 2 | log_return,1.0,0.40772032791484353,-0.6065216902300024,0.3236860540770087 3 | log_return_0,0.40772032791484353,1.0,-0.45879668178550886,0.07407664102120662 4 | wap_log_return,-0.6065216902300024,-0.45879668178550886,1.0,0.03232056831367304 5 | label_1,0.3236860540770087,0.07407664102120662,0.03232056831367304,1.0 6 | -------------------------------------------------------------------------------- /conf/test.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | volitility = 18 5 | k1 = 0.2 6 | k2 = 0.2 7 | stop_profit = 3 8 | stop_loss = 10 9 | fee = 1.51 10 | open_fee = 1.51 11 | close_t0_fee = 0.0 12 | start_tick = 2 13 | long_lots_limit = 1 14 | short_lots_limit = 1 15 | slope_upper = 1.5 16 | slope_lower = -1.5 17 | start_timestamp = 21:00:30 18 | end_timestamp = 14:59:30 19 | delay_sec = 3 20 | uqer_token = aaa 21 | -------------------------------------------------------------------------------- /.idea/option_future_research.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /conf/strategy.ini: -------------------------------------------------------------------------------- 1 | [strategy] 2 | stop_profit = 2.0 3 | stop_loss = 5.0 4 | close_type = 0 5 | vol_limit = 2 6 | init_cash = 1000000 7 | risk_ratio = 0.3 8 | order_duration = 20 9 | signal_delay = 5 10 | risk_duration = 10 11 | cancel_order_delay = 30 12 | up_limit_price = 5000 13 | down_limit_price = 4900 14 | reg_long_threshold = 0.0008 15 | reg_short_threshold = 0.0008 16 | 17 | [models] 18 | log_return = 0.9368 19 | log_return_0 = 0.0174 20 | wap_log_return = 0.8194 21 | intercept = 000.00000119 22 | a = 3 23 | b = 2 24 | 25 | -------------------------------------------------------------------------------- /research/ml_models/xgb_demo.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2021/10/25 22:14 4 | # @Author : rpyxqi@gmail.com 5 | # @Site : 6 | # @File : xgb_demo.py 7 | 8 | import xgboost as xgb 9 | 10 | # read in data 11 | dtrain = xgb.DMatrix('demo/data/agaricus.txt.train') 12 | dtest = xgb.DMatrix('demo/data/agaricus.txt.test') 13 | # specify parameters via map 14 | param = {'max_depth': 2, 'eta': 1, 'objective': 'binary:logistic'} 15 | num_round = 2 16 | bst = xgb.train(param, dtrain, num_round) 17 | # make prediction 18 | preds = bst.predict(dtest) 19 | -------------------------------------------------------------------------------- /research/ml_models/reg_demos.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2021/10/26 10:41 4 | # @Author : rpyxqi@gmail.com 5 | # @Site : 6 | # @File : reg_demos.py 7 | 8 | from sklearn.datasets import load_boston 9 | from sklearn.model_selection import cross_val_score 10 | from sklearn.ensemble import RandomForestRegressor 11 | 12 | boston = load_boston() 13 | re_model = RandomForestRegressor(n_estimators=100, n_jobs=1, random_state=0) 14 | ret = cross_val_score(re_model, boston.data, boston.target, cv=10, scoring='neg_mean_squared_error') 15 | print(ret) 16 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # option_future_research 2 | This repo will focus on hf factor research, clf/reg model research, and backtester for T0 strategy. 3 | Structures: 4 | - conf: conf file 5 | - global: global define across the projects 6 | - cache: some historical tick mkt data for testing purpose 7 | - research: factor and model research, mainly focus on tick model for short-terms prediction,e.g. 5s,10s 8 | - results: model evaluation results and backtesting results for strategy 9 | - strategy: strategies are kept here. To create a new strategy, you need to define a new signal class which inherit 10 | from base class Signal -------------------------------------------------------------------------------- /research/daily_models/back_test_result.csv: -------------------------------------------------------------------------------- 1 | acc_return,std,sharp_ratio,max_drawdown,max_risk_ratio,long_sig,short_sig,long_holding,short_holding,open_trans,close_trans,bc_return,bc_sharp_ratio,product_id,start_date,end_date,strategy_name 2 | -0.23787499999999995,0.16767478666046562,-1.418668869289733,0.4603076049741889,0.3767597220529075,8,10,18,35,13,13,0.656737998843262,3.3987740584734762,P,20210403,20220419,P_5_0.2_0.2 3 | 0.11937500000000001,0.0513078005944206,2.326644264945968,0.17483882285864655,0.30732824427480915,3,2,11,6,5,4,0.5417115177610334,3.0321702441026046,P,20210403,20220419,P_12_0.2_0.2 4 | 0.06637500000000007,0.030318123265707042,2.189284587911057,0.03615,0.2401431758053639,1,1,8,4,2,2,0.40451090953665103,2.477749378945712,P,20210403,20220419,P_26_0.2_0.2 5 | 0.017077899999999646,0.0922639596050876,0.18509827751916622,0.21842811871510265,0.32375218159944424,4,4,25,33,6,5,-0.10954253037884198,-1.4519837164434737,rb,20210403,20220419,rb_5_0.2_0.2 6 | -0.17420240000000053,0.08520115789967338,-2.044601321089186,0.17959126154724436,0.23843601136932108,1,0,11,0,1,1,-0.10954253037884198,-1.4519837164434737,rb,20210403,20220419,rb_12_0.2_0.2 7 | 0.0,0.0,,0.0,0.0,0,0,0,0,0,0,-0.10954253037884198,-1.4519837164434737,rb,20210403,20220419,rb_26_0.2_0.2 8 | -------------------------------------------------------------------------------- /strategy/Signal.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2021/7/26 16:28 4 | # @Author : rpyxqi@gmail.com 5 | # @Site : 6 | # @File : Signal.py 7 | 8 | import abc 9 | import utils.define as define 10 | 11 | 12 | class SignalField(object): 13 | def __init__(self): 14 | self._signal_type = define.NO_SIGNAL 15 | self._vol = 0 16 | self._price = 0 17 | self._direction = define.LONG 18 | 19 | @property 20 | def signal_type(self): 21 | return self._signal_type 22 | 23 | @signal_type.setter 24 | def signal_type(self, val): 25 | self._signal_type = val 26 | 27 | @property 28 | def vol(self): 29 | return self._vol 30 | 31 | @vol.setter 32 | def vol(self, val): 33 | self._vol = val 34 | 35 | @property 36 | def price(self): 37 | return self._price 38 | 39 | @price.setter 40 | def price(self, val): 41 | self._price = val 42 | 43 | @property 44 | def direction(self): 45 | return self._direction 46 | 47 | @direction.setter 48 | def direction(self, val): 49 | self._direction = val 50 | 51 | 52 | class Signal(object): 53 | def __init__(self, factor, position): 54 | self.factor = factor 55 | self.position = position 56 | 57 | @abc.abstractmethod 58 | def __call__(self, *args, **kwargs): 59 | pass 60 | -------------------------------------------------------------------------------- /research/ml_models/select_features.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2021/9/14 10:05 4 | # @Author : rpyxqi@gmail.com 5 | # @Site : 6 | # @File : select_features.py 7 | 8 | from sklearn.feature_selection import SelectKBest 9 | from sklearn.feature_selection import chi2 10 | from sklearn.feature_selection import mutual_info_classif 11 | from sklearn.datasets import load_iris 12 | import pandas as pd 13 | import numpy as np 14 | from scipy.stats import chi2_contingency 15 | from sklearn.decomposition import PCA 16 | 17 | 18 | def iris_feature_test(): 19 | iris = load_iris() 20 | # mutual_info_classif, chi2 21 | model1 = SelectKBest(mutual_info_classif, k=3) 22 | ret = model1.fit_transform(iris.data, iris.target) 23 | 24 | print(model1.scores_) 25 | print(model1.pvalues_) 26 | # chi2test1(iris.target, iris.data[:,0]) 27 | print(ret.shape) 28 | 29 | 30 | def chi2test1(y, x): 31 | con_table = pd.crosstab(y, x) 32 | chi2, p, df, ex = chi2_contingency(con_table) 33 | print(chi2, p, df) 34 | 35 | 36 | def pca_demo(): 37 | X = np.array([[-1, -1], [-2, -1], [-3, -2], [1, 1], [2, 1], [3, 2]]) 38 | pca = PCA(n_components=1, svd_solver='arpack') 39 | pca.fit(X) 40 | print(pca.explained_variance_ratio_) 41 | print(pca.singular_values_) 42 | print(pca.transform([[2, 3]])) 43 | 44 | 45 | if __name__ == '__main__': 46 | # iris_feature_test() 47 | # iris = load_iris() 48 | # (iris.target, iris.data[:, 0]) 49 | pca_demo() 50 | -------------------------------------------------------------------------------- /backtester/Account.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2021/7/26 14:02 4 | # @Author : rpyxqi@gmail.com 5 | # @Site : 6 | # @File : Account.py 7 | 8 | class Account(object): 9 | def __init__(self, init_cash=100000): 10 | self.transaction = list() 11 | self.fee = 0.0 12 | self.risk_ratio = [0.0] 13 | self.occupied_margin = list() 14 | self.market_value = init_cash 15 | self.available_margin = list() 16 | self.trade_market_values = [init_cash] 17 | self.settle_market_values = [init_cash] 18 | 19 | def add_transaction(self, val=[]): 20 | self.transaction.append(val) 21 | 22 | def cache_transaction(self): 23 | pass 24 | 25 | def update_fee(self, val): 26 | self.fee += val 27 | 28 | def update_risk_ratio(self, val): 29 | self.risk_ratio.append(val) 30 | 31 | def update_occupied_margin(self, val): 32 | self.occupied_margin.append(val) 33 | 34 | def update_market_value(self, trade_val, settle_val, fee): 35 | _trade_val = self.trade_market_values[-1] 36 | # print("old mkt val", _trade_val) 37 | self.trade_market_values.append(_trade_val - fee + trade_val) 38 | # print("new mkt val", self.trade_market_values[-1]) 39 | _settle_val = self.settle_market_values[-1] 40 | # print("old mkt val 1", _settle_val) 41 | self.settle_market_values.append(_settle_val - fee + settle_val) 42 | # print("new mkt val 1", self.settle_market_values[-1]) 43 | 44 | def update_available_margin(self, val): 45 | self.available_margin.append(val) 46 | 47 | def get_trade_market_values(self): 48 | return self.trade_market_values 49 | 50 | def get_settle_market_values(self): 51 | return self.settle_market_values 52 | -------------------------------------------------------------------------------- /research/daily_models/com_analysis.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2022/3/15 13:33 4 | # @Author : rpyxqi@gmail.com 5 | # @Site : 6 | # @File : com_analysis.py 7 | 8 | import matplotlib.pyplot as plt 9 | import uqer 10 | import pprint 11 | import numpy as np 12 | from uqer import DataAPI 13 | 14 | uqer_client = uqer.Client(token="6aa0df8d4eec296e0c25fac407b332449112aad6c717b1ada315560e9aa0a311") 15 | 16 | 17 | # 返回相应品种主力合约阶段的日频率 价格波动，收盘价涨跌幅波动，结算价涨跌幅波动，分钟收盘价波动均值 18 | def get_daily_vol(start_date='', end_date='', product_id='RB'): 19 | df = DataAPI.MktFutdGet(secID=u"", ticker=u"", tradeDate=u"", beginDate=start_date, endDate=end_date, exchangeCD="", 20 | field=u"", pandas="1") 21 | df = df[df.mainCon == 1] 22 | # df_m = df[df.ticker == 'M2205'] 23 | df_rb = df[df.contractObject == product_id] 24 | df_hc = df[df.contractObject == 'HC'] 25 | # _flag = [item.startswith('m') for item in df['ticker']] 26 | # df_m = df[_flag] 27 | # print(df_hc.head().T) 28 | # print(df_hc['CHG1'].mean(), df_hc['CHG1'].std()) 29 | # print(df_hc['closePrice'].mean(), df_hc['closePrice'].std()) 30 | # plt.plot(df_rb['CHG']) 31 | # plt.show() 32 | # plt.plot(df_rb['closePrice']) 33 | # # df_hc['ClosePrice'] 34 | # plt.show() 35 | diff = df_rb['closePrice'] - df_hc['closePrice'] 36 | diff = [item - list(df_hc['closePrice'])[idx] for idx, item in enumerate(df_rb['closePrice'])] 37 | plt.plot(diff) 38 | plt.show() 39 | min_std = [] 40 | for d in df_rb['tradeDate']: 41 | df_min = DataAPI.FutBarHistOneDay2Get(instrumentID=u"rb2205", date=d, unit=u"1", field=u"", pandas="1") 42 | # print(d, df_min['closePrice'].std()) 43 | min_std.append(df_min['closePrice'].std()) 44 | return df_rb['closePrice'].std(), df_rb['CHG'].std(), df_rb['CHG1'].std(), sum(min_std) / len(min_std) 45 | 46 | 47 | if __name__ =="__main__": 48 | daily_close_std, daily_chg_std, daily_chg1_std, min_std = get_daily_vol("20220208", "20220314", 'RB') 49 | print('daily close std:',daily_close_std,'daily close chg std:', daily_chg_std,'daily clear chg std:', daily_chg1_std, 'min_std:', min_std) 50 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | cache/ 131 | results/ 132 | strategy/ 133 | conf/.editorconfig 134 | logs/ 135 | -------------------------------------------------------------------------------- /research/data_process/DataProcess.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2021/7/29 10:33 4 | # @Author : rpyxqi@gmail.com 5 | # @Site : 6 | # @File : DataProcess.py 7 | 8 | from ...utils.define import * 9 | import os 10 | import pandas as pd 11 | from datetime import datetime 12 | import shutil 13 | 14 | 15 | def cache_depth_mkt(): 16 | lst = os.listdir('C:\projects\pycharm\option_future_research\cache\mkt_pycache') 17 | for item in lst: 18 | _tmp = item.split('.')[0].split('_') 19 | try: 20 | os.system('mkdir C:\projects\pycharm\option_future_research\cache\\{0}'.format(_tmp[-1])) 21 | shutil.copy('C:\projects\pycharm\option_future_research\cache\mkt_pycache\{0}'.format(item), 22 | 'C:\projects\pycharm\option_future_research\cache\{0}\{1}.csv'.format(_tmp[-1], _tmp[0])) 23 | except Exception as ex: 24 | print(ex) 25 | # os.system('rm -r C:\projects\pycharm\option_future_research\cache\\mkt_pycache') 26 | 27 | 28 | def prepare_test_mkt(): 29 | with open('../../utils/trade_dates.txt', 'r') as f: 30 | lines = f.readlines() 31 | dates = [item.strip() for item in lines] 32 | # create folders 33 | for item in dates: 34 | try: 35 | os.system('mkdir C:\projects\pycharm\option_future_research\cache\\{0}'.format(item)) 36 | except Exception as ex: 37 | print("folder exist", item) 38 | 39 | 40 | def transaction_analysis(): 41 | df = pd.read_csv('results/trans_m2105_20210104.csv') 42 | long_open_ts = list(df[df.direction == 0]['timestamp']) 43 | long_close_ts = list(df[df.direction == 1]['timestamp']) 44 | short_open_ts = list(df[df.direction == 10]['timestamp']) 45 | short_close_ts = list(df[df.direction == 11]['timestamp']) 46 | 47 | assert len(long_open_ts) == len(long_close_ts) 48 | assert len(short_open_ts) == len(short_close_ts) 49 | 50 | long_tran_time = [] 51 | for idx, item in enumerate(long_open_ts): 52 | _sec = datetime.strptime(long_close_ts[idx], '%H:%M:%S') - datetime.strptime(item, '%H:%M:%S') 53 | long_tran_time.append(_sec.seconds) 54 | short_tran_time = [] 55 | for idx, item in enumerate(short_open_ts): 56 | _sec = datetime.strptime(short_close_ts[idx], '%H:%M:%S') - datetime.strptime(item, '%H:%M:%S') 57 | print(idx, _sec.seconds, item, short_close_ts[idx]) 58 | short_tran_time.append(_sec.seconds) 59 | # print(long_tran_time) 60 | # print(short_tran_time) 61 | # print(max(long_tran_time)/60, min(long_tran_time)) 62 | # print(max(short_tran_time)/60, min(short_tran_time)) 63 | 64 | 65 | if __name__ == '__main__': 66 | cache_depth_mkt() 67 | -------------------------------------------------------------------------------- /backtester/Position.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2021/7/26 14:03 4 | # @Author : rpyxqi@gmail.com 5 | # @Site : 6 | # @File : Position.py 7 | import copyreg 8 | from collections import defaultdict 9 | import utils.define as define 10 | import copy 11 | 12 | 13 | class Position(object): 14 | def __init__(self): 15 | self.position = defaultdict(list) 16 | 17 | def open_position(self, instrument_id, long_short, price, timestamp, vol): 18 | self.position[instrument_id].append([long_short, price, timestamp, vol]) 19 | 20 | def close_position(self, instrument_id, long_short, price, timestamp): 21 | _lst = self.position.get(instrument_id) or [] 22 | for item in _lst: 23 | if item[0] == long_short: 24 | _lst.remove(item) 25 | 26 | def update_position(self, instrument_id, long_short, price, timestamp, vol, order_type): 27 | print('update position:long_short=>', long_short, 'price=>', price, 'ts=>', timestamp, 'vol=>', vol, 28 | 'order type=>', order_type) 29 | _lst = self.position.get(instrument_id) or [] 30 | update_lst = [] 31 | 32 | print("before update position:", self.position[instrument_id]) 33 | if not _lst: 34 | print('pos not exist,add possition') 35 | # self.open_position(instrument_id, long_short, price, timestamp, vol) 36 | update_lst.append([long_short, price, timestamp, vol]) 37 | else: 38 | for item in _lst: 39 | print("before update position:", item, instrument_id, long_short, price, vol, order_type) 40 | _direction, _price, _ts, _vol = item 41 | if _direction == define.LONG and order_type == define.LONG_OPEN: 42 | item[0] = define.LONG 43 | item[1] = (_price * _vol + price * vol) / (_vol + vol) 44 | item[2] = timestamp 45 | item[3] = _vol + vol 46 | update_lst.append(item) 47 | elif _direction == define.LONG and order_type == define.LONG_CLOSE: 48 | assert _vol >= vol 49 | if _vol == vol: 50 | continue 51 | item[1] = 0.0 if _vol == vol else (_price * _vol - price * vol) / (_vol - vol) 52 | item[2] = timestamp 53 | item[3] = _vol - vol 54 | elif _direction == define.SHORT and order_type == define.SHORT_OPEN: 55 | item[0] = define.SHORT 56 | item[1] = (_price * _vol + price * vol) / (_vol + vol) 57 | item[2] = timestamp 58 | item[3] = _vol + vol 59 | update_lst.append(item) 60 | elif _direction == define.SHORT and order_type == define.SHORT_CLOSE: 61 | assert _vol >= vol 62 | if _vol == vol: 63 | continue 64 | item[1] = 0.0 if _vol == vol else (_price * _vol - price * vol) / (_vol - vol) 65 | item[2] = timestamp 66 | item[3] = _vol - vol 67 | else: 68 | update_lst.append(item) 69 | self.position[instrument_id] = update_lst 70 | print("after update position:", self.position[instrument_id]) 71 | 72 | def get_position(self, instrument_id): 73 | _lst = self.position.get(instrument_id) or [] 74 | ret = [] 75 | for item in _lst: 76 | if item[-1] > 0: 77 | ret.append(item) 78 | return ret 79 | 80 | def get_position_side(self, instrument_id, side): 81 | _pos_lst = self.position.get(instrument_id) 82 | if _pos_lst: 83 | for item in _pos_lst: 84 | if item[0] == side: 85 | return item 86 | return 87 | -------------------------------------------------------------------------------- /strategy/T0Signal.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2021/7/26 16:32 4 | # @Author : rpyxqi@gmail.com 5 | # @Site : 6 | # @File : T0Signal.py 7 | 8 | from strategy.Signal import Signal 9 | import utils.define as define 10 | from backtester.Factor import Factor 11 | from backtester.Position import Position 12 | 13 | 14 | class TOSignal(Signal): 15 | def __init__(self, factor, position): 16 | super().__init__(factor, position) 17 | self.is_available = True 18 | 19 | def __call__(self, *args, **kwargs): 20 | assert isinstance(self.factor, Factor) 21 | assert isinstance(self.position, Position) 22 | params = kwargs.get('params') 23 | volitility = float(params.get('volitility')) or 20.0 24 | k1 = float(params.get('k1')) or 1.0 25 | k2 = float(params.get('k2')) or 1.0 26 | instrument_id = params.get('instrument_id') 27 | # adjust by multiplier 28 | stop_profit = float(params.get('stop_profit')) or 5.0 29 | stop_loss = float(params.get('stop_loss')) or 20.0 30 | multiplier = int(params.get('multiplier')) or 10 31 | 32 | open_fee = float(params.get('open_fee')) or 1.51 33 | close_to_fee = float(params.get('close_t0_fee')) or 0.0 34 | fee = (open_fee + close_to_fee) / multiplier 35 | start_tick = int(params.get('start_tick')) or 2 36 | long_lots_limit = int(params.get('long_lots_limit')) or 1 37 | short_lots_limit = int(params.get('short_lots_limit')) or 1 38 | slope_upper = float(params.get('slope_upper')) or 1.0 39 | slope_lower = float(params.get('slope_lower')) or -1.0 40 | _position = self.position.get_position(instrument_id) 41 | 42 | if params.get('tick')[2].split()[1]<'21:00:00.000': 43 | print('check') 44 | 45 | if len(self.factor.last_price) < start_tick: 46 | return define.NO_SIGNAL 47 | _long, _short, long_price, short_price = 0, 0, 0.0, 0.0 48 | 49 | if _position: 50 | for item in _position: 51 | if item[0] == define.LONG: 52 | long_price = item[1] 53 | _long += 1 54 | elif item[0] == define.SHORT: 55 | short_price = item[1] 56 | _short += 1 57 | # if factor.last_price[-1] > factor.last_price[-2] and factor.last_price[-1] > factor.vwap[ 58 | # -1] + k1 * volitility and not _long: 59 | if (self.factor.slope[-1] > self.factor.slope[-2]) and (self.factor.slope[-2] < self.factor.slope[-3]) and abs( 60 | self.factor.slope[-1]) > slope_upper and _short < short_lots_limit: 61 | # if (factor.slope[-1] > factor.slope[-2]) and abs( 62 | # factor.slope[-1]) > 0.8 and not _long: 63 | # print(factor.last_price, factor.slope) 64 | # print(_short, short_lots_limit) 65 | return define.SHORT_OPEN 66 | # if factor.last_price[-1] < factor.last_price[-2] and factor.last_price[-1] < factor.vwap[ 67 | # -1] + k2 * volitility and not _short: 68 | if (self.factor.slope[-1] < self.factor.slope[-2]) and (self.factor.slope[-2] > self.factor.slope[-3]) and abs( 69 | self.factor.slope[-1]) > slope_upper and _long < long_lots_limit: 70 | # if (factor.slope[-1] < factor.slope[-2]) and abs( 71 | # factor.slope[-1]) > 0.8 and not _short: 72 | 73 | # print(factor.last_price,factor.slope) 74 | return define.LONG_OPEN 75 | # if factor.last_price[-1] < factor.last_price[-2] and _long and factor.last_price[-1] > long_price + stop_profit: 76 | if _long and (self.factor.last_price[-1] > long_price + stop_profit + fee or 77 | self.factor.last_price[-1] < long_price - stop_loss - fee): 78 | return define.LONG_CLOSE 79 | # if factor.last_price[-1] > factor.last_price[-2] and _short and short_price > factor.last_price[-1] + stop_profit: 80 | if _short and ( 81 | short_price > self.factor.last_price[-1] + stop_profit + fee or self.factor.last_price[ 82 | -1] > short_price + stop_loss + fee): 83 | return define.SHORT_CLOSE 84 | return define.NO_SIGNAL 85 | -------------------------------------------------------------------------------- /utils/utils.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2021/7/29 15:50 4 | # @Author : rpyxqi@gmail.com 5 | # @Site : 6 | # @File : utils.py 7 | 8 | import time 9 | import json 10 | import math 11 | import uqer 12 | import pprint 13 | import numpy as np 14 | from uqer import DataAPI 15 | import matplotlib.pyplot as plt 16 | from sklearn.linear_model import LinearRegression 17 | from sklearn import metrics 18 | import utils.define as define 19 | from sklearn.feature_selection import SelectKBest 20 | from sklearn.feature_selection import mutual_info_classif 21 | from sklearn.linear_model import LogisticRegression 22 | from scipy.stats import rankdata 23 | import talib as ta 24 | import pandas as pd 25 | import statsmodels.api as sm 26 | from editorconfig import get_properties, EditorConfigError 27 | import logging 28 | import os 29 | import gc 30 | 31 | try: 32 | print("current path in utils:", os.getcwd()) 33 | # _conf_file = os.path.join(os.path.abspath(os.pardir), define.CONF_DIR, 34 | # define.CONF_FILE_NAME) 35 | _conf_file = os.path.join(os.path.abspath(os.getcwd()), define.CONF_DIR, define.CONF_FILE_NAME) 36 | options = get_properties(_conf_file) 37 | except EditorConfigError: 38 | logging.warning("Error getting EditorConfig propterties", exc_info=True) 39 | else: 40 | for key, value in options.items(): 41 | # _config = '{0},{1}:{2}'.format(_config, key, value) 42 | print("{0}:{1}".format(key, value)) 43 | print(options.get('uqer_token')) 44 | uqer_client = uqer.Client(token=options.get('uqer_token')) 45 | 46 | 47 | def get_instrument_ids(start_date='20210901', end_date='20210930', product_id='RB'): 48 | df = DataAPI.MktFutdGet(endDate=end_date, beginDate=start_date, pandas="1") 49 | df = df[df.mainCon == 1] 50 | df = df[df.contractObject == product_id.upper()][['ticker', 'tradeDate', 'exchangeCD']] 51 | return df 52 | 53 | 54 | def get_trade_dates(start_date='20110920', end_date='20210921'): 55 | df = DataAPI.TradeCalGet(exchangeCD=u"XSHG,XSHE", beginDate=start_date, endDate=end_date, isOpen=u"1", 56 | field=u"", 57 | pandas="1") 58 | df = df[df.isOpen == 1] 59 | return df 60 | 61 | 62 | def timeit(func): 63 | def timed(*args, **kwargs): 64 | ts = time.time() 65 | result = func(*args, **kwargs) 66 | te = time.time() 67 | # logger.info('%r (%r, %r) %2.2f sec' % (func.__name__, args, kwargs, te - ts)) 68 | print('%r (%r, %r) %2.2f sec' % (func.__name__, args, kwargs, te - ts)) 69 | return result 70 | 71 | return 72 | 73 | 74 | def test_time(): 75 | for item in range(10): 76 | time.sleep(1) 77 | 78 | 79 | def is_trade(start_timestamp=None, end_timestamp=None, update_time=None): 80 | update_time = update_time.split()[1].split('.')[0] 81 | if update_time > start_timestamp and update_time < '22:59:30': 82 | return True 83 | elif update_time > '09:00:30' and update_time < end_timestamp: 84 | return True 85 | else: 86 | return False 87 | return False 88 | 89 | 90 | def get_path(ref_path_lst=[]): 91 | _path = os.path.join(os.path.abspath(os.getcwd())) 92 | # print(_path) 93 | for p in ref_path_lst: 94 | _path = os.path.join(_path, p) 95 | return _path 96 | 97 | 98 | def get_contract(instrument_id=''): 99 | df = DataAPI.FutuGet(secID=u"", ticker=instrument_id, exchangeCD=u"", contractStatus="", contractObject=u"", 100 | prodID="", 101 | field=u"", pandas="1") 102 | return df 103 | 104 | 105 | def get_mul_num(instrument_id=''): 106 | df = get_contract(instrument_id=instrument_id) 107 | return df['contMultNum'].values[0] 108 | 109 | 110 | def write_json_file(file_path='', data=None): 111 | # if data == None: 112 | # return 113 | with open(file_path, 'w') as outfile: 114 | j_data = json.dumps(data) 115 | outfile.write(j_data) 116 | 117 | 118 | def load_json_file(filepath=''): 119 | with open(filepath) as infile: 120 | contents = infile.read() 121 | return json.loads(contents) 122 | 123 | 124 | def get_trade_dates(start_date='20110920', end_date='20210921'): 125 | df = DataAPI.TradeCalGet(exchangeCD=u"XSHG,XSHE", beginDate=start_date, endDate=end_date, isOpen=u"1", field=u"", 126 | pandas="1") 127 | df = df[df.isOpen == 1] 128 | return df 129 | 130 | 131 | def _is_trading_time(time_str=''): 132 | if time_str > '09:00:00.000' and time_str <= '15:00:00.000' or time_str > '21:00:00.000' and time_str < '23:00:01.000': 133 | return True 134 | else: 135 | return False 136 | 137 | 138 | if __name__ == "__main__": 139 | start_ts = time.time() 140 | test_time() 141 | end_ts = time.time() 142 | print(end_ts - start_ts) 143 | # df = get_instrument_ids(start_date='20210901', end_date='20210930', product_id='RB') 144 | # print(df) 145 | df = get_contract(instrument_id='rb2201', exchangeCD=u"XSGE") 146 | print(df) 147 | -------------------------------------------------------------------------------- /strategy/RegSignal.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2021/9/28 16:15 4 | # @Author : rpyxqi@gmail.com 5 | # @Site : 6 | # @File : RegSignal.py 7 | 8 | import numpy as np 9 | import pandas as pd 10 | from strategy.Signal import Signal 11 | import utils.define as define 12 | import utils.utils as utils 13 | from backtester.Factor import Factor 14 | from backtester.Position import Position 15 | import os 16 | 17 | 18 | class RegSignal(Signal): 19 | def __init__(self, factor, position, instrument_id=None, trade_date=None): 20 | super().__init__(factor, position) 21 | self.is_available = True 22 | trade_date_df = utils.get_trade_dates(start_date='20210101', end_date=trade_date) 23 | trade_date_df = trade_date_df[trade_date_df.exchangeCD == 'XSHE'] 24 | trade_dates = list(trade_date_df['calendarDate']) 25 | prev_date = trade_dates[-2] 26 | _evalute_path = utils.get_path([define.RESULT_DIR, define.TICK_MODEL_DIR, 27 | 'model_evaluate.json']) 28 | _model_evaluate = utils.load_json_file(_evalute_path) 29 | _ret_lst = _model_evaluate.get('{0}_{1}'.format(instrument_id, trade_date.replace('-', ''))) or [] 30 | if not _ret_lst: 31 | self.is_available = False 32 | return 33 | _ret_lst.sort(key=lambda x: x[1], reverse=True) 34 | mse, r2, date, predict_window, lag_window = _ret_lst[0] 35 | _path = utils.get_path( 36 | [define.RESULT_DIR, define.TICK_MODEL_DIR, 37 | 'pred_{0}_{1}_{2}_{3}.csv'.format(instrument_id, date.replace('-', ''), predict_window, lag_window)]) 38 | df = pd.read_csv(_path) 39 | _pred_lst = sorted(list(df['pred'])) 40 | self.map_dict = dict(zip([item.split()[1].split('.')[0] for item in df['UpdateTime']], df['pred'])) 41 | _prev_path = utils.get_path( 42 | [define.RESULT_DIR, define.TICK_MODEL_DIR, 43 | 'pred_{0}_{1}_{2}_{3}.csv'.format(instrument_id, prev_date.replace('-', ''), predict_window, lag_window)]) 44 | self.df_prev = pd.read_csv(_prev_path) 45 | _label_lst = sorted(list(self.df_prev['pred'])) 46 | # self._ret_up = self.df_prev['label'].quantile(1 - 0.0001) 47 | # self._ret_down = self.df_prev['label'].quantile(0.002) 48 | # TODO CHECK HARDCODE 49 | self._ret_up = _label_lst[-50] 50 | self._ret_down = _label_lst[20] 51 | self._ret_up = 0.0005 52 | self._ret_down = -0.0005 53 | print('up ret:{0}, down ret:{1},up pred:{2}, down pred:{3}'.format(self._ret_up, _pred_lst[-50], self._ret_down, 54 | _pred_lst[50])) 55 | 56 | def __call__(self, *args, **kwargs): 57 | if not self.is_available: 58 | return define.NO_SIGNAL 59 | 60 | params = kwargs.get('params') 61 | _k = params.get('tick')[2].split()[1].split('.')[0] 62 | _v = self.map_dict.get(_k) or 0.0 63 | _up_ratio = float(params.get('ret_up_ratio')) or 0.0015 64 | _down_ratio = float(params.get('ret_down_ratio')) or 0.001 65 | 66 | # _ret_down = float(params.get('ret_down_ratio')) or 0.001 67 | _long, _short, long_price, short_price = 0, 0, 0.0, 0.0 68 | instrument_id = params.get('instrument_id') 69 | start_tick = int(params.get('start_tick')) or 2 70 | stop_profit = float(params.get('stop_profit')) or 5.0 71 | stop_loss = float(params.get('stop_loss')) or 20.0 72 | multiplier = int(params.get('multiplier')) or 10 73 | long_lots_limit = int(params.get('long_lots_limit')) or 1 74 | short_lots_limit = int(params.get('short_lots_limit')) or 1 75 | 76 | open_fee = float(params.get('open_fee')) or 1.51 77 | close_to_fee = float(params.get('close_t0_fee')) or 0.0 78 | fee = (open_fee + close_to_fee) / multiplier 79 | _position = self.position.get_position(instrument_id) 80 | if _position: 81 | for item in _position: 82 | if item[0] == define.LONG: 83 | long_price = item[1] 84 | _long += 1 85 | elif item[0] == define.SHORT: 86 | short_price = item[1] 87 | _short += 1 88 | if _v >= self._ret_up and _long < long_lots_limit and _v > 0: 89 | return define.LONG_OPEN 90 | elif _v <= self._ret_down and _short < short_lots_limit and _v < 0: 91 | return define.SHORT_OPEN 92 | 93 | if len(self.factor.last_price) < start_tick: 94 | return define.NO_SIGNAL 95 | 96 | if _long and (self.factor.last_price[-1] > long_price + stop_profit + fee or 97 | self.factor.last_price[-1] < long_price - stop_loss - fee): 98 | return define.LONG_CLOSE 99 | if _short and ( 100 | short_price > self.factor.last_price[-1] + stop_profit + fee or self.factor.last_price[ 101 | -1] > short_price + stop_loss + fee): 102 | return define.SHORT_CLOSE 103 | return define.NO_SIGNAL 104 | -------------------------------------------------------------------------------- /utils/define.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2021/7/26 14:04 4 | # @Author : rpyxqi@gmail.com 5 | # @Site : 6 | # @File : define.py 7 | 8 | 9 | NO_SIGNAL = -1 10 | LONG_OPEN = 0 11 | LONG_CLOSE = 1 12 | SHORT_OPEN = 2 13 | SHORT_CLOSE = 3 14 | LONG = 4 15 | SHORT = 5 16 | STOP = 6 17 | PLT_START = 3 18 | PLT_END = -10 19 | TICK_SIZE = 41400 20 | TICK = 1 21 | 22 | cols = ["InstrumentID", "LastPrice", "OpenPrice", "HighestPrice", "LowestPrice", "Volume", "Turnover", "OpenInterest", 23 | "UpperLimitPrice", "LowerLimitPrice", "UpdateTime", 24 | "UpdateMillisec", 25 | "BidPrice1", "BidVolume1", "AskPrice1", "AskVolume1", "BidPrice2", "BidVolume2", "AskPrice2", "AskVolume2", 26 | "BidPrice3", "BidVolume3", 27 | "AskPrice3", "AskVolume3", "BidPrice4", "BidVolume4", "AskPrice4", "AskVolume4", "BidPrice5", "BidVolume5", 28 | "AskPrice5", "AskVolume5"] 29 | 30 | selected_cols = ['InstrumentID', 'UpdateTime', 'Turnover', 'Volume', 'LastPrice', 'AskPrice1', 'AskVolume1', 31 | 'BidPrice1', 'BidVolume1'] 32 | 33 | tb_cols = ["Exchange", "InstrumentID", "UpdateTime", "LastPrice", "OpenInterest", "InterestDiff", "Turnover", 34 | "Volume", "OpenVolume", "CloseVolume", "TransactionType", "Direction", "BidPrice1", "AskPrice1", 35 | "BidVolume1", 36 | "AskVolume1"] 37 | 38 | # skip_raw_cols = ['Exchange', 'InstrumentID', 'LastPrice', 'OpenInterest', 'InterestDiff', 39 | # 'Turnover', 'Volume', 'OpenVolume', 'CloseVolume', 'TransactionType', 'Direction', 40 | # 'BidPrice1', 'AskPrice1', 'BidVolume1', 'AskVolume1', 'vwap', 'vol_short', 'vol_long', 41 | # 'turnover_short', 'turnover_long', 'vwap_short', 'vwap_long', 'bs_vol', 'bs_vol_long', 42 | # 'bs_vol_short', 'bs_vol_diff', 'bs_tag', 'wap', 43 | # 'trenddiff', 'trend_long', 'dif', 'dea', 'turnover_ls_ratio', 44 | # 'vwap_ls_ratio', 'aoi', 'oi'] 45 | # remove from VIF, to be added with other calculation 46 | 47 | 48 | # skip_raw_cols = ['Exchange', 'InstrumentID'] 49 | 50 | # skip_raw_cols_normalized = ['Exchange', 'InstrumentID', 'OpenInterest', 'InterestDiff', 51 | # 'Turnover', 'Volume', 'OpenVolume', 'CloseVolume', 'TransactionType', 'Direction', 52 | # 'BidPrice1', 'AskPrice1', 'BidVolume1', 'AskVolume1', 'vwap', 'vol_short', 'vol_long', 53 | # 'turnover_short', 'turnover_long', 'vwap_short', 'vwap_long', 'bs_vol', 'bs_vol_long', 54 | # 'bs_vol_short', 'bs_vol_diff', 'bs_tag'] 55 | 56 | skip_raw_cols = ['Exchange', 'InstrumentID', 'TransactionType', 'Direction', 'BidPrice1', 57 | 'bs_tag', 58 | 'LastPrice', 'InterestDiff', 'OpenInterest', 'Turnover', 'Volume', 'OpenVolume', 'CloseVolume', 59 | 'AskPrice1', 60 | # norm value 61 | 'AskVolume1', 'BidVolume1', 'vwap', 'wap', 'volume_ls_diff', 'turnover_ls_diff', 'bs_vol_ls_diff', 62 | 'norm_turnover', 63 | # norm value 64 | 'norm_openvolume', 'norm_closevolume', 'norm_turnover_ls_diff', ] # norm value 65 | 66 | # normalized_cols = [('LastPrice', 'LastPrice'), ('wap', 'LastPrice'), ('OpenInterest', 'OpenInterest'), 67 | # ('vwap', 'LastPrice'), ('Volume', 'Volume'), ('BidPrice1', 'LastPrice'), ('AskPrice1', 'LastPrice')] 68 | 69 | 70 | BASE_DIR = 'option_future_research' 71 | RESULT_DIR = 'results' 72 | CONF_DIR = 'conf' 73 | FACTOR_DIR = 'factors' 74 | CONF_FILE_NAME = '.editorconfig' 75 | TICK_MODEL_DIR = 'tickmodels' 76 | TICK_MKT_DIR = 'C:\projects\l2mkt\FutAC_TickKZ_PanKou_Daily_202107' 77 | FACTOR_DIR = 'factors' 78 | CACHE_DIR = 'cache' 79 | BT_DIR = 't0backtest' 80 | daily_cache_name = 'cache/future_20210101_20210804.csv' 81 | 82 | MKT_MISSING_SKIP = 0.3 83 | 84 | exchange_map = {'XZCE': 'zc', 'XSGE': 'sc', 'XSIE': 'ine', 'XDCE': 'dc'} 85 | 86 | normalized_vals = {'LastPrice': 0, 'Volume': 0, 'OpenInterest': 0} 87 | normalized_refs = [('wap', 'LastPrice')] 88 | # normalized_refs = [('LastPrice', 'LastPrice'), ('OpenInterest', 'OpenInterest'), ('InterestDiff', 'Volume'), 89 | # ('Turnover', 'Turnover'), ('Volume', 'Volume'), ('OpenVolume', 'Volume'), ('CloseVolume', 'Volume'), 90 | # ('BidVolume1', 'Volume'), ('AskVolume1', 'Volume'), ('vwap', 'LastPrice'), ('wap', 'LastPrice'), 91 | # ('bs_tag', 'Volume'), 92 | # ('volume_ls_diff', 'Volume'), ('turnover_ls_diff', 'Turnover')] 93 | 94 | CLF_LABEL_NAME = 'label_clf_1' 95 | REG_LABEL_NAME = 'label_1' 96 | 97 | # train_cols = ['open_close_ratio', 'oi', 'oir', 'aoi', 'slope', 'cos', 'macd', 'dif', 'dea', 'bsvol_volume_0', 98 | # 'trend_1', 'bsvol_volume_1', 99 | # 'trend_ls_diff', 'trend_ls_ratio', 'volume_ls_ratio', 'turnover_ls_ratio', 'bs_vol_ls_ratio', 100 | # 'bsvol_volume_ls_ratio'] 101 | 102 | # train_cols = ['slope'] 103 | # train_cols = ['cos', 'macd', 'aoi', 'trend_0', 'log_return_0', 'bsvol_volume_0', 'trend_1', 104 | # 'bsvol_volume_1', 'trend_ls_ratio', 'volume_ls_ratio', 'log_return'] 105 | # train_cols = ['log_return_0', 'log_return_1', 'slope'] 106 | # train_cols = ['cos'] 107 | train_cols = ['log_return', 'log_return_0', 'wap_log_return'] 108 | -------------------------------------------------------------------------------- /research/daily_models/basis_index.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2021/9/3 9:53 4 | # @Author : rpyxqi@gmail.com 5 | # @Site : 6 | # @File : daily_models.py 7 | 8 | import math 9 | import uqer 10 | import datetime 11 | import pprint 12 | import pandas as pd 13 | import numpy as np 14 | from uqer import DataAPI 15 | import matplotlib.pyplot as plt 16 | from sklearn.model_selection import train_test_split 17 | from sklearn.linear_model import LinearRegression 18 | from sklearn import metrics 19 | 20 | uqer_client = uqer.Client(token="") 21 | 22 | 23 | # TODO line correlation 24 | 25 | 26 | def fetch_data(start_date='', end_date=''): 27 | DataAPI.TradeCalGet(exchangeCD=u"XSHG,XSHE", beginDate=start_date, endDate=end_date, isOpen=u"1", 28 | field=u"", pandas="1") 29 | df = DataAPI.MktMFutdGet(mainCon=u"1", contractMark=u"", contractObject=u"", tradeDate=u"", startDate=start_date, 30 | endDate=end_date, field=u"", pandas="1") 31 | df = df[df.exchangeCD == 'CCFX'] 32 | df = df[df.mainCon == 1] 33 | ic_df = df[df.contractObject == 'IC'][['tradeDate', 'ticker']] 34 | if_df = df[df.contractObject == 'IF'][['tradeDate', 'ticker']] 35 | ih_df = df[df.contractObject == 'IH'][['tradeDate', 'ticker']] 36 | 37 | df_basis = DataAPI.MktFutIdxBasisGet(secID=u"", ticker=u"", beginDate=start_date, endDate=end_date, field=u"", 38 | pandas="1") 39 | ic_mkt = ic_df.set_index(['tradeDate', 'ticker']).join(df_basis.set_index(['tradeDate', 'ticker']), 40 | how='left').reset_index() 41 | if_mkt = if_df.set_index(['tradeDate', 'ticker']).join(df_basis.set_index(['tradeDate', 'ticker']), 42 | how='left').reset_index() 43 | ih_mkt = ih_df.set_index(['tradeDate', 'ticker']).join(df_basis.set_index(['tradeDate', 'ticker']), 44 | how='left').reset_index() 45 | 46 | icf_mkt = ic_mkt.set_index('tradeDate').join(other=if_mkt.set_index('tradeDate'), on='tradeDate', lsuffix='_ic', 47 | rsuffix='_if') 48 | df_mkt = icf_mkt.join(ih_mkt.set_index('tradeDate')) 49 | df_mkt = df_mkt.sort_values(by='tradeDate', ascending=True) 50 | df_mkt.reset_index().to_csv('fut_index_mkt_{0}_{1}.csv'.format(start_date, end_date), index=False) 51 | 52 | 53 | def factor_calculate(predict_windows=20, lag_windows=20): 54 | df_factor = pd.read_csv('fut_index_mkt.csv') 55 | df_factor['basis_ic_mean'] = df_factor[['basis_ic']].rolling(lag_windows).mean() 56 | df_factor['basis_if_mean'] = df_factor[['basis_if']].rolling(lag_windows).mean() 57 | df_factor['basis_ih_mean'] = df_factor[['basis']].rolling(lag_windows).mean() 58 | df_factor['basis_ic_pct_annual'] = (df_factor['basis_ic_mean'] / df_factor['closeIndex_ic']) * 12 59 | df_factor['basis_if_pct_annual'] = (df_factor['basis_if_mean'] / df_factor['closeIndex_if']) * 12 60 | df_factor['basis_ih_pct_annual'] = (df_factor['basis_ih_mean'] / df_factor['closeIndex']) * 12 61 | 62 | df_factor['index_return_ic'] = df_factor['closeIndex_ic'].rolling(2).apply( 63 | lambda x: math.log(list(x)[-1] / list(x)[0])) 64 | df_factor['index_return_if'] = df_factor['closeIndex_if'].rolling(2).apply( 65 | lambda x: math.log(list(x)[-1] / list(x)[0])) 66 | df_factor['index_return'] = df_factor['closeIndex'].rolling(2).apply(lambda x: math.log(list(x)[-1] / list(x)[0])) 67 | df_factor['index_return_ic_win'] = df_factor['index_return_ic'].rolling(predict_windows).sum().shift( 68 | 1 - predict_windows) 69 | df_factor['index_return_if_win'] = df_factor['index_return_if'].rolling(predict_windows).sum().shift( 70 | 1 - predict_windows) 71 | df_factor['index_return_win'] = df_factor['index_return'].rolling(predict_windows).sum().shift(1 - predict_windows) 72 | df_factor['icf_index_return_diff_win'] = df_factor['index_return_ic_win'] - df_factor['index_return_if_win'] 73 | df_factor['icf_basis_annual_diff'] = df_factor['basis_ic_pct_annual'] - df_factor['basis_if_pct_annual'] 74 | lst_rolling_corr = list( 75 | df_factor[['basis_ic_pct_annual', 'basis_if_pct_annual']].rolling(lag_windows).corr().reset_index()[ 76 | 'basis_ic_pct_annual']) 77 | 78 | df_factor['icf_corr_win'] = lst_rolling_corr[1::2] 79 | df_factor[['basis_ic_pct_annual', 'basis_if_pct_annual', 'icf_corr_win']].rolling(lag_windows).corr() 80 | print(df_factor[['icf_index_return_diff_win', 'icf_basis_annual_diff', 'icf_corr_win']].corr()) 81 | factor_cols = ['tradeDate', 'basis_ic_mean', 'basis_if_mean', 'basis_ic_pct_annual', 'basis_if_pct_annual', 82 | 'index_return_ic', 83 | 'index_return_if', 'index_return_ic_win', 84 | 'index_return_if_win', 'icf_index_return_diff_win', 'icf_basis_annual_diff', 'icf_corr_win'] 85 | df_factor[factor_cols].to_csv('fut_index_factor_{0}_{1}.csv'.format(predict_windows, lag_windows), index=False) 86 | 87 | 88 | def model_process(predict_windows=40, lag_windows=20): 89 | df_factor = pd.read_csv('fut_index_factor_{0}_{1}.csv'.format(predict_windows, lag_windows)) 90 | df_factor = df_factor.dropna(axis=0) 91 | cols = list(df_factor.columns) 92 | trade_dates = list(df_factor['tradeDate']) 93 | cols.remove('index_return_ic_win') 94 | cols.remove('index_return_if_win') 95 | cols.remove('tradeDate') 96 | df_corr = df_factor[cols].corr() 97 | df_corr.index = list(df_corr.columns) 98 | df_corr.to_csv('corr_{0}_{1}.csv'.format(predict_windows, lag_windows)) 99 | cols.remove('icf_index_return_diff_win') 100 | X = df_factor[cols] 101 | y = df_factor['icf_index_return_diff_win'] 102 | X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=None) 103 | 104 | with open('model_evaluation.txt', 'a') as f: 105 | print('Test results for pred_win:{0}, lag_win:{1}'.format(predict_windows, lag_windows)) 106 | f.write('Test results for pred_win:{0}, lag_win:{1}\n'.format(predict_windows, lag_windows)) 107 | lin_reg_model = LinearRegression() 108 | lin_reg_model.fit(X_train, y_train) 109 | print('intercept:', lin_reg_model.intercept_) 110 | f.write('coef_:{0}\n'.format(list(zip(cols, lin_reg_model.coef_)))) 111 | print('coef_:{0}', list(zip(cols, lin_reg_model.coef_))) 112 | f.write('coef_:{0}\n'.format(list(zip(cols, lin_reg_model.coef_)))) 113 | y_pred = lin_reg_model.predict(X_test) 114 | print('rmse:', np.sqrt(metrics.mean_squared_error(y_test, y_pred)), 'r2:', metrics.r2_score(y_test, y_pred)) 115 | f.write('rmse:{0}\n r2:{1}\n'.format(np.sqrt(metrics.mean_squared_error(y_test, y_pred)), 116 | metrics.r2_score(y_test, y_pred))) 117 | fig = plt.figure() 118 | ax1 = fig.add_subplot(111) 119 | ax1.plot(df_factor['icf_index_return_diff_win'], 120 | label="IC IF index log return diff in predict windows {0}".format(predict_windows)) 121 | ax1.legend() 122 | 123 | ax2 = ax1.twinx() 124 | ax2.plot(df_factor['icf_basis_annual_diff'], 'r', 125 | label='IC IF annual basis pct diff in lag windows {0}'.format(lag_windows)) 126 | ax2.legend() 127 | _idx_lst = list(range(df_factor.shape[0])) 128 | ax1.set_xticks(_idx_lst[::120]) 129 | trade_date_str = [item.replace('-', '') for item in trade_dates] 130 | ax1.set_xticklabels(trade_date_str[::120], rotation=30) 131 | plt.savefig('icf return diff.jpg') 132 | 133 | 134 | if __name__ == '__main__': 135 | # fetch_data('20150103', '20210902') 136 | predict_windows_lst = [60] 137 | lag_windows_lst = [120] 138 | for pred_win in predict_windows_lst: 139 | for lag_win in lag_windows_lst: 140 | factor_calculate(predict_windows=pred_win, lag_windows=lag_win) 141 | model_process(predict_windows=pred_win, lag_windows=lag_win) 142 | -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2021/6/22 21:00 4 | # @Author : rpyxqi@gmail.com 5 | # @Site : 6 | # @File : main.py 7 | 8 | 9 | from backtester.BackTester import backtesting 10 | from editorconfig import get_properties, EditorConfigError 11 | import utils.define as define 12 | import utils.utils as utils 13 | from research.tick_models.ModelProcess import * 14 | import logging 15 | import hashlib 16 | import time 17 | import gc 18 | import os 19 | import pandas as pd 20 | import matplotlib.pyplot as plt 21 | import pprint 22 | 23 | logging.basicConfig(filename='logs/{0}.txt'.format(os.path.split(__file__)[-1].split('.')[0]), level=logging.DEBUG) 24 | logger = logging.getLogger() 25 | 26 | 27 | def data_visialize(n_records=100): 28 | df = pd.read_csv('C:\\projects\\pycharm\\option_future_research\\cache\\factors\\factor_rb2110_20210706.csv') 29 | print(df.shape) 30 | # df[['oir', 'label_1']].plot() 31 | # plt.show() 32 | # df.hist(bins=80, figsize=(9, 6)) 33 | # df[['oir','oi','cos','label_0']].hist(bins=80, figsize=(9, 6)) 34 | cols = list(df.columns) 35 | cols.remove('norm_bs_tag') 36 | 37 | 38 | # df[cols].hist() 39 | 40 | # sns.pairplot(df_final_features.iloc[:,2:], height=1.5) 41 | 42 | from random import sample 43 | # df0 = df[df.label_clf_1 == 0] 44 | # df1 = df[df.label_clf_1 == 1] 45 | # 46 | # num0 = df0.shape[0] 47 | # sample_0 = sample(list(range(num0)), int(num0 * 0.1)) 48 | # df = df1.append(df0.iloc[sample_0]) 49 | cols = df.columns 50 | for c in cols: 51 | df.plot.scatter(c, 'label_1') 52 | plt.show() 53 | # df.plot.scatter('realized_vol', 'future_vol') 54 | # df.plot.scatter('aratio', 'bratio') 55 | # df = df[df.label_clf_1 != 0] 56 | # df.plot.scatter('oir', 'label_1') 57 | # plt.xlim(-50,50) 58 | plt.show() 59 | 60 | # x_idx = list(range(n_records)) 61 | # 62 | # fig = plt.figure() 63 | # ax = fig.add_subplot(111) 64 | # ax.plot(x_idx, list(df['cos'])[:n_records], '-', label='oir') 65 | # # ax.plot(time, Rn, '-', label='Rn') 66 | # ax2 = ax.twinx() 67 | # ax2.plot(x_idx, list(df['price_chg_1'])[:n_records], '-r', label='label') 68 | # ax.legend(loc=0) 69 | # ax.grid() 70 | # # ax.set_xlabel("Time (h)") 71 | # # ax.set_ylabel(r"Radiation ($MJ\,m^{-2}\,d^{-1}$)") 72 | # # ax2.set_ylabel(r"Temperature ($^\circ$C)") 73 | # ax2.set_ylim(-15, 15) 74 | # ax.set_ylim(-2, 2) 75 | # ax2.legend(loc=0) 76 | # # plt.savefig('0.png') 77 | # plt.show() 78 | 79 | 80 | def train_models(start_date: str = '20210701', end_date: str = '20210730', product_id: str = 'rb'): 81 | """ 82 | 83 | :param start_date: 84 | :param end_date: 85 | :param product_id: 86 | :return: 87 | """ 88 | trade_date_df = utils.get_trade_dates(start_date=start_date, end_date=end_date) 89 | trade_date_df = trade_date_df[trade_date_df.exchangeCD == 'XSHE'] 90 | trade_dates = list(trade_date_df['calendarDate']) 91 | train_days = 4 92 | num_date = len(trade_dates) 93 | idx = 0 94 | predict_window_lst = [10,20] # 10s, 30s,1min,5min,10min-- 10s 95 | lag_window_lst = [10, 60] # 10s, 30s,1min,5min, 96 | 97 | print( 98 | 'train for predict windows:{0} and lag_windows:{1}'.format(predict_window_lst, lag_window_lst)) 99 | lst_score = [] 100 | while idx < num_date - train_days: 101 | print(trade_dates[idx], trade_dates[idx + train_days]) 102 | # train_model_reg_without_feature_preselect(predict_windows=predict_window_lst, 103 | # lag_windows=lag_window_lst, 104 | # top_k_features=20, 105 | # start_date=trade_dates[idx], 106 | # end_date=trade_dates[idx + train_days], 107 | # train_days=train_days, product_id=product_id.upper()) 108 | 109 | ret_scores = train_model_reg(predict_windows=predict_window_lst, lag_windows=lag_window_lst, 110 | start_date=trade_dates[idx], 111 | end_date=trade_dates[idx + train_days], 112 | top_k_features=-10, train_days=train_days, product_id=product_id.upper()) 113 | # ret_scores = train_model_clf(predict_windows=predict_window_lst, lag_windows=lag_window_lst, 114 | # start_date=trade_dates[idx], 115 | # end_date=trade_dates[idx + train_days], 116 | # top_k_features=-10, train_days=train_days, product_id=product_id.upper()) 117 | pprint.pprint(ret_scores) 118 | lst_score.append(ret_scores) 119 | idx += 1 120 | gc.collect() 121 | # df_score = pd.DataFrame(lst_score, columns=list(lst_score[0].keys())) 122 | # _train_results_file = os.path.join(os.path.abspath(os.pardir), define.BASE_DIR, define.RESULT_DIR, 123 | # define.TICK_MODEL_DIR, 124 | # 'model_evaluation_{0}.csv'.format(product_id)) 125 | # df_score.to_csv(_train_results_file, index=False) 126 | 127 | 128 | def backtest(start_date: str = '20210707', end_date: str = '20210709', product_id: str = 'rb', 129 | strategy_name: str = 'T0Signal'): 130 | """ 131 | :param start_date: 132 | :param end_date: 133 | :param product_id: 134 | :return: 135 | """ 136 | trade_date_df = utils.get_trade_dates(start_date=start_date, end_date=end_date) 137 | trade_date_df = trade_date_df[trade_date_df.exchangeCD == 'XSHE'] 138 | trade_dates = list(trade_date_df['calendarDate']) 139 | 140 | total_return, total_fee, precision = 0.0, 0.0, [] 141 | backtesting_config = '' 142 | multiplier = 10 143 | margin_rate = 0.1 144 | try: 145 | _conf_file = utils.get_path([define.CONF_DIR, define.CONF_FILE_NAME]) 146 | print("conf file in main:", _conf_file) 147 | options = get_properties(_conf_file) 148 | except EditorConfigError: 149 | logging.warning("Error getting EditorConfig propterties", exc_info=True) 150 | else: 151 | for key, value in options.items(): 152 | backtesting_config = '{0},{1}:{2}'.format(backtesting_config, key, value) 153 | _ret_path = utils.get_path([define.RESULT_DIR, define.BT_DIR, 'results_{0}.txt'.format(product_id)]) 154 | f = open(_ret_path, "a") 155 | result_fname_digest = hashlib.sha256(bytes(backtesting_config, encoding='utf-8')).hexdigest() 156 | print('digest', result_fname_digest) 157 | f.write("{0}:{1}\n".format(backtesting_config, result_fname_digest)) 158 | dates = ['20210708'] 159 | trans_lst = [] 160 | for idx, trade_date in enumerate(trade_dates): 161 | print('start back test for date:{0}'.format(trade_date)) 162 | start_ts = time.time() 163 | ret = backtesting(product_id=product_id, trade_date=trade_date.strip().replace('-', ''), 164 | signal_name=strategy_name, result_fname_digest=result_fname_digest, options=options) 165 | end_ts = time.time() 166 | print("back testing time:", end_ts - start_ts) 167 | if ret: 168 | _total_return, _total_fee, _precision, _transaction = ret 169 | total_return += _total_return 170 | total_fee += _total_fee 171 | [item.append(trade_date) for item in _transaction] 172 | trans_lst.extend(_transaction) 173 | 174 | trans_df = pd.DataFrame(trans_lst, 175 | columns=['idx', 'instrument_id', 'direction', 'last_price', 'filled_price', 'filled_vol', 176 | 'open_price', 177 | 'fee', 'open_ts', 178 | 'close_ts', 'holding_time', 179 | 'return', 'trade_date']) 180 | 181 | trans_df['open_cost'] = trans_df['filled_vol'] * trans_df['open_price'] * multiplier * margin_rate 182 | trans_df['return_ratio'] = (trans_df['return'] / trans_df['open_cost']) * 10000 183 | trans_df['return_cut'] = pd.cut(trans_df['return_ratio'], 5) 184 | group_cnt = trans_df[trans_df.holding_time > 0].groupby('return_cut')[['idx']].count() 185 | total_trans_num = len(group_cnt) 186 | group_ratio = [item / sum(group_cnt['idx']) * 100 for item in group_cnt['idx']] 187 | # trans_df['return_count'] = list(group_cnt['idx']) 188 | _idx_lst = list(range(len(group_cnt))) 189 | xtick_labels = group_cnt.index 190 | fig = plt.figure() 191 | ax1 = fig.add_subplot(111) 192 | ax1.bar(_idx_lst, group_ratio) 193 | ax1.set_xticks(_idx_lst[::1]) 194 | plt.xticks(rotation=75) 195 | plt.subplots_adjust(left=0.1, right=0.95, top=0.95, bottom=0.3) 196 | min_lst = [] 197 | ax1.set_xticklabels(xtick_labels[::1]) 198 | _bt_evaluation_path = utils.get_path( 199 | [define.RESULT_DIR, define.BT_DIR, '{0}_{1}_{2}.jpg'.format(product_id, start_date, end_date)]) 200 | plt.savefig(_bt_evaluation_path) 201 | _trans_path = utils.get_path( 202 | [define.RESULT_DIR, define.BT_DIR, 'trans_{0}_{1}_{2}.csv'.format(product_id, start_date, end_date)]) 203 | trans_df.to_csv(_trans_path, index=False) 204 | print(total_return, total_fee, precision) 205 | 206 | 207 | if __name__ == "__main__": 208 | # data_visialize() 209 | start_date = '20210701' 210 | bt_start_date = '20210707' 211 | end_date = '20210709' 212 | product_id = 'm' 213 | # train_models(start_date=start_date, end_date=end_date, product_id=product_id) 214 | backtest(start_date=bt_start_date, end_date=end_date, product_id=product_id, strategy_name='ClfSignal') 215 | # data_visialize() 216 | -------------------------------------------------------------------------------- /research/tick_models/FactorProcess.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2021/10/25 14:11 4 | # @Author : rpyxqi@gmail.com 5 | # @Site : 6 | # @File : FactorProcess.py 7 | 8 | import math 9 | import numpy as np 10 | import talib as ta 11 | import pandas as pd 12 | import logging 13 | import os 14 | import matplotlib.pyplot as plt 15 | 16 | import utils.define as define 17 | import utils.utils as utils 18 | 19 | logging.basicConfig(filename='logs/{0}.txt'.format(os.path.split(__file__)[-1].split('.')[0]), level=logging.DEBUG) 20 | logger = logging.getLogger() 21 | 22 | 23 | def _normalized_by_base(x: list, normalized_dict: dict, key: str) -> list: 24 | """ 25 | 26 | :param x: 27 | :param normalized_dict: 28 | :param key: 29 | :return: 30 | """ 31 | _normalized_base = normalized_dict.get(key) 32 | if not _normalized_base: 33 | return [] 34 | else: 35 | try: 36 | return [float(item) / _normalized_base for item in x] 37 | except Exception as ex: 38 | print('error for norm cal for col:{0} with error:{1}'.format(key, ex)) 39 | return [] 40 | 41 | 42 | def _cal_turning_item(x: list) -> tuple: 43 | if not x: 44 | return 0, np.nan 45 | if len(x) == 1: 46 | return 0, np.nan 47 | if len(x) == 2: 48 | return -1, x[0] 49 | try: 50 | if (x[-1] - x[-2]) * (x[-2] - x[-3]) > 0: 51 | return -2, x[0] 52 | else: 53 | return -1, x[1] 54 | except Exception as ex: 55 | raise ValueError("Error to cal turning with error:{0}".format(ex)) 56 | 57 | 58 | def log_return(x): 59 | return np.log(x).diff() 60 | 61 | 62 | def realized_volatility(x): 63 | return np.sqrt(np.sum(x ** 2)) 64 | 65 | 66 | def count_unique(x): 67 | return len(np.unique(x)) 68 | 69 | 70 | def cal_turning(x: list) -> tuple: 71 | idx_lst = [] 72 | val_lst = [] 73 | if len(x) == 1: 74 | idx_lst.append(0) 75 | val_lst.append(np.nan) 76 | elif len(x) == 2: 77 | _idx, _val = _cal_turning_item(x) 78 | return [0], [_val] 79 | else: 80 | _len = len(x) 81 | idx_lst.append(0) 82 | val_lst.append(np.nan) 83 | idx_lst.append(0) 84 | val_lst.append(x[0]) 85 | _idx, _val = _cal_turning_item(x) 86 | idx_lst.append(2 + _idx) 87 | val_lst.append(_val) 88 | for idx in range(3, _len, 1): 89 | _idx, _val = _cal_turning_item(x[idx - 3:idx]) 90 | idx_lst.append(idx + _idx) 91 | val_lst.append(_val) 92 | return idx_lst, val_lst 93 | 94 | 95 | def cal_slope(x: list, turn_idx: list, turn_val: list) -> list: 96 | assert len(x) == len(turn_idx) 97 | assert len(x) == len(turn_val) 98 | ret = [] 99 | for idx, item in enumerate(x): 100 | if idx == 0: 101 | ret.append(np.nan) 102 | continue 103 | ret.append((x[idx] - turn_val[idx]) / (idx - turn_idx[idx])) 104 | return ret 105 | 106 | 107 | def cal_cos(x: list, turn_idx: list, turn_val: list) -> list: 108 | assert len(x) == len(turn_idx) 109 | assert len(x) == len(turn_val) 110 | ret = [] 111 | for idx, item in enumerate(x): 112 | if idx == 0: 113 | ret.append(0) 114 | continue 115 | a = np.array([idx - turn_idx[idx], item - turn_val[idx]]) 116 | b = np.array([0, 1]) 117 | ret.append(a.dot(b) / (np.linalg.norm(a) * np.linalg.norm(b))) 118 | return ret 119 | 120 | 121 | def cal_oir(bid_price: list, bid_vol: list, ask_price: list, ask_vol: list, n_rows: int) -> tuple: 122 | """ 123 | calculate order imbalance factors 124 | :param bid_price: 125 | :param bid_vol: 126 | :param ask_price: 127 | :param ask_vol: 128 | :param n_rows: 129 | :return: 130 | """ 131 | v_b = [0] 132 | v_a = [0] 133 | 134 | for i in range(1, n_rows): 135 | v_b.append( 136 | 0 if bid_price[i] < bid_price[i - 1] else bid_vol[i] - bid_vol[i - 1] if bid_price[i] == bid_price[ 137 | i - 1] else bid_vol[i]) 138 | 139 | v_a.append( 140 | 0 if ask_price[i] < ask_price[i - 1] else ask_vol[i] - ask_vol[i - 1] if ask_price[i] == ask_price[ 141 | i - 1] else ask_vol[i]) 142 | lst_oi = [] 143 | lst_oir = [] 144 | lst_aoi = [] 145 | for idx, item in enumerate(v_a): 146 | lst_oi.append(v_b[idx] - item) 147 | lst_oir.append((v_b[idx] - item) / (v_b[idx] + item) if v_b[idx] + item != 0 else 0.0) 148 | lst_aoi.append((v_b[idx] - item) / (ask_price[idx] - bid_price[idx])) 149 | return lst_oi, lst_oir, lst_aoi 150 | 151 | 152 | def plot_factor(x=[], y=[], labels=[], tick_step=120): 153 | fig = plt.figure() 154 | ax1 = fig.add_subplot(111) 155 | ax1.plot(x, label="factor") 156 | ax1.legend() 157 | 158 | ax2 = ax1.twinx() 159 | ax2.plot(y, 'r', label='log return') 160 | ax2.legend() 161 | if labels: 162 | _idx_lst = list(range(len(x))) 163 | ax1.set_xticks(_idx_lst[::120]) 164 | trade_date_str = [item.replace('-', '') for item in labels] 165 | ax1.set_xticklabels(trade_date_str[::tick_step], rotation=30) 166 | plt.show() 167 | 168 | 169 | def get_factor(trade_date: str = "20210701", predict_windows: list = [1200], lag_windows: list = [10, 600], 170 | instrument_id: str = '', exchange_cd: str = '') -> pd.DataFrame: 171 | """ 172 | 173 | :param trade_date: 174 | :param predict_windows: 175 | :param lag_long: 176 | :param lag_short: 177 | :param stop_profit: 178 | :param stop_loss: 179 | :param instrument_id: 180 | :param exchange_cd: 181 | :return: 182 | """ 183 | _mul_num = utils.get_mul_num(instrument_id) or 1 184 | print(instrument_id, trade_date, define.exchange_map, exchange_cd, define.TICK_MODEL_DIR) 185 | _tick_mkt_path = os.path.join(define.TICK_MKT_DIR, define.exchange_map.get(exchange_cd), 186 | '{0}_{1}.csv'.format(instrument_id, trade_date.replace('-', ''))) 187 | tick_mkt = pd.read_csv(_tick_mkt_path, encoding='gbk') 188 | tick_mkt.columns = define.tb_cols 189 | 190 | # filter trading timestamp 191 | _flag = [utils._is_trading_time(item.split()[-1]) for item in list(tick_mkt['UpdateTime'])] 192 | tick_mkt = tick_mkt[_flag] 193 | 194 | # for price/volume, if not define the normorlized base or defined as 0, we will ret it as the first tick 195 | open_dict = tick_mkt.head(1).to_dict('record')[0] 196 | 197 | tick_mkt['vwap'] = (tick_mkt['Turnover'] / tick_mkt['Volume']) / _mul_num 198 | tick_mkt['wap'] = (tick_mkt['BidPrice1'] * tick_mkt['AskVolume1'] + tick_mkt['AskPrice1'] * tick_mkt[ 199 | 'BidVolume1']) / (tick_mkt['AskVolume1'] + tick_mkt['BidVolume1']) 200 | # tick_mkt['log_return'] = tick_mkt['LastPrice'].rolling(2).apply(lambda x: math.log(list(x)[-1] / list(x)[0])) 201 | tick_mkt['log_return'] = np.log(tick_mkt['LastPrice']).diff() 202 | tick_mkt['wap_log_return'] = np.log(tick_mkt['wap']) - np.log(tick_mkt['LastPrice']) 203 | 204 | # tick_mkt['log_return'] = tick_mkt['wap'].rolling(2).apply(lambda x: math.log(list(x)[-1] / list(x)[0])) 205 | 206 | # tick_mkt['label_reg'] = tick_mkt['log_return'].rolling(predict_windows).sum().shift(1 - predict_windows) 207 | # lst_ret_max = list(tick_mkt['label_reg'].rolling(predict_windows).max()) 208 | # lst_ret_min = list(tick_mkt['label_reg'].rolling(predict_windows).min()) 209 | # 1: profit for long;2: profit for short; 0: not profit for the predict_windows, based on stop_profit threshold 210 | # tick_mkt['label_clf'] = [1 if item > math.log(1 + stop_profit) else 2 if item < math.log(1 - stop_profit) else 0 for 211 | # item in 212 | # tick_mkt['label_reg']] 213 | sum_max = [] 214 | sum_min = [] 215 | 216 | _x, _y = tick_mkt.shape 217 | _diff = list(tick_mkt['InterestDiff']) 218 | _vol = list(tick_mkt['Volume']) 219 | 220 | open_close_ratio = [] 221 | for idx, item in enumerate(_diff): 222 | try: 223 | open_close_ratio.append(item / (_vol[idx] - item)) 224 | except Exception as ex: 225 | open_close_ratio.append(open_close_ratio[-1]) 226 | 227 | # cal ori factor(paper factor) 228 | lst_oi, lst_oir, lst_aoi = cal_oir(list(tick_mkt['BidPrice1']), list(tick_mkt['BidVolume1']), 229 | list(tick_mkt['AskPrice1']), list(tick_mkt['AskVolume1']), n_rows=_x) 230 | 231 | # cal cos factor(market factor) 232 | _lst_last_price = list(tick_mkt['LastPrice']) 233 | _lst_turn_idx, _lst_turn_val = cal_turning(_lst_last_price) 234 | _lst_slope = cal_slope(_lst_last_price, _lst_turn_idx, _lst_turn_val) 235 | _lst_cos = cal_cos(_lst_last_price, _lst_turn_idx, _lst_turn_val) 236 | 237 | dif, dea, macd = ta.MACD(tick_mkt['LastPrice'], fastperiod=12, slowperiod=26, signalperiod=9) 238 | 239 | tick_mkt['open_close_ratio'] = open_close_ratio 240 | tick_mkt['price_spread'] = (tick_mkt['BidPrice1'] - tick_mkt['AskPrice1']) / ( 241 | tick_mkt['BidPrice1'] + tick_mkt['AskPrice1'] / 2) 242 | tick_mkt['buy_sell_spread'] = abs(tick_mkt['BidPrice1'] - tick_mkt['AskPrice1']) 243 | tick_mkt['oi'] = lst_oi 244 | tick_mkt['oir'] = lst_oir 245 | tick_mkt['aoi'] = lst_aoi 246 | tick_mkt['slope'] = _lst_slope 247 | tick_mkt['cos'] = _lst_cos 248 | tick_mkt['macd'] = macd 249 | tick_mkt['dif'] = dif 250 | tick_mkt['dea'] = dea 251 | tick_mkt['bs_tag'] = tick_mkt['LastPrice'].rolling(2).apply(lambda x: 1 if list(x)[-1] > list(x)[0] else -1) 252 | tick_mkt['bs_vol'] = tick_mkt['bs_tag'] * tick_mkt['Volume'] 253 | 254 | def _cal_clf_label(x): 255 | # FIXME remove clf bc hardcode, log(1.001)= 0.0009988(0.1%), log(0.999)=-0.001(0.1%) 256 | # return [1 if item > 0.001 else 2 if item < -0.001 else 0 for item in x] 257 | # return [1 if item > 2 else 2 if item < -2 else 0 for item in x] 258 | # ret = [] 259 | # for item in x: 260 | # if item > 5: 261 | # ret.append(2) 262 | # elif item < 5 and item >= 2: 263 | # ret.append(1) 264 | # elif item < 2 and item > -2: 265 | # ret.append(0) 266 | # elif item <= -2 and item > -5: 267 | # ret.append(-1) 268 | # else: 269 | # ret.append(-2) 270 | # return ret 271 | return [1 if item > 2 else -1 if item < -2 else 0 for item in x] 272 | 273 | # agg by windows,agg_dict={factor_name:(executor, new_factor_name)}, if None for new_factor_name, then use the 274 | # raw name by default 275 | windows_agg_dict = { 276 | 'LastPrice': [(lambda x: (list(x)[-1] - list(x)[0]) / len(x), 'trend')], 277 | 'Volume': [(np.sum, 'volume')], 278 | 'Turnover': [(np.sum, 'turnover')], 279 | 'bs_vol': [(np.sum, None)], 280 | 'log_return': [(np.sum, None)], 281 | } 282 | for idx, _lag in enumerate(lag_windows): 283 | for key, val in windows_agg_dict.items(): 284 | for _func, rename in val: 285 | factor_name = rename or key 286 | tick_mkt['{0}_{1}'.format(factor_name, idx)] = list(tick_mkt[key].rolling(_lag).apply(_func)) 287 | 288 | tick_mkt['bsvol_volume_{0}'.format(idx)] = tick_mkt['bs_vol_{0}'.format(idx)] / tick_mkt[ 289 | 'volume_{0}'.format(idx)] 290 | 291 | for idx, _pred in enumerate(predict_windows): 292 | tick_mkt['label_{0}'.format(idx)] = tick_mkt['log_return'].rolling(_pred).sum().shift(1 - _pred) 293 | tick_mkt['rv_{0}'.format(idx)] = tick_mkt['log_return'].rolling(_pred).apply(realized_volatility).shift( 294 | 1 - _pred) 295 | # tick_mkt['label_clf_{0}'.format(idx)] = _cal_clf_label(tick_mkt['label_{0}'.format(idx)]) 296 | 297 | tick_mkt['price_chg_{0}'.format(idx)] = tick_mkt['LastPrice'].rolling(_pred).apply( 298 | lambda x: list(x)[-1] - list(x)[0]).shift(1 - _pred) 299 | tick_mkt['vwap_chg_{0}'.format(idx)] = tick_mkt['LastPrice'].rolling(_pred).apply( 300 | lambda x: x.mean() - list(x)[0]).shift(1 - _pred) 301 | tick_mkt['label_clf_{0}'.format(idx)] = _cal_clf_label(tick_mkt['vwap_chg_{0}'.format(idx)]) 302 | 303 | # tick_mkt['log_return_short'] = tick_mkt['log_return'].rolling(lag_short).sum() 304 | # tick_mkt['log_return_long'] = tick_mkt['log_return'].rolling(lag_long).sum() 305 | # tick_mkt['trend_short'] = tick_mkt['LastPrice'].rolling(lag_short).apply( 306 | # lambda x: (list(x)[-1] - list(x)[0]) / lag_short) 307 | # tick_mkt['trend_long'] = tick_mkt['LastPrice'].rolling(lag_long).apply( 308 | # lambda x: (list(x)[-1] - list(x)[0]) / lag_long) 309 | # tick_mkt['vol_short'] = tick_mkt['Volume'].rolling(lag_short).sum() 310 | # tick_mkt['vol_long'] = tick_mkt['Volume'].rolling(lag_long).sum() 311 | # tick_mkt['turnover_short'] = tick_mkt['Turnover'].rolling(lag_short).sum() 312 | # tick_mkt['turnover_long'] = tick_mkt['Turnover'].rolling(lag_long).sum() 313 | # tick_mkt['bs_vol_long'] = tick_mkt['bs_vol'].rolling(lag_long).sum() 314 | # tick_mkt['bs_vol_short'] = tick_mkt['bs_vol'].rolling(lag_short).sum() 315 | 316 | # agg by factor,remove vwap 317 | factor_agg_dict = {'trend': ['minus', 'divide'], 'volume': ['minus', 'divide'], 'turnover': ['minus', 'divide'], 318 | 'bs_vol': ['minus', 'divide'], 'bsvol_volume': ['divide']} 319 | lag_win_size = len(lag_windows) 320 | if lag_win_size >= 2: 321 | for key, val in factor_agg_dict.items(): 322 | for _func_name in val: 323 | if _func_name == 'minus': 324 | tick_mkt['{0}_ls_diff'.format(key)] = tick_mkt['{0}_{1}'.format(key, 0)] - tick_mkt[ 325 | '{0}_{1}'.format(key, lag_win_size - 1)] 326 | if _func_name == 'divide': 327 | tick_mkt['{0}_ls_ratio'.format(key)] = tick_mkt['{0}_{1}'.format(key, 0)] / tick_mkt[ 328 | '{0}_{1}'.format(key, lag_win_size - 1)] 329 | 330 | # tick_mkt['trenddiff'] = tick_mkt['trend_short'] - tick_mkt['trend_long'] 331 | # tick_mkt['trend_ls_ratio'] = tick_mkt['trend_short'] / tick_mkt['trend_long'] 332 | # tick_mkt['vol_ls_ratio'] = tick_mkt['vol_short'] / tick_mkt['vol_long'] 333 | # tick_mkt['turnover_ls_ratio'] = tick_mkt['turnover_short'] / tick_mkt['turnover_long'] 334 | # tick_mkt['vwap_short'] = tick_mkt['turnover_short'] / tick_mkt['vol_short'] 335 | # tick_mkt['vwap_long'] = tick_mkt['turnover_long'] / tick_mkt['vol_long'] 336 | # tick_mkt['vwap_ls_ratio'] = tick_mkt['vwap_short'] / tick_mkt['vwap_long'] 337 | # tick_mkt['bs_vol_diff'] = tick_mkt['bs_vol_short'] - tick_mkt['bs_vol_long'] 338 | # tick_mkt['bs_vol_ls_ratio'] = tick_mkt['bs_vol_short'] / tick_mkt['bs_vol_long'] 339 | 340 | # TODO ADD norm factor 341 | # norm factor processing, calculate the norm here or in model processing. norm with the value of the current date 342 | # or summary of the prev date 343 | normalized = define.normalized_vals 344 | # TODO open_dict is the open value of the current date, this could be normalized by other ref,e.g. the prev date 345 | for k, v in normalized.items(): 346 | if not v: 347 | normalized.update({k: (open_dict.get(k) or 1.0)}) 348 | for _base_col, _ref_col in define.normalized_refs: 349 | _arr = _normalized_by_base(tick_mkt[_base_col], normalized, _ref_col) 350 | if not _arr: # FIXME 351 | continue 352 | tick_mkt['norm_{0}'.format(_base_col.lower())] = _arr 353 | 354 | _factor_lst = list(tick_mkt.columns) 355 | print(_factor_lst) 356 | 357 | for col in define.skip_raw_cols: 358 | try: 359 | _factor_lst.remove(col) 360 | except Exception as ex: 361 | print('col:{0} not exist with error:{1}'.format(col, ex)) 362 | 363 | # FIXME replace this with better solution 364 | for idx, _lag in enumerate(lag_windows): 365 | _factor_lst.remove('volume_{0}'.format(idx)) 366 | _factor_lst.remove('turnover_{0}'.format(idx)) 367 | _factor_lst.remove('bs_vol_{0}'.format(idx)) 368 | 369 | # handle exception values 370 | tick_mkt = tick_mkt.replace(np.inf, np.nan) 371 | tick_mkt = tick_mkt.replace(-np.inf, np.nan) 372 | tick_mkt = tick_mkt.dropna() 373 | _factor_path = utils.get_path([define.CACHE_DIR, define.FACTOR_DIR, 374 | 'factor_{0}_{1}.csv'.format(instrument_id, trade_date.replace('-', ''))]) 375 | tick_mkt[_factor_lst].to_csv(_factor_path, index=False) 376 | return tick_mkt[_factor_lst] 377 | -------------------------------------------------------------------------------- /backtester/Factor.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2021/7/26 14:03 4 | # @Author : rpyxqi@gmail.com 5 | # @Site : 6 | # @File : Factor.py 7 | 8 | import numpy as np 9 | import pandas 10 | import math 11 | import utils.utils as utils 12 | import utils.define as define 13 | import collections 14 | import copy 15 | 16 | 17 | class Factor(object): 18 | def __init__(self, product_id='m', instrument_id='', trade_date='', long_windows=600, short_windows=120): 19 | # same variables in live 20 | self.last_price = collections.deque(long_windows) # xxx_circular_ptr 21 | self.mid_price = collections.deque(long_windows) 22 | self.spread = collections.deque(long_windows) 23 | self.buy_vol = collections.deque(long_windows) 24 | self.sell_vol = collections.deque(long_windows) 25 | self.turnover = collections.deque(long_windows) 26 | self.vol = collections.deque(long_windows) 27 | self.long_windows = long_windows 28 | self.short_windows = short_windows 29 | self.curr_factor = [] 30 | self.last_factor = [] 31 | self.last_mkt = [] 32 | 33 | self.product_id = product_id 34 | self.instrument_id = instrument_id 35 | self.trade_date = trade_date 36 | 37 | self.vwap = [] 38 | self.upper_bound = [] 39 | self.lower_bound = [] 40 | self.open_turnover = -np.inf 41 | self.open_vol = -np.inf 42 | self.product_id = product_id 43 | self.update_time = [] 44 | self.turning = [] 45 | self.turning_idx = [] 46 | self.slope = [] 47 | # this only calculate by time stamp 48 | 49 | self.b2s_vol = [] 50 | self.b2s_turnover = [] 51 | self.turnover = [] 52 | self.price_highest = [] 53 | self.price_lowest = [] 54 | self.ma10 = [] 55 | self.ma20 = [] 56 | self.ma60 = [] 57 | self.ma120 = [] 58 | self.mid_price = [] 59 | self.trend_ls_ratio = [] 60 | self.cos = [] 61 | self.trend_long = [] 62 | self.trend_short = [] 63 | self.log_return = [] 64 | self.log_return10 = [] 65 | self.log_return20 = [] 66 | self.log_return40 = [] 67 | self.log_return60 = [] 68 | self.mid_log_return = [] 69 | self.vwap_ls_diff = [] 70 | 71 | def update_factor(self, tick=[], *args, **kwa): 72 | ''' 73 | 74 | :param tick: live的depth market 75 | :param args: 76 | :param kwa: 77 | :return: 78 | ''' 79 | last_price = tick[3] 80 | vol = tick[7] # 当前成交量，和live 不同 81 | turnover = tick[6] # 当前成交额，和live不同 82 | _update_time = tick[2] 83 | bid_price1, ask_price1, bid_vol1, ask_vol1 = tick[-4:] 84 | 85 | # same var with live 86 | _curr_last = tick[3] 87 | _curr_vol = tick[7] # 当前成交量，和CTP live不同 88 | _curr_interest = tick[4] # 当前持仓，和CTP live 不同 89 | _curr_turnover = tick[6] # 当前成交额，和CTP live不同 90 | _prev_max = _curr_last 91 | _prev_min = _curr_last 92 | _prev_vwap = _curr_last 93 | _log_return = 0.0 94 | _mid_log_return = 0.0 95 | _log_return_long = 0.0 96 | _log_return_short = 0.0 97 | _mid_log_return_short = 0.0 98 | _mid_log_return_long = 0.0 99 | _ma_short = 0.0 100 | _ma_long = 0.0 101 | _ma_ls_diff_last = 0.0 102 | _ma_ls_diff_curr = 0.0 103 | # already included in the hist data 104 | tick_type = tick[10] 105 | tick_direction = tick[11] 106 | _vol_buy = 0.0 107 | _vol_sell = 0.0 108 | _interest_open = 0.0 109 | _interest_close = 0.0 110 | _turnover_long = 0.0 111 | _turnover_short = 0.0 112 | _vol_long = 0.0 113 | _vol_short = 0.0 114 | 115 | _curr_spread = ask_price1 - bid_price1 116 | _curr_vwap = turnover / vol # FIXME check the calculation with live 117 | _curr_mid = (ask_price1 * bid_vol1 + bid_price1 * ask_vol1) / (ask_vol1 + bid_vol1) 118 | 119 | # update from last factor, start from the 2nd tick 120 | if len(self.last_factor) > 0: 121 | _prev_max = self.last_factor[1] 122 | _prev_min = self.last_factor[2] 123 | _prev_vwap = self.last_factor[5] 124 | _log_return = math.log(_curr_last) - math.log(self.last_factor[0]) 125 | _mid_log_return = math.log(_curr_mid) - math.log(self.last_factor[4]) 126 | 127 | cir_size = len(self.last_price) 128 | _curr_max = max(_prev_max, _curr_last) 129 | _curr_min = min(_prev_min, _curr_last) 130 | 131 | if len(self.last_mkt) > 0: 132 | pass 133 | 134 | if cir_size >= self.long_windows: 135 | _log_return_long = self.last_factor[8] + _log_return - math.log(self.last_price[1]) - math.log( 136 | self.last_price[1]) 137 | _mid_log_return_long = self.last_factor[10] + _mid_log_return - math.log(self.mid_price[1]) - math.log( 138 | self.mid_price[1]) 139 | _log_return_short = self.last_factor[7] + _log_return - math.log(self.last_price[1]) - math.log( 140 | self.last_price[0]) 141 | _mid_log_return_short = self.last_factor[9] + _mid_log_return - math.log(self.mid_price[1]) - math.log( 142 | self.mid_price[0]) 143 | _turnover_long = sum(self.turnover[-self.long_windows:]) # TODO different from live 144 | _turnover_short = sum(self.turnover[-self.short_windows:]) # TODO 145 | _vol_long = sum(self.vol[-self.long_windows:]) 146 | _vol_short = sum(self.vol[-self.short_windows:]) 147 | _ma_long = _turnover_long / _vol_long 148 | _ma_short = _turnover_short / _vol_short 149 | _ma_ls_diff_curr = _ma_short - _ma_long 150 | _ma_ls_diff_last = self.last_factor[11] - self.last_factor[12] 151 | elif cir_size >= self.short_windows: 152 | _ma_long = _curr_vwap 153 | _log_return_short = self.last_factor[7] + _log_return - math.log(self.last_price[1]) - math.log( 154 | self.last_price[0]) 155 | 156 | _mid_log_return_short = self.last_factor[9] + _mid_log_return - math.log(self.mid_price[1]) - math.log( 157 | self.mid_price[0]) 158 | _ma_short = _curr_vwap 159 | else: 160 | _ma_long = _curr_vwap 161 | _ma_short = _curr_vwap 162 | 163 | # time series cached factor 164 | self.last_price.append(_curr_last) 165 | self.spread.append(_curr_spread) 166 | self.mid_price.append(_curr_mid) 167 | self.turnover.append(_curr_turnover) 168 | self.vol.append(_curr_vol) 169 | 170 | ret_factor = copy.deepcopy(self.curr_factor) 171 | # currentfactorupdate 172 | self.curr_factor.clear() 173 | self.curr_factor.append(_curr_last) # factor vector:0 174 | self.curr_factor.append(_curr_max) # vector: 1;factor: 5 175 | self.curr_factor.append(_curr_min) # vector: 2;factor: 6 176 | self.curr_factor.append(_curr_spread) # vector: 3;factor: 7 177 | self.curr_factor.append(_curr_mid) # vector: 4;factor: 8 178 | self.curr_factor.append(_curr_vwap) # vector: 5;factor: 9 179 | self.curr_factor.append(_log_return) # vector: 6;factor: 10 180 | self.curr_factor.append(_log_return_short) # vector: 7;factor: 11 181 | self.curr_factor.append(_log_return_long) # vector: 8;factor: 12 182 | self.curr_factor.append(_mid_log_return_short) # vector: 9;factor: 13 183 | self.curr_factor.append(_mid_log_return_long) # vector: 10;factor: 14 184 | self.curr_factor.append(_ma_short) # vector: 11; factor: 15 185 | self.curr_factor.append(_ma_long) # // vector: 12; factor: 16 186 | self.curr_factor.append(_curr_vol) # vector: 13;factor: 17 187 | self.curr_factor.append(_curr_interest) # vector14; factor: 18 188 | 189 | self.last_factor.clear() 190 | for item in self.curr_factor: 191 | self.last_factor.append(item) 192 | self.last_mkt.clear() 193 | self.last_mkt.append(ask_price1) # 0: askprice 1 194 | self.last_mkt.append(bid_price1) # 1:bidprice 1 195 | self.last_mkt.append(_curr_vol) # 2 vol TODO VOL is different 196 | self.last_mkt.append(_curr_interest) # 3, TODO interest is diff 197 | self.last_mkt.append(_vol_buy) 198 | self.last_mkt.append(_vol_sell) 199 | self.last_mkt.append(_curr_interest) # TODO interest is diff 200 | return ret_factor 201 | 202 | # @timeit 203 | def update_factor_bt(self, tick=[], *args, **kwargs): 204 | # values from tick 205 | last_price = tick[3] 206 | vol = tick[7] 207 | turnover = tick[6] 208 | _update_time = tick[2] 209 | bid_price1, ask_price1, bid_vol1, ask_vol1 = tick[-4:] 210 | 211 | _mid = (bid_price1 * ask_vol1 + ask_price1 * bid_vol1) / (ask_vol1 + bid_vol1) 212 | self.mid_price.append(_mid) 213 | _range = kwargs.get('range') or 20 214 | k1 = kwargs.get('k1') or 0.2 215 | k2 = kwargs.get('k2') or 0.2 216 | idx = kwargs.get('idx') or 0 217 | lag_long = kwargs.get('lag_long') or 60 218 | lag_short = kwargs.get('lag_short') or 20 219 | _mul_num = kwargs.get('multiplier') or 1.0 220 | self.open_turnover = turnover if self.open_turnover < 0 else self.open_turnover 221 | self.open_vol = vol if self.open_vol < 0 else self.open_vol 222 | self.last_price.append(last_price) 223 | self.update_time.append(_update_time) 224 | # "BidPrice1", "BidVolume1", "AskPrice1", "AskVolume1", 225 | 226 | self.spread.append(ask_price1 - bid_price1) 227 | self.b2s_vol.append(bid_vol1 / ask_vol1 if ask_vol1 > 0 else 1.0) 228 | self.b2s_turnover.append( 229 | ((bid_vol1 * bid_price1) / (ask_vol1 * ask_price1) if ask_vol1 * ask_price1 > 0 else 1.0)) 230 | # self.price_highest.append(max(self.last_price)) 231 | # self.price_lowest.append(min(self.last_price)) 232 | _vwap_val = None 233 | try: 234 | # if _update_time >= '21:00:00': 235 | _tmp_vol = vol 236 | _tmp_turnover = turnover 237 | self.vol.append(_tmp_vol) 238 | self.turnover.append(_tmp_turnover) 239 | _vwap_val = (_tmp_turnover) / (_mul_num * (_tmp_vol)) 240 | 241 | except Exception as ex: 242 | if self.vwap: 243 | _vwap_val = self.vwap[-1] 244 | else: 245 | _vwap_val = last_price 246 | self.vwap.append(_vwap_val) 247 | self.upper_bound.append(_vwap_val + k1 * _range) 248 | self.lower_bound.append(_vwap_val - k2 * _range) 249 | 250 | # ma for fixed length 251 | self.ma10.append(sum(self.last_price[-10:]) / len(self.last_price[-10:])) 252 | self.ma20.append(sum(self.last_price[-20:]) / len(self.last_price[-20:])) 253 | self.ma60.append(sum(self.last_price[-60:]) / len(self.last_price[-60:])) 254 | self.ma120.append(sum(self.last_price[-120:]) / len(self.last_price[-120:])) 255 | 256 | if len(self.turning) <= 1: 257 | self.turning.append(last_price) 258 | self.turning_idx.append(idx) 259 | else: 260 | _last_turn = self.turning[-1] 261 | _last_turn1 = self.turning[-2] 262 | _last_price = self.last_price[-2] 263 | _last_price1 = self.last_price[-3] 264 | if (last_price - _last_price) * (_last_price - _last_price1) <= 0: # new turning 265 | self.turning.append(_last_price) 266 | self.turning_idx.append(idx - 1) 267 | else: # last turning 268 | self.turning.append(_last_turn) 269 | self.turning_idx.append(self.turning_idx[-1]) 270 | # print(self.turning) 271 | if not self.slope: 272 | self.slope.append(0.5) 273 | else: 274 | if idx == self.turning_idx[-1]: 275 | self.slope.append(self.slope[-1]) 276 | else: 277 | self.slope.append((last_price - self.turning[-1]) / (idx - self.turning_idx[-1])) 278 | 279 | # factor for lag windows, 280 | try: 281 | if len(self.last_price) >= lag_long: # long windows 282 | _trend_long = (self.last_price[-1] - self.last_price[-lag_long]) / lag_long 283 | _trend_short = (self.last_price[-1] - self.last_price[-lag_short]) / lag_short 284 | self.trend_long.append(_trend_long) 285 | self.trend_short.append(_trend_short) 286 | self.trend_ls_ratio.append( 287 | self.trend_short[-1] / self.trend_long[-1] if self.trend_long[-1] != 0 else np.nan) 288 | 289 | _ls_diff = sum(self.turnover[lag_long:]) / sum(self.vol[lag_long:]) - sum( 290 | self.turnover[lag_short:]) / sum( 291 | self.vol[lag_short:]) 292 | # _ls_diff = np.array(self.last_price[lag_long:]).mean() - np.array(self.last_price[lag_short:]).mean() 293 | self.vwap_ls_diff.append(_ls_diff) 294 | 295 | 296 | elif len(self.last_price) >= lag_short: # short windows 297 | _trend_short = (self.last_price[-1] - self.last_price[-lag_short]) / lag_short 298 | self.trend_short.append(_trend_short) 299 | self.trend_long.append(np.nan) 300 | self.trend_ls_ratio.append(np.nan) 301 | self.vwap_ls_diff.append(np.nan) 302 | 303 | else: 304 | self.trend_long.append(np.nan) 305 | self.trend_short.append(np.nan) 306 | self.trend_ls_ratio.append(np.nan) 307 | self.vwap_ls_diff.append(np.nan) 308 | 309 | except Exception as ex: 310 | print(ex) 311 | 312 | a = np.array([idx - self.turning_idx[-1], self.last_price[-1] - self.last_price[self.turning_idx[-1]]]) 313 | b = np.array([0, 1]) 314 | self.cos.append(a.dot(b) / (np.linalg.norm(a) * np.linalg.norm(b))) 315 | if len(self.last_price) >= 2: 316 | self.log_return.append(math.log(self.last_price[-1]) - math.log(self.last_price[-2])) 317 | else: 318 | self.log_return.append(np.nan) 319 | if len(self.mid_price) >= 2: 320 | self.mid_log_return.append(math.log(self.mid_price[-1]) - math.log(self.mid_price[-2])) 321 | else: 322 | self.mid_log_return.append(np.nan) 323 | if len(self.log_return) > 10: 324 | self.log_return10.append(sum(self.log_return[-10:])) 325 | else: 326 | self.log_return10.append(np.nan) 327 | if len(self.last_price) != len(self.trend_long): 328 | print('check') 329 | 330 | def load_factor(self): 331 | pass 332 | 333 | def get_factor(self): 334 | return {'slope': self.slope[-1], 335 | 'vwap': self.vwap[-1], 336 | 'spread': self.spread[-1], 337 | 'b2s_vol': self.b2s_vol[-1], 338 | 'cos': self.cos[-1], 339 | 'trend_ls_ratio': self.trend_ls_ratio[-1], 340 | 'log_return': self.log_return[-1], 341 | 'log_return_0': self.log_return10[-1], 342 | 'wap_log_return': self.mid_log_return[-1], 343 | 344 | } 345 | 346 | def cache_factor(self): 347 | # factor_df = pandas.DataFrame({'last_price': self.last_price, 'vwap': self.vwap, 'upper_bound': self.upper_bound, 348 | # 'lower_bound': self.lower_bound, 'update_time': self.update_time, 349 | # 'turning': self.turning, 350 | # 'slope': self.slope, 'turning_idx': self.turning_idx, 'spread': self.spread, 351 | # 'vol_ratio': self.vol_ratio, 'price_highest': self.price_highest, 352 | # 'price_lowest': self.price_lowest, 353 | # 'curr_vol': self.curr_vol, 'curr_turnover': self.curr_turnover}) 354 | factor_df = pandas.DataFrame({'last_price': self.last_price, 'vwap': self.vwap, 'upper_bound': self.upper_bound, 355 | 'lower_bound': self.lower_bound, 'update_time': self.update_time, 356 | 'turning': self.turning, 357 | 'slope': self.slope, 'turning_idx': self.turning_idx, 'spread': self.spread, 358 | 'b2s_vol': self.b2s_vol, 359 | 'b2s_turnover': self.b2s_turnover, 'vol': self.vol, 360 | 'turnover': self.turnover, 361 | # 'price_high': self.price_highest, 362 | # 'price_low': self.price_lowest 363 | }) 364 | _factor_path = utils.get_path([define.CACHE_DIR, define.FACTOR_DIR, 365 | 'factor_{0}_{1}.csv'.format(self.instrument_id, self.trade_date)]) 366 | factor_df.to_csv(_factor_path, index=False) 367 | -------------------------------------------------------------------------------- /backtester/backtest.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | from collections import defaultdict, deque 3 | import os 4 | import pickle as pk 5 | 6 | from sqlalchemy import create_engine 7 | import numpy as np 8 | import pandas as pd 9 | from tqdm import tqdm 10 | 11 | import core 12 | import core.theo as T 13 | import core.fitting as F 14 | 15 | 16 | def get_daily_info(startdate, enddate): 17 | cache_file = os.path.join(os.sep.join(os.path.abspath(__file__).split(os.sep)[:-3]), 'data', 'stock_daily_info.csv') 18 | daily_info = pd.read_csv(cache_file, dtype={'date': str, 'symbol': str}) 19 | daily_info['refPrice'] = round((daily_info['price_limit_up'] + daily_info['price_limit_down']) / 2 * 100 - 0.1)/ 100 20 | daily_info = daily_info.set_index(['symbol', 'date']) 21 | return daily_info 22 | 23 | 24 | def cross_edge_notional_multilevel(book_with_valuation, symbol, posNotionals, edges, daily_info, use_ratio=True, start_extime=93200000000): 25 | results = defaultdict(lambda: defaultdict(dict)) 26 | for date, sub_df in sorted(book_with_valuation.groupby('date')): 27 | if (symbol, date) in daily_info.index: 28 | refPrice = daily_info.loc[(symbol, date), 'refPrice'] 29 | else: 30 | print('%s: %s not in daily info. skip.' % (date, symbol)) 31 | continue 32 | 33 | for posNotional in posNotionals: 34 | position_normal = int(posNotional / refPrice // 100 * 100) 35 | # print(date, '%d / %.2f = %d' % (posNotional, refPrice, position_normal)) 36 | if position_normal == 0: 37 | # print('posNormal = 0. Skip') 38 | continue 39 | 40 | if sub_df.shape[0] - sub_df.dropna().shape[0] > 50 or sub_df.dropna().shape[0] < 50: 41 | pass 42 | else: 43 | print(date, sub_df.dropna().shape[0], sub_df.shape[0]) 44 | book_events = sub_df.to_dict('records') 45 | for edge in edges: 46 | trades = [] 47 | buyprice = defaultdict(int) 48 | sellprice = defaultdict(int) 49 | cur_position = 0 50 | for event in book_events: 51 | if event['ExchangeTime'] < start_extime: 52 | continue 53 | if not np.isnan(event['value']) and event['Bid1'] != 0 and event['Ask1'] != 0: 54 | midpt = (event['Bid1'] + event['Ask1']) / 2 55 | fv = midpt + event['value'] 56 | for level in range(5): 57 | i = level + 1 58 | if (not use_ratio and (fv - event['Ask%d' % i]) > edge) or (use_ratio and (fv - event['Ask%d' % i]) / midpt > edge): 59 | new_buy_qty = min(max(0, event['Ask%dSize' % i] - buyprice[event['Ask%d' % i]]), position_normal - cur_position) 60 | if new_buy_qty > 0: 61 | trades.append({'dir': False, 'price': event['Ask%d' % i], 'qty': new_buy_qty, 'extime': event['ExchangeTime'], 'date': event['date'], 'TimeStamp': event['TimeStamp']}) 62 | buyprice[event['Ask%d' % i]] += new_buy_qty 63 | cur_position += new_buy_qty 64 | 65 | if (not use_ratio and (event['Bid%d' % i] - fv)) > edge or (use_ratio and (event['Bid%d' % i] - fv) / midpt > edge): 66 | new_sell_qty = min(max(0, event['Bid%dSize' % i] - sellprice[event['Bid%d' % i]]), cur_position + position_normal) 67 | if new_sell_qty: 68 | trades.append({'dir': True, 'price': event['Bid%d' % i], 'qty': new_sell_qty, 'extime': event['ExchangeTime'], 'date': event['date'], 'TimeStamp': event['TimeStamp']}) 69 | sellprice[event['Bid%d' % i]] += new_sell_qty 70 | cur_position -= new_sell_qty 71 | 72 | buyprice_new = defaultdict(int) 73 | sellprice_new = defaultdict(int) 74 | buyprice_new.update({k: v for k, v in buyprice.items() if k > event['Bid1']}) 75 | sellprice_new.update({k: v for k, v in sellprice.items() if k < event['Ask1']}) 76 | buyprice = buyprice_new 77 | sellprice = sellprice_new 78 | results[posNotional][edge][date] = trades 79 | 80 | return results 81 | 82 | 83 | 84 | def constraint_trades_time(trades, start_extime, end_extime): 85 | trades = trades.copy() 86 | for date in trades: 87 | tmp = trades[date] 88 | trades[date] = [v for v in tmp if v['extime'] >= start_extime and v['extime'] < end_extime] 89 | 90 | return trades 91 | 92 | 93 | def constraint_trades_maxvolume(trades_date, maxvolume): 94 | pos_total = 0 95 | neg_total = 0 96 | new_trades = [] 97 | for t in trades_date: 98 | if t['dir'] and t['qty'] and neg_total < maxvolume: 99 | new_trades.append(t.copy()) 100 | new_trades[-1]['qty'] = min(maxvolume - neg_total, new_trades[-1]['qty']) 101 | neg_total += new_trades[-1]['qty'] 102 | if not t['dir'] and t['qty'] and pos_total < maxvolume: 103 | new_trades.append(t.copy()) 104 | new_trades[-1]['qty'] = min(maxvolume - pos_total, new_trades[-1]['qty']) 105 | pos_total += new_trades[-1]['qty'] 106 | 107 | return new_trades 108 | 109 | 110 | def handle_match(fills, t): 111 | matches = [] 112 | t = t.copy() 113 | t['remain_qty'] = t['qty'] 114 | if not fills or (fills[0]['dir'] and t['dir']) or (not fills[0]['dir'] and not t['dir']): 115 | fills.append(t) 116 | else: 117 | while fills: 118 | fill = fills[0] 119 | match_qty = min(fill['remain_qty'], t['remain_qty']) 120 | fill['remain_qty'] -= match_qty 121 | t['remain_qty'] -= match_qty 122 | if fill['remain_qty'] == 0: 123 | fills.popleft() 124 | holdtime = t['TimeStamp'] - fill['TimeStamp'] 125 | # print(fill) 126 | # print(t) 127 | matches.append({'qty': match_qty, 128 | 'entry_dir': fill['dir'], 129 | 'entryprice': fill['price'], 130 | 'exitprice': t['price'], 131 | 'entrytime': datetime.fromtimestamp(fill['TimeStamp'] / 1e6), 132 | 'exittime': datetime.fromtimestamp(t['TimeStamp'] / 1e6)}) 133 | 134 | if t['remain_qty'] == 0: 135 | break 136 | 137 | if t['remain_qty'] > 0: 138 | fills.append(t) 139 | 140 | return matches 141 | 142 | 143 | def match(trades_date): 144 | matches = [] 145 | fills = deque() 146 | for t in trades_date: 147 | pos = t['qty'] * (1.0 if not t['dir'] else -1.0) 148 | if abs(pos) != 0: 149 | tmp_matches = handle_match(fills, t) 150 | matches.extend(tmp_matches) 151 | 152 | return matches, fills 153 | 154 | 155 | def get_open_pnl(lastbook_date, remain_fills): 156 | open_pnl = 0 157 | for f in remain_fills: 158 | open_pnl += (1.0 if f['dir'] else -1.0) * f['remain_qty'] * (f['price'] - lastbook_date['midpt']) 159 | return open_pnl 160 | 161 | 162 | def get_open_notional(remain_fills): 163 | open_notional = 0 164 | for f in remain_fills: 165 | open_notional += f['remain_qty'] * f['price'] * (1.0 if not f['dir'] else -1.0) 166 | return open_notional 167 | 168 | 169 | def get_remain_qty(remain_fills): 170 | remain_qty = 0 171 | for f in remain_fills: 172 | remain_qty += f['remain_qty'] * (1.0 if not f['dir'] else -1.0) 173 | return remain_qty 174 | 175 | 176 | def get_lastbook(book_with_valuation, end_extime): 177 | book_with_valuation = book_with_valuation[book_with_valuation['ExchangeTime'] >= end_extime] 178 | lastbook = {} 179 | for date, sub_df in book_with_valuation.groupby('date'): 180 | sub_df = sub_df.sort_values('ExchangeTime') 181 | lastbook[date] = sub_df.iloc[0] 182 | return lastbook 183 | 184 | 185 | def end_penalty_half_spread(last_book_date, remain_qty): 186 | if last_book_date['Ask1'] == 0 or last_book_date['Bid1'] == 0: 187 | return 0 188 | else: 189 | return abs(remain_qty) * 1 / 2 * (last_book_date['Ask1'] - last_book_date['Bid1']) 190 | 191 | 192 | def end_penalty_whole_book(last_book_date, remain_qty): 193 | if last_book_date['Ask1'] == 0 or last_book_date['Bid1'] == 0: 194 | return 0 195 | else: 196 | if remain_qty > 0: 197 | total_qty = 0 198 | total_notional = 0 199 | first_bid = last_book_date['Bid1'] 200 | last_bid = 0 201 | remain_qty = abs(remain_qty) 202 | for level in range(1, 6): 203 | if last_book_date['Bid%d' % level] != 0: 204 | bid = last_book_date['Bid%d' % level] 205 | bidsize = last_book_date['Bid%dSize' % level] 206 | matchqty = min(bidsize, remain_qty - total_qty) 207 | total_qty += matchqty 208 | total_notional += matchqty * bid 209 | if total_qty == remain_qty: 210 | break 211 | last_bid = bid 212 | else: 213 | break 214 | 215 | if remain_qty > total_qty: 216 | total_notional += (remain_qty - total_qty) * last_bid 217 | total_qty = remain_qty 218 | 219 | return last_book_date['midpt'] * remain_qty - total_notional 220 | 221 | if remain_qty < 0: 222 | total_qty = 0 223 | total_notional = 0 224 | first_ask = last_book_date['Ask1'] 225 | last_ask = 0 226 | remain_qty = abs(remain_qty) 227 | for level in range(1, 6): 228 | if last_book_date['Ask%d' % level] != 0: 229 | ask = last_book_date['Ask%d' % level] 230 | asksize = last_book_date['Ask%dSize' % level] 231 | matchqty = min(asksize, remain_qty - total_qty) 232 | total_qty += matchqty 233 | total_notional += matchqty * ask 234 | if total_qty == remain_qty: 235 | break 236 | last_ask = ask 237 | else: 238 | break 239 | 240 | if remain_qty > total_qty: 241 | total_notional += (remain_qty - total_qty) * last_ask 242 | total_qty = remain_qty 243 | 244 | return total_notional - last_book_date['midpt'] * remain_qty 245 | 246 | 247 | def get_holdtime(matches_date): 248 | total_qty = 0 249 | total_holdtime = 0 250 | for m in matches_date: 251 | total_qty += m['qty'] 252 | holdtime = (m['exittime'] - m['entrytime']).total_seconds() 253 | if (m['entrytime'].hour + m['entrytime'].minute / 60) <= 11.7 and (m['exittime'].hour + m['exittime'].minute / 60) >= 12.8: 254 | holdtime = max(0, holdtime - 5400) 255 | total_holdtime += m['qty'] * holdtime 256 | 257 | return total_holdtime / total_qty if total_qty > 0 else 0.0, total_qty 258 | 259 | 260 | def get_holdtime2(matches_date, remain_fills, lastbook_date): 261 | total_holdtime, total_qty = get_holdtime(matches_date) 262 | total_qty2 = 0 263 | total_holdtime2 = 0 264 | 265 | exit_time = datetime.fromtimestamp(lastbook_date['TimeStamp'] / 1e6) 266 | for f in remain_fills: 267 | total_qty2 += f['remain_qty'] 268 | entry_time = datetime.fromtimestamp(f['TimeStamp'] / 1e6) 269 | holdtime = (exit_time - entry_time).total_seconds() 270 | if (entry_time.hour + entry_time.minute / 60) <= 11.7 and (exit_time.hour + exit_time.minute / 60) >= 12.8: 271 | holdtime = max(0, holdtime - 5400) 272 | 273 | total_holdtime2 += f['remain_qty'] * holdtime 274 | 275 | combined_qty = total_qty + total_qty2 276 | combined_holdtime = total_holdtime * total_qty + total_holdtime2 277 | return combined_holdtime / combined_qty if combined_qty > 0 else 0.0, combined_qty 278 | 279 | 280 | def get_summary(daily_info, symbol, trades, book_with_valuation, maxNotional, start_extime, end_extime, fee_ratio=0.0014, startdate=None, enddate=None): 281 | summary = defaultdict(lambda: defaultdict(dict)) 282 | lastbook = get_lastbook(book_with_valuation, end_extime) 283 | for posnotional in trades: 284 | trades_posnotional = trades[posnotional] 285 | for edge_ratio in trades_posnotional: 286 | trades_edge = trades_posnotional[edge_ratio] 287 | trades_time = constraint_trades_time(trades_edge, start_extime, end_extime) 288 | for date in trades_time: 289 | if (startdate is not None and startdate > date) or (enddate is not None and enddate < date): 290 | continue 291 | try: 292 | lastbook_date = lastbook[date] 293 | except Exception as e: 294 | print(symbol, 'lastbook error', e) 295 | continue 296 | maxvolume = maxNotional / daily_info.loc[(symbol, date), 'refPrice'] // 100 * 100 297 | trades_date = trades_time[date] 298 | trades_date_maxvolume = constraint_trades_maxvolume(trades_date, maxvolume) 299 | matches_date, remaining_fills = match(trades_date_maxvolume) 300 | holdtime, matchqty = get_holdtime(matches_date) 301 | holdtime2, matchqty2 = get_holdtime2(matches_date, remaining_fills, lastbook_date) 302 | try: 303 | if len(matches_date): 304 | df = pd.DataFrame(matches_date) 305 | # df['entryTimeBin'] = df['entrytime'].apply(lambda x: x.hour + x.minute // 30 * 0.5) 306 | df['pnl'] = df['qty'] * (df['entryprice'] - df['exitprice']) * df['entry_dir'].apply(lambda x: 1.0 if x else -1.0) 307 | df['notional'] = df['qty'] * (df['entryprice'] + df['exitprice']) / 2 308 | df['fee'] = df['notional'] * fee_ratio 309 | sum_ = df[['fee', 'pnl', 'notional']].sum() 310 | else: 311 | sum_ = { 312 | 'fee': 0, 313 | 'pnl': 0, 314 | 'notional': 0, 315 | } 316 | open_pnl = get_open_pnl(lastbook_date, remaining_fills) 317 | open_notional = get_open_notional(remaining_fills) 318 | remaining_qty = get_remain_qty(remaining_fills) 319 | date_perf = { 320 | 'closed_pnl': sum_['pnl'], 321 | 'closed_fee': sum_['fee'], 322 | 'maxNotional': maxNotional, 323 | 'closed_notional': sum_['notional'], 324 | 'open_qty': remaining_qty, 325 | 'open_pnl': open_pnl, 326 | 'open_notional': open_notional, 327 | 'open_fee': abs(open_notional) * fee_ratio, 328 | 'holdtime': holdtime, 329 | 'holdtime2': holdtime2, 330 | 'open_penalty_half_spread': end_penalty_half_spread(lastbook_date, remaining_qty), 331 | 'open_penalty_whole_book': end_penalty_whole_book(lastbook_date, remaining_qty), 332 | } 333 | summary[posnotional][edge_ratio][date] = date_perf 334 | except Exception as e: 335 | print('except', e) 336 | import pdb; pdb.set_trace() 337 | return summary 338 | 339 | 340 | def gen_valuation(symbols: list, make_theolist, weights, date_first: str, date_last: str) -> pd.DataFrame: 341 | """ 342 | Returns: 343 | DataFrame: book and 'value' column 344 | """ 345 | quote_src = core.quote.load_data(symbols, date_first=date_first, date_last=date_last) 346 | quotes = [quote_src.get_quote(symbol) for symbol in symbols] 347 | quotes[0].alias = 'self' 348 | markup = core.markup.gen_markup(quotes[0], alpha=0.5, lag=3) 349 | 350 | theofunc_list = [T.MakeTimeTheo(quotes[0])] + make_theolist(quotes) 351 | 352 | theomatrix = T.make_theomatrix(theofunc_list, quotes[0].sampler, quote_src.get_data_num()) 353 | 354 | # Sampling 355 | sampling_param = {'sample_type': 'none'} 356 | sampler = F.FitSampling(sampling_param) 357 | 358 | contract = quote_src.dict_sym2contract[symbols[0]] 359 | book_with_val = [] 360 | 361 | for idx in range(len(quote_src.dateStr)): 362 | day = quote_src.dateStr[idx] 363 | raw = quote_src.dfdict[contract][idx] 364 | thmatrix_samp, markup_samp = sampler(quotes[0], theomatrix, markup, 0, selectedDays=[idx]) 365 | valuation = np.dot(weights, thmatrix_samp[1:,]) 366 | df_val = pd.DataFrame({'TimeStamp': thmatrix_samp[0,:], 'value': valuation}) 367 | book_with_val.append(raw.merge(df_val, on='TimeStamp')) 368 | 369 | 370 | return pd.concat(book_with_val) 371 | 372 | 373 | class BasicCrossSimulationParam: 374 | def __init__(self, date_first, date_last, position_notionals, edge_ratios): 375 | self.position_notionals = position_notionals 376 | self.edge_ratios = edge_ratios 377 | self.date_first = date_first 378 | self.date_last = date_last 379 | 380 | 381 | def run_basic_cross_strategy(book_with_valuation, symbol: str, param: BasicCrossSimulationParam, daily_info): 382 | book_with_valuation['midpt'] = (book_with_valuation['Bid1'] * (1 + (book_with_valuation['Ask1Size'] == 0)) + book_with_valuation['Ask1'] * (1 + (book_with_valuation['Bid1Size'] == 0))) / 2 383 | book_with_valuation['date'] = book_with_valuation['TimeStamp'].apply(lambda x: datetime.fromtimestamp(x / 1e6).strftime('%Y%m%d')) 384 | resultsMultiLevel = cross_edge_notional_multilevel( 385 | book_with_valuation, symbol, param.position_notionals, param.edge_ratios, daily_info) 386 | print(resultsMultiLevel) 387 | 388 | max_notional = 2000000 389 | start_extime = 93200000000 390 | end_extime = 145000000000 391 | summary = get_summary(daily_info, symbol, resultsMultiLevel, book_with_valuation, max_notional, start_extime, end_extime) 392 | return summary 393 | 394 | 395 | if __name__ == '__main__': 396 | # book_parquet_dir = '/home/mhyang/Data/Experiment/DynamicEdgeIntraDay/book_with_valuation_GBDT_SH' 397 | # trade_dir = '/home/mhyang/Data/Experiment/PythonSim/GBDT_SH_Trades' 398 | # output_dir = 'GBDT_SH_Summary_0932_1450' 399 | 400 | book_parquet_dir = 'example/valuation/' 401 | trade_dir = 'example/trades' 402 | output_dir = 'example/summary' 403 | 404 | symbols = sorted([_.split('.p')[0] for _ in os.listdir(trade_dir)]) 405 | symbols = ['600519'] 406 | os.makedirs(output_dir, exist_ok=True) 407 | max_notional = 2000000 408 | start_extime = 93200000000 409 | end_extime = 145000000000 410 | 411 | daily_info = get_daily_info('20210801', '20210831') 412 | 413 | def default_to_regular(d): 414 | if isinstance(d, defaultdict): 415 | d = {k: default_to_regular(v) for k, v in d.items()} 416 | return d 417 | 418 | def run(symbol): 419 | print(symbol) 420 | output_path = os.path.join(output_dir, '%s.p' % symbol) 421 | if not os.path.exists(output_path): 422 | try: 423 | book_fpath = os.path.join(book_parquet_dir, '%s.parquet.gzip' % symbol) 424 | book_with_valuation = pd.read_parquet(book_fpath) 425 | book_with_valuation['date'] = book_with_valuation['TimeStamp'].apply(lambda x: datetime.fromtimestamp(x / 1e6).strftime('%Y%m%d')) 426 | book_with_valuation['midpt'] = (book_with_valuation['Bid1'] * (1 + (book_with_valuation['Ask1Size'] == 0)) + book_with_valuation['Ask1'] * (1 + (book_with_valuation['Bid1Size'] == 0))) / 2 427 | trade_fpath = os.path.join(trade_dir, '%s.p' % symbol) 428 | trades = pk.load(open(trade_fpath, 'rb')) 429 | summary = get_summary(daily_info, symbol, trades, book_with_valuation, max_notional, start_extime, end_extime) 430 | pk.dump(default_to_regular(summary), open(output_path, 'wb')) 431 | except Exception as e: 432 | print(symbol, e) 433 | else: 434 | print('%s exists' % output_path) 435 | 436 | import multiprocessing 437 | 438 | n = 8 439 | pool = multiprocessing.Pool(n) 440 | pool.map(run, symbols) 441 | -------------------------------------------------------------------------------- /backtester/BackTester.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2021/7/26 14:05 4 | # @Author : rpyxqi@gmail.com 5 | # @Site : 6 | # @File : BackTester.py 7 | import time 8 | from .Factor import Factor 9 | from .Position import Position 10 | from backtester.Account import Account 11 | from strategy.T0Signal import TOSignal 12 | from strategy.ClfSignal import ClfSignal 13 | from strategy.RegSignal import RegSignal 14 | from editorconfig import get_properties, EditorConfigError 15 | import pandas as pd 16 | import logging 17 | import matplotlib.pyplot as plt 18 | import numpy as np 19 | import hashlib 20 | from datetime import datetime 21 | import os 22 | import utils.utils as utils 23 | import utils.define as define 24 | from copy import deepcopy 25 | import configparser 26 | 27 | 28 | def get_fill_ret(order=[], tick=1, mkt=[]): 29 | _order_type, _price, _lot = order 30 | bid_price1, ask_price1, bid_vol1, ask_vol1 = mkt[-4:] 31 | _last_price = mkt[3] 32 | if _order_type == define.LONG: 33 | if _lot <= ask_vol1 and (not _price or _price >= ask_price1): 34 | return [ask_price1, _lot] 35 | else: 36 | return [0, 0] 37 | if _order_type == define.SHORT: 38 | if _lot <= bid_vol1 and (not _price or _price <= bid_price1): 39 | return [bid_price1, _lot] 40 | else: 41 | return [0, 0] 42 | return [0, 0] 43 | 44 | 45 | def stop_profit_loss(risk_input: dict = {}, risk_conf: dict = {}) -> tuple: 46 | ''' 47 | # the range in the past 1 min is laess than 2(N) ticks, stop profit/loss will be downgraded,then tend to close the position 48 | #easier; if not, then the oppisite side, try to earn more in the order, at the same time, it may lose more, so it should 49 | #improve the strategy/signal 50 | :param risk_input: 51 | :param risk_conf: 52 | :return: 53 | ''' 54 | _last_price = risk_input.get('last_price')[-120:] 55 | _range = max(_last_price) - min(_last_price) 56 | _stop_lower = risk_conf.get('stop_lower') 57 | _stop_upper = risk_conf.get('stop_upper') 58 | if _range < 2: 59 | return _stop_lower, _stop_lower 60 | else: 61 | return _stop_upper, _stop_upper 62 | 63 | 64 | def backtesting(product_id: str = 'm', trade_date: str = '20210401', signal_name: str = 'RegSignal', 65 | result_fname_digest: str = '', options: dict = {}, 66 | plot_mkt: bool = True) -> tuple: 67 | ''' 68 | 69 | :param product_id: product id to be tested, lower case 70 | :param trade_date: backtest trade, 'yyyymmdd' 71 | :param signal_name: signal type 72 | :param result_fname_digest: file name to cache test result, which the digest of back test params 73 | :param options: other params to be included in the back test 74 | :param plot_mkt: whether to plot and save jpg of the mkt and trade signal 75 | :return: (total return, total fee, precision, list of transaction) 76 | ''' 77 | 78 | # load back test config 79 | backtesting_config = '' 80 | if not options: 81 | try: 82 | _strategy_conf = utils.get_path([define.CONF_DIR, 83 | define.STRATEGY_CONF_NAME]) 84 | config = configparser.ConfigParser() 85 | config.read(_strategy_conf) 86 | signal_lst = [] 87 | for k, v in config['strategy'].items(): 88 | if k == 'signal_names': 89 | options.update({k: v}) 90 | signal_lst = v.split(',') 91 | elif k == 'start_timestamp' or k == 'end_timestamp': 92 | options.update({k: v}) 93 | else: 94 | options.update({k: float(v)}) 95 | for sig_name in signal_lst: 96 | options[sig_name] = {} 97 | for k1, v1 in config[sig_name].items(): 98 | options[sig_name].update({k1: float(v1)}) 99 | except Exception as ex: 100 | print(ex) 101 | 102 | for key, value in options.items(): 103 | backtesting_config = '{0},{1}:{2}'.format(backtesting_config, key, value) 104 | 105 | # get contract and daily cache 106 | instrument_id_df = utils.get_instrument_ids(start_date=trade_date, end_date=trade_date, product_id=product_id) 107 | instrument_id, trade_date, exchange_cd = instrument_id_df.values[0] 108 | _mul_num = utils.get_mul_num(instrument_id=instrument_id) 109 | 110 | # Load depth markets 111 | _tick_mkt_path = os.path.join(define.TICK_MKT_DIR, define.exchange_map.get(exchange_cd), 112 | '{0}_{1}.csv'.format(instrument_id, trade_date.replace('-', ''))) 113 | tick_mkt = pd.read_csv(_tick_mkt_path, encoding='gbk') 114 | tick_mkt.columns = define.tb_cols 115 | logging.info('trade_date:{0}, instrument_id:{1}, shape:{2}'.format(trade_date, instrument_id, tick_mkt.shape)) 116 | values = tick_mkt.values 117 | 118 | # init factor, position account signal 119 | factor = Factor(product_id=product_id, instrument_id=instrument_id, trade_date=trade_date) 120 | position = Position() 121 | account = Account() 122 | signal = ClfSignal(factor=factor, position=position, instrument_id=instrument_id, 123 | trade_date=trade_date, product_id=product_id) 124 | 125 | # Init backtest params 126 | open_fee = float(options.get('open_fee')) or 3.0 127 | close_t0_fee = float(options.get('close_t0_fee')) or 0.0 128 | fee = open_fee + close_t0_fee 129 | start_timestamp = options.get('start_timestamp') or '09:05:00' 130 | start_datetime = '{0} {1}'.format(trade_date, start_timestamp) 131 | end_timestamp = options.get('end_timestamp') or '22:50:00' 132 | # end_datetime = '{0} {1}'.format(trade_date, end_timestamp) 133 | # delay_sec = int(options.get('delay_sec')) or 5 134 | total_return = 0.0 135 | close_price = 0.0 # not the true close price, to close the position 136 | _text_lst = ['long_open', 'long_close', 'short_open', 'short_close'] 137 | update_factor_time = 0.0 138 | get_signal_time = 0.0 139 | signal_delay = 0 140 | conf_signal_delay = int(options.get('signal_delay')) 141 | 142 | # robust handling, skip dates with records missing more than threshold(e.g. 0.3 here) 143 | if len(values) < define.TICK_SIZE * define.MKT_MISSING_SKIP: 144 | logging.warning("miss mkt for trade_date:{0} with len:{1} and tick size:{2}".format(trade_date, len(values), 145 | define.TICK_SIZE 146 | )) 147 | return 148 | _size = len(values) 149 | print('size is:', _size) 150 | 151 | dt_last_trans_time = None 152 | try: 153 | dt_last_trans_time = datetime.strptime(start_datetime, '%H:%M:%S') 154 | except Exception as ex: 155 | dt_last_trans_time = datetime.strptime(start_datetime.split('.')[0], '%Y-%m-%d %H:%M:%S') 156 | else: 157 | pass 158 | 159 | signal_num = 0 160 | print("**************Start Ticks with size:{0} and trade_date:{1}****************".format(_size, trade_date)) 161 | for idx, item in enumerate(values): 162 | _last = item[3] 163 | _update_time = item[2] 164 | start_ts = time.time() 165 | curr_factor =factor.update_factor(item, idx=idx, multiplier=_mul_num, lag_long=int(options.get('long_windows')), 166 | lag_short=int(options.get('short_windows'))) 167 | end_ts = time.time() 168 | update_factor_time += (end_ts - start_ts) 169 | 170 | # FIXME double check 171 | if not utils.is_trade(start_timestamp, end_timestamp, _update_time): 172 | # print(_update_time, 'not trade time--------------') 173 | continue 174 | 175 | if signal_delay % conf_signal_delay != 0: 176 | signal_delay += 1 177 | continue 178 | close_price = _last 179 | close_timestamp = _update_time 180 | start_ts = time.time() 181 | 182 | options.update({'instrument_id': instrument_id}) 183 | options.update({'multiplier': _mul_num}) 184 | options.update({'fee': fee}) 185 | options.update({'factor': curr_factor}) 186 | options.update({'trade_date': trade_date}) 187 | s1 = time.time() 188 | options.update({'stop_lower': 2.0}) 189 | options.update({'stop_upper': 20.0}) 190 | stop_profit, stop_loss = stop_profit_loss({'last_price': factor.last_price}, options) 191 | options.update({'stop_profit': stop_profit, 'stop_loss': stop_loss}) 192 | 193 | # Get signal 194 | _signal = signal(params=options) 195 | if _signal.signal_type != define.NO_SIGNAL: 196 | signal_num += 1 197 | print('Get Signal=>', _signal.signal_type, 'vol=>', _signal.vol, 'price=>', _signal.price, 'direction=>', 198 | _signal.direction, 'update time=>', _update_time) 199 | 200 | e1 = time.time() 201 | end_ts = time.time() 202 | get_signal_time += (end_ts - start_ts) 203 | _pos = position.get_position(instrument_id) 204 | _last_trans_time = _pos[-1][2] if _pos else start_datetime 205 | dt_curr_time = datetime.strptime(_update_time.split('.')[0], '%Y-%m-%d %H:%M:%S') 206 | _trans_gap_time = (dt_curr_time - dt_last_trans_time).seconds 207 | s2 = time.time() 208 | # handle signal, update account and position 209 | if _signal.signal_type == define.LONG_OPEN: 210 | print("Long Open with update time:{0}------".format(_update_time)) 211 | _fill_price, _fill_lot = get_fill_ret(order=[define.LONG, _signal.price, _signal.vol], tick=1, mkt=item) 212 | if _fill_lot > 0: 213 | dt_last_trans_time = datetime.strptime(_update_time.split('.')[0], '%Y-%m-%d %H:%M:%S') 214 | account.add_transaction( 215 | [idx, instrument_id, _text_lst[define.LONG_OPEN], _last, _fill_price, _fill_lot, _fill_price, 216 | open_fee * _fill_lot, 217 | _update_time.split(' ')[-1], 218 | _update_time.split(' ')[-1], 219 | 0.0, 0.0]) 220 | position.update_position(instrument_id=instrument_id, long_short=define.LONG, price=_fill_price, 221 | timestamp=_update_time, vol=_fill_lot, order_type=define.LONG_OPEN) 222 | account.update_fee(open_fee * _fill_lot) 223 | 224 | elif _signal.signal_type == define.SHORT_OPEN: 225 | print("Short Open with update time:{0}------".format(_update_time)) 226 | _fill_price, _fill_lot = get_fill_ret(order=[define.SHORT, _signal.price, _signal.vol], tick=1, mkt=item) 227 | if _fill_lot > 0: 228 | dt_last_trans_time = datetime.strptime(_update_time.split('.')[0], '%Y-%m-%d %H:%M:%S') 229 | account.add_transaction( 230 | [idx, instrument_id, _text_lst[define.SHORT_OPEN], _last, _fill_price, _fill_lot, _fill_price, 231 | open_fee * _fill_lot, 232 | _update_time.split(' ')[-1], 233 | _update_time.split(' ')[-1], 234 | 0.0, 0.0]) 235 | position.update_position(instrument_id=instrument_id, long_short=define.SHORT, price=_fill_price, 236 | timestamp=_update_time, 237 | vol=_fill_lot, order_type=define.SHORT_OPEN) 238 | account.update_fee(open_fee * _fill_lot) 239 | 240 | elif _signal.signal_type == define.LONG_CLOSE: 241 | print("Long Close with update time:{0}------".format(_update_time)) 242 | _pos = position.get_position_side(instrument_id, define.LONG) 243 | if _pos: 244 | dt_curr_time = datetime.strptime(_update_time.split('.')[0], '%Y-%m-%d %H:%M:%S') 245 | try: 246 | dt_last_trans_time = datetime.strptime(_pos[2], '%H:%M:%S') 247 | except Exception as ex: 248 | dt_last_trans_time = datetime.strptime(_pos[2].split('.')[0], '%Y-%m-%d %H:%M:%S') 249 | else: 250 | pass 251 | holding_time = (dt_curr_time - dt_last_trans_time).seconds 252 | 253 | _fill_price, _fill_lot = get_fill_ret(order=[define.SHORT, _signal.price, _signal.vol], tick=1, 254 | mkt=item) 255 | if _fill_lot > 0: 256 | dt_last_trans_time = datetime.strptime(_update_time.split('.')[0], '%Y-%m-%d %H:%M:%S') 257 | # print('transaction time=>', dt_last_trans_time) 258 | curr_return = (((_fill_price - _pos[1]) * _mul_num) - fee) * _fill_lot 259 | total_return += curr_return 260 | 261 | account.add_transaction( 262 | [idx, instrument_id, _text_lst[define.LONG_CLOSE], _last, _fill_price, _fill_lot, _pos[1], 263 | close_t0_fee * _fill_lot, 264 | _pos[2].split(' ')[-1], 265 | _update_time.split(' ')[-1], 266 | holding_time, 267 | curr_return]) 268 | position.update_position(instrument_id=instrument_id, long_short=define.SHORT, price=_fill_price, 269 | timestamp=_update_time, vol=_fill_lot, order_type=define.LONG_CLOSE) 270 | account.update_fee(close_t0_fee * _fill_lot) 271 | elif _signal.signal_type == define.SHORT_CLOSE: 272 | print("Short Close with update time:{0}------".format(_update_time)) 273 | _pos = position.get_position_side(instrument_id, define.SHORT) 274 | if _pos: 275 | dt_curr_time = datetime.strptime(_update_time.split('.')[0], '%Y-%m-%d %H:%M:%S') 276 | try: 277 | dt_last_trans_time = datetime.strptime(_pos[2], '%H:%M:%S') 278 | except Exception as ex: 279 | dt_last_trans_time = datetime.strptime(_pos[2].split('.')[0], '%Y-%m-%d %H:%M:%S') 280 | else: 281 | pass 282 | holding_time = (dt_curr_time - dt_last_trans_time).seconds 283 | 284 | _fill_price, _fill_lot = get_fill_ret(order=[define.LONG, _signal.price, _signal.vol], tick=1, mkt=item) 285 | if _fill_lot > 0: 286 | dt_last_trans_time = datetime.strptime(_update_time.split('.')[0], '%Y-%m-%d %H:%M:%S') 287 | curr_return = ((_pos[1] - _fill_price) * _mul_num - fee) * _fill_lot 288 | total_return += curr_return 289 | account.add_transaction( 290 | [idx, instrument_id, _text_lst[define.SHORT_CLOSE], _last, _fill_price, _fill_lot, _pos[1], 291 | close_t0_fee * _fill_lot, 292 | _pos[2].split(' ')[-1], 293 | _update_time, 294 | holding_time, 295 | curr_return]) 296 | position.update_position(instrument_id=instrument_id, long_short=define.LONG, price=_fill_price, 297 | timestamp=_update_time, vol=_fill_lot, order_type=define.SHORT_CLOSE) 298 | account.update_fee(close_t0_fee * _fill_lot) 299 | else: # NO_SIGNAL 300 | pass 301 | e2 = time.time() 302 | # print('handle signal time:', idx, idx/_size, e2-s2) 303 | signal_delay += 1 304 | _pos = position.get_position(instrument_id) 305 | total_return_risk = total_return 306 | total_risk = 0.0 307 | print("***************complete tick with signal num:{0} *************************".format(signal_num)) 308 | if _pos: 309 | _tmp_pos = deepcopy(_pos) 310 | for item in _tmp_pos: 311 | if item[0] == define.LONG: 312 | dt_curr_time = datetime.strptime(close_timestamp.split('.')[0], '%Y-%m-%d %H:%M:%S') 313 | try: 314 | dt_last_trans_time = datetime.strptime(item[2], '%H:%M:%S') 315 | except Exception as ex: 316 | dt_last_trans_time = datetime.strptime(item[2].split('.')[0], '%Y-%m-%d %H:%M:%S') 317 | else: 318 | pass 319 | holding_time = (dt_curr_time - dt_last_trans_time).seconds 320 | # TODO to apply fill ??now assume all fill with latest price with one tick down 321 | _return = ((close_price - define.TICK - item[1]) * item[-1] - fee) * _mul_num 322 | total_return += _return 323 | total_risk += item[1] * item[-1] 324 | print('final long close with return:{0},total return after:{1} for trade_date:{2}'.format(_return, 325 | total_return, 326 | trade_date)) 327 | 328 | account.add_transaction( 329 | [idx, instrument_id, _text_lst[define.LONG_CLOSE], close_price, close_price - define.TICK, item[-1], 330 | item[1], 331 | close_t0_fee * item[-1], 332 | item[2], 333 | close_timestamp, holding_time, _return]) 334 | account.update_fee(close_t0_fee * item[-1]) 335 | position.update_position(instrument_id=instrument_id, long_short=define.SHORT, price=close_price, 336 | timestamp=close_timestamp, 337 | vol=item[-1], order_type=define.LONG_CLOSE) 338 | else: 339 | 340 | dt_curr_time = datetime.strptime(close_timestamp.split('.')[0], '%Y-%m-%d %H:%M:%S') 341 | try: 342 | dt_last_trans_time = datetime.strptime(item[2], '%H:%M:%S') 343 | except Exception as ex: 344 | dt_last_trans_time = datetime.strptime(item[2].split('.')[0], '%Y-%m-%d %H:%M:%S') 345 | else: 346 | pass 347 | holding_time = (dt_curr_time - dt_last_trans_time).seconds 348 | # TODO to apply fill ??now assume all fill with latest price with one tick up, tick hardcode 349 | _return = ((item[1] - close_price - define.TICK) * _mul_num - fee) * item[-1] 350 | total_risk += item[1] * item[-1] 351 | total_return += _return 352 | print('final short close with return:{0},total return after:{1}'.format(_return, total_return)) 353 | 354 | account.add_transaction( 355 | [idx, instrument_id, _text_lst[define.SHORT_CLOSE], close_price, 356 | close_price + define.TICK, item[-1], item[1], 357 | close_t0_fee * item[-1], item[2], 358 | close_timestamp, holding_time, _return]) 359 | account.update_fee(close_t0_fee * item[-1]) 360 | position.update_position(instrument_id=instrument_id, long_short=define.LONG, price=close_price, 361 | timestamp=close_timestamp, 362 | vol=item[-1], order_type=define.SHORT_CLOSE) 363 | 364 | if options.get('cache_factor') == '1': 365 | logging.info('Start cache factor') 366 | factor.cache_factor() 367 | logging.info('Complete cache factor') 368 | else: 369 | logging.info('Stip cache factor') 370 | if plot_mkt: 371 | _idx_lst = list(range(len(factor.last_price))) 372 | fig = plt.figure() 373 | ax1 = fig.add_subplot(111) 374 | ax1.plot(_idx_lst[define.PLT_START:define.PLT_END], factor.last_price[define.PLT_START:define.PLT_END]) 375 | ax1.plot(_idx_lst[define.PLT_START:define.PLT_END], factor.vwap[define.PLT_START:define.PLT_END]) 376 | ax1.plot(_idx_lst[define.PLT_START:define.PLT_END], factor.turning[define.PLT_START:define.PLT_END]) 377 | print(np.array(factor.last_price).std()) 378 | ax1.grid(True) 379 | ax1.set_title('{0}_{1}'.format(instrument_id, trade_date)) 380 | xtick_labels = [item[:-3] for item in factor.update_time] 381 | ax1.set_xticks(_idx_lst[::3600]) 382 | min_lst = [] 383 | ax1.set_xticklabels(xtick_labels[::3600]) 384 | # ax1.set_xticks(factor.update_time[::3600]) 385 | # ax1.set_xticks(x_idx, xtick_labels, rotation=60, FontSize=6) 386 | for item in account.transaction: 387 | _t_lst = ['lo', 'lc', 'so', 'sc'] 388 | ax1.text(item[0], item[3], s='{0}'.format(item[2])) 389 | 390 | ax2 = ax1.twinx() 391 | ax2.plot(_idx_lst[define.PLT_START:define.PLT_END], factor.trend_short[define.PLT_START:define.PLT_END], 'r') 392 | 393 | _ret_path = utils.get_path([define.RESULT_DIR, define.BT_DIR, '{0}_{1}.jpg'.format(instrument_id, trade_date)]) 394 | plt.savefig(_ret_path) 395 | long_open, short_open, correct_long_open, wrong_long_open, correct_short_open, wrong_short_open = 0, 0, 0, 0, 0, 0 396 | total_fee = 0.0 397 | total_holding_time = 0.0 398 | max_holding_time = -np.inf 399 | min_holding_time = np.inf 400 | for item in account.transaction: 401 | if item[2] == 'long_open': 402 | long_open += 1 403 | elif item[2] == 'short_open': 404 | short_open += 1 405 | elif item[2] == 'long_close': 406 | total_holding_time += item[-2] 407 | max_holding_time = max(max_holding_time, item[-2]) 408 | min_holding_time = min(min_holding_time, item[-2]) 409 | if item[-1] > 0: 410 | correct_long_open += 1 411 | else: 412 | wrong_long_open += 1 413 | else: # short close 414 | total_holding_time += item[-2] 415 | max_holding_time = max(max_holding_time, item[-2]) 416 | min_holding_time = min(min_holding_time, item[-2]) 417 | if item[-1] > 0: 418 | correct_short_open += 1 419 | else: 420 | wrong_short_open += 1 421 | average_holding_time = total_holding_time / (long_open + short_open) if long_open + short_open > 0 else 0.0 422 | print("******************back test results for date:{0}*********************".format(trade_date)) 423 | print('trade date', trade_date) 424 | print(long_open, short_open, correct_long_open, wrong_long_open, correct_short_open, wrong_short_open) 425 | print('total return:', total_return) 426 | print('total fee:', account.fee) 427 | print('total risk:', total_risk) 428 | print('update factor time:', update_factor_time) 429 | print('get signal time:', get_signal_time) 430 | print("average_holding_time:", average_holding_time) 431 | print("max_holding_time:", max_holding_time) 432 | print("min_holding_time:", min_holding_time) 433 | print("******************back test results for date:{0}*********************".format(trade_date)) 434 | 435 | precision = (correct_long_open + correct_short_open) / ( 436 | long_open + short_open) if long_open + short_open > 0 else 0.0 437 | # f = open("results/results_{0}.txt".format(product_id), "a") 438 | # _key = '{0}_{1}'.format(trade_date, instrument_id) 439 | # result_fname_digest = hashlib.sha256(bytes(backtesting_config, encoding='utf-8')).hexdigest() 440 | # f.write("{0}:{1}\n".format(backtesting_config, result_fname_digest)) 441 | _ret_path = utils.get_path([define.RESULT_DIR, define.BT_DIR, '{0}.csv'.format(result_fname_digest)]) 442 | try: 443 | result_df = pd.read_csv(_ret_path) 444 | except Exception as ex: 445 | result_df = pd.DataFrame( 446 | {'trade_date': [], 'product_id': [], 'instrument_id': [], 'total_return_final': [], 447 | 'total_return_unclose': [], 448 | 'total_fee': [], 449 | 'unclosed_value': [], 'precision': [], 'long_open': [], 'short_open': [], 450 | 'correct_long_open': [], 'wrong_long_open': [], 'correct_short_open': [], 'wrong_short_open': [], 451 | 'average_holding_time': [], 'max_holding_time': [], 'min_holding_time': [] 452 | }) 453 | 454 | result_df = result_df.append( 455 | {'trade_date': trade_date, 'product_id': product_id, 'instrument_id': instrument_id, 456 | 'total_return_final': total_return, 'total_return_unclose': total_return_risk, 457 | 'total_fee': account.fee, 458 | 'unclosed_value': total_risk, 'precision': precision, 459 | 'long_open': long_open, 'short_open': short_open, 'correct_long_open': correct_long_open, 460 | 'wrong_long_open': wrong_long_open, 'correct_short_open': correct_short_open, 461 | 'wrong_short_open': wrong_short_open, 'average_holding_time': average_holding_time, 462 | 'max_holding_time': max_holding_time, 'min_holding_time': min_holding_time 463 | }, ignore_index=True) 464 | 465 | result_df.to_csv(_ret_path, index=False) 466 | ret = (total_return, account.fee, precision, account.transaction) 467 | return ret 468 | -------------------------------------------------------------------------------- /research/daily_models/com_daily.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2022/4/14 14:19 4 | # @Author : rpyxqi@gmail.com 5 | # @Site : 6 | # @File : com_daily.py 7 | 8 | import math 9 | import uqer 10 | import datetime 11 | import pprint 12 | import pandas as pd 13 | import numpy as np 14 | from uqer import DataAPI 15 | import matplotlib.pyplot as plt 16 | import backtester.Account as Account 17 | import backtester.Position as Position 18 | from sklearn.model_selection import train_test_split 19 | from sklearn.linear_model import LinearRegression 20 | from sklearn import metrics 21 | import utils.define as define 22 | 23 | uqer_client = uqer.Client(token="6aa0df8d4eec296e0c25fac407b332449112aad6c717b1ada315560e9aa0a311") 24 | 25 | margin_overwrite = {'p': 11} 26 | 27 | ['P', 'rb', 'm', 'eg', 'fu'] 28 | 29 | open_fee_overwrite = {'p': 2.5, 'rb': 5.08, 'm': 2.5} 30 | close_fee_overwrite = {'p': 2.5, 'rb': 5.08, 'm': 2.5} # rb is close today 31 | 32 | 33 | def select_product(product_lst: list = ['rb', 'm'], start_date: str = '', end_date: str = '', 34 | trade_date: str = '') -> pd.DataFrame: 35 | sum_lst = [] 36 | for pid in product_lst: 37 | df = DataAPI.MktMFutdGet(mainCon=u"1", contractMark=u"", contractObject=pid, tradeDate=u"", 38 | startDate=start_date, 39 | endDate=end_date, field=u"", pandas="1") 40 | trade_dates = list(df['tradeDate'])[-5:] 41 | vol_std_lst = [] 42 | instrument_id_lst = list(df['ticker'])[-5:] 43 | 44 | for idx, trade_date in enumerate(trade_dates): 45 | df_intraday = DataAPI.FutBarHistOneDay2Get(instrumentID=instrument_id_lst[idx], date=trade_date, unit=u"1", 46 | field=u"", 47 | pandas="1") 48 | # print(instrument_id_lst[idx], trade_date) 49 | vol_std_lst.append(df_intraday['closePrice'].std()) 50 | # print(vol_std_lst) 51 | _liquid = df['turnoverVol'].mean() 52 | _vol = sum(vol_std_lst) / len(vol_std_lst) 53 | # print(pid, _liquid, _vol) 54 | _fee = (open_fee_overwrite.get(pid.lower()) or 0.0) + (close_fee_overwrite.get(pid.lower()) or 0.0) 55 | sum_lst.append([pid, _liquid, _vol, _fee]) 56 | df_sum = pd.DataFrame(sum_lst, columns=['product_id', 'liquid', 'vol', 'fee']) 57 | df_sum['liquid_rank'] = df_sum['liquid'].rank() 58 | df_sum['vol_rank'] = df_sum['vol'].rank() 59 | df_sum['score'] = df_sum['liquid_rank'] + df_sum['vol_rank'] 60 | df_sum = df_sum.sort_values(by='score', ascending=False) 61 | print(df_sum) 62 | 63 | 64 | def get_trend_signal(df_mkt: pd.DataFrame, k1: float = 0.2, k2: float = 0.2, std_weight=0.2, sod: bool = True): 65 | # ['secID', 'ticker', 'exchangeCD', 'secShortName', 'secShortNameEN', 66 | # 'tradeDate', 'contractObject', 'contractMark', 'preSettlePrice', 67 | # 'preClosePrice', 'openPrice', 'highestPrice', 'lowestPrice', 68 | # 'settlePrice', 'closePrice', 'turnoverVol', 'turnoverValue', 'openInt', 69 | # 'chg', 'chg1', 'chgPct', 'mainCon', 'smainCon'] 70 | # df_mkt = df_mkt.dropna() 71 | sig = [] 72 | upbound = [] 73 | lowerbound = [] 74 | ref_price = 'preClosePrice' if sod else 'closePrice' # FIXME should be open price 75 | for item in df_mkt[[ref_price, 'hh', 'lc', 'hc', 'll', 'accstd']].values: 76 | _range = max(item[1] - item[2], item[3] - item[4]) 77 | upbound.append(item[0] + k1 * _range + std_weight * item[5]) 78 | lowerbound.append(item[0] - k2 * _range - std_weight * item[5]) 79 | if item[-1] > item[0] + k1 * _range: 80 | sig.append(define.LONG) 81 | elif item[-1] < item[0] - k2 * _range: 82 | sig.append(define.SHORT) 83 | else: 84 | sig.append(define.NO_SIGNAL) 85 | df_mkt['upbound'] = upbound 86 | df_mkt['lowbound'] = lowerbound 87 | df_mkt['signal'] = sig 88 | return df_mkt 89 | 90 | 91 | def get_revert_signal(df_mkt: pd.DataFrame, k1: float = 0.2, k2: float = 0.2, sod: bool = True): 92 | pass 93 | 94 | 95 | def get_signal(start_date: str = '', end_date: str = '', product_id: str = '', options: dict = {}, 96 | sod: bool = True) -> pd.DataFrame: 97 | ''' 98 | 99 | :param start_date: 100 | :param end_date: 101 | :param product_id: 102 | :param options: 103 | :param sod: True, the signal is available start of the day(for backtest mode);False:eod of the day(for live ) 104 | :return: 105 | ''' 106 | df = DataAPI.MktMFutdGet(mainCon=u"1", contractMark=u"", contractObject=product_id, tradeDate=u"", 107 | startDate=start_date, 108 | endDate=end_date, field=u"", pandas="1") 109 | # ['secID', 'ticker', 'exchangeCD', 'secShortName', 'secShortNameEN', 110 | # 'tradeDate', 'contractObject', 'contractMark', 'preSettlePrice', 111 | # 'preClosePrice', 'openPrice', 'highestPrice', 'lowestPrice', 112 | # 'settlePrice', 'closePrice', 'turnoverVol', 'turnoverValue', 'openInt', 113 | # 'chg', 'chg1', 'chgPct', 'mainCon', 'smainCon'] 114 | lag_window = options.get('lag_window') or 5 115 | k1 = options.get('k1') or 0.2 116 | k2 = options.get('k2') or 0.2 117 | if sod: # ignore the latest eod signal, for backtest; for the last date, got the previous market data 118 | df['hh'] = df['highestPrice'].rolling(lag_window).max().shift() 119 | df['lc'] = df['closePrice'].rolling(lag_window).min().shift() 120 | df['hc'] = df['closePrice'].rolling(lag_window).max().shift() 121 | df['ll'] = df['lowestPrice'].rolling(lag_window).min().shift() 122 | df['accstd'] = df['settlePrice'].rolling(lag_window).std().shift() 123 | else: # the the signal when the date end, signal for the next day, for live trade 124 | df['hh'] = df['highestPrice'].rolling(lag_window).max() 125 | df['lc'] = df['closePrice'].rolling(lag_window).min() 126 | df['hc'] = df['closePrice'].rolling(lag_window).max() 127 | df['ll'] = df['lowestPrice'].rolling(lag_window).min() 128 | df['accstd'] = df['settlePrice'].rolling(lag_window).std() 129 | 130 | df = df.dropna() 131 | df = get_trend_signal(df_mkt=df, k1=k1, k2=k2, sod=sod) 132 | # ls_tag = [] 133 | # upbound = [] 134 | # lowbound = [] 135 | # for item in df.values: 136 | # _max_min_range = item[-3] - item[-2] # acchigh-acclow 137 | # _std = item[-1] # acc std 138 | # _min_mix_weight = options.get('min_max_weight') or 0.2 139 | # _std_weight = options.get('std_weight') or 0.2 140 | # _up_range = item[8] + _min_mix_weight * _max_min_range + _std_weight * _std 141 | # _low_range = item[8] - _min_mix_weight * _max_min_range - _std_weight * _std 142 | # upbound.append(_up_range) 143 | # lowbound.append(_low_range) 144 | # if item[9] > _up_range: # open > acchigh 145 | # ls_tag.append(define.SHORT) # short 146 | # elif item[9] < _low_range: # open < acclow 147 | # ls_tag.append(define.LONG) # long 148 | # else: 149 | # ls_tag.append(define.NO_SIGNAL) 150 | # 151 | # df['upbound'] = upbound 152 | # df['lowbound'] = lowbound 153 | # df['signal'] = ls_tag 154 | return df 155 | 156 | 157 | def handle_bar(signal_df: pd.DataFrame, strategy_option: dict, strategy_name: str = '', save_cache: bool = False): 158 | init_cash = strategy_option.get('init_cash') or 100000 159 | stop_profit = strategy_option.get('stop_profit') or 0.2 160 | stop_loss = strategy_option.get('stop_loss') or 0.2 161 | account = Account.Account(init_cash) 162 | pos = Position.Position() 163 | init_risk_ratio = strategy_option.get('inti_risk_ratio') or 0.3 164 | 165 | mkt_val1 = [] 166 | mkt_val2 = [] 167 | 168 | product_id = list(signal_df['contractObject'])[0] 169 | 170 | df_contract = DataAPI.FutuGet(secID=u"", ticker=u"", exchangeCD=u"", contractStatus="", contractObject=product_id, 171 | prodID="", field=u"", pandas="1") 172 | 173 | max_drawdown = [] 174 | risk_ratio = [] 175 | pos_vol_lst = [] 176 | transac_lst = [] 177 | for item in signal_df.values: 178 | instrument_id = item[1] 179 | open_price = item[10] 180 | close_price = item[14] 181 | settle_price = item[13] 182 | chg = item[19] # close price 183 | chg1 = item[20] # settle price 184 | pre_settle_price = item[8] 185 | pre_close = item[9] 186 | _contract_info = df_contract[df_contract.ticker == instrument_id].to_dict('record') 187 | margin_val = 0.0 188 | pos_vol = 0 189 | transac_vol = 0 190 | if _contract_info: 191 | open_fee = open_fee_overwrite.get(product_id.lower()) or _contract_info[0].get('tradeCommiNum') # 元 192 | close_fee = close_fee_overwrite.get(product_id.lower()) or _contract_info[0].get('tradeCommiNum') # % 193 | multiplier = _contract_info[0].get('contMultNum') 194 | tick_val = _contract_info[0].get('minChgPriceNum') 195 | margin_num = margin_overwrite.get(product_id.lower()) or _contract_info[0].get('tradeMarginRatio') 196 | else: 197 | open_fee = 2 198 | close_fee = 2 199 | multiplier = 10 200 | tick_val = 1 201 | margin_num = 12 202 | _pos = pos.get_position(product_id) 203 | # position.update_position(instrument_id=instrument_id, long_short=define.SHORT, price=_fill_price, 204 | # timestamp=_update_time, vol=_fill_lot, order_type=define.LONG_CLOSE) 205 | # update_lst.append([long_short, price, timestamp, vol]) 206 | # print('start of check pos', item[5], _pos) 207 | if _pos: # has position 208 | for long_short, price, ts, vol in _pos: 209 | if item[-1] == long_short or item[ 210 | -1] == define.NO_SIGNAL: # hv pos and same signal direction or no signal, pos not change,update account fee and mv 211 | if long_short == define.LONG: 212 | _profit = (pre_close / price - 1) * (100 / margin_num) 213 | _profit1 = (pre_settle_price / price - 1) * (100 / margin_num) 214 | if _profit > stop_profit or _profit1 < -stop_loss: # TODO use close price, close pos, stop profit/loss 215 | print("stop profit/loss for long, profit=>", _profit) 216 | _fee = close_fee * vol 217 | _val = (open_price - pre_close - tick_val) * vol * multiplier 218 | _val1 = (open_price - pre_settle_price - tick_val) * vol * multiplier 219 | account.update_fee(_fee) 220 | account.update_market_value(_val, _val1, _fee) 221 | pos.update_position(instrument_id=product_id, long_short=define.SHORT, price=open_price, 222 | timestamp='', vol=vol, order_type=define.LONG_CLOSE) 223 | margin_val = 0.0 224 | pos_vol = 0 225 | transac_vol = -vol 226 | # print('pos vol:', pos_vol, 'get_position vol:', pos.get_position(instrument_id)) 227 | 228 | else: 229 | _val = chg * vol * multiplier 230 | _val1 = chg1 * vol * multiplier 231 | _val = _val if long_short == define.LONG else -_val 232 | _val1 = _val1 if long_short == define.LONG else -_val1 233 | account.update_fee(_fee) 234 | account.update_market_value(_val, _val1, _fee) 235 | margin_val = settle_price * vol * multiplier * margin_num / 100 236 | pos_vol = vol 237 | transac_vol = 0 238 | # print('pos vol:', pos_vol, 'get_position vol:', pos.get_position(instrument_id)) 239 | elif long_short == define.SHORT: 240 | _profit = (price / pre_close - 1) * (100 / margin_num) 241 | _profit1 = (price / pre_close - 1) * (100 / margin_num) 242 | if _profit > stop_profit or _profit1 < -stop_loss: # TODO use close price, close pos, stop profit/loss 243 | print("stop profit/loss for short=>", _profit) 244 | _fee = close_fee * vol 245 | _val = (pre_close - open_price - tick_val) * vol * multiplier 246 | _val1 = (pre_settle_price - open_price - tick_val) * vol * multiplier 247 | account.update_fee(_fee) 248 | account.update_market_value(_val, _val1, _fee) 249 | pos.update_position(instrument_id=product_id, long_short=define.LONG, price=open_price, 250 | timestamp='', vol=vol, order_type=define.SHORT_CLOSE) 251 | margin_val = 0.0 252 | pos_vol = 0 253 | transac_vol = -vol 254 | # print(item[5], 'pos vol:', pos_vol, 'get_position vol:', pos.get_position(instrument_id)) 255 | else: 256 | _val = chg * vol * multiplier 257 | _val1 = chg1 * vol * multiplier 258 | _val = _val if long_short == define.LONG else -_val 259 | _val1 = _val1 if long_short == define.LONG else -_val1 260 | account.update_fee(_fee) 261 | account.update_market_value(_val, _val1, _fee) 262 | margin_val = settle_price * vol * multiplier * margin_num / 100 263 | pos_vol = -vol 264 | transac_vol = 0 265 | # print('pos vol:', pos_vol, 'get_position vol:', pos.get_position(instrument_id)) 266 | elif long_short == define.LONG: # long pos and short signal, close long；remarks要不要反手？？ 267 | _fee = close_fee * vol 268 | _val = (open_price - pre_close - tick_val) * vol * multiplier 269 | _val1 = (open_price - pre_settle_price - tick_val) * vol * multiplier 270 | account.update_fee(_fee) 271 | account.update_market_value(_val, _val1, _fee) 272 | pos.update_position(instrument_id=product_id, long_short=define.SHORT, price=open_price, 273 | timestamp='', vol=vol, order_type=define.LONG_CLOSE) 274 | 275 | margin_val = 0.0 276 | pos_vol = 0 277 | transac_vol = -vol 278 | # print('close pos vol:', pos_vol, 'get_position vol:', pos.get_position(instrument_id)) 279 | elif long_short == define.SHORT: # short pos and long signal, close short，remarks 要不要反手？ 280 | _fee = close_fee * vol 281 | _val = (pre_close - open_price - tick_val) * vol * multiplier 282 | _val1 = (pre_settle_price - open_price - tick_val) * vol * multiplier 283 | account.update_fee(_fee) 284 | account.update_market_value(_val, _val1, _fee) 285 | pos.update_position(instrument_id=product_id, long_short=define.LONG, price=open_price, 286 | timestamp='', vol=vol, order_type=define.SHORT_CLOSE) 287 | 288 | margin_val = 0.0 289 | pos_vol = 0 290 | transac_vol = -vol 291 | # print('pos vol:', pos_vol, 'get_position vol:', pos.get_position(instrument_id)) 292 | else: 293 | pass 294 | else: # no position 295 | _mkt_val1 = account.get_trade_market_values() 296 | available_cash = _mkt_val1[-1] if _mkt_val1 else init_cash 297 | order_vol = int(available_cash * init_risk_ratio / (open_price * multiplier * margin_num / 100)) 298 | if item[-1] == define.LONG: # no pos and long signal,open long 299 | _fee = open_fee * order_vol 300 | _val = ( 301 | close_price - open_price - tick_val) * order_vol * multiplier # assume open price is open_price - 1tick 302 | _val1 = ( 303 | settle_price - open_price - tick_val) * order_vol * multiplier # assume open price is open_price - 1tick 304 | account.update_fee(_fee) 305 | account.update_market_value(_val, _val1, _fee) 306 | pos.update_position(instrument_id=product_id, long_short=define.LONG, price=open_price + tick_val, 307 | timestamp='', 308 | vol=order_vol, order_type=define.LONG_OPEN) 309 | 310 | margin_val = settle_price * order_vol * multiplier * margin_num / 100 311 | pos_vol = order_vol 312 | transac_vol = order_vol 313 | # print('pos vol:', pos_vol, 'get_position vol:', pos.get_position(instrument_id)) 314 | elif item[-1] == define.SHORT: # no pos and short signal, open short 315 | _fee = open_fee * order_vol 316 | _val = ( 317 | open_price - close_price - tick_val) * order_vol * multiplier # assume open price is open_price - 1tick 318 | _val1 = ( 319 | open_price - settle_price - tick_val) * order_vol * multiplier # assume open price is open_price - 1tick 320 | account.update_fee(_fee) 321 | account.update_market_value(_val, _val1, _fee) 322 | pos.update_position(instrument_id=product_id, long_short=define.SHORT, price=open_price - tick_val, 323 | timestamp='', 324 | vol=order_vol, order_type=define.SHORT_OPEN) 325 | 326 | margin_val = settle_price * order_vol * multiplier * margin_num / 100 327 | pos_vol = -order_vol 328 | transac_vol = order_vol 329 | # print('pos vol:', pos_vol, 'get_position vol:', pos.get_position(instrument_id)) 330 | else: # no actions needed 331 | pass 332 | # print(pos.get_position(product_id), item[5], "??????") 333 | mkt_val1.append(account.get_trade_market_values()[-1]) 334 | mkt_val2.append(account.get_settle_market_values()[-1]) 335 | _max_his_mv = max(account.get_trade_market_values()) 336 | _drawdown_val = max(_max_his_mv - account.get_trade_market_values()[-1], 0) 337 | # print(_max_his_mv, account.get_trade_market_values()[-1], _drawdown_val) 338 | _tmp_drawdown_ratio = _drawdown_val / _max_his_mv 339 | max_drawdown.append(_tmp_drawdown_ratio) 340 | risk_ratio.append(margin_val / mkt_val1[-1]) 341 | pos_vol_lst.append(pos_vol) 342 | transac_lst.append(transac_vol) 343 | # print(pos.get_position(instrument_id), item[5], "??????") 344 | 345 | signal_df['mkt_val1'] = mkt_val1 346 | signal_df['mkt_val2'] = mkt_val2 347 | signal_df['max_drawdown'] = max_drawdown 348 | signal_df['pos'] = pos_vol_lst 349 | signal_df['trans'] = transac_lst 350 | signal_df['risk_ratio'] = risk_ratio 351 | bc_start = list(signal_df['preClosePrice'])[0] 352 | bc_return = [item / bc_start - 1 for item in signal_df['closePrice']] 353 | 354 | if save_cache: 355 | signal_df.to_csv('{0}.csv'.format(strategy_name), index=False) 356 | signal_lst = [1 if item == define.LONG else -1 if item == define.SHORT else 0 for item in signal_df['signal']] 357 | evaluate_ret = [] 358 | 359 | # trade_mkt_values = account.get_trade_market_values() 360 | # settle_mkt_values = account.get_settle_market_values() 361 | net_val1 = [item / init_cash for item in signal_df['mkt_val1']] 362 | net_val2 = [item / init_cash for item in signal_df['mkt_val2']] 363 | _idx_lst = list(range(signal_df.shape[0])) 364 | trade_date_str = [item.replace('-', '') for item in signal_df['tradeDate']] 365 | 366 | return_lst = np.array([item - 1 for item in net_val1]) 367 | sharp_ratio = (net_val1[-1] / net_val1[0] - 1) / return_lst.std() 368 | long_sig = len([item for item in signal_lst if item == 1]) 369 | short_sig = len([item for item in signal_lst if item == -1]) 370 | long_holding = len([item for item in pos_vol_lst if item > 0]) 371 | short_holding = len([item for item in pos_vol_lst if item < 0]) 372 | open_trans = len([item for item in transac_lst if item > 0]) 373 | close_trans = len([item for item in transac_lst if item < 0]) 374 | bc_sharp = bc_return[-1] / np.array(bc_return).std() 375 | evaluate_ret.append(net_val1[-1] / net_val1[0] - 1) 376 | evaluate_ret.append(return_lst.std()) 377 | evaluate_ret.append(sharp_ratio) 378 | evaluate_ret.append(max(max_drawdown)) 379 | evaluate_ret.append(max(risk_ratio)) 380 | evaluate_ret.append(long_sig) 381 | evaluate_ret.append(short_sig) 382 | evaluate_ret.append(long_holding) 383 | evaluate_ret.append(short_holding) 384 | evaluate_ret.append(open_trans) 385 | evaluate_ret.append(close_trans) 386 | evaluate_ret.append(bc_return[-1]) 387 | evaluate_ret.append(bc_sharp) 388 | tick_interval = 30 389 | fig = plt.figure() 390 | fig.tight_layout() 391 | plt.title('{0}'.format(strategy_name)) 392 | plt.axis('off') 393 | plt.subplots_adjust(wspace=0.5, hspace=0.5) 394 | ax1 = fig.add_subplot(221) 395 | ax1.plot(net_val1, label='net value(by close)') 396 | # ax1.plot(df['acclow'], label='acclow') 397 | ax1.legend() 398 | 399 | # ax2 = ax1.twinx() 400 | # # ax2.plot(net_val2, 'r', label='nv2') 401 | # ax2.legend() 402 | ax1.set_xticks(_idx_lst[::tick_interval]) 403 | ax1.set_xticklabels(trade_date_str[::tick_interval], rotation=45, fontsize=5) 404 | 405 | ax3 = fig.add_subplot(222) 406 | ax3.plot(signal_df['closePrice'], 'r', label='closePrice') 407 | ax3.legend() 408 | ax3.set_xticks(_idx_lst[::tick_interval]) 409 | ax3.set_xticklabels(trade_date_str[::tick_interval], rotation=45, fontsize=5) 410 | 411 | ax8 = ax3.twinx() 412 | ax8.bar(_idx_lst, transac_lst, label='pos_vol') 413 | 414 | # ax4 = ax3.twinx() 415 | ax4 = fig.add_subplot(223) 416 | ax4.bar(_idx_lst, signal_lst, label='signal') 417 | ax4.legend() 418 | ax4.set_xticks(_idx_lst[::tick_interval]) 419 | ax4.set_xticklabels(trade_date_str[::tick_interval], rotation=45, fontsize=5) 420 | 421 | ax6 = ax4.twinx() 422 | ax6.plot(signal_df['upbound'], 'r', label='upbound') 423 | ax6.plot(signal_df['lowbound'], 'r', label='lowbound') 424 | ax6.plot(signal_df['preClosePrice'], 'g', label='pre_close') 425 | 426 | ax5 = fig.add_subplot(224) 427 | ax5.plot(max_drawdown, label='max_drawdown') 428 | ax5.legend() 429 | ax5.set_xticks(_idx_lst[::tick_interval]) 430 | ax5.set_xticklabels(trade_date_str[::tick_interval], rotation=45, fontsize=5) 431 | 432 | ax7 = ax5.twinx() 433 | ax7.plot(risk_ratio, 'r', label='risk ratio') 434 | if save_cache: 435 | plt.savefig('{0}.jpg'.format(strategy_name)) 436 | # plt.show() 437 | return evaluate_ret 438 | 439 | 440 | def backtest(start_date: str = '20210403', end_date: str = '20220419', save_factor: bool = False): 441 | min_max_weight_lst = [0.2, 0.3, 0.5] 442 | # min_max_weight_lst = [0.2] 443 | std_weight_lst = [-0.1, 0, 0.2, 0.3, 0.5] 444 | # std_weight_lst = [0.2] 445 | lag_windows_lst = [5, 12, 26] 446 | # lag_windows_lst = [5] 447 | # product_id_lst = ['P', 'rb', 'm', 'eg', 'fu'] 448 | product_id_lst = ['P', 'rb'] 449 | k1_lst = [0.2] 450 | k2_lst = [0.2] 451 | back_test_ret = [] 452 | strategy_option = {'stop_profit': 0.3, 'stop_loss': 0.5, 'init_cash': 100000, 'inti_risk_ratio': 0.3} 453 | for w1 in k1_lst: 454 | for w2 in k2_lst: 455 | for std_weight in std_weight_lst: 456 | for product_id in product_id_lst: 457 | for lag_window in lag_windows_lst: 458 | strategy_name = "{0}_{1}_{2}_{3}".format(product_id, lag_window, w1, w2) 459 | signal_option = {'k1': w1, 'std_weight': std_weight, 'k2': w2, 460 | 'lag_window': lag_window} 461 | df = get_signal(start_date=start_date, end_date=end_date, product_id=product_id, 462 | options=signal_option, sod=True) 463 | ret = handle_bar(signal_df=df, strategy_option=strategy_option, strategy_name=strategy_name, 464 | save_cache=save_factor) 465 | # select_product(start_date='20220401', end_date='20220415', product_lst=['rb', 'm', ]) 466 | ret.append(product_id) 467 | ret.append(start_date) 468 | ret.append(end_date) 469 | ret.append(strategy_name) 470 | back_test_ret.append(ret) 471 | bt_df = pd.DataFrame(back_test_ret, 472 | columns=['acc_return', 'std', 'sharp_ratio', 'max_drawdown', 'max_risk_ratio', 'long_sig', 473 | 'short_sig', 'long_holding', 'short_holding', 'open_trans', 'close_trans', 474 | 'bc_return', 475 | 'bc_sharp_ratio', 'product_id', 476 | 'start_date', 'end_date', 'strategy_name']) 477 | bt_df.to_csv('back_test_result.csv', index=False) 478 | 479 | 480 | def get_eod_signal(): 481 | pass 482 | 483 | 484 | if __name__ == '__main__': 485 | backtest(start_date='20210403', end_date='20220419', save_factor=False) 486 | -------------------------------------------------------------------------------- /corr.csv: -------------------------------------------------------------------------------- 1 | ,log_return,open_close_ratio,price_spread,buy_sell_spread,oi,oir,aoi,slope,cos,macd,dif,dea,bs_vol,trend_0,log_return_0,bsvol_volume_0,trend_1,log_return_1,bsvol_volume_1,label_0,rv_0,price_chg_0,label_clf_0,label_1,rv_1,price_chg_1,label_clf_1,trend_ls_diff,trend_ls_ratio,volume_ls_ratio,turnover_ls_ratio,bs_vol_ls_ratio,bsvol_volume_ls_ratio,norm_lastprice,norm_openinterest,norm_interestdiff,norm_volume,norm_bidvolume1,norm_askvolume1,norm_vwap,norm_wap,norm_bs_tag,norm_volume_ls_diff 2 | log_return,1.0,-0.001909949153267583,-0.011322941331217851,0.01151094444604926,-0.04042932986770494,-0.028231821391445637,-0.04092879521069447,0.5432372099461207,0.5256817719727389,0.15709365258539715,0.07036867889680094,0.02590204426329577,0.36642862863579895,0.28935679860375996,0.28462427236158183,0.09070336289428182,0.18327645850335914,0.18169826891415178,0.058694026936791946,0.36939565843947203,0.016299696091755373,-0.2452747285184763,0.05501590071220928,0.2789478255086415,0.011232752126037282,-0.18275654382672596,0.04637147346560956,0.2219724586039065,0.02248117677720777,0.006144528876721859,0.006163724060260921,-0.0077051058846247045,-0.008400306094597675,0.025550012849563108,0.0016377857662318784,0.0017756125544669292,0.023739479157777535,-0.05793142753675455,0.05041786715463712,0.01174928073368846,0.00831067569562216,0.7799302620131483,-0.0033881076144420025 3 | open_close_ratio,-0.001909949153267583,1.0,0.010759143271798416,-0.010169305844265158,0.0027401594939323783,-0.004075399522453756,0.0019995351406224726,-0.0018034783842755797,-0.0028909726488215203,-0.003561554244411846,0.0153497252515745,0.017369729680281443,-0.02345186856495664,0.007402878472484531,0.007574577866273225,-0.0016315989094493385,0.02401897375457497,0.022655239228405677,0.0011161409214845252,-0.008157562027375371,-0.012382822766602079,-0.007283501168432657,-0.010436211117863211,-0.006262491961302007,-0.016064995876277362,-0.005490545466873095,-0.009093195973181612,-0.008251418824299068,-0.009572022204702133,0.006351558096494069,0.0063610224548742095,0.002429401854186942,0.0012108893752758108,0.041089524033103036,0.02709960272594908,0.23309250267300968,0.01699043437126748,0.0017671644254506874,0.00482710498620479,0.04112423683053382,0.04064287131517503,-0.013868880619242673,0.013794460670124622 4 | price_spread,-0.011322941331217851,0.010759143271798416,1.0,-0.9998907053244683,0.08805672687849317,0.02786474446755065,0.03337186764364601,-0.006026991930247215,-0.003382078702084296,0.0011553227238044806,0.009023794838967054,0.009204724500023265,0.03675925697594281,0.008271409412813133,0.009923362938337841,-0.025503873981625926,0.0018937766020748583,0.0034540050556136734,-0.022576964019916608,-0.0210841355653561,-0.18293246643979927,-0.014830977523974125,-0.06867130750973274,-0.017372080936605125,-0.17059873416850516,-0.012462046829067219,-0.04045205138270358,0.008755990348824322,-0.012713313015204014,-0.07153820161594453,-0.07153775107589254,0.0005745450245647402,0.001884262418507718,0.013426474043960327,0.017222421644095497,0.031080289670741546,-0.17103032842120514,-0.048721028082614144,-0.014040665235812664,0.013130830466571871,0.01339544510464434,-0.039652281053981615,0.04553697472655794 5 | buy_sell_spread,0.01151094444604926,-0.010169305844265158,-0.9998907053244683,1.0,-0.0886072572923769,-0.02790900249689699,-0.033838731197169146,0.006255551277796912,0.0035616345861296615,-0.0010383545962504416,-0.007971827995708449,-0.008126176906679158,-0.03673993852253436,-0.007495170372572372,-0.009148896319394575,0.0257278830582537,-0.0006456330663136456,-0.0021681995558814203,0.023035088857052015,0.020870635127052856,0.1828137190974216,0.014490471970812543,0.06890639960328447,0.01699192634228204,0.17044513627417834,0.011986134395992568,0.04088763300403798,-0.00870568159244484,0.012645761420629969,0.07167242859740797,0.07167245044534787,-0.0004771819932714005,-0.0017958067638953763,0.0009898114853633783,-0.010478300047777384,-0.03016456149377878,0.17116359808955356,0.04839132326040783,0.015071571174353565,0.0012876978139854374,0.0010255800932542295,0.039826838470452056,-0.04563182007460571 6 | oi,-0.04042932986770494,0.0027401594939323783,0.08805672687849317,-0.0886072572923769,1.0,0.035974571943457635,0.9778707639079527,-0.01669337804471738,-0.01035335152741857,0.020050578380352103,-0.004031536274617544,-0.010484088456814372,-0.021345041358703905,0.0014923586263610785,-1.736166064074487e-05,-0.003996286919903948,-0.016483405681227,-0.01480259409550331,-0.00988390389626022,0.010168212937281255,-0.026845873331659564,0.03609804472367022,-0.012923469828317862,0.006231535622128172,-0.026495754452218176,0.025323440743364754,-0.008470170386257529,0.01370491460513496,-0.007314846746996655,-0.016238447726833836,-0.016244292667572123,-0.005796848587927473,-0.00414045202908549,-0.02865597314141963,-0.012727075045276521,-0.015974228593088276,-0.10024699701677081,0.07709366043393663,-0.3106827457049558,-0.02736404606009161,-0.02803940514393768,-0.04076870508108915,0.03596876463416743 7 | oir,-0.028231821391445637,-0.004075399522453756,0.02786474446755065,-0.02790900249689699,0.035974571943457635,1.0,0.03641392813559959,-0.009489349974593138,-0.0103304940650022,0.02150014317213812,-0.002090455580844521,-0.00887618121954512,-0.004167233919382794,-0.003330313483323745,-0.004777296895976673,-0.005690294220441272,-0.012413830934520182,-0.009641099134626938,-0.013867319696322364,0.004110417844789159,-0.0070866017394002985,0.022098208898505688,-0.012172936028767551,0.005659089746495046,-0.006517528371290065,0.01903495847674398,-0.01131629762716183,0.004871206101749936,-0.006440267708494729,-0.004425701795913912,-0.004431856875027815,4.778794005972989e-05,0.0008139952524411973,-0.006115703790106687,-0.0110037009776697,-0.010307054707560943,-0.021387832036459276,-0.02666985918834899,-0.0019040195940504326,-0.006420202135596084,-0.0049491226997694275,-0.019872900348486773,0.007771569962883365 8 | aoi,-0.04092879521069447,0.0019995351406224726,0.03337186764364601,-0.033838731197169146,0.9778707639079527,0.03641392813559959,1.0,-0.015452111519395765,-0.010311577801716452,0.02264237164110034,-0.0034207241410678705,-0.010639757204914446,-0.04135889531388325,0.0019278610658433261,6.666946153895985e-05,-0.0023617427190496137,-0.016363128451382367,-0.014359777431331685,-0.009769023546103987,0.01537761796261592,-0.01884972361485903,0.04185140013258146,-0.01026405080992445,0.00900330401677855,-0.020234707090195786,0.02839780685833531,-0.007609580148801688,0.014151109230898634,-0.006481657826853009,-0.013256869542197095,-0.013262300622624854,-0.006474631201690894,-0.004718902367383397,-0.026952728653736453,-0.010125678610840797,-0.006069948548535816,-0.07672208143973462,0.08359749696233801,-0.2990214444394452,-0.025415427079063066,-0.026428545839160263,-0.04095325791346074,0.03488353637379769 9 | slope,0.5432372099461207,-0.0018034783842755797,-0.006026991930247215,0.006255551277796912,-0.01669337804471738,-0.009489349974593138,-0.015452111519395765,1.0,0.9766755216712971,0.3019671357942443,0.13253360247108506,0.04689633873058686,0.2235246581443786,0.35015683389062807,0.344506609128991,0.12022479036201782,0.22105556212388625,0.2199022440236993,0.0812839590029103,0.12831864075488286,0.022915508260917934,-0.20875703752895422,0.03913245615089788,0.09460903950800492,0.01327145536167107,-0.15757758485384937,0.031139294804627828,0.2691405440098944,0.01565345987903653,0.009142012905006706,0.009173870520016869,-8.942032308483025e-05,-0.00041299255164008126,0.025144763862424112,-8.644076148797822e-05,-0.0031480786108221233,0.04057284529970334,-0.09589511083007193,0.06898893507551075,0.013467134615391093,0.009719650807471879,0.4335951829290541,-0.005177202396516902 10 | cos,0.5256817719727389,-0.0028909726488215203,-0.003382078702084296,0.0035616345861296615,-0.01035335152741857,-0.0103304940650022,-0.010311577801716452,0.9766755216712971,1.0,0.28657617934852947,0.12431945373732596,0.04295990676933593,0.19656090093239098,0.3351418290767474,0.32958570707278345,0.11428076139850706,0.21347247319218543,0.2115033015155072,0.07510243455885618,0.11818401127838393,0.01848536539050914,-0.20825318410182936,0.03444695061686324,0.08771329165476037,0.009370385055645514,-0.15641323400667875,0.033998018871790804,0.25623325168025046,0.010840224434811169,0.00786953590375899,0.007899239690598147,-3.5808092042107104e-05,-0.0003199635755873146,0.02352537823817993,0.002109128668451936,0.0032476791912068306,0.029556972592232875,-0.09546516032471163,0.06642260282288691,0.012131318075888074,0.008087414559543402,0.4341009467743041,-0.001908131550121311 11 | macd,0.15709365258539715,-0.003561554244411846,0.0011553227238044806,-0.0010383545962504416,0.020050578380352103,0.02150014317213812,0.02264237164110034,0.3019671357942443,0.28657617934852947,1.0,0.33349325754068126,0.04360087999533756,0.08248734255792559,0.577797505248084,0.5455202834418865,0.20108627782306446,0.04470457521578339,0.043089881365906364,0.03033081901066395,0.02755586090078684,0.0002258313165772644,-0.07040137085781173,0.001928842131675332,0.03127998095835881,-0.0066502218606768635,-0.04162970243138074,-0.0036045265494982193,0.6747664291809042,0.032539584326466156,0.021106325516925283,0.021057716210632082,-0.00868461468880142,-0.012617976836497307,0.003296579595369577,-0.0026757717763379275,-0.0062780901356971255,0.019813330176989545,-0.1011358829231641,0.07286120425010958,-0.0009043689929572019,-0.0018659150683488952,0.1278477283251027,-0.00794392385915063 12 | dif,0.07036867889680094,0.0153497252515745,0.009023794838967054,-0.007971827995708449,-0.004031536274617544,-0.002090455580844521,-0.0034207241410678705,0.13253360247108506,0.12431945373732596,0.33349325754068126,1.0,0.9563965542376675,0.033369987002097,0.8350328813592699,0.8491344882533649,0.3278114259771233,0.7908395225631711,0.7861286476589804,0.33451812542906006,0.018233532876856615,0.01276671926057492,-0.025325238219704525,-0.00823563575448219,0.018222131727650068,0.021152660784757065,-0.0142203939047115,-0.018018254265371448,0.4518074659732228,0.018235355858930116,0.03847263893140304,0.03888229149853291,0.0013715995613127739,-0.0004451952738086288,0.07142295288883949,-0.007448899871702797,0.03351573820118008,0.025266336342045243,-0.11891065263037233,0.12062077130971308,0.0691896384867596,0.0638135948622747,0.06098510292148342,-0.039037837886383875 13 | dea,0.02590204426329577,0.017369729680281443,0.009204724500023265,-0.008126176906679158,-0.010484088456814372,-0.00887618121954512,-0.010639757204914446,0.04689633873058686,0.04295990676933593,0.04360087999533756,0.9563965542376675,1.0,0.00980752612607598,0.7058912547874612,0.7308346481677316,0.28508878560123174,0.824214783627037,0.8197228418130464,0.3450972050651053,0.010785352464526976,0.013459120875470125,-0.0050267025856654286,-0.009324996153806473,0.00961951219660556,0.02447607871460727,-0.0021723945557887573,-0.017977508455464704,0.2697398539614809,0.00924329212300576,0.03423114938397332,0.034680323890234385,0.004144058785043478,0.0034373568447483077,0.07466667731255003,-0.007064738853267061,0.0374621293546739,0.020636810261246172,-0.09467881236441009,0.10525074305258557,0.07360148344448206,0.06820230257973207,0.025018718014714055,-0.03890790283145778 14 | bs_vol,0.36642862863579895,-0.02345186856495664,0.03675925697594281,-0.03673993852253436,-0.021345041358703905,-0.004167233919382794,-0.04135889531388325,0.2235246581443786,0.19656090093239098,0.08248734255792559,0.033369987002097,0.00980752612607598,1.0,0.12265687075492804,0.11920062528263338,0.15812472238824013,0.06943607447839686,0.07286212030780148,0.1024778550108714,0.16775181079875867,0.01636558771771087,-0.056151315129988576,-0.004647393535255153,0.12257505581952625,-0.01547009838862226,-0.046174325985232585,-0.010565694118499669,0.10004121437457769,0.003594716878716654,-0.07088554921621376,-0.07087457626932733,-0.007090165649876515,-0.007012253069885544,0.0023556796258078355,-0.028488352951787147,-0.006200854813225596,-0.2567478013508999,-0.06661803562337204,-0.03189559972988895,-0.0038778337151370392,-0.002159175215825415,0.3956841939637449,0.09173223322578739 15 | trend_0,0.28935679860375996,0.007402878472484531,0.008271409412813133,-0.007495170372572372,0.0014923586263610785,-0.003330313483323745,0.0019278610658433261,0.35015683389062807,0.3351418290767474,0.577797505248084,0.8350328813592699,0.7058912547874612,0.12265687075492804,1.0,0.9072431536793563,0.34172271154073125,0.576389478176685,0.5728176383754342,0.24552128400376083,0.0876179250829926,0.011120598131294554,-0.09108751740958566,0.0048893549093152544,0.0772174669081728,0.016175918625896603,-0.05643901749824275,-0.005672375013977172,0.8082034390970928,0.0449124086725976,0.03966092134532006,0.03993292004215339,-0.004589419005076029,-0.006870863276785661,0.05428484906657767,-0.0053540979560117025,0.020583604092535367,0.026919171706112615,-0.12699627693903612,0.1117177313108476,0.048425585537789584,0.043564699863022235,0.23178589033488695,-0.028454659410071657 16 | log_return_0,0.28462427236158183,0.007574577866273225,0.009923362938337841,-0.009148896319394575,-1.736166064074487e-05,-0.004777296895976673,6.666946153895985e-05,0.344506609128991,0.32958570707278345,0.5455202834418865,0.8491344882533649,0.7308346481677316,0.11920062528263338,0.9072431536793563,1.0,0.3671649165908091,0.5904670175342477,0.5872504628552454,0.25058870248179893,0.08815654748424076,0.011279064875382175,-0.08753103048563439,0.00720847481743129,0.07639181911276105,0.018098871887724435,-0.05506226791567972,-0.0033918492869243486,0.6845626683256297,0.03244993929637206,0.04057772750919149,0.04086594315396657,-0.005100408640173139,-0.00756266178867319,0.054149605252608354,-0.0060603430584108605,0.022910870570040414,0.026807065190963746,-0.12592117467294603,0.11076592498477474,0.04836636253874355,0.043393423729295603,0.2290337708176124,-0.029192276232836302 17 | bsvol_volume_0,0.09070336289428182,-0.0016315989094493385,-0.025503873981625926,0.0257278830582537,-0.003996286919903948,-0.005690294220441272,-0.0023617427190496137,0.12022479036201782,0.11428076139850706,0.20108627782306446,0.3278114259771233,0.28508878560123174,0.15812472238824013,0.34172271154073125,0.3671649165908091,1.0,0.24071138093989328,0.23937188938424436,0.5878589775440052,0.04254103180539655,0.09278423380214099,-0.01286597650920801,0.025664856149499902,0.03182370654866379,0.1034053693248603,-0.009968187336035286,0.028905753544332398,0.24465558300670953,0.020618797460469053,0.048782564053904715,0.04890615251492233,-0.015945979274746263,-0.020588329229078987,0.01746362032655039,-0.02919093880909888,-0.006468517772344754,0.0485201821949584,-0.07235620695740264,0.017915190388773487,0.01589868055771091,0.014890559170437518,0.13777538854382776,-0.07059150166749972 18 | trend_1,0.18327645850335914,0.02401897375457497,0.0018937766020748583,-0.0006456330663136456,-0.016483405681227,-0.012413830934520182,-0.016363128451382367,0.22105556212388625,0.21347247319218543,0.04470457521578339,0.7908395225631711,0.824214783627037,0.06943607447839686,0.576389478176685,0.5904670175342477,0.24071138093989328,1.0,0.9693498710991307,0.40235441697265284,0.052133424171559146,0.022849798104399903,-0.061134540251971424,0.004641044514580806,0.03640227049388742,0.03071667647798733,-0.04844585092925512,-0.007299187772135927,-0.015397236044889328,0.002230442214599289,0.02911316872136266,0.02952101174263018,0.002986905037955258,0.0031320746435373366,0.08787642571372858,-0.006595132460655768,0.042898605797340714,0.036918920155100426,-0.11179702084845447,0.12216033556844769,0.08396656715226664,0.07805657941770541,0.14758374475352554,-0.04579519025316414 19 | log_return_1,0.18169826891415178,0.022655239228405677,0.0034540050556136734,-0.0021681995558814203,-0.01480259409550331,-0.009641099134626938,-0.014359777431331685,0.2199022440236993,0.2115033015155072,0.043089881365906364,0.7861286476589804,0.8197228418130464,0.07286212030780148,0.5728176383754342,0.5872504628552454,0.23937188938424436,0.9693498710991307,1.0,0.4089533787311917,0.05145489209740354,0.02325360413697211,-0.06085599861707065,0.004684497849879221,0.036845256318698616,0.031057627164509858,-0.04725280043941611,-0.007082492075118972,0.002320567782241834,0.003425503505403228,0.028056008589512,0.028461021811054175,0.00320954186940537,0.003359058335616809,0.09027399340924488,-0.005750751006494651,0.04216374724755004,0.036857385711182154,-0.11079092665173343,0.11937317789143154,0.08636860686832198,0.08053588311631943,0.14616333030985673,-0.046870696474985665 20 | bsvol_volume_1,0.058694026936791946,0.0011161409214845252,-0.022576964019916608,0.023035088857052015,-0.00988390389626022,-0.013867319696322364,-0.009769023546103987,0.0812839590029103,0.07510243455885618,0.03033081901066395,0.33451812542906006,0.3450972050651053,0.1024778550108714,0.24552128400376083,0.25058870248179893,0.5878589775440052,0.40235441697265284,0.4089533787311917,1.0,0.026419010418722674,0.10588699701814405,-0.0095192263327721,0.022017725400942354,0.02515015389454617,0.12817648028767664,-0.0018336255928404366,0.019506246589418276,0.010455897522520897,0.009075403791258083,0.008190495191106144,0.008358507903475287,0.01564832148004116,0.016868362028471005,0.03398571338647485,-0.050624654489106316,-0.00450927232594373,0.05727546071332129,-0.062121378401127136,0.014548854299839393,0.03281813632427243,0.031195824156018317,0.0830097662737387,-0.12894132355353266 21 | label_0,0.36939565843947203,-0.008157562027375371,-0.0210841355653561,0.020870635127052856,0.010168212937281255,0.004110417844789159,0.01537761796261592,0.12831864075488286,0.11818401127838393,0.02755586090078684,0.018233532876856615,0.010785352464526976,0.16775181079875867,0.0876179250829926,0.08815654748424076,0.04254103180539655,0.052133424171559146,0.05145489209740354,0.026419010418722674,1.0,0.05646093255167114,0.8102774370308368,-0.1881836744079126,0.6647371071115457,0.04095802118968368,0.5075365125071036,-0.1398684660580214,0.06963744640588676,-0.010507875695530019,-0.008164553330016155,-0.008149429041154274,-0.008684069051029721,-0.008873873591315333,-0.01667095049772346,-0.0005750080209854852,0.012583724068644293,0.004489172872992793,-0.026688541404795072,0.016518767801762202,-0.016685103109304916,-0.014988885490955856,0.2851905407240717,-0.022181015671228946 22 | rv_0,0.016299696091755373,-0.012382822766602079,-0.18293246643979927,0.1828137190974216,-0.026845873331659564,-0.0070866017394002985,-0.01884972361485903,0.022915508260917934,0.01848536539050914,0.0002258313165772644,0.01276671926057492,0.013459120875470125,0.01636558771771087,0.011120598131294554,0.011279064875382175,0.09278423380214099,0.022849798104399903,0.02325360413697211,0.10588699701814405,0.05646093255167114,1.0,0.04862971188016301,0.19052162492595137,0.058456608672606575,0.7787820034241565,0.05215966904713087,0.10344164691173774,-0.002859905595181167,0.015686672074114137,0.056279343946433295,0.05628790138329882,0.008281351364745844,0.00806242644100997,-0.0016568941571958333,-0.04583158522619454,-0.038919195409231305,0.1704089701968651,-0.018635206417055232,-0.009744178297340856,-0.0017392704257459988,-0.0023177709578571246,0.17217298348812116,-0.23650595214169454 23 | price_chg_0,-0.2452747285184763,-0.007283501168432657,-0.014830977523974125,0.014490471970812543,0.03609804472367022,0.022098208898505688,0.04185140013258146,-0.20875703752895422,-0.20825318410182936,-0.07040137085781173,-0.025325238219704525,-0.0050267025856654286,-0.056151315129988576,-0.09108751740958566,-0.08753103048563439,-0.01286597650920801,-0.061134540251971424,-0.06085599861707065,-0.0095192263327721,0.8102774370308368,0.04862971188016301,1.0,-0.23101882096057486,0.5174989932332564,0.03567352882084029,0.6446772681216031,-0.17520804685645272,-0.06739605811598848,-0.02513349882120021,-0.01234543599667244,-0.012341735136786268,-0.004217006306927853,-0.003974399005642217,-0.03345508684644617,-0.0015555244086645082,0.012003262032685194,-0.010235454039429792,0.00875096511539969,-0.014571007165970633,-0.024766083113741188,-0.020827420339148552,-0.19435742139021403,-0.020988224744090827 24 | label_clf_0,0.05501590071220928,-0.010436211117863211,-0.06867130750973274,0.06890639960328447,-0.012923469828317862,-0.012172936028767551,-0.01026405080992445,0.03913245615089788,0.03444695061686324,0.001928842131675332,-0.00823563575448219,-0.009324996153806473,-0.004647393535255153,0.0048893549093152544,0.00720847481743129,0.025664856149499902,0.004641044514580806,0.004684497849879221,0.022017725400942354,-0.1881836744079126,0.19052162492595137,-0.23101882096057486,1.0,-0.12274759266114206,0.20015549143891284,-0.15141830995166553,0.34271438780689123,0.0026379271974671446,0.018014306205290622,0.04320584570445539,0.04320252628271997,-0.0056613091821502695,-0.005059186778788717,0.019882684459645243,-0.02145922764922648,-0.02368872517107596,0.10580335754605173,0.0017041162602890777,-0.0041909752432901844,0.017787152606758855,0.016752114392454494,0.06661730596897232,-0.08827715153442882 25 | label_1,0.2789478255086415,-0.006262491961302007,-0.017372080936605125,0.01699192634228204,0.006231535622128172,0.005659089746495046,0.00900330401677855,0.09460903950800492,0.08771329165476037,0.03127998095835881,0.018222131727650068,0.00961951219660556,0.12257505581952625,0.0772174669081728,0.07639181911276105,0.03182370654866379,0.03640227049388742,0.036845256318698616,0.02515015389454617,0.6647371071115457,0.058456608672606575,0.5174989932332564,-0.12274759266114206,1.0,0.06673668034161448,0.8931480076285518,-0.25663208977048213,0.06824839829356014,-0.006131971093202885,-0.011257839891772242,-0.01124836461539646,-0.0042133994212118515,-0.0006862049293191473,-0.02741749606915113,-0.001128961242683365,0.0018492339619891598,0.010592857555095498,-0.027434370111809575,0.010441737087579426,-0.02742916942894154,-0.026174052801068676,0.2169026799501304,-0.033097301293992215 26 | rv_1,0.011232752126037282,-0.016064995876277362,-0.17059873416850516,0.17044513627417834,-0.026495754452218176,-0.006517528371290065,-0.020234707090195786,0.01327145536167107,0.009370385055645514,-0.0066502218606768635,0.021152660784757065,0.02447607871460727,-0.01547009838862226,0.016175918625896603,0.018098871887724435,0.1034053693248603,0.03071667647798733,0.031057627164509858,0.12817648028767664,0.04095802118968368,0.7787820034241565,0.03567352882084029,0.20015549143891284,0.06673668034161448,1.0,0.06307065015257253,0.14694430071875036,-0.0023436271685679825,0.003177195993483266,0.06361630972864837,0.06363102121774865,0.009792018240210327,0.007675974996375772,-0.002293873842707399,-0.06251206647829781,-0.05009725158450896,0.19642558228066176,-0.030673839396512203,-0.01570134599105522,-0.0023902834050890174,-0.0029973116451321702,0.11516980057806929,-0.3038610994307427 27 | price_chg_1,-0.18275654382672596,-0.005490545466873095,-0.012462046829067219,0.011986134395992568,0.025323440743364754,0.01903495847674398,0.02839780685833531,-0.15757758485384937,-0.15641323400667875,-0.04162970243138074,-0.0142203939047115,-0.0021723945557887573,-0.046174325985232585,-0.05643901749824275,-0.05506226791567972,-0.009968187336035286,-0.04844585092925512,-0.04725280043941611,-0.0018336255928404366,0.5075365125071036,0.05215966904713087,0.6446772681216031,-0.15141830995166553,0.8931480076285518,0.06307065015257253,1.0,-0.2845207302306171,-0.03414494722349296,-0.016806002725056366,-0.01435022085888908,-0.014349444940272758,-0.0007085251727418098,0.0032422922263392003,-0.03994812201905904,-0.0018010387290632222,0.0011135777265215863,-0.0002405894099277259,-0.0008989105146791506,-0.012906736358955982,-0.03349554864941822,-0.03060030065954934,-0.14322992175916546,-0.032274217249379185 28 | label_clf_1,0.04637147346560956,-0.009093195973181612,-0.04045205138270358,0.04088763300403798,-0.008470170386257529,-0.01131629762716183,-0.007609580148801688,0.031139294804627828,0.033998018871790804,-0.0036045265494982193,-0.018018254265371448,-0.017977508455464704,-0.010565694118499669,-0.005672375013977172,-0.0033918492869243486,0.028905753544332398,-0.007299187772135927,-0.007082492075118972,0.019506246589418276,-0.1398684660580214,0.10344164691173774,-0.17520804685645272,0.34271438780689123,-0.25663208977048213,0.14694430071875036,-0.2845207302306171,1.0,-0.0016804062515182179,-0.002394807357909251,0.026917038230414355,0.026910739339090866,0.004140697410819052,-0.0010915054561001422,0.035690384254963736,-0.019161453969645373,-0.01437336341277636,0.07203721081755816,-0.0061892794991880475,-0.011099557984717888,0.03385356039399726,0.032929275370472466,0.0620132157255632,-0.07549681467533133 29 | trend_ls_diff,0.2219724586039065,-0.008251418824299068,0.008755990348824322,-0.00870568159244484,0.01370491460513496,0.004871206101749936,0.014151109230898634,0.2691405440098944,0.25623325168025046,0.6747664291809042,0.4518074659732228,0.2697398539614809,0.10004121437457769,0.8082034390970928,0.6845626683256297,0.24465558300670953,-0.015397236044889328,0.002320567782241834,0.010455897522520897,0.06963744640588676,-0.002859905595181167,-0.06739605811598848,0.0026379271974671446,0.06824839829356014,-0.0023436271685679825,-0.03414494722349296,-0.0016804062515182179,1.0,0.05334666798872013,0.02754779927043832,0.027586697072235343,-0.007768069942760115,-0.010664223590577964,0.0030932036165731433,-0.001798350653929668,-0.005729457748309333,0.00633193951753354,-0.07482310298601748,0.0486601359672552,-0.0012584217846168604,-0.0029470410466851324,0.17725190509184077,-0.0018139851797676263 30 | trend_ls_ratio,0.02248117677720777,-0.009572022204702133,-0.012713313015204014,0.012645761420629969,-0.007314846746996655,-0.006440267708494729,-0.006481657826853009,0.01565345987903653,0.010840224434811169,0.032539584326466156,0.018235355858930116,0.00924329212300576,0.003594716878716654,0.0449124086725976,0.03244993929637206,0.020618797460469053,0.002230442214599289,0.003425503505403228,0.009075403791258083,-0.010507875695530019,0.015686672074114137,-0.02513349882120021,0.018014306205290622,-0.006131971093202885,0.003177195993483266,-0.016806002725056366,-0.002394807357909251,0.05334666798872013,1.0,0.19194399240348828,0.19194344852915612,0.002897529465602425,-0.0029711791178919666,-0.0036078785153296094,-0.011465222729724481,-0.011658683316863084,0.046113961929263945,0.060614372345330465,0.032796093897679325,-0.004463716144893776,-0.004256243507965783,0.03437174393065234,0.11653108621581809 31 | volume_ls_ratio,0.006144528876721859,0.006351558096494069,-0.07153820161594453,0.07167242859740797,-0.016238447726833836,-0.004425701795913912,-0.013256869542197095,0.009142012905006706,0.00786953590375899,0.021106325516925283,0.03847263893140304,0.03423114938397332,-0.07088554921621376,0.03966092134532006,0.04057772750919149,0.048782564053904715,0.02911316872136266,0.028056008589512,0.008190495191106144,-0.008164553330016155,0.056279343946433295,-0.01234543599667244,0.04320584570445539,-0.011257839891772242,0.06361630972864837,-0.01435022085888908,0.026917038230414355,0.02754779927043832,0.19194399240348828,1.0,0.9999998826525042,0.019764649778022,-0.003385323971168232,0.013462463313919653,0.01304878736881774,-0.037141645539322574,0.21131072652141056,0.02684725455948848,0.06253056679469954,0.013493617845747735,0.013020952652011133,0.012137718458712183,0.25729561907984766 32 | turnover_ls_ratio,0.006163724060260921,0.0063610224548742095,-0.07153775107589254,0.07167245044534787,-0.016244292667572123,-0.004431856875027815,-0.013262300622624854,0.009173870520016869,0.007899239690598147,0.021057716210632082,0.03888229149853291,0.034680323890234385,-0.07087457626932733,0.03993292004215339,0.04086594315396657,0.04890615251492233,0.02952101174263018,0.028461021811054175,0.008358507903475287,-0.008149429041154274,0.05628790138329882,-0.012341735136786268,0.04320252628271997,-0.01124836461539646,0.06363102121774865,-0.014349444940272758,0.026910739339090866,0.027586697072235343,0.19194344852915612,0.9999998826525042,1.0,0.019764818794206735,-0.0033844880597027776,0.013495611056904031,0.0130447842173238,-0.03712285344424824,0.2113205903654169,0.02680792553413025,0.06257984024819177,0.013526481692114595,0.013051329574837513,0.012155158410349072,0.2572715047882559 33 | bs_vol_ls_ratio,-0.0077051058846247045,0.002429401854186942,0.0005745450245647402,-0.0004771819932714005,-0.005796848587927473,4.778794005972989e-05,-0.006474631201690894,-8.942032308483025e-05,-3.5808092042107104e-05,-0.00868461468880142,0.0013715995613127739,0.004144058785043478,-0.007090165649876515,-0.004589419005076029,-0.005100408640173139,-0.015945979274746263,0.002986905037955258,0.00320954186940537,0.01564832148004116,-0.008684069051029721,0.008281351364745844,-0.004217006306927853,-0.0056613091821502695,-0.0042133994212118515,0.009792018240210327,-0.0007085251727418098,0.004140697410819052,-0.007768069942760115,0.002897529465602425,0.019764649778022,0.019764818794206735,1.0,0.910564401721571,0.007335097897226584,-0.011431469213875,0.004849156463890722,0.004466113094543014,-0.002277453039196899,0.0008035528161606313,0.007348170442475548,0.007371431923765388,-0.0076822421929908,-0.008425543287590728 34 | bsvol_volume_ls_ratio,-0.008400306094597675,0.0012108893752758108,0.001884262418507718,-0.0017958067638953763,-0.00414045202908549,0.0008139952524411973,-0.004718902367383397,-0.00041299255164008126,-0.0003199635755873146,-0.012617976836497307,-0.0004451952738086288,0.0034373568447483077,-0.007012253069885544,-0.006870863276785661,-0.00756266178867319,-0.020588329229078987,0.0031320746435373366,0.003359058335616809,0.016868362028471005,-0.008873873591315333,0.00806242644100997,-0.003974399005642217,-0.005059186778788717,-0.0006862049293191473,0.007675974996375772,0.0032422922263392003,-0.0010915054561001422,-0.010664223590577964,-0.0029711791178919666,-0.003385323971168232,-0.0033844880597027776,0.910564401721571,1.0,0.006342871393432024,-0.010018025930311958,0.00486505686500601,0.0024060199550001308,-0.0008857445398116769,-0.0005344922113364504,0.0063689999529933285,0.006509275347651979,-0.007377982428478173,-0.015573187854846567 35 | norm_lastprice,0.025550012849563108,0.041089524033103036,0.013426474043960327,0.0009898114853633783,-0.02865597314141963,-0.006115703790106687,-0.026952728653736453,0.025144763862424112,0.02352537823817993,0.003296579595369577,0.07142295288883949,0.07466667731255003,0.0023556796258078355,0.05428484906657767,0.054149605252608354,0.01746362032655039,0.08787642571372858,0.09027399340924488,0.03398571338647485,-0.01667095049772346,-0.0016568941571958333,-0.03345508684644617,0.019882684459645243,-0.02741749606915113,-0.002293873842707399,-0.03994812201905904,0.035690384254963736,0.0030932036165731433,-0.0036078785153296094,0.013462463313919653,0.013495611056904031,0.007335097897226584,0.006342871393432024,1.0,0.46907003849498463,0.05516895412105506,0.01772320389430136,-0.019192397605104182,0.0707390807255493,0.9996643169889884,0.9994751729390902,0.023567125220449542,-0.01679734578115597 36 | norm_openinterest,0.0016377857662318784,0.02709960272594908,0.017222421644095497,-0.010478300047777384,-0.012727075045276521,-0.0110037009776697,-0.010125678610840797,-8.644076148797822e-05,0.002109128668451936,-0.0026757717763379275,-0.007448899871702797,-0.007064738853267061,-0.028488352951787147,-0.0053540979560117025,-0.0060603430584108605,-0.02919093880909888,-0.006595132460655768,-0.005750751006494651,-0.050624654489106316,-0.0005750080209854852,-0.04583158522619454,-0.0015555244086645082,-0.02145922764922648,-0.001128961242683365,-0.06251206647829781,-0.0018010387290632222,-0.019161453969645373,-0.001798350653929668,-0.011465222729724481,0.01304878736881774,0.0130447842173238,-0.011431469213875,-0.010018025930311958,0.46907003849498463,1.0,0.04996527255215003,0.026461339653671093,0.0794310829832814,0.02743003515127373,0.46919973031408657,0.4701532328509145,-0.001541806601678097,-0.0347744351714621 37 | norm_interestdiff,0.0017756125544669292,0.23309250267300968,0.031080289670741546,-0.03016456149377878,-0.015974228593088276,-0.010307054707560943,-0.006069948548535816,-0.0031480786108221233,0.0032476791912068306,-0.0062780901356971255,0.03351573820118008,0.0374621293546739,-0.006200854813225596,0.020583604092535367,0.022910870570040414,-0.006468517772344754,0.042898605797340714,0.04216374724755004,-0.00450927232594373,0.012583724068644293,-0.038919195409231305,0.012003262032685194,-0.02368872517107596,0.0018492339619891598,-0.05009725158450896,0.0011135777265215863,-0.01437336341277636,-0.005729457748309333,-0.011658683316863084,-0.037141645539322574,-0.03712285344424824,0.004849156463890722,0.00486505686500601,0.05516895412105506,0.04996527255215003,1.0,-0.17043742585694788,-0.007502388047832082,-0.01474561599659727,0.0552690437042062,0.05528161802613098,0.004362877678196357,0.025483461049101774 38 | norm_volume,0.023739479157777535,0.01699043437126748,-0.17103032842120514,0.17116359808955356,-0.10024699701677081,-0.021387832036459276,-0.07672208143973462,0.04057284529970334,0.029556972592232875,0.019813330176989545,0.025266336342045243,0.020636810261246172,-0.2567478013508999,0.026919171706112615,0.026807065190963746,0.0485201821949584,0.036918920155100426,0.036857385711182154,0.05727546071332129,0.004489172872992793,0.1704089701968651,-0.010235454039429792,0.10580335754605173,0.010592857555095498,0.19642558228066176,-0.0002405894099277259,0.07203721081755816,0.00633193951753354,0.046113961929263945,0.21131072652141056,0.2113205903654169,0.004466113094543014,0.0024060199550001308,0.01772320389430136,0.026461339653671093,-0.17043742585694788,1.0,0.07431065178449221,0.1462915656574945,0.017827305853692663,0.016544076225834974,0.04741515623980669,-0.2443153320149407 39 | norm_bidvolume1,-0.05793142753675455,0.0017671644254506874,-0.048721028082614144,0.04839132326040783,0.07709366043393663,-0.02666985918834899,0.08359749696233801,-0.09589511083007193,-0.09546516032471163,-0.1011358829231641,-0.11891065263037233,-0.09467881236441009,-0.06661803562337204,-0.12699627693903612,-0.12592117467294603,-0.07235620695740264,-0.11179702084845447,-0.11079092665173343,-0.062121378401127136,-0.026688541404795072,-0.018635206417055232,0.00875096511539969,0.0017041162602890777,-0.027434370111809575,-0.030673839396512203,-0.0008989105146791506,-0.0061892794991880475,-0.07482310298601748,0.060614372345330465,0.02684725455948848,0.02680792553413025,-0.002277453039196899,-0.0008857445398116769,-0.019192397605104182,0.0794310829832814,-0.007502388047832082,0.07431065178449221,1.0,0.028530113616259938,-0.01820403088297347,-0.013877248648944998,-0.058648302792326495,-0.059375401658092054 40 | norm_askvolume1,0.05041786715463712,0.00482710498620479,-0.014040665235812664,0.015071571174353565,-0.3106827457049558,-0.0019040195940504326,-0.2990214444394452,0.06898893507551075,0.06642260282288691,0.07286120425010958,0.12062077130971308,0.10525074305258557,-0.03189559972988895,0.1117177313108476,0.11076592498477474,0.017915190388773487,0.12216033556844769,0.11937317789143154,0.014548854299839393,0.016518767801762202,-0.009744178297340856,-0.014571007165970633,-0.0041909752432901844,0.010441737087579426,-0.01570134599105522,-0.012906736358955982,-0.011099557984717888,0.0486601359672552,0.032796093897679325,0.06253056679469954,0.06257984024819177,0.0008035528161606313,-0.0005344922113364504,0.0707390807255493,0.02743003515127373,-0.01474561599659727,0.1462915656574945,0.028530113616259938,1.0,0.07008324340591686,0.06648497745295315,0.03685180715174309,-0.06534545744761575 41 | norm_vwap,0.01174928073368846,0.04112423683053382,0.013130830466571871,0.0012876978139854374,-0.02736404606009161,-0.006420202135596084,-0.025415427079063066,0.013467134615391093,0.012131318075888074,-0.0009043689929572019,0.0691896384867596,0.07360148344448206,-0.0038778337151370392,0.048425585537789584,0.04836636253874355,0.01589868055771091,0.08396656715226664,0.08636860686832198,0.03281813632427243,-0.016685103109304916,-0.0017392704257459988,-0.024766083113741188,0.017787152606758855,-0.02742916942894154,-0.0023902834050890174,-0.03349554864941822,0.03385356039399726,-0.0012584217846168604,-0.004463716144893776,0.013493617845747735,0.013526481692114595,0.007348170442475548,0.0063689999529933285,0.9996643169889884,0.46919973031408657,0.0552690437042062,0.017827305853692663,-0.01820403088297347,0.07008324340591686,1.0,0.9996851492425094,0.012809232864344171,-0.016519512372277653 42 | norm_wap,0.00831067569562216,0.04064287131517503,0.01339544510464434,0.0010255800932542295,-0.02803940514393768,-0.0049491226997694275,-0.026428545839160263,0.009719650807471879,0.008087414559543402,-0.0018659150683488952,0.0638135948622747,0.06820230257973207,-0.002159175215825415,0.043564699863022235,0.043393423729295603,0.014890559170437518,0.07805657941770541,0.08053588311631943,0.031195824156018317,-0.014988885490955856,-0.0023177709578571246,-0.020827420339148552,0.016752114392454494,-0.026174052801068676,-0.0029973116451321702,-0.03060030065954934,0.032929275370472466,-0.0029470410466851324,-0.004256243507965783,0.013020952652011133,0.013051329574837513,0.007371431923765388,0.006509275347651979,0.9994751729390902,0.4701532328509145,0.05528161802613098,0.016544076225834974,-0.013877248648944998,0.06648497745295315,0.9996851492425094,1.0,0.009432817879780515,-0.01584170826115794 43 | norm_bs_tag,0.7799302620131483,-0.013868880619242673,-0.039652281053981615,0.039826838470452056,-0.04076870508108915,-0.019872900348486773,-0.04095325791346074,0.4335951829290541,0.4341009467743041,0.1278477283251027,0.06098510292148342,0.025018718014714055,0.3956841939637449,0.23178589033488695,0.2290337708176124,0.13777538854382776,0.14758374475352554,0.14616333030985673,0.0830097662737387,0.2851905407240717,0.17217298348812116,-0.19435742139021403,0.06661730596897232,0.2169026799501304,0.11516980057806929,-0.14322992175916546,0.0620132157255632,0.17725190509184077,0.03437174393065234,0.012137718458712183,0.012155158410349072,-0.0076822421929908,-0.007377982428478173,0.023567125220449542,-0.001541806601678097,0.004362877678196357,0.04741515623980669,-0.058648302792326495,0.03685180715174309,0.012809232864344171,0.009432817879780515,1.0,-0.02135184570787218 44 | norm_volume_ls_diff,-0.0033881076144420025,0.013794460670124622,0.04553697472655794,-0.04563182007460571,0.03596876463416743,0.007771569962883365,0.03488353637379769,-0.005177202396516902,-0.001908131550121311,-0.00794392385915063,-0.039037837886383875,-0.03890790283145778,0.09173223322578739,-0.028454659410071657,-0.029192276232836302,-0.07059150166749972,-0.04579519025316414,-0.046870696474985665,-0.12894132355353266,-0.022181015671228946,-0.23650595214169454,-0.020988224744090827,-0.08827715153442882,-0.033097301293992215,-0.3038610994307427,-0.032274217249379185,-0.07549681467533133,-0.0018139851797676263,0.11653108621581809,0.25729561907984766,0.2572715047882559,-0.008425543287590728,-0.015573187854846567,-0.01679734578115597,-0.0347744351714621,0.025483461049101774,-0.2443153320149407,-0.059375401658092054,-0.06534545744761575,-0.016519512372277653,-0.01584170826115794,-0.02135184570787218,1.0 45 | --------------------------------------------------------------------------------