├── mas_tools ├── README.md ├── __init__.py ├── api │ ├── __init__.py │ └── common.py ├── layers │ ├── data_to_img.py │ ├── __init__.py │ ├── attention_weighted_average.py │ ├── attention_with_context.py │ └── Attention.py ├── models │ ├── __init__.py │ ├── models.py │ ├── convolutions.py │ └── autoencoders.py ├── os.py ├── ml.py ├── classes.py ├── data.py ├── trade.py └── envs │ └── envs.py ├── docs ├── PULL_REQUEST_TEMPLATE.md ├── CONTRIBUTING.md └── CODE_OF_CONDUCT.md ├── data ├── normalized │ └── README.md ├── windowed │ └── README.md ├── pictured │ └── README.md ├── exported │ └── README.md ├── transformed │ └── README.md ├── files.py ├── crypto │ ├── _concatenate.py │ ├── _prices.py │ ├── _tickerbook.py │ ├── _prices+tickerbook.py │ └── _crypto_collector.py ├── to_normalized.py ├── to_windowed.py ├── transform.py ├── create_picture.py ├── to_pictures.py └── README.md ├── mas_cryptobot ├── vae │ ├── img │ │ ├── ae.png │ │ ├── ae_dec.png │ │ ├── ae_enc.png │ │ ├── 1_Animation.gif │ │ └── 4_Animation.gif │ ├── 2_8-24_80 │ │ ├── 10.png │ │ ├── 11.png │ │ ├── 21.png │ │ ├── 30.png │ │ ├── 31.png │ │ ├── 40.png │ │ ├── 41.png │ │ ├── 50.png │ │ ├── 51.png │ │ ├── 60.png │ │ └── 61.png │ ├── 4_3-12_60 │ │ ├── 0vae_img20_flt3-12_code60_origin_pics.png │ │ ├── 1vae_img20_flt3-12_code60_origin_pics.png │ │ ├── 2vae_img20_flt3-12_code60_origin_pics.png │ │ ├── 3vae_img20_flt3-12_code60_origin_pics.png │ │ ├── 4vae_img20_flt3-12_code60_origin_pics.png │ │ ├── 5vae_img20_flt3-12_code60_origin_pics.png │ │ ├── vae_img20_flt3-12_code60_latent-distr.png │ │ ├── 0vae_img20_flt3-12_code60_restored_from_z.png │ │ ├── 1vae_img20_flt3-12_code60_restored_from_z.png │ │ ├── 2vae_img20_flt3-12_code60_restored_from_z.png │ │ ├── 3vae_img20_flt3-12_code60_restored_from_z.png │ │ ├── 4vae_img20_flt3-12_code60_restored_from_z.png │ │ ├── 5vae_img20_flt3-12_code60_restored_from_z.png │ │ ├── vae_img20_flt3-12_code60_latent-distr_log.png │ │ └── vae_img20_flt3-12_code60_latent-distr__mean.png │ ├── model.py │ └── train_autoencoders.py ├── img │ ├── arch_bot_02a.png │ ├── arch_bot_00_cnn2in.png │ └── arch_bot_01_cnn2in_feedback.png ├── README.md ├── bot-rl_v0.0.py ├── bot-rl_v0.1.py ├── train_1.2_cnn_ae+lstm.py ├── bot-rl_v0.2.py ├── train_1.1_cnn_ae_for_candles.py └── train_1.3_ae_for_tickerbook.py ├── mas_mt ├── v0.5_deep_regul │ ├── predicted.png │ └── classifier_1.py ├── v0.6_deep_recurrent │ ├── good.png │ └── classifier_2.py ├── v0.8_functional_models │ ├── simple.png │ └── simple_arch1.py ├── README.md ├── v0.4_simple_classifier │ ├── classifier_0.py │ ├── classifier_0_activation.py │ └── classifier_0_feature_selection.py └── v0.7_conv+rnn │ └── classifier_3.py ├── .gitignore ├── .github └── ISSUE_TEMPLATE │ ├── feature_request.md │ └── bug_report.md ├── setup.py ├── LICENSE └── readme.md /mas_tools/README.md: -------------------------------------------------------------------------------- 1 | ## MAS_TOOLS PACKAGE 2 | -------------------------------------------------------------------------------- /docs/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | 1. One 2 | 2. Two 3 | -------------------------------------------------------------------------------- /mas_tools/__init__.py: -------------------------------------------------------------------------------- 1 | __version__ = '0.8.10-alpha' -------------------------------------------------------------------------------- /data/normalized/README.md: -------------------------------------------------------------------------------- 1 | ## Directory for data normalized to 1.0. -------------------------------------------------------------------------------- /data/windowed/README.md: -------------------------------------------------------------------------------- 1 | ## Data in the form of a sequence of observation windows -------------------------------------------------------------------------------- /data/pictured/README.md: -------------------------------------------------------------------------------- 1 | ## Directory with windows of time series converted into images. -------------------------------------------------------------------------------- /data/exported/README.md: -------------------------------------------------------------------------------- 1 | ## Exported from Alpari servers 2 | [Link](https://goo.gl/q4S5Qu "GoogleDrive") -------------------------------------------------------------------------------- /mas_tools/api/__init__.py: -------------------------------------------------------------------------------- 1 | from mas_tools.api.binance import Binance 2 | # from mas_tools.api.exmo import Exmo 3 | -------------------------------------------------------------------------------- /mas_cryptobot/vae/img/ae.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tau-lex/market-analysis-system/HEAD/mas_cryptobot/vae/img/ae.png -------------------------------------------------------------------------------- /data/transformed/README.md: -------------------------------------------------------------------------------- 1 | ## Transformed data into the format used in the project 2 | [Link](https://goo.gl/vCJNj6 "GoogleDrive") -------------------------------------------------------------------------------- /mas_cryptobot/vae/img/ae_dec.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tau-lex/market-analysis-system/HEAD/mas_cryptobot/vae/img/ae_dec.png -------------------------------------------------------------------------------- /mas_cryptobot/vae/img/ae_enc.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tau-lex/market-analysis-system/HEAD/mas_cryptobot/vae/img/ae_enc.png -------------------------------------------------------------------------------- /mas_cryptobot/img/arch_bot_02a.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tau-lex/market-analysis-system/HEAD/mas_cryptobot/img/arch_bot_02a.png -------------------------------------------------------------------------------- /mas_cryptobot/vae/2_8-24_80/10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tau-lex/market-analysis-system/HEAD/mas_cryptobot/vae/2_8-24_80/10.png -------------------------------------------------------------------------------- /mas_cryptobot/vae/2_8-24_80/11.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tau-lex/market-analysis-system/HEAD/mas_cryptobot/vae/2_8-24_80/11.png -------------------------------------------------------------------------------- /mas_cryptobot/vae/2_8-24_80/21.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tau-lex/market-analysis-system/HEAD/mas_cryptobot/vae/2_8-24_80/21.png -------------------------------------------------------------------------------- /mas_cryptobot/vae/2_8-24_80/30.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tau-lex/market-analysis-system/HEAD/mas_cryptobot/vae/2_8-24_80/30.png -------------------------------------------------------------------------------- /mas_cryptobot/vae/2_8-24_80/31.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tau-lex/market-analysis-system/HEAD/mas_cryptobot/vae/2_8-24_80/31.png -------------------------------------------------------------------------------- /mas_cryptobot/vae/2_8-24_80/40.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tau-lex/market-analysis-system/HEAD/mas_cryptobot/vae/2_8-24_80/40.png -------------------------------------------------------------------------------- /mas_cryptobot/vae/2_8-24_80/41.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tau-lex/market-analysis-system/HEAD/mas_cryptobot/vae/2_8-24_80/41.png -------------------------------------------------------------------------------- /mas_cryptobot/vae/2_8-24_80/50.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tau-lex/market-analysis-system/HEAD/mas_cryptobot/vae/2_8-24_80/50.png -------------------------------------------------------------------------------- /mas_cryptobot/vae/2_8-24_80/51.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tau-lex/market-analysis-system/HEAD/mas_cryptobot/vae/2_8-24_80/51.png -------------------------------------------------------------------------------- /mas_cryptobot/vae/2_8-24_80/60.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tau-lex/market-analysis-system/HEAD/mas_cryptobot/vae/2_8-24_80/60.png -------------------------------------------------------------------------------- /mas_cryptobot/vae/2_8-24_80/61.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tau-lex/market-analysis-system/HEAD/mas_cryptobot/vae/2_8-24_80/61.png -------------------------------------------------------------------------------- /mas_mt/v0.5_deep_regul/predicted.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tau-lex/market-analysis-system/HEAD/mas_mt/v0.5_deep_regul/predicted.png -------------------------------------------------------------------------------- /mas_mt/v0.6_deep_recurrent/good.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tau-lex/market-analysis-system/HEAD/mas_mt/v0.6_deep_recurrent/good.png -------------------------------------------------------------------------------- /mas_cryptobot/vae/img/1_Animation.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tau-lex/market-analysis-system/HEAD/mas_cryptobot/vae/img/1_Animation.gif -------------------------------------------------------------------------------- /mas_cryptobot/vae/img/4_Animation.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tau-lex/market-analysis-system/HEAD/mas_cryptobot/vae/img/4_Animation.gif -------------------------------------------------------------------------------- /mas_cryptobot/img/arch_bot_00_cnn2in.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tau-lex/market-analysis-system/HEAD/mas_cryptobot/img/arch_bot_00_cnn2in.png -------------------------------------------------------------------------------- /mas_mt/v0.8_functional_models/simple.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tau-lex/market-analysis-system/HEAD/mas_mt/v0.8_functional_models/simple.png -------------------------------------------------------------------------------- /docs/CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | ### Hello. 2 | Please describe the main idea or essence of the changes. If necessary, leave a more detailed description below. 3 | -------------------------------------------------------------------------------- /mas_cryptobot/img/arch_bot_01_cnn2in_feedback.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tau-lex/market-analysis-system/HEAD/mas_cryptobot/img/arch_bot_01_cnn2in_feedback.png -------------------------------------------------------------------------------- /mas_cryptobot/vae/4_3-12_60/0vae_img20_flt3-12_code60_origin_pics.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tau-lex/market-analysis-system/HEAD/mas_cryptobot/vae/4_3-12_60/0vae_img20_flt3-12_code60_origin_pics.png -------------------------------------------------------------------------------- /mas_cryptobot/vae/4_3-12_60/1vae_img20_flt3-12_code60_origin_pics.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tau-lex/market-analysis-system/HEAD/mas_cryptobot/vae/4_3-12_60/1vae_img20_flt3-12_code60_origin_pics.png -------------------------------------------------------------------------------- /mas_cryptobot/vae/4_3-12_60/2vae_img20_flt3-12_code60_origin_pics.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tau-lex/market-analysis-system/HEAD/mas_cryptobot/vae/4_3-12_60/2vae_img20_flt3-12_code60_origin_pics.png -------------------------------------------------------------------------------- /mas_cryptobot/vae/4_3-12_60/3vae_img20_flt3-12_code60_origin_pics.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tau-lex/market-analysis-system/HEAD/mas_cryptobot/vae/4_3-12_60/3vae_img20_flt3-12_code60_origin_pics.png -------------------------------------------------------------------------------- /mas_cryptobot/vae/4_3-12_60/4vae_img20_flt3-12_code60_origin_pics.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tau-lex/market-analysis-system/HEAD/mas_cryptobot/vae/4_3-12_60/4vae_img20_flt3-12_code60_origin_pics.png -------------------------------------------------------------------------------- /mas_cryptobot/vae/4_3-12_60/5vae_img20_flt3-12_code60_origin_pics.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tau-lex/market-analysis-system/HEAD/mas_cryptobot/vae/4_3-12_60/5vae_img20_flt3-12_code60_origin_pics.png -------------------------------------------------------------------------------- /mas_cryptobot/vae/4_3-12_60/vae_img20_flt3-12_code60_latent-distr.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tau-lex/market-analysis-system/HEAD/mas_cryptobot/vae/4_3-12_60/vae_img20_flt3-12_code60_latent-distr.png -------------------------------------------------------------------------------- /mas_cryptobot/vae/4_3-12_60/0vae_img20_flt3-12_code60_restored_from_z.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tau-lex/market-analysis-system/HEAD/mas_cryptobot/vae/4_3-12_60/0vae_img20_flt3-12_code60_restored_from_z.png -------------------------------------------------------------------------------- /mas_cryptobot/vae/4_3-12_60/1vae_img20_flt3-12_code60_restored_from_z.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tau-lex/market-analysis-system/HEAD/mas_cryptobot/vae/4_3-12_60/1vae_img20_flt3-12_code60_restored_from_z.png -------------------------------------------------------------------------------- /mas_cryptobot/vae/4_3-12_60/2vae_img20_flt3-12_code60_restored_from_z.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tau-lex/market-analysis-system/HEAD/mas_cryptobot/vae/4_3-12_60/2vae_img20_flt3-12_code60_restored_from_z.png -------------------------------------------------------------------------------- /mas_cryptobot/vae/4_3-12_60/3vae_img20_flt3-12_code60_restored_from_z.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tau-lex/market-analysis-system/HEAD/mas_cryptobot/vae/4_3-12_60/3vae_img20_flt3-12_code60_restored_from_z.png -------------------------------------------------------------------------------- /mas_cryptobot/vae/4_3-12_60/4vae_img20_flt3-12_code60_restored_from_z.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tau-lex/market-analysis-system/HEAD/mas_cryptobot/vae/4_3-12_60/4vae_img20_flt3-12_code60_restored_from_z.png -------------------------------------------------------------------------------- /mas_cryptobot/vae/4_3-12_60/5vae_img20_flt3-12_code60_restored_from_z.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tau-lex/market-analysis-system/HEAD/mas_cryptobot/vae/4_3-12_60/5vae_img20_flt3-12_code60_restored_from_z.png -------------------------------------------------------------------------------- /mas_cryptobot/vae/4_3-12_60/vae_img20_flt3-12_code60_latent-distr_log.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tau-lex/market-analysis-system/HEAD/mas_cryptobot/vae/4_3-12_60/vae_img20_flt3-12_code60_latent-distr_log.png -------------------------------------------------------------------------------- /mas_cryptobot/vae/4_3-12_60/vae_img20_flt3-12_code60_latent-distr__mean.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tau-lex/market-analysis-system/HEAD/mas_cryptobot/vae/4_3-12_60/vae_img20_flt3-12_code60_latent-distr__mean.png -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | build*/ 2 | dist/ 3 | *.egg-info/ 4 | __pycache__/ 5 | wgts/ 6 | arbitrage/ 7 | *.autosave 8 | *.hdf5 9 | *.h5f 10 | *.model 11 | *.csv 12 | *.txt 13 | *.xlsx 14 | *.user 15 | *.pyc 16 | *.rar 17 | *.npz 18 | api.conf 19 | -------------------------------------------------------------------------------- /mas_tools/layers/data_to_img.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from keras import backend as K, initializers, regularizers, constraints 3 | from keras.engine.topology import Layer 4 | 5 | 6 | class Price_to_img(Layer): 7 | pass 8 | 9 | 10 | class Tickers_to_img(Layer): 11 | pass 12 | -------------------------------------------------------------------------------- /mas_tools/layers/__init__.py: -------------------------------------------------------------------------------- 1 | from mas_tools.layers.attention import Attention 2 | from mas_tools.layers.attention_weighted_average import AttentionWeightedAverage 3 | from mas_tools.layers.attention_with_context import AttentionWithContext 4 | from mas_tools.layers.data_to_img import Price_to_img, Tickers_to_img 5 | -------------------------------------------------------------------------------- /mas_tools/models/__init__.py: -------------------------------------------------------------------------------- 1 | from mas_tools.models.models import save_model, load_model, simple_model 2 | 3 | from mas_tools.models.convolutions import cnn_model_2in, cnn_model_2in_with_feedback 4 | from mas_tools.models.autoencoders import dense_ae, deep_ae 5 | from mas_tools.models.autoencoders import deep_conv_ae, deep_conv2d_vae -------------------------------------------------------------------------------- /mas_cryptobot/README.md: -------------------------------------------------------------------------------- 1 | ## MAS CryptoBot En 2 | 3 | ## MAS CryptoBot Ru 4 | Каталог содержит проект торгового робота основанного на глубоких нейронных сетях и обучении с подкреплением. 5 | Архитектура нейронной сети достаточно сложная. Обучается поэтапно, блоками логически выделенных слоев. Идеи архитектуры и методов обучения подчерпнуты из последних научных исследований в области исскуственного интеллекта. -------------------------------------------------------------------------------- /data/files.py: -------------------------------------------------------------------------------- 1 | 2 | # from files import FILES, PERIODS, CSV 3 | 4 | FILES = ['AUDJPY', 'AUDUSD', 'CHFJPY', 'EURAUD', 5 | 'EURCAD', 'EURCHF', 'EURGBP', 'EURJPY', 6 | 'EURRUB', 'EURUSD', 'GBPAUD', 'GBPCAD', 7 | 'GBPCHF', 'GBPJPY', 'GBPUSD', 'NZDJPY', 8 | 'NZDUSD', 'USDCAD', 'USDCHF', 'USDJPY', 9 | 'USDRUB', 'XAGUSD', 'XAUUSD'] 10 | 11 | PERIODS = ['1', '5', '15', '30', '60', '240', '1440', '10080', '43200'] 12 | 13 | CSV = '.csv' 14 | -------------------------------------------------------------------------------- /mas_cryptobot/vae/model.py: -------------------------------------------------------------------------------- 1 | from mas_tools.models.autoencoders import deep_conv2d_vae 2 | from mas_tools.ml import save_model_arch 3 | 4 | 5 | if __name__ == "__main__": 6 | path = 'E:/Projects/market-analysis-system/mas_vae/' 7 | 8 | enc, dec, ae, _ = deep_conv2d_vae((80, 80, 3), latent_dim=60, filters_count=(3, 15), dropout=0.3) 9 | save_model_arch(enc, path+'ae_enc') 10 | enc.summary() 11 | save_model_arch(dec, path+'ae_dec') 12 | dec.summary() 13 | save_model_arch(ae, path+'ae') 14 | ae.summary() 15 | -------------------------------------------------------------------------------- /data/crypto/_concatenate.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import numpy as np 3 | 4 | 5 | path = 'E:/Projects/market-analysis-system/data/crypto/' 6 | files = ['BNBETH', 'BNBUSDT', 'BTCUSDT', 'ETHUSDT']#, 'BCCUSDT'] 7 | 8 | data = list() 9 | 10 | for fn in files: 11 | for idx in range(11, 15): 12 | fname = '{path}{fn}/{nb}.csv'.format(path=path, fn=fn, nb=idx) 13 | data.append(np.genfromtxt(fname, delimiter=';', dtype=np.float)) 14 | 15 | data = np.vstack(data) 16 | fname = '{path}{fn}{nb}.csv'.format(path=path, fn=fn, nb='03') 17 | np.savetxt(fname, data, fmt='%.8f', delimiter=';') 18 | data = list() 19 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | 5 | --- 6 | 7 | **Is your feature request related to a problem? Please describe.** 8 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 9 | 10 | **Describe the solution you'd like** 11 | A clear and concise description of what you want to happen. 12 | 13 | **Describe alternatives you've considered** 14 | A clear and concise description of any alternative solutions or features you've considered. 15 | 16 | **Additional context** 17 | Add any other context or screenshots about the feature request here. 18 | -------------------------------------------------------------------------------- /data/to_normalized.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from files import FILES, PERIODS, CSV 4 | 5 | 6 | print('Warning! Process may be very long.') 7 | lpath = 'E:/Projects/market-analysis-system/data/transformed/' 8 | spath = 'E:/Projects/market-analysis-system/data/normalized/' 9 | 10 | for symbol in FILES: 11 | for tf in PERIODS: 12 | ## Read 13 | data = np.genfromtxt(lpath+symbol+tf+CSV, delimiter=';')[:, 1:5] 14 | ## Normalize 15 | data /= max(np.max(data[:, 0]), np.max(data[:, 1]), np.max(data[:, 2]), np.max(data[:, 3])) 16 | ## Save 17 | np.savetxt(spath+symbol+tf+CSV, data, fmt='%0.6f', delimiter=';') 18 | data = None 19 | -------------------------------------------------------------------------------- /data/crypto/_prices.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import numpy as np 3 | 4 | 5 | path = 'E:/Projects/market-analysis-system/data/crypto/' 6 | files = ['BNBETH', 'BNBUSDT', 'BTCUSDT', 'ETHUSDT', 'BCCUSDT'] 7 | nb = '04' 8 | data = None 9 | 10 | for fn in files: 11 | fname = '{path}{name}{nb}.csv'.format(path=path, name=fn, nb=nb) 12 | newfname = '{path}{name}{nb}_candles.csv'.format(path=path, name=fn, nb=nb) 13 | data = np.genfromtxt(fname, delimiter=';', dtype=np.float) 14 | 15 | data = np.reshape(data, (len(data), 20, 15)) 16 | data = data[:, :, 0:4] 17 | data = np.reshape(data, (len(data), 80)) 18 | 19 | np.savetxt(newfname, data, fmt='%.8f', delimiter=';') 20 | -------------------------------------------------------------------------------- /data/crypto/_tickerbook.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import numpy as np 3 | 4 | 5 | path = 'E:/Projects/market-analysis-system/data/crypto/' 6 | files = ['BNBETH', 'BNBUSDT', 'BTCUSDT', 'ETHUSDT', 'BCCUSDT'] 7 | nb = '04' 8 | data = None 9 | 10 | for fn in files: 11 | fname = '{path}{name}{nb}.csv'.format(path=path, name=fn, nb=nb) 12 | newfname = '{path}{name}{nb}_tickers.csv'.format(path=path, name=fn, nb=nb) 13 | data = np.genfromtxt(fname, delimiter=';', dtype=np.float) 14 | 15 | data = np.reshape(data, (len(data), 20, 15)) 16 | data = data[:, :, 10:14] 17 | data = np.reshape(data, (len(data), 80)) 18 | 19 | np.savetxt(newfname, data, fmt='%.8f', delimiter=';') 20 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | 5 | --- 6 | 7 | **Describe the bug** 8 | A clear and concise description of what the bug is. 9 | 10 | **To Reproduce** 11 | Steps to reproduce the behavior: 12 | 1. Go to '...' 13 | 2. Click on '....' 14 | 3. Scroll down to '....' 15 | 4. See error 16 | 17 | **Expected behavior** 18 | A clear and concise description of what you expected to happen. 19 | 20 | **Screenshots** 21 | If applicable, add screenshots to help explain your problem. 22 | 23 | **Desktop (please complete the following information):** 24 | - OS: [e.g. Windows 10 x64] 25 | - Version python and packages [e.g. python 3.5, tf 1.5] 26 | 27 | **Additional context** 28 | Add any other context about the problem here. 29 | -------------------------------------------------------------------------------- /data/crypto/_prices+tickerbook.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import numpy as np 3 | 4 | 5 | path = 'E:/Projects/market-analysis-system/data/crypto/' 6 | files = ['BNBETH', 'BNBUSDT', 'BTCUSDT', 'ETHUSDT', 'BCCUSDT'] 7 | nb = '04' 8 | data = None 9 | 10 | for fn in files: 11 | fname = '{path}{name}{nb}.csv'.format(path=path, name=fn, nb=nb) 12 | newfname = '{path}{name}{nb}_candles+tickers.csv'.format(path=path, name=fn, nb=nb) 13 | data = np.genfromtxt(fname, delimiter=';', dtype=np.float) 14 | 15 | data = np.reshape(data, (len(data), 20, 15)) 16 | 17 | data_p = data[:, :, 0:4] 18 | data_p = np.reshape(data_p, (len(data), 80)) 19 | data_t = data[:, :, 10:14] 20 | data_t = np.reshape(data_t, (len(data), 80)) 21 | 22 | data = np.hstack((data_p, data_t)) 23 | 24 | np.savetxt(newfname, data, fmt='%.8f', delimiter=';') 25 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | 3 | import mas_tools 4 | 5 | 6 | setup( 7 | name='mas_tools', 8 | version=mas_tools.__version__, 9 | packages=find_packages(), 10 | long_description=open('mas_tools/README.md', encoding='utf-8').read(), 11 | keywords=['trade', 'ml'], 12 | url='https://github.com/terentjew-alexey/market-analysis-system', 13 | author='Terentyev Aleksey', 14 | author_email='terentjew.alexey@ya.ru', 15 | license='MIT', 16 | # include_package_data=True, 17 | install_requires=[ 18 | 'numpy', 'pandas', 'matplotlib', 19 | 'scikit-learn', 'keras>=2.1', 20 | 'tensorflow>=1.8', 21 | # 'cntk>=2.5', 22 | 'gym', 'keras-rl', 23 | 'plaidml-keras', 'plaidbench', # for OpenCL calculation 24 | # 'ccxt>=1.16', # universal api for crypto exchanges 25 | ] 26 | ) 27 | -------------------------------------------------------------------------------- /data/to_windowed.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from mas_tools.data import create_timeseries_matrix 3 | 4 | from files import FILES, PERIODS, CSV 5 | 6 | 7 | print('Warning! Process may be very long.') 8 | # lpath = 'E:/Projects/market-analysis-system/data/transformed/' 9 | lpath = 'E:/Projects/market-analysis-system/data/normalized/' 10 | spath = 'E:/Projects/market-analysis-system/data/windowed/' 11 | 12 | window = 20 # warning! size of file multiply in to window size 13 | 14 | for symbol in FILES: 15 | for tf in PERIODS: 16 | ## Optimize skip 17 | if tf == '1' or tf == '5' or tf =='15': 18 | continue 19 | ## Read 20 | data = np.genfromtxt(lpath+symbol+tf+CSV, delimiter=';') 21 | ## To windowed 22 | data, _ = create_timeseries_matrix(data, look_back=window) 23 | ## Save 24 | np.savetxt(spath+'norm_w_'+str(window)+symbol+tf+CSV, data, fmt='%0.6f', delimiter=';') 25 | data = None 26 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright © 2018 Terentyev Aleksey 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 6 | 7 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 8 | 9 | THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 10 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | # The Market Analysis System 2 | 3 | ## The Market analysis system 4 | MAS - a set of tools and examples of their use, the purpose of which is to analyze time series, build models based on machine learning, forecast signals for entering the market, automate trade. 5 | * /data - Data sets for learning models, scripts for processing and data collection. 6 | * /mas_cbot - Directory for experiments on crypto exchanges. 7 | * /mas_mt - Directory for experiments with the MetaTrader platform. 8 | * /mas_tools - A library in Python 3 that provides tools for experiments. 9 | 10 | ## Система анализа рынков 11 | MAS - набор инструментов и примеров их использования, предназначением которых является анализ временных рядов, построение моделей на основе машинного обучения, прогнозирование сигналов для входа в рынок, автоматизация торговли. 12 | * /data - Наборы данных для обучения моделей, скрипты по обработке и сбору данных. 13 | * /mas_cbot - Директория для экспериментов на криптовалютных биржах. 14 | * /mas_mt - Директория для экспериментов с платформой MetaTrader. 15 | * /mas_tools - Библиотека на Python 3, предоставляющая инструменты для экспериментов. 16 | -------------------------------------------------------------------------------- /data/transform.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import pandas as pd 4 | 5 | 6 | def get_script_dir(follow_symlinks=True): 7 | """https://stackoverflow.com/questions/3718657/how-to-properly-determine-current-script-directory/22881871#22881871""" 8 | import inspect 9 | import os 10 | import sys 11 | if getattr(sys, 'frozen', False): # py2exe, PyInstaller, cx_Freeze 12 | path = os.path.abspath(sys.executable) 13 | else: 14 | path = inspect.getabsfile(get_script_dir) 15 | if follow_symlinks: 16 | path = os.path.realpath(path) 17 | return os.path.dirname(path) 18 | 19 | ## Files list 20 | directory = get_script_dir() 21 | files = list(filter(lambda x: x.endswith('.csv'), os.listdir(directory+'/exported'))) 22 | 23 | for filename in files: 24 | ## Read 25 | table = pd.read_csv(directory + '/exported/' + filename, sep=',', header=None) 26 | ## Format time 27 | table[1] = pd.to_datetime(table[0] + ' ' + table[1]) 28 | table.drop([0], axis='columns') 29 | ## Save 30 | table.to_csv(directory + '/transformed/' + filename, sep=';', 31 | header=False, index=False, 32 | date_format="%Y.%m.%d %H:%M") 33 | -------------------------------------------------------------------------------- /mas_mt/README.md: -------------------------------------------------------------------------------- 1 | ## MAS MetaTrader En 2 | Directory with scripts of examples and experiments with the MetaTrader platform. 3 | The scripts take data from the directory of the MetaTrader terminal, from the sub-directory where the script saves the data [ML-Assistant](https://github.com/terentjew-alexey/mql_projects/blob/master/MASi_ML-Assistant.mq4) or [ML-DataSaver](https://github.com/terentjew-alexey/mql_projects/blob/master/MASi_ML-DataSaver.mq4). 4 | With these scripts, the adviser's script [ML-Trader](https://github.com/terentjew-alexey/mql_projects/blob/master/MASx_ML-Trader.mq4) also works, which automatically trades under the forecasts of signals. Trade can be done both by withdrawing transactions to the broker's server, and virtually, displaying transactions on the chart. 5 | 6 | ## MAS MetaTrader Ru 7 | Каталог со скриптами примеров и экспериментов с платформой MetaTrader. 8 | Скрипты берут данные из дериктории терминала MetaTrader, из под-дериктории куда сохраняет эти данные скрипт [ML-Assistant](https://github.com/terentjew-alexey/mql_projects/blob/master/MASi_ML-Assistant.mq4) или [ML-DataSaver](https://github.com/terentjew-alexey/mql_projects/blob/master/MASi_ML-DataSaver.mq4). 9 | С этими скриптами также работает скрипт советника [ML-Trader](https://github.com/terentjew-alexey/mql_projects/blob/master/MASx_ML-Trader.mq4), который автоматически торгует по прогнозам сигналов. Торговлю может совершать как выводя сделки на сервер брокера, так и виртуально, отображая сделки на графике. -------------------------------------------------------------------------------- /mas_tools/os.py: -------------------------------------------------------------------------------- 1 | import inspect 2 | import os 3 | import sys 4 | 5 | 6 | def get_home(): 7 | """ 8 | Returns the path to the user directory. 9 | """ 10 | 11 | return os.path.expanduser("~") 12 | 13 | 14 | def get_parameters(): 15 | """ 16 | Returns a list of parameters (without filename). 17 | """ 18 | 19 | return sys.argv[1:] 20 | 21 | 22 | def get_script_dir(follow_symlinks=True): 23 | """ 24 | Return script file directory. 25 | from: https://stackoverflow.com/questions/3718657/how-to-properly-determine-current-script-directory/22881871#22881871 26 | 27 | PS. Return this script path. =) 28 | Use inline. 29 | """ 30 | 31 | if getattr(sys, 'frozen', False): # py2exe, PyInstaller, cx_Freeze 32 | path = os.path.abspath(sys.executable) 33 | else: 34 | path = inspect.getabsfile(get_script_dir) 35 | if follow_symlinks: 36 | path = os.path.realpath(path) 37 | 38 | return os.path.dirname(path) 39 | 40 | def get_api_pair(filename): 41 | """ 42 | Return api key-secret pair from file. 43 | """ 44 | 45 | key, secret = '', '' 46 | 47 | ## Read file in to lines list 48 | f = open(filename, 'r') 49 | lines = f.readlines() 50 | f.close() 51 | 52 | for line in lines: 53 | if line.find('KEY', 0, 7) > 0: 54 | key = line.split('=')[1] 55 | if line.find('SECRET', 0, 10) > 0: 56 | secret = line.split('=')[1] 57 | 58 | return key, secret -------------------------------------------------------------------------------- /data/create_picture.py: -------------------------------------------------------------------------------- 1 | import time 2 | import numpy as np 3 | import matplotlib.pyplot as plt 4 | plt.style.use('dark_background') 5 | 6 | from mas_tools.data import timeseries_to_img 7 | 8 | 9 | lpath = 'E:/Projects/market-analysis-system/data/transformed/' 10 | spath = 'E:/Projects/market-analysis-system/data/test/' 11 | fn = 'GBPUSD240' 12 | window = 50 13 | 14 | new_data = np.genfromtxt(lpath+fn+'.csv', delimiter=';') 15 | 16 | # data = new_data[800:920, 1:5] 17 | # plt.plot(data[:, 0]) 18 | # plt.plot(data[:, 1]) 19 | # plt.plot(data[:, 2]) 20 | # plt.plot(data[:, 3]) 21 | # plt.legend(['open', 'high', 'low', 'close']) 22 | # plt.savefig(spath + fn + '_plt.png') 23 | # plt.close() 24 | # plt.show() 25 | 26 | imgs = np.array([]) 27 | 28 | for idx in range(800, 900): 29 | image = timeseries_to_img(new_data[idx:idx+window, 1:5]) 30 | # image.save(spath + fn + '_pil_' + str(idx) + '.png', 'PNG') 31 | imgs = np.append(imgs, image) 32 | 33 | imgs = imgs.reshape((100, window*4, window*4, 3)) 34 | print(imgs.shape) 35 | print(imgs[2, :, :, 1]) 36 | print(imgs.dtype) 37 | 38 | # imgs = imgs.reshape((100, (window*4+1) * (window*4+1) * 3)) 39 | # np.savetxt(spath + '_numpy_array.csv', imgs, 40 | # delimiter=';', fmt='%.0f') 41 | # np.savez_compressed(spath + '_numpy_array.npz', imgs) 42 | 43 | # print('Saved\nNow read and save as image') 44 | 45 | # loaded_imgs = np.load(spath + '_numpy_array.npz') 46 | # print(loaded_imgs.f.arr_0.shape) 47 | 48 | # from PIL import Image 49 | # pic = np.reshape(imgs[99], (window*4+1, window*4+1, 3)).astype(np.float) 50 | # pic = Image.fromarray(pic) 51 | # pic.show() 52 | -------------------------------------------------------------------------------- /data/to_pictures.py: -------------------------------------------------------------------------------- 1 | import os 2 | import numpy as np 3 | 4 | from mas_tools.data import timeseries_to_img 5 | 6 | from files import FILES, PERIODS, CSV 7 | 8 | 9 | print('Warning! Process may be very long.') 10 | lpath = 'E:/Projects/market-analysis-system/data/transformed/' 11 | spath = 'E:/Projects/market-analysis-system/data/pictured/' 12 | window = 50 13 | img_size = window*4 * window*4 * 3 14 | count = 1000 # count images of last bars 15 | 16 | if not os.path.exists(spath): 17 | os.makedirs(spath) 18 | 19 | for symbol in FILES[:]: 20 | for tf in PERIODS[3:]: 21 | ## Load price history 22 | new_data = np.array([]) 23 | data = np.genfromtxt(lpath+symbol+tf+CSV, delimiter=';') 24 | data = data[:, 1:5] 25 | ## Map range 26 | len_data = len(data) - window + 1 27 | if len_data < count: 28 | continue 29 | 30 | for idx in range(len_data-count, len_data): 31 | ## Get image from window in history 32 | image = timeseries_to_img(data[idx:idx+window]) 33 | new_data = np.append(new_data, image) 34 | ## Print and save checkpoints 35 | if idx % 100 == 0: print('{} / {}'.format(idx, len_data)) 36 | if idx % 500 == 0: 37 | np.savez_compressed(spath+symbol+tf+'_w'+str(window)+'.npz', 38 | new_data.reshape((idx+1-len_data-count, img_size))) 39 | ## Save image packet in to numpy archive 40 | new_data = new_data.reshape((count, img_size)) 41 | print('{}{} : {}'.format(symbol, tf, new_data.shape)) 42 | np.savez_compressed(spath+symbol+tf+'_w'+str(window)+'.npz', new_data) 43 | data = None 44 | new_data = None 45 | -------------------------------------------------------------------------------- /mas_cryptobot/bot-rl_v0.0.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from keras.optimizers import Adam 4 | 5 | from rl.agents.dqn import DQNAgent 6 | from rl.policy import BoltzmannQPolicy 7 | from rl.memory import SequentialMemory 8 | 9 | from mas_tools.envs import MarketEnv 10 | from mas_tools.markets import AbstractMarket 11 | from mas_tools.api import Binance 12 | from mas_tools.models import simple_model 13 | 14 | 15 | MY_API_KEY = '---' 16 | MY_API_SECRET = '---' 17 | 18 | ENV_NAME = 'Binance' 19 | 20 | 21 | ## Init exchange api 22 | api = Binance(API_KEY=MY_API_KEY, API_SECRET=MY_API_SECRET) 23 | 24 | ## Init market environment 25 | connector = AbstractMarket(api, symbols=['ETHUSDT'], periods='5m', balance=1000.0) 26 | market = MarketEnv(connector) 27 | print('connector shape', connector.shape) 28 | 29 | ## Environment parameters 30 | observation_shape = market.observation_space.shape 31 | nb_actions = market.action_space.n 32 | print('state =', observation_shape, '| actions =', nb_actions) 33 | 34 | ## Init ML-model for agent 35 | model = simple_model(observation_shape, nb_actions) 36 | 37 | ## Init RL-metod parameters 38 | memory = SequentialMemory(limit=10000, window_length=1) 39 | policy = BoltzmannQPolicy() 40 | 41 | ## Init RL agent 42 | agent = DQNAgent(model=model, nb_actions=nb_actions, 43 | memory=memory, nb_steps_warmup=1000, 44 | target_model_update=1e-2, policy=policy, 45 | # enable_dueling_network=True, dueling_type='avg' 46 | ) 47 | agent.compile(Adam(lr=1e-3), metrics=['mae']) 48 | 49 | ## Train and evaluation 50 | # agent.load_weights('dqn_{}_weights.h5f'.format(ENV_NAME)) 51 | 52 | agent.fit(market, nb_steps=100000, visualize=False, verbose=2) 53 | 54 | agent.save_weights('dqn_{}_weights.h5f'.format(ENV_NAME), overwrite=True) 55 | 56 | agent.test(market, nb_episodes=5, visualize=False) 57 | -------------------------------------------------------------------------------- /data/README.md: -------------------------------------------------------------------------------- 1 | ## Data En 2 | Directory for storing and transforming data necessary for training models. 3 | * /exported - Directory of original data exported from Alpari servers. [Link](https://goo.gl/q4S5Qu "GoogleDrive") to download the entire data in one archive. 4 | * /transformed - Directory of the original data exported from Alpari servers, with the format `[datetime; open; high; low; close; volume]`. The following format of csv files is used by default in [ML-Assistant](https://github.com/terentjew-alexey/mql_projects/blob/master/MASi_ML-Assistant.mq4 "Github") and [ML-DataSaver](https://github.com/terentjew-alexey/mql_projects/blob/master/MASi_ML-DataSaver.mq4 "Github") programs to save current quotes from the MetaTrader terminal. [Link](https://goo.gl/vCJNj6 "GoogleDrive") to download the entire data in one archive. 5 | * /normalized - Directory for data normalized to 1.0. 6 | * /windowed - Directory of data in the form of a sequence of observation windows. 7 | * /pictured - Directory with windows of time series converted into images. 8 | 9 | ## Data Ru 10 | Каталог для хранения и трансформации данных необходимых для обучения моделей. 11 | * /exported - Директория оригинальных данных экспортированных с серверов Alpari. [Ссылка](https://goo.gl/q4S5Qu "GoogleDrive") на скачивание всего датасета одним архивом. 12 | * /transformed - Директория оригинальных данных экспортированных с серверов Alpari, с форматом `[datetime;open;high;low;close;volume]`. Следующий формат csv файлов используется по умолчанию в программах [ML-Assistant](https://github.com/terentjew-alexey/mql_projects/blob/master/MASi_ML-Assistant.mq4 "Github") и [ML-DataSaver](https://github.com/terentjew-alexey/mql_projects/blob/master/MASi_ML-DataSaver.mq4 "Github") для сохранения текущих котировок из терминала MetaTrader. [Ссылка](https://goo.gl/vCJNj6 "GoogleDrive") на скачивание всего датасета одним архивом. 13 | * /normalized - Директория для данных нормированных к 1.0. 14 | * /windowed - Директория данных в виде последовательности окон наблюдения. 15 | * /pictured - Директория с окнами временных рядов преобразованных в изображения. 16 | -------------------------------------------------------------------------------- /mas_tools/api/common.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Exchanges API's base class. 4 | https://bablofil.ru 5 | """ 6 | 7 | class BaseApi(): 8 | """Exchanges API's base class.""" 9 | 10 | server = 'https://127.0.0.1/' 11 | methods = { 12 | # public methods ## {'url': '', 'method': 'GET', 'private': False, 'args': []} 13 | 'ping': {}, 14 | 'server_time': {}, 15 | 'exchange_info': {}, 16 | 'candlesticks': {}, 17 | 'tickers': {}, 18 | 'ticker_24': {}, 19 | 'ticker_price': {}, 20 | 'ticker_book_price':{}, 21 | 'trades': {}, 22 | 'h_trades': {}, 23 | 'agr_trades': {}, 24 | # private methods ## {'url': '', 'method': 'GET', 'private': True, 'args': []} 25 | 'account': {}, 26 | 'new_order': {}, 27 | 'cancel_order': {}, 28 | 'order_info': {}, 29 | 'test_order': {}, 30 | 'open_orders': {}, 31 | 'all_orders': {}, 32 | 'my_trades': {}, 33 | } 34 | periods = [] 35 | 36 | def __init__(self, API_KEY, API_SECRET): 37 | """Constructor 38 | 39 | Arguments: 40 | API_KEY: 41 | API_SECRET: 42 | """ 43 | 44 | self.API_KEY = API_KEY 45 | self.API_SECRET = bytearray(API_SECRET, encoding='utf-8') 46 | self.shift_seconds = 0 47 | 48 | def set_shift_seconds(self, seconds): 49 | """""" 50 | self.shift_seconds = seconds 51 | 52 | def __getattr__(self, name): 53 | def wrapper(*args, **kwargs): 54 | kwargs.update(command=name) 55 | return self.call_api(**kwargs) 56 | return wrapper 57 | 58 | def call_api(self, **kwargs): 59 | """""" 60 | # command = kwargs.pop('command') 61 | # api_url = self.server + self.methods[command]['url'] 62 | 63 | # payload = kwargs 64 | # headers = {} 65 | raise NotImplementedError() 66 | 67 | -------------------------------------------------------------------------------- /mas_tools/models/models.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from mas_tools.ml import save_model_arch 3 | 4 | from keras.models import model_from_json 5 | 6 | from keras.models import Model, Sequential 7 | 8 | from keras.layers import Input, concatenate, add 9 | from keras.layers import Dense, Activation 10 | from keras.layers import LSTM, GRU 11 | from keras.layers import BatchNormalization, Dropout 12 | from keras.layers import Flatten, Reshape 13 | 14 | from mas_tools.layers import Attention, AttentionWithContext, AttentionWeightedAverage 15 | 16 | from keras.activations import relu 17 | 18 | 19 | def save_model(model: Model, filename: str): 20 | """Writes the model to a text file. 21 | 22 | Arguments 23 | model (keras.Model): Model of the neural network to save. 24 | file (str): Path and filename.""" 25 | 26 | json_string = model.to_json() 27 | 28 | with open(filename, 'w') as file: 29 | file.write(json_string) 30 | 31 | 32 | def load_model(filename: str): 33 | """Loads the model from a text file. 34 | 35 | Arguments 36 | file (str): Path and filename. 37 | 38 | Returns 39 | model (keras.Model): Model of neural network.""" 40 | 41 | json_string = '' 42 | try: 43 | file = open(filename, 'r') 44 | except IOError as e: 45 | print('Model file not found', e) 46 | else: 47 | json_string = file.read() 48 | file.close() 49 | if len(json_string) > 0: 50 | model = model_from_json(json_string) 51 | return model 52 | 53 | 54 | def simple_model(input_shape, nb_output, act='linear'): 55 | """Simple model for RL. 56 | 57 | Returns 58 | model (keras.Model): Model of neural network.""" 59 | 60 | model = Sequential() 61 | # model.add(Reshape((input_shape[1], input_shape[2]), 62 | # batch_input_shape=(None, 1, input_shape[0], input_shape[1], input_shape[2]))) 63 | # model.add(BatchNormalization()) 64 | model.add(BatchNormalization(batch_input_shape=(None, input_shape[0], input_shape[1]))) 65 | model.add(LSTM(input_shape[1] * input_shape[2])) 66 | model.add(Activation('relu')) 67 | model.add(Dropout(0.4)) 68 | model.add(Dense(64)) 69 | model.add(Activation('relu')) 70 | model.add(Dropout(0.4)) 71 | model.add(Dense(32)) 72 | model.add(Activation('relu')) 73 | model.add(Dropout(0.4)) 74 | model.add(Dense(nb_output, activation=act)) 75 | 76 | return model 77 | 78 | 79 | if __name__ == "__main__": 80 | path = 'E:/Projects/market-analysis-system/' 81 | 82 | model = simple_model((100, 4, 10), 3) 83 | save_model_arch(model, path+'simple') 84 | model.summary() 85 | -------------------------------------------------------------------------------- /mas_tools/layers/attention_weighted_average.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # From: https://github.com/bfelbo/DeepMoji/blob/master/deepmoji/attlayer.py 3 | from __future__ import absolute_import, division 4 | 5 | import sys 6 | from os.path import dirname 7 | sys.path.append(dirname(dirname(__file__))) 8 | from keras import initializers 9 | from keras.engine import InputSpec, Layer 10 | from keras import backend as K 11 | 12 | 13 | class AttentionWeightedAverage(Layer): 14 | """ 15 | Computes a weighted average of the different channels across timesteps. 16 | Uses 1 parameter pr. channel to compute the attention value for a single timestep. 17 | """ 18 | 19 | def __init__(self, return_attention=False, **kwargs): 20 | self.init = initializers.get('uniform') 21 | self.supports_masking = True 22 | self.return_attention = return_attention 23 | super(AttentionWeightedAverage, self).__init__(** kwargs) 24 | 25 | def build(self, input_shape): 26 | self.input_spec = [InputSpec(ndim=3)] 27 | assert len(input_shape) == 3 28 | 29 | self.W = self.add_weight(shape=(input_shape[2], 1), 30 | name='{}_W'.format(self.name), 31 | initializer=self.init) 32 | self.trainable_weights = [self.W] 33 | super(AttentionWeightedAverage, self).build(input_shape) 34 | 35 | def call(self, x, mask=None): 36 | # computes a probability distribution over the timesteps 37 | # uses 'max trick' for numerical stability 38 | # reshape is done to avoid issue with Tensorflow 39 | # and 1-dimensional weights 40 | logits = K.dot(x, self.W) 41 | x_shape = K.shape(x) 42 | logits = K.reshape(logits, (x_shape[0], x_shape[1])) 43 | ai = K.exp(logits - K.max(logits, axis=-1, keepdims=True)) 44 | 45 | # masked timesteps have zero weight 46 | if mask is not None: 47 | mask = K.cast(mask, K.floatx()) 48 | ai = ai * mask 49 | att_weights = ai / (K.sum(ai, axis=1, keepdims=True) + K.epsilon()) 50 | weighted_input = x * K.expand_dims(att_weights) 51 | result = K.sum(weighted_input, axis=1) 52 | if self.return_attention: 53 | return [result, att_weights] 54 | return result 55 | 56 | def get_output_shape_for(self, input_shape): 57 | return self.compute_output_shape(input_shape) 58 | 59 | def compute_output_shape(self, input_shape): 60 | output_len = input_shape[2] 61 | if self.return_attention: 62 | return [(input_shape[0], output_len), (input_shape[0], input_shape[1])] 63 | return (input_shape[0], output_len) 64 | 65 | def compute_mask(self, input, input_mask=None): 66 | if isinstance(input_mask, list): 67 | return [None] * len(input_mask) 68 | else: 69 | return None 70 | -------------------------------------------------------------------------------- /docs/CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and orientation. 6 | 7 | ## Our Standards 8 | 9 | Examples of behavior that contributes to creating a positive environment include: 10 | 11 | * Using welcoming and inclusive language 12 | * Being respectful of differing viewpoints and experiences 13 | * Gracefully accepting constructive criticism 14 | * Focusing on what is best for the community 15 | * Showing empathy towards other community members 16 | 17 | Examples of unacceptable behavior by participants include: 18 | 19 | * The use of sexualized language or imagery and unwelcome sexual attention or advances 20 | * Trolling, insulting/derogatory comments, and personal or political attacks 21 | * Public or private harassment 22 | * Publishing others' private information, such as a physical or electronic address, without explicit permission 23 | * Other conduct which could reasonably be considered inappropriate in a professional setting 24 | 25 | ## Our Responsibilities 26 | 27 | Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior. 28 | 29 | Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful. 30 | 31 | ## Scope 32 | 33 | This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers. 34 | 35 | ## Enforcement 36 | 37 | Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at terentjew.alexey@ya.ru. The project team will review and investigate all complaints, and will respond in a way that it deems appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately. 38 | 39 | Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership. 40 | 41 | ## Attribution 42 | 43 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at [http://contributor-covenant.org/version/1/4][version] 44 | 45 | [homepage]: http://contributor-covenant.org 46 | [version]: http://contributor-covenant.org/version/1/4/ 47 | -------------------------------------------------------------------------------- /mas_tools/ml.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | 4 | from sklearn.metrics import confusion_matrix 5 | from sklearn.metrics import classification_report 6 | from sklearn.metrics import matthews_corrcoef 7 | 8 | from keras.utils import plot_model 9 | 10 | 11 | def plot_history(history, acc='accuracy'): 12 | """ 13 | Plot history train graph. 14 | """ 15 | 16 | plt.plot(history.history['loss']) 17 | plt.plot(history.history['val_loss']) 18 | plt.title('Model loss') 19 | plt.ylabel('loss') 20 | plt.xlabel('epoch') 21 | plt.legend(['train', 'test'], loc='upper left') 22 | plt.show() 23 | 24 | if acc: 25 | plt.plot(history.history[acc]) 26 | plt.plot(history.history['val_'+acc]) 27 | plt.title('Model accuracy') 28 | plt.ylabel('accuracy') 29 | plt.xlabel('epoch') 30 | plt.legend(['train', 'test'], loc='upper left') 31 | plt.show() 32 | 33 | 34 | def classification_scores(true_y, test_y, n=3): 35 | """""" 36 | 37 | if n <= 1: 38 | raise ValueError("classification scores") 39 | 40 | labels = range(n) 41 | if n == 2: 42 | names = ['pass', 'signal'] 43 | elif n == 3: 44 | names = ['hold', 'buy', 'sell'] 45 | 46 | # TODO check data 47 | 48 | result = '-' * 20 49 | result += '\n\nMATTHEWS CORRELATION:\n' 50 | result += str(matthews_corrcoef(true_y, test_y)) 51 | 52 | result += '\n\nCONFUSION MATRIX:\n' 53 | cm = confusion_matrix(true_y, test_y, labels=labels) 54 | result += str(cm / cm.astype(np.float).sum(axis=1)) 55 | 56 | result += '\n\nCLASSIFICATION REPORT:\n' 57 | result += classification_report(true_y, test_y, 58 | labels=labels, 59 | target_names=names) 60 | result += '\n' + '-' * 20 61 | 62 | return result 63 | 64 | 65 | def plot_confusion_matrix(cm, classes, 66 | normalize=False, 67 | title='Confusion matrix', 68 | cmap=plt.cm.Blues): 69 | """ 70 | This function prints and plots the confusion matrix. 71 | Normalization can be applied by setting `normalize=True`. 72 | """ 73 | if normalize: 74 | cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis] 75 | print("Normalized confusion matrix") 76 | else: 77 | print('Confusion matrix, without normalization') 78 | 79 | plt.figure() 80 | 81 | print(cm) 82 | 83 | plt.imshow(cm, interpolation='nearest', cmap=cmap) 84 | plt.title(title) 85 | plt.colorbar() 86 | tick_marks = np.arange(classes) 87 | plt.xticks(tick_marks, classes, rotation=45) 88 | plt.yticks(tick_marks, classes) 89 | 90 | fmt = '.2f' if normalize else 'd' 91 | thresh = cm.max() / 2. 92 | for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])): 93 | plt.text(j, i, format(cm[i, j], fmt), 94 | horizontalalignment="center", 95 | color="white" if cm[i, j] > thresh else "black") 96 | 97 | plt.tight_layout() 98 | plt.ylabel('True label') 99 | plt.xlabel('Predicted label') 100 | 101 | 102 | def save_model_arch(model, name): 103 | """ 104 | Save model architecture. 105 | """ 106 | 107 | plot_model(model, to_file=name+'.png', 108 | show_shapes=True, 109 | show_layer_names=True) 110 | 111 | text = '' 112 | for item in model.layers: 113 | text = text + str(item.name) + '\n' 114 | text = text + str(item.input_shape) + '\n' 115 | text = text + str(item.output_shape) + '\n' 116 | # text = text + str(item.param_count) + '\n' 117 | # text = text + str(item.get_weights()) + '\n' 118 | text = text + '\n=====================================\n' 119 | 120 | f = open(name+'.txt', 'w') 121 | f.write(text) 122 | f.close() 123 | 124 | -------------------------------------------------------------------------------- /mas_cryptobot/bot-rl_v0.1.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | import os 4 | import time 5 | import logging 6 | 7 | import numpy as np 8 | 9 | from keras.optimizers import Adam 10 | 11 | from rl.agents.dqn import DQNAgent 12 | from rl.policy import BoltzmannQPolicy 13 | from rl.memory import SequentialMemory 14 | from rl.processors import MultiInputProcessor 15 | 16 | from mas_tools.api import Binance 17 | from mas_tools.markets import VirtualExchange 18 | from mas_tools.envs import MarketEnv 19 | from mas_tools.models import simple_model, cnn_model_2in 20 | 21 | 22 | MY_API_KEY = '---' 23 | MY_API_SECRET = '---' 24 | 25 | PATH = os.path.dirname(os.path.abspath(__file__)) 26 | ENV_NAME = 'cb_Binance_3' 27 | 28 | SLEEP = 4 29 | TRAIN = True 30 | tickcount = 0 31 | 32 | logging.basicConfig(level=logging.INFO, 33 | handlers=[logging.FileHandler("{p}/logs/{fn}.log".format(p=PATH, fn=ENV_NAME)), 34 | logging.StreamHandler()] 35 | ) 36 | 37 | log = logging.getLogger() 38 | 39 | 40 | ## Init exchange api 41 | api = Binance(API_KEY=MY_API_KEY, API_SECRET=MY_API_SECRET) 42 | 43 | ## Init market environment 44 | market_conn = VirtualExchange(api, symbols=['ETHUSDT'], period='5m', 45 | balance=1000.0, lot_size=0.1) 46 | market = MarketEnv(market_conn) 47 | 48 | ## Environment parameters 49 | observation_shape = market.observation_space.shape 50 | nb_actions = market.action_space.n 51 | log.info('State shape = {a} | actions = {b}'.format(a=observation_shape, b=nb_actions)) 52 | 53 | ## Init ML-model for agent 54 | limit = observation_shape[1] 55 | model = cnn_model_2in((limit, 4), (limit, 4), nb_actions, 'softmax') 56 | 57 | ## Init RL-metod parameters 58 | memory = SequentialMemory(limit=10000, window_length=1) 59 | # TODO implement policies for multiply symbols 60 | policy = BoltzmannQPolicy() 61 | 62 | ## Init RL agent 63 | agent = DQNAgent(model=model, nb_actions=nb_actions, 64 | memory=memory, nb_steps_warmup=1000, 65 | target_model_update=1e-2, policy=policy, 66 | processor=MultiInputProcessor(2), 67 | # enable_dueling_network=True, dueling_type='avg' 68 | ) 69 | agent.compile(Adam(lr=1e-3), metrics=['mae']) 70 | 71 | ## Comment this row if you want to start learning again 72 | agent.load_weights('{p}/dqn_{fn}_weights.h5f'.format(p=PATH, fn=ENV_NAME)) 73 | 74 | ## Train or evaluate 75 | if TRAIN: 76 | agent.training = True 77 | 78 | observation = market.reset() 79 | 80 | while True: 81 | try: 82 | # TODO add callbacks? 83 | 84 | ## Agent vybiraet dejstvie 85 | # (candles=9(mb=>(2,4)?), tickers=4, trades=2) 86 | # TODO actions for multy symbols market 87 | action = agent.forward(observation) 88 | 89 | ## Execute action 90 | observation, reward, done, info = market.step([action]) 91 | 92 | ## Poluchaem otvet ot sredy 93 | agent.backward(reward, terminal=done) 94 | 95 | ## Esli dostigli konca 96 | if done: 97 | observation = market.reset() 98 | agent.reset_states() 99 | done = False 100 | log.info('Is terminal state. Reset..') 101 | log.info('='*40) 102 | 103 | log.info('Tick: {t} | {info}'.format( 104 | t=tickcount, info=info 105 | )) 106 | 107 | ## Check point 108 | if tickcount % 100 == 0: 109 | agent.save_weights('{p}/dqn_{fn}_weights.h5f'.format(p=PATH, fn=ENV_NAME), overwrite=True) 110 | 111 | ## Time shift and counter 112 | time.sleep(SLEEP) 113 | tickcount += 1 114 | 115 | except ConnectionError as e: 116 | log.exception(e) 117 | 118 | # TODO not working 119 | except KeyboardInterrupt as e: 120 | # We catch keyboard interrupts here so that training can be be safely aborted. 121 | # This is so common that we've built this right into this function, which ensures that 122 | # the `on_train_end` method is properly called. 123 | log.info('Aborted by user. {} \nExit...'.format(e)) 124 | agent.save_weights('{p}/dqn_{fn}_weights.h5f'.format(p=PATH, fn=ENV_NAME), overwrite=True) 125 | break 126 | 127 | except RuntimeError as e: 128 | log.exception(e) 129 | break 130 | 131 | # except Exception as e: 132 | # print(e) 133 | # # log.exception(e) 134 | # break 135 | -------------------------------------------------------------------------------- /mas_cryptobot/train_1.2_cnn_ae+lstm.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | import random 4 | import numpy as np 5 | 6 | # # OpenCL GPU accelerating 7 | # # http://github.com/plaidml 8 | import plaidml.keras 9 | plaidml.keras.install_backend() 10 | 11 | from mas_tools.models.autoencoders import deep_conv2d_vae 12 | from keras.callbacks import ModelCheckpoint, ReduceLROnPlateau 13 | 14 | from keras.models import Model 15 | from keras.layers import Input, LSTM, Dense, Reshape 16 | 17 | from sklearn.model_selection import train_test_split 18 | from mas_tools.ml import plot_history 19 | import matplotlib.pyplot as plt 20 | 21 | 22 | # data 23 | dt_path = 'E:/Projects/market-analysis-system/data/pictured/' 24 | symbols = ['AUDJPY', 'AUDUSD', 'CHFJPY', 'EURAUD', 25 | 'EURCAD', 'EURCHF', 'EURGBP', 'EURJPY', 26 | 'EURRUB', 'EURUSD', 'GBPAUD', 'GBPCAD', 27 | 'GBPCHF', 'GBPJPY', 'GBPUSD', 'NZDJPY', 28 | 'NZDUSD', 'USDCAD', 'USDCHF', 'USDJPY', 29 | 'USDRUB', 'XAGUSD', 'XAUUSD'] 30 | tf = 240 # timeframe (period) 31 | window = 20 # size history window (bar count) 32 | 33 | # model 34 | action = 'train1' # train1, train2, predict 35 | wgt_path1 = 'E:/Projects/market-analysis-system/mas_vae/' 36 | wgt_path2 = 'E:/Projects/market-analysis-system/mas_cryptobot/' 37 | code = 60 # latent tensor size 38 | filters = (3, 12) # convolution filters count 39 | dropout = 0.4 # inside dropout 40 | lstm_size = 64 41 | 42 | epochs = 10 43 | batch = 128 44 | img_width = window * 4 45 | img_size = img_width * img_width * 3 46 | 47 | 48 | #====== load data ====== 49 | print('Load data...') 50 | random.seed(666) 51 | symbols = random.sample(symbols, 6) 52 | filename = str(tf) + '_w' + str(window) + '.npz' 53 | # create form 54 | data = np.ones((img_size,), dtype=np.float) 55 | data = np.reshape(data, (1, img_width, img_width, 3)) 56 | # load 57 | for symbol in symbols: 58 | # compressed npz 59 | npz_file = np.load(dt_path + symbol + filename) 60 | new_data = npz_file.f.arr_0 61 | new_data = np.reshape(new_data, (len(new_data), img_width, img_width, 3)) 62 | data = np.vstack((data, new_data)) 63 | # clean first row 64 | data = data[1:] 65 | # normalize images data 66 | data = data.astype('float32') / 255 67 | 68 | print('Data shape: {}'.format(data.shape)) 69 | 70 | 71 | #====== Build encoder ====== 72 | print('Build encoder layers...') 73 | model_name = 'vae_img{}_flt{}-{}_code{}_enc'.format(window, filters[0], filters[1], code) 74 | encoder, _, _, _ = deep_conv2d_vae((img_width, img_width, 3), 75 | filters_count=filters, 76 | latent_dim=code, 77 | dropout=dropout) 78 | 79 | encoder.compile(optimizer='rmsprop', loss='mse') 80 | encoder.load_weights(wgt_path1 + model_name + '.hdf5', by_name=True) 81 | encoder.trainable = False 82 | 83 | # generate output 84 | _, _, y_data = encoder.predict(data, batch_size=batch) 85 | data = data[:-1] 86 | y_data = y_data[1:] 87 | x_train, x_test, y_train, y_test = train_test_split(data, y_data, shuffle=False, test_size=0.1) 88 | # clear memory 89 | data = None 90 | y_data = None 91 | 92 | #====== Build lstm ====== 93 | print('Build lstm layers...') 94 | model_name = 'istm_conv{}-{}-{}-{}_lstm{}'.format(window, filters[0], filters[1], code, lstm_size) 95 | picture_tensor = Input(shape=(img_width, img_width, 3), name='encoder_input') 96 | 97 | input_tensor = Input(shape=(code,), name='lstm_input') 98 | lstm = Reshape((1, code))(input_tensor) 99 | lstm = LSTM(lstm_size, return_sequences=True)(lstm) 100 | lstm = LSTM(lstm_size)(lstm) 101 | lstm = Dense(code, activation='linear')(lstm) 102 | 103 | lstm = Model(input_tensor, lstm, name='lstm_layers') 104 | model = Model(picture_tensor, lstm(encoder(picture_tensor)[2]), name='conv-lstm_layers') 105 | 106 | model.compile(optimizer='rmsprop', loss='mse', metrics=['acc']) 107 | 108 | if action in ['train2', 'predict']: 109 | model.load_weights(wgt_path2 + model_name + '.hdf5', by_name=True) 110 | if action in ['train1', 'train2']: 111 | reduce_lr = ReduceLROnPlateau(factor=0.1, patience=3, min_lr=0.00001, verbose=1) 112 | 113 | history = model.fit(x_train, y_train, 114 | epochs=epochs, 115 | batch_size=batch, 116 | shuffle=False, 117 | callbacks=[reduce_lr], 118 | validation_data=(x_test, y_test)) 119 | 120 | model.save_weights(wgt_path2 + model_name + '.hdf5') 121 | plot_history(history, acc='acc') 122 | print('Weight saved.') 123 | 124 | -------------------------------------------------------------------------------- /mas_tools/layers/attention_with_context.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # From: https://gist.github.com/cbaziotis/7ef97ccf71cbc14366835198c09809d2 3 | from keras import backend as K, initializers, regularizers, constraints 4 | from keras.engine.topology import Layer 5 | 6 | 7 | def dot_product(x, kernel): 8 | """ 9 | Wrapper for dot product operation, in order to be compatible with both 10 | Theano and Tensorflow 11 | Args: 12 | x (): input 13 | kernel (): weights 14 | Returns: 15 | """ 16 | if K.backend() == 'tensorflow': 17 | return K.squeeze(K.dot(x, K.expand_dims(kernel)), axis=-1) 18 | else: 19 | return K.dot(x, kernel) 20 | 21 | 22 | class AttentionWithContext(Layer): 23 | """ 24 | Attention operation, with a context/query vector, for temporal data. 25 | Supports Masking. 26 | Follows the work of Yang et al. [https://www.cs.cmu.edu/~diyiy/docs/naacl16.pdf] 27 | "Hierarchical Attention Networks for Document Classification" 28 | by using a context vector to assist the attention 29 | # Input shape 30 | 3D tensor with shape: `(samples, steps, features)`. 31 | # Output shape 32 | 2D tensor with shape: `(samples, features)`. 33 | 34 | How to use: 35 | Just put it on top of an RNN Layer (GRU/LSTM/SimpleRNN) with return_sequences=True. 36 | The dimensions are inferred based on the output shape of the RNN. 37 | 38 | Note: The layer has been tested with Keras 2.0.6 39 | 40 | Example: 41 | model.add(LSTM(64, return_sequences=True)) 42 | model.add(AttentionWithContext()) 43 | # next add a Dense layer (for classification/regression) or whatever... 44 | """ 45 | 46 | def __init__(self, 47 | W_regularizer=None, u_regularizer=None, b_regularizer=None, 48 | W_constraint=None, u_constraint=None, b_constraint=None, 49 | bias=True, **kwargs): 50 | 51 | self.supports_masking = True 52 | self.init = initializers.get('glorot_uniform') 53 | 54 | self.W_regularizer = regularizers.get(W_regularizer) 55 | self.u_regularizer = regularizers.get(u_regularizer) 56 | self.b_regularizer = regularizers.get(b_regularizer) 57 | 58 | self.W_constraint = constraints.get(W_constraint) 59 | self.u_constraint = constraints.get(u_constraint) 60 | self.b_constraint = constraints.get(b_constraint) 61 | 62 | self.bias = bias 63 | super(AttentionWithContext, self).__init__(**kwargs) 64 | 65 | def build(self, input_shape): 66 | assert len(input_shape) == 3 67 | 68 | self.W = self.add_weight((input_shape[-1], input_shape[-1],), 69 | initializer=self.init, 70 | name='{}_W'.format(self.name), 71 | regularizer=self.W_regularizer, 72 | constraint=self.W_constraint) 73 | if self.bias: 74 | self.b = self.add_weight((input_shape[-1],), 75 | initializer='zero', 76 | name='{}_b'.format(self.name), 77 | regularizer=self.b_regularizer, 78 | constraint=self.b_constraint) 79 | 80 | self.u = self.add_weight((input_shape[-1],), 81 | initializer=self.init, 82 | name='{}_u'.format(self.name), 83 | regularizer=self.u_regularizer, 84 | constraint=self.u_constraint) 85 | 86 | super(AttentionWithContext, self).build(input_shape) 87 | 88 | def compute_mask(self, input, input_mask=None): 89 | # do not pass the mask to the next layers 90 | return None 91 | 92 | def call(self, x, mask=None): 93 | uit = dot_product(x, self.W) 94 | 95 | if self.bias: 96 | uit += self.b 97 | 98 | uit = K.tanh(uit) 99 | ait = K.dot(uit, self.u) 100 | 101 | a = K.exp(ait) 102 | 103 | # apply mask after the exp. will be re-normalized next 104 | if mask is not None: 105 | # Cast the mask to floatX to avoid float64 upcasting in theano 106 | a *= K.cast(mask, K.floatx()) 107 | 108 | # in some cases especially in the early stages of training the sum may be almost zero 109 | # and this results in NaN's. A workaround is to add a very small positive number ε to the sum. 110 | # a /= K.cast(K.sum(a, axis=1, keepdims=True), K.floatx()) 111 | a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx()) 112 | 113 | a = K.expand_dims(a) 114 | weighted_input = x * a 115 | return K.sum(weighted_input, axis=1) 116 | 117 | def compute_output_shape(self, input_shape): 118 | return input_shape[0], input_shape[-1] 119 | -------------------------------------------------------------------------------- /mas_tools/layers/Attention.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # From: https://gist.github.com/cbaziotis/6428df359af27d58078ca5ed9792bd6d 3 | from keras import backend as K, initializers, regularizers, constraints 4 | from keras.engine.topology import Layer 5 | 6 | 7 | def dot_product(x, kernel): 8 | """ 9 | Wrapper for dot product operation, in order to be compatible with both 10 | Theano and Tensorflow 11 | Args: 12 | x (): input 13 | kernel (): weights 14 | Returns: 15 | """ 16 | if K.backend() == 'tensorflow': 17 | # todo: check that this is correct 18 | return K.squeeze(K.dot(x, K.expand_dims(kernel)), axis=-1) 19 | else: 20 | return K.dot(x, kernel) 21 | 22 | 23 | class Attention(Layer): 24 | def __init__(self, 25 | W_regularizer=None, b_regularizer=None, 26 | W_constraint=None, b_constraint=None, 27 | bias=True, 28 | return_attention=False, 29 | **kwargs): 30 | """ 31 | Keras Layer that implements an Attention mechanism for temporal data. 32 | Supports Masking. 33 | Follows the work of Raffel et al. [https://arxiv.org/abs/1512.08756] 34 | # Input shape 35 | 3D tensor with shape: `(samples, steps, features)`. 36 | # Output shape 37 | 2D tensor with shape: `(samples, features)`. 38 | :param kwargs: 39 | Just put it on top of an RNN Layer (GRU/LSTM/SimpleRNN) with return_sequences=True. 40 | The dimensions are inferred based on the output shape of the RNN. 41 | 42 | 43 | Note: The layer has been tested with Keras 1.x 44 | 45 | Example: 46 | 47 | # 1 48 | model.add(LSTM(64, return_sequences=True)) 49 | model.add(Attention()) 50 | # next add a Dense layer (for classification/regression) or whatever... 51 | 52 | # 2 - Get the attention scores 53 | hidden = LSTM(64, return_sequences=True)(words) 54 | sentence, word_scores = Attention(return_attention=True)(hidden) 55 | 56 | """ 57 | self.supports_masking = True 58 | self.return_attention = return_attention 59 | self.init = initializers.get('glorot_uniform') 60 | 61 | self.W_regularizer = regularizers.get(W_regularizer) 62 | self.b_regularizer = regularizers.get(b_regularizer) 63 | 64 | self.W_constraint = constraints.get(W_constraint) 65 | self.b_constraint = constraints.get(b_constraint) 66 | 67 | self.bias = bias 68 | super(Attention, self).__init__(**kwargs) 69 | 70 | def build(self, input_shape): 71 | assert len(input_shape) == 3 72 | 73 | self.W = self.add_weight((input_shape[-1],), 74 | initializer=self.init, 75 | name='{}_W'.format(self.name), 76 | regularizer=self.W_regularizer, 77 | constraint=self.W_constraint) 78 | if self.bias: 79 | self.b = self.add_weight((input_shape[1],), 80 | initializer='zero', 81 | name='{}_b'.format(self.name), 82 | regularizer=self.b_regularizer, 83 | constraint=self.b_constraint) 84 | else: 85 | self.b = None 86 | 87 | self.built = True 88 | 89 | def compute_mask(self, input, input_mask=None): 90 | # do not pass the mask to the next layers 91 | return None 92 | 93 | def call(self, x, mask=None): 94 | eij = dot_product(x, self.W) 95 | 96 | if self.bias: 97 | eij += self.b 98 | 99 | eij = K.tanh(eij) 100 | 101 | a = K.exp(eij) 102 | 103 | # apply mask after the exp. will be re-normalized next 104 | if mask is not None: 105 | # Cast the mask to floatX to avoid float64 upcasting in theano 106 | a *= K.cast(mask, K.floatx()) 107 | 108 | # in some cases especially in the early stages of training the sum may be almost zero 109 | # and this results in NaN's. A workaround is to add a very small positive number ε to the sum. 110 | # a /= K.cast(K.sum(a, axis=1, keepdims=True), K.floatx()) 111 | a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx()) 112 | 113 | weighted_input = x * K.expand_dims(a) 114 | 115 | result = K.sum(weighted_input, axis=1) 116 | 117 | if self.return_attention: 118 | return [result, a] 119 | return result 120 | 121 | def compute_output_shape(self, input_shape): 122 | if self.return_attention: 123 | return [(input_shape[0], input_shape[-1]), 124 | (input_shape[0], input_shape[1])] 125 | else: 126 | return input_shape[0], input_shape[-1] 127 | -------------------------------------------------------------------------------- /mas_cryptobot/bot-rl_v0.2.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | import os 4 | import time 5 | import logging 6 | from requests.exceptions import ConnectionError 7 | 8 | import numpy as np 9 | 10 | from keras.optimizers import Adam 11 | 12 | from rl.agents.dqn import DQNAgent 13 | from rl.policy import BoltzmannQPolicy 14 | from rl.memory import SequentialMemory 15 | from rl.processors import MultiInputProcessor 16 | 17 | from mas_tools.api import Binance 18 | from mas_tools.markets import VirtualExchange 19 | from mas_tools.envs import MarketEnv 20 | from mas_tools.models import cnn_model_2in_with_feedback 21 | 22 | 23 | MY_API_KEY = '---' 24 | MY_API_SECRET = '---' 25 | 26 | PATH = os.path.dirname(os.path.abspath(__file__)) 27 | ENV_NAME = 'cb_Binance_5' 28 | 29 | SLEEP = 5 30 | TRAIN = True 31 | tickcount = 0 32 | 33 | logging.basicConfig(level=logging.INFO, 34 | handlers=[logging.FileHandler("{p}/logs/{fn}.log".format(p=PATH, fn=ENV_NAME)), 35 | logging.StreamHandler()] 36 | ) 37 | 38 | log = logging.getLogger() 39 | 40 | 41 | ## Init exchange api 42 | api = Binance(API_KEY=MY_API_KEY, API_SECRET=MY_API_SECRET) 43 | 44 | ## Init market environment 45 | market_conn = VirtualExchange(api, symbols=['ETHUSDT'], period='1m', 46 | balance=1000.0, lot_size=0.1) 47 | market = MarketEnv(market_conn, True, True) 48 | 49 | ## Environment parameters 50 | observation_shape = market.observation_space.shape 51 | nb_actions = market.action_space.n 52 | log.info('State shape = {a} | actions = {b}'.format(a=observation_shape, b=nb_actions)) 53 | 54 | ## Init ML-model for agent 55 | limit = observation_shape[1] 56 | model = cnn_model_2in_with_feedback( 57 | (limit, 4), (limit, 4), 58 | market.feedback_shape, nb_actions, 'softmax') 59 | 60 | ## Init RL-metod parameters 61 | memory = SequentialMemory(limit=10000, window_length=1) 62 | # TODO implement policies for multiply symbols 63 | policy = BoltzmannQPolicy() 64 | 65 | ## Init RL agent 66 | agent = DQNAgent(model=model, nb_actions=nb_actions, 67 | memory=memory, nb_steps_warmup=1000, 68 | target_model_update=1e-2, policy=policy, 69 | processor=MultiInputProcessor(3), 70 | enable_dueling_network=True, dueling_type='avg' 71 | ) 72 | agent.compile(Adam(lr=1e-3), metrics=['mae']) 73 | 74 | try: 75 | ## Comment here if you want to start learning again 76 | agent.load_weights('{p}/dqn_{fn}_weights.h5f'.format(p=PATH, fn=ENV_NAME)) 77 | pass 78 | except OSError as e: 79 | print(e) 80 | except ValueError as e: 81 | print(e) 82 | 83 | # agent.fit(market, nb_steps=100000, visualize=False, verbose=2) 84 | # agent.save_weights('dqn_{}_weights.h5f'.format(ENV_NAME), overwrite=True) 85 | # agent.test(market, nb_episodes=5, visualize=False) 86 | 87 | ## Train or evaluate 88 | if TRAIN: 89 | agent.training = True 90 | 91 | observation = market.reset() 92 | 93 | while True: 94 | try: 95 | # TODO add callbacks? 96 | 97 | ## Agent vybiraet dejstvie 98 | # (candles=9(mb=>(2,4)?), tickers=4, trades=2) 99 | # TODO actions for multy symbols market 100 | action = agent.forward(observation) 101 | 102 | ## Execute action 103 | observation, reward, done, info = market.step([action]) 104 | 105 | ## Poluchaem otvet ot sredy 106 | agent.backward(reward, terminal=done) 107 | 108 | ## Esli dostigli konca 109 | if done: 110 | observation = market.reset() 111 | agent.reset_states() 112 | done = False 113 | log.info('Is terminal state. Reset..') 114 | log.info('='*40) 115 | 116 | log.info('Tick: {t} | {info}'.format( 117 | t=tickcount, info=info 118 | )) 119 | 120 | ## Check point 121 | if tickcount % 100 == 0: 122 | agent.save_weights('{p}/dqn_{fn}_weights.h5f'.format(p=PATH, fn=ENV_NAME), overwrite=True) 123 | 124 | ## Time shift and counter 125 | time.sleep(SLEEP) 126 | tickcount += 1 127 | 128 | except ConnectionError as e: 129 | log.exception(e) 130 | 131 | # TODO not working 132 | except KeyboardInterrupt as e: 133 | ## https://stackoverflow.com/questions/15457786/ctrl-c-crashes-python-after-importing-scipy-stats 134 | # We catch keyboard interrupts here so that training can be be safely aborted. 135 | # This is so common that we've built this right into this function, which ensures that 136 | # the `on_train_end` method is properly called. 137 | log.info('Aborted by user. {} \nExit...'.format(e)) 138 | agent.save_weights('{p}/dqn_{fn}_weights.h5f'.format(p=PATH, fn=ENV_NAME), overwrite=True) 139 | break 140 | 141 | except RuntimeError as e: 142 | log.exception(e) 143 | break 144 | 145 | # except Exception as e: 146 | # print(e) 147 | # # log.exception(e) 148 | # break 149 | -------------------------------------------------------------------------------- /mas_tools/classes.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import numpy as np 3 | 4 | 5 | def signal_to_class(data, n=2, normalize=True): 6 | """ 7 | Converts a list of signals to a n-dimensional list of classes [buy, .., sell]. 8 | 9 | Arguments 10 | n (int): Number of classes. 11 | normalize (bool): It normalizes to unity. False - the signal changes only the sign. 12 | 13 | Returns 14 | Array of classes. 15 | """ 16 | 17 | result = np.array([]) 18 | data = np.array(data) 19 | 20 | if len(data.shape) > 1: 21 | raise ValueError("The array must be one-dimensional.") 22 | 23 | if n == 2: 24 | if normalize: 25 | for item in data: 26 | if item > 0: # buy 27 | result = np.append(result, [1.0, 0.0]) 28 | if item <= 0: # sell 29 | result = np.append(result, [0.0, 1.0]) 30 | else: 31 | for item in data: 32 | result = np.append(result, [0.5+item/2.0, 0.5-item/2.0]) 33 | elif n == 3: 34 | if normalize: 35 | for item in data: 36 | if item > 0: # buy 37 | result = np.append(result, [1.0, 0.0, 0.0]) 38 | if item < 0: # sell 39 | result = np.append(result, [0.0, 0.0, 1.0]) 40 | if item == 0: # pass 41 | result = np.append(result, [0.0, 1.0, 0.0]) 42 | else: 43 | for item in data: 44 | if item > 0: # buy 45 | result = np.append(result, [abs(item), (1.0-abs(item)), 0.0]) 46 | if item < 0: # sell 47 | result = np.append(result, [0.0, (1.0-abs(item)), abs(item)]) 48 | if item == 0: # pass 49 | result = np.append(result, [0.0, 1.0, 0.0]) 50 | elif n == 6: 51 | for item in data: 52 | if item >= 0.8 and item <= 1.0: 53 | result = np.append(result, [1.0, 0.0, 0.0, 0.0, 0.0, 0.0]) 54 | elif item >= 0.4 and item < 0.8: 55 | result = np.append(result, [0.0, 1.0, 0.0, 0.0, 0.0, 0.0]) 56 | elif item >= 0.0 and item < 0.4: 57 | result = np.append(result, [0.0, 0.0, 1.0, 0.0, 0.0, 0.0]) 58 | elif item > -0.4 and item < 0.0: 59 | result = np.append(result, [0.0, 0.0, 0.0, 1.0, 0.0, 0.0]) 60 | elif item > -0.8 and item <= 0.4: 61 | result = np.append(result, [0.0, 0.0, 0.0, 0.0, 1.0, 0.0]) 62 | elif item >= -1.0 and item <= 0.8: 63 | result = np.append(result, [0.0, 0.0, 0.0, 0.0, 0.0, 1.0]) 64 | 65 | return result.reshape((data.shape[0], n)) 66 | 67 | 68 | def class_to_signal(data, n=2, normalized=True): 69 | """ 70 | Converts a n-dimensional list of classes to a list of signals. 71 | """ 72 | 73 | result = np.array([]) 74 | 75 | if n == 2: 76 | if normalized: 77 | for item in data: 78 | result = np.append(result, 1 if item[0] > item[1] else -1) 79 | else: 80 | for item in data: 81 | result = np.append(result, item[0] * 2 - 1.0) 82 | elif n == 3: 83 | if normalized: 84 | for item in data: 85 | _class = np.argmax(item) 86 | if _class == 0: 87 | result = np.append(result, 1.0) 88 | elif _class == 1: 89 | result = np.append(result, 0.0) 90 | elif _class == 2: 91 | result = np.append(result, -1.0) 92 | else: 93 | for item in data: 94 | _class = np.argmax(item) 95 | if _class == 0: 96 | result = np.append(result, item[0]) 97 | elif _class == 1: 98 | result = np.append(result, 0.0) 99 | elif _class == 2: 100 | result = np.append(result, -item[2]) 101 | elif n == 6: 102 | for item in data: 103 | _class = np.argmax(item) 104 | if _class == 0: 105 | result = np.append(result, 1.0) 106 | elif _class == 1: 107 | result = np.append(result, 0.66) 108 | elif _class == 2: 109 | result = np.append(result, 0.33) 110 | elif _class == 3: 111 | result = np.append(result, -0.33) 112 | elif _class == 4: 113 | result = np.append(result, -0.66) 114 | elif _class == 5: 115 | result = np.append(result, -1.0) 116 | 117 | return result 118 | 119 | 120 | def prepare_target(data, close_index=3, classes=6): 121 | """ 122 | Hello (= 123 | uniform classes 124 | """ 125 | # TODO 126 | # while const 127 | classes = 6 128 | 129 | data = np.array(data) 130 | new_target = data[1:, close_index] / data[:-1, close_index] 131 | new_target = np.insert(new_target, obj=0, values=[1.0]) 132 | 133 | n, bins = np.histogram(new_target, bins=200, range=(0.99, 1.01)) 134 | 135 | sixth = sum(n) / classes 136 | 137 | points = [0., 0., 1., 0., 0.] 138 | _sum = n[100]/2 139 | p_idx = 1 140 | for idx in range(99, -1): 141 | _sum += n[idx] 142 | if _sum >= sixth: 143 | points[p_idx] = (idx - 100) / 10**4 + 1 144 | p_idx -= 1 145 | if p_idx < 0: 146 | break 147 | _sum = n[100]/2 148 | p_idx = 3 149 | for idx in range(101, 201): 150 | _sum += n[idx] 151 | if _sum >= sixth: 152 | points[p_idx] = (idx - 100) / 10**4 + 1 153 | p_idx += 1 154 | if p_idx > 4: 155 | break 156 | # TODO 157 | def select(a): 158 | a > points[2] 159 | return 1 160 | new_target = [select(x) for x in new_target] 161 | 162 | return new_target 163 | -------------------------------------------------------------------------------- /mas_cryptobot/train_1.1_cnn_ae_for_candles.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | import random 4 | import numpy as np 5 | 6 | # OpenCL GPU accelerating 7 | # http://github.com/plaidml 8 | import plaidml.keras 9 | plaidml.keras.install_backend() 10 | 11 | from mas_tools.models.autoencoders import deep_conv2d_vae 12 | from keras.callbacks import ModelCheckpoint, ReduceLROnPlateau, TensorBoard 13 | 14 | from mas_tools.ml import plot_history 15 | import matplotlib.pyplot as plt 16 | 17 | 18 | random.seed(666) 19 | 20 | ## Data 21 | # 1 22 | dt_path = 'E:/Projects/market-analysis-system/data/pictured/' 23 | symbols = [ 24 | 'AUDJPY', 'AUDUSD', 'CHFJPY', 'EURAUD', 25 | 'EURCAD', 'EURCHF', 'EURGBP', 'EURJPY', 26 | 'EURRUB', 'EURUSD', 'GBPAUD', 'GBPCAD', 27 | 'GBPCHF', 'GBPJPY', 'GBPUSD', 'NZDJPY', 28 | 'NZDUSD', 'USDCAD', 'USDCHF', 'USDJPY', 29 | 'USDRUB', 'XAGUSD', 'XAUUSD' 30 | ] 31 | tfs = [ 32 | '1', '5', '15', '30', 33 | '60', '240', '1440' 34 | ] 35 | tf = 240 # timeframe (period) 36 | window = 20 # size history window (bar count) 37 | # .. from random files 38 | rnd = random.sample 39 | filenames = ['{}{}_w{}.npz'.format(symbol, rnd(tfs, 1)[0], window) for symbol in rnd(symbols, 6)] 40 | # # 2 41 | # dt_path = 'E:/Projects/market-analysis-system/data/crypto/' 42 | # symbols = [ 43 | # 'BCCUSDT', 'BNBETH', 'BNBUSDT', 44 | # 'BTCUSDT', 'ETHUSDT' 45 | # ] 46 | # postfix = '{n}_candles.csv' 47 | # window = 20 # size history window (bar count) 48 | 49 | 50 | ## Model 51 | wgt_path = 'E:/Projects/market-analysis-system/mas_cryptobot/wgts/' 52 | # Select mode: 53 | # Training from scratch - train1, 54 | # Training a trained model - train2, 55 | # Predicting - predict 56 | action = 'train1' 57 | # latent tensor size 58 | code = 60 59 | # convolution filters count 60 | filters = (4, 12) 61 | # inside dropout 62 | dropout = 0.5 63 | 64 | model_name = 'vae_w{}_flt{}-{}_code{}'.format(window, filters[0], filters[1], code) 65 | epochs = 10 66 | batch = 128 67 | # image sizes 68 | img_width = window * 4 69 | img_size = img_width * img_width * 3 70 | 71 | 72 | ## Load data 73 | print('Load data...') 74 | # Create form 75 | x_data = np.ones((img_size,), dtype=np.float) 76 | x_data = np.reshape(x_data, (1, img_width, img_width, 3)) 77 | # Load 78 | for filename in filenames: 79 | # compressed npz 80 | npz_file = np.load(dt_path + filename) 81 | new_data = npz_file.f.arr_0 82 | new_data = np.reshape(new_data, (len(new_data), img_width, img_width, 3)) 83 | x_data = np.vstack((x_data, new_data)) 84 | # clean first row 85 | x_data = x_data[1:] 86 | # normalize imagess data 87 | x_data = x_data.astype('float32') / 255 88 | print('New data shape:', x_data.shape) 89 | 90 | 91 | ## Build VAE 92 | print('Build autoencoder...') 93 | encoder, decoder, autoencoder, vae_loss = deep_conv2d_vae((img_width, img_width, 3), 94 | filters_count=filters, 95 | latent_dim=code, 96 | dropout=dropout) 97 | # TODO How to use vae_loss? 98 | # print(type(vae_loss)) 99 | autoencoder.compile(optimizer='rmsprop', loss='mse', metrics=['acc']) 100 | 101 | 102 | ## Train or do prediction 103 | if action in ['train2', 'predict']: 104 | autoencoder.load_weights(wgt_path + model_name + '.hdf5', by_name=True) 105 | if action in ['train1', 'train2']: 106 | print('Train model...') 107 | # reduce_lr = ReduceLROnPlateau(factor=0.1, patience=3, min_lr=0.00001, verbose=1) 108 | # chpt = ModelCheckpoint(wgt_path + model_name + '_{}.hdf5') 109 | # tb = TensorBoard() 110 | 111 | # Train 112 | history = autoencoder.fit( 113 | x_data, x_data, 114 | epochs=epochs, 115 | batch_size=batch, 116 | validation_split=0.1, 117 | shuffle=True, 118 | # callbacks=[tb] 119 | ) 120 | 121 | # Save weights 122 | autoencoder.save_weights(wgt_path + model_name + '.hdf5') 123 | encoder.save_weights(wgt_path + model_name + '_enc.hdf5') 124 | plot_history(history, acc='acc') 125 | 126 | 127 | ## Visual evaluation 128 | # Number of examples = n^2 129 | n = 10 130 | test_z_mean, test_z_log_var, test_z = encoder.predict(x_data[-100:], batch_size=batch) 131 | 132 | # Save images of origin and decoded data 133 | if action in ['train1', 'train2', 'predict']: 134 | def plot_data(fname, plot_data): 135 | figure = np.zeros((img_width * n, img_width * n, 3)) 136 | plot_data = plot_data[-(n*n):] 137 | for idx in range(n): 138 | for jdx in range(n): 139 | digit = plot_data[idx*n+jdx].reshape(img_width, img_width, 3) 140 | figure[(idx * img_width): ((idx + 1) * img_width), 141 | (jdx * img_width): ((jdx + 1) * img_width)] = digit 142 | 143 | fig = plt.figure() 144 | img = plt.imshow(figure) 145 | plt.colorbar(img) 146 | plt.savefig(fname) 147 | plt.close() 148 | 149 | # Origin test data 150 | plot_data(wgt_path+model_name+'_origin_pics.png', x_data) 151 | # Decoded test data 152 | plot_data(wgt_path+model_name+'_restored_from_z_1.png', decoder.predict(test_z[:100, :])) 153 | # plot_data(wgt_path+model_name+'_restored_from_z_mean.png', decoder.predict(test_z_mean[:100, :])) 154 | # plot_data(wgt_path+model_name+'_restored_from_z_log.png', decoder.predict(test_z_log_var[:100, :])) 155 | 156 | # Save a histogram of the digit classes in the latent space 157 | if action in ['train2']: 158 | def plot_hist(fname, hist_data): 159 | for idx in range(n): 160 | plt.hist(hist_data[idx], bins=100) 161 | plt.xlabel('z size') 162 | plt.savefig(fname) 163 | plt.close() 164 | 165 | plot_hist(wgt_path+model_name+'_latent-distr.png', test_z) 166 | # plot_hist(wgt_path+model_name+'_latent-distr-mean.png', test_z_mean) 167 | # plot_hist(wgt_path+model_name+'_latent-distr--log.png', test_z_log_var) 168 | -------------------------------------------------------------------------------- /data/crypto/_crypto_collector.py: -------------------------------------------------------------------------------- 1 | import time 2 | import requests 3 | from threading import Thread 4 | 5 | import numpy as np 6 | import pandas as pd 7 | 8 | from mas_tools.api import Binance 9 | 10 | api = Binance('', '') 11 | symbols = ['BTCUSDT', 'ETHUSDT', 'BNBETH', 'BNBUSDT', 'BCCUSDT'] 12 | period = '1m' 13 | 14 | limit = 20 15 | ticks = 0 16 | save_period = 1000 17 | path = 'E:/Projects/market-analysis-system/data/crypto/' 18 | 19 | data = dict(zip(symbols, [dict({'data':[]}) for i in symbols])) 20 | 21 | 22 | while True: 23 | try: 24 | start_time = time.time() 25 | for symbol in symbols: 26 | candles = pd.DataFrame(api.candlesticks(symbol=symbol, interval=period, limit=limit), dtype=np.float) 27 | tickers = pd.DataFrame(api.tickers(symbol=symbol, limit=limit)) 28 | trades = pd.DataFrame(api.aggr_trades(symbol=symbol, limit=limit), dtype=np.float) 29 | # data[symbol]['candles'] = np.column_stack((candles.values[:, 1:6], # o,h,l,c,v 30 | # candles.values[:, 7:11])) # qav, nt, bv, qv 31 | # data[symbol]['tickers'] = np.column_stack(([np.array([x[0:2] for x in tickers['bids'].values], dtype=np.float), 32 | # np.array([x[0:2] for x in tickers['asks'].values], dtype=np.float)])) 33 | # data[symbol]['trades'] = trades[['p', 'q']].values 34 | data[symbol]['data'] = np.append(data[symbol]['data'], np.column_stack((np.column_stack((candles.values[:, 1:6], # o,h,l,c,v 35 | candles.values[:, 7:11])), # qav, nt, bv, qv 36 | np.column_stack(([np.array([x[0:2] for x in tickers['bids'].values], dtype=np.float), 37 | np.array([x[0:2] for x in tickers['asks'].values], dtype=np.float)])), 38 | trades[['p', 'q']].values))).reshape(len(data[symbol]['data'])+1, limit*15) 39 | 40 | if ticks % save_period == 0: 41 | for symbol in symbols: 42 | fname = path + symbol + '/' + str(round(time.time())) + '.csv' 43 | np.savetxt(fname, data[symbol]['data'], delimiter=';', fmt='%.8f') 44 | print('{} saved.'.format(fname)) 45 | data[symbol]['data'] = np.array([]) 46 | 47 | ticks += 1 48 | # print('tick. {} s.'.format(time.time()-start_time)) 49 | time.sleep(5) 50 | 51 | except requests.exceptions.ConnectionError as e: 52 | print(e) 53 | 54 | except TimeoutError as e: 55 | print('TimeoutError:', e) 56 | 57 | # except Exception as e: 58 | # print('Error:', e) 59 | # break 60 | 61 | except KeyboardInterrupt: 62 | print('Exit...') 63 | for symbol in symbols: 64 | fname = path + symbol + '/' + str(round(time.time())) + '.csv' 65 | np.savetxt(fname, data[symbol]['data'], delimiter=';', fmt='%.8f') 66 | data[symbol]['data'] = np.array([]) 67 | print('{} saved.'.format(fname)) 68 | break 69 | 70 | 71 | # def get_candles(symbol): 72 | # while True: 73 | # try: 74 | # candles = pd.DataFrame(api.candlesticks(symbol=symbol, interval=period, limit=limit), dtype=np.float) 75 | # data[symbol]['candles'] = np.column_stack((candles.values[:, 1:6], # o,h,l,c,v 76 | # candles.values[:, 7:11])) # qav, nt, bv, qv 77 | # except Exception as e: 78 | # print('get_candles:', e) 79 | # except KeyboardInterrupt: 80 | # print('Exit...') 81 | # break 82 | # time.sleep(.5) 83 | 84 | # def get_tickers(symbol): 85 | # while True: 86 | # try: 87 | # tickers = pd.DataFrame(api.tickers(symbol=symbol, limit=limit)) 88 | # data[symbol]['tickers'] = np.column_stack(([np.array([x[0:2] for x in tickers['bids'].values], dtype=np.float), 89 | # np.array([x[0:2] for x in tickers['asks'].values], dtype=np.float)])) 90 | # except Exception as e: 91 | # print('get_tickers:', e) 92 | # except KeyboardInterrupt: 93 | # print('Exit...') 94 | # break 95 | # time.sleep(.5) 96 | 97 | # def get_trades(symbol): 98 | # while True: 99 | # try: 100 | # trades = pd.DataFrame(api.aggr_trades(symbol=symbol, limit=limit), dtype=np.float) 101 | # data[symbol]['trades'] = trades[['p', 'q']].values 102 | # except Exception as e: 103 | # print('get_trades:', e) 104 | # except KeyboardInterrupt: 105 | # print('Exit...') 106 | # break 107 | # time.sleep(.5) 108 | 109 | # def saver(): 110 | # ticks = 1 111 | # while True: 112 | # try: 113 | # start_time = time.time() 114 | # if ticks % save_period == 0: 115 | # print('Save...') 116 | # for symbol in symbols: 117 | # np.savetxt(path+symbol+'.csv', np.column_stack((data[symbol]['candles'], 118 | # data[symbol]['tickers'], 119 | # data[symbol]['trades'])), delimiter=';', fmt='%.8f') 120 | 121 | # ticks += 1 122 | # time.sleep(1.0) 123 | # print('tick. {} s.'.format(time.time()-start_time)) 124 | 125 | # # Подготавливаем потоки, складываем их в массив 126 | # threads = [] 127 | # for symbol in symbols: 128 | # threads.append(Thread(target=get_candles, args=(symbol,))) 129 | # threads.append(Thread(target=get_tickers, args=(symbol,))) 130 | # threads.append(Thread(target=get_trades, args=(symbol,))) 131 | # threads.append(Thread(target=saver)) 132 | 133 | # # Запускаем каждый поток 134 | # for thread in threads: 135 | # if not thread.is_alive(): 136 | # thread.start() 137 | 138 | # # Ждем завершения каждого потока 139 | # for thread in threads: 140 | # thread.join() 141 | 142 | -------------------------------------------------------------------------------- /mas_tools/data.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from math import exp 3 | 4 | import numpy as np 5 | from numpy.random import shuffle 6 | from PIL import Image, ImageDraw 7 | 8 | 9 | def create_timeseries_matrix(data_x, data_y=[], look_back=3): 10 | """ 11 | Converts a dataset into a time series matrix. 12 | 13 | Arguments 14 | data_x (array like): Features data. 15 | data_y (array like): Target data. 16 | look_back (int): size of minibatches. 17 | 18 | Returns 19 | data_x (array): Transformed features data. 20 | data_y (array): Transformed target data. 21 | """ 22 | 23 | if look_back <= 1: 24 | return np.array(data_x), np.array(data_y) 25 | 26 | if look_back >= data_x.shape[0]: 27 | print('create_timeseries_matrix() error = look back size is large') 28 | return np.array(data_x), np.array(data_y) 29 | 30 | back = look_back - 1 31 | len_x = len(data_x) 32 | data_x = np.array(data_x) 33 | data_y = np.array(data_y) 34 | 35 | lshape = len(data_x.shape) 36 | if lshape > 1: 37 | result = np.array(data_x[:-back, :]) 38 | else: 39 | result = np.array(data_x[:-back]) 40 | 41 | for i in range(1, look_back): 42 | j = len_x - back + i 43 | if lshape > 1: 44 | result = np.hstack((result, data_x[i:j, :])) 45 | else: 46 | result = np.vstack((result, data_x[i:j])) 47 | 48 | if lshape > 1: 49 | new_shape = (data_x.shape[0] - look_back + 1, data_x.shape[1] * look_back) 50 | else: 51 | new_shape = (data_x.shape[0] - look_back + 1, look_back) 52 | result = result.T 53 | result = np.reshape(result, new_shape) 54 | 55 | return result, data_y[back:] 56 | 57 | 58 | def shuffle_xy(data_a = [], data_b = []): 59 | """ 60 | Shuffle data sets. 61 | """ 62 | 63 | data_a = np.array(data_a) 64 | data_b = np.array(data_b) 65 | try: 66 | width_a = data_a.shape[1] 67 | temp = np.hstack((data_a, data_b)) 68 | shuffle(temp) 69 | except: 70 | print('Exception: non equal shapes. A:', data_a.shape, 'B:', data_b.shape) 71 | return data_a, data_b 72 | 73 | return np.hsplit(temp, np.array([width_a])) 74 | 75 | 76 | def timeseries_to_img(data): 77 | """ 78 | Creates an image of a time series window of the 'ohlc' type. 79 | 80 | Arguments 81 | data (array like): Input array size (window_size, 4). 82 | 83 | Returns 84 | img (Image object): PIL module image object. 85 | """ 86 | 87 | width = len(data) * 4 88 | height = width 89 | 90 | mn = min(min(data[:, 0]), min(data[:, 1]), min(data[:, 2]), min(data[:, 3])) 91 | mx = max(max(data[:, 0]), max(data[:, 1]), max(data[:, 2]), max(data[:, 3])) 92 | 93 | img = Image.new("RGB", (width, height)) 94 | draw = ImageDraw.Draw(img) 95 | 96 | def norm_height(value): 97 | val = (value - mn) / (mx - mn) * height # scale 98 | return height - val # invert 99 | 100 | pix = img.load() 101 | for idx in range(len(data)): 102 | bar = data[idx] 103 | x = (idx + 1) * 4 - 2 104 | o = norm_height(bar[0]) 105 | h = norm_height(bar[1]) 106 | l = norm_height(bar[2]) 107 | c = norm_height(bar[3]) 108 | clr_bar = 'red' if o < c else 'green' 109 | clr_line = (255, 0, 127) if o < c else (0, 255, 127) 110 | draw.rectangle((x-1, o, x+1, c), fill=clr_bar) 111 | draw.line((x, h, x, l), fill=clr_line, width=1) 112 | del draw 113 | 114 | return img 115 | 116 | 117 | def get_delta(data, index1=0, index2=1): 118 | """ 119 | Returns the difference between [,index1] and [,index2] in 2-D array. 120 | """ 121 | 122 | return data[:, index1] - data[:, index2] 123 | 124 | 125 | def get_deltas_from_ohlc(data, index1=0): 126 | """ 127 | Calculates the delta prices (open, high, low, close) between index1 and index2. 128 | Returns the numpy array with the shape (:, 6): [O-C, H-L, H-O, H-C, O-L, C-L] 129 | """ 130 | 131 | return np.column_stack((get_delta(data, index1, index1 + 3), # Open - Close 132 | get_delta(data, index1 + 1, index1 + 2),# High - Low 133 | get_delta(data, index1 + 1, index1), # High - Open 134 | get_delta(data, index1 + 1, index1 + 3),# High - Close 135 | get_delta(data, index1, index1 + 2), # Open - Low 136 | get_delta(data, index1 + 3, index1 + 2) # Close - Low 137 | )) 138 | 139 | 140 | def get_diff(data, rate=1): 141 | """ 142 | Computes a derivative and returns an array equal to 143 | the length of the original array. 144 | """ 145 | 146 | result = np.array([]) 147 | for idx in range(rate): 148 | result = np.append(result, 0.0) 149 | 150 | if rate == 1: 151 | return np.append(result, np.diff(data)) 152 | 153 | for idx in range(rate, len(data)): 154 | result = np.append(result, data[idx] - data[idx-rate]) 155 | 156 | return result 157 | 158 | 159 | def get_log_diff(data, rate=1): 160 | """ 161 | Computes the log-differential and returns an array equal to 162 | the length of the original array. 163 | """ 164 | 165 | result = np.array([]) 166 | for idx in range(rate): 167 | result = np.append(result, 0.0) 168 | 169 | for idx in range(rate, len(data)): 170 | ld = np.log(data[idx] / data[idx-rate]) 171 | result = np.append(result, ld) 172 | 173 | return result 174 | 175 | 176 | def get_sigmoid(data): 177 | """ 178 | Sigmoid function. 179 | """ 180 | 181 | return 1 / (1 + np.exp(-data)) 182 | # return exp(-np.logaddexp(0, -data)) 183 | # return 0.5 * (1 + data / (1 + abs(data))) 184 | 185 | 186 | def get_sigmoid_to_zero(data): 187 | """ 188 | Sigmoid function. 189 | """ 190 | 191 | return 1 / (1 + np.exp(-data)) - 0.5 192 | 193 | 194 | def get_sigmoid_stable(data): 195 | """ 196 | Numerically-stable sigmoid function. 197 | """ 198 | 199 | result = np.array([]) 200 | z = 0.0 201 | 202 | for item in data: 203 | if item >= 0.0: 204 | z = 1.0 / (1 + exp(-data)) 205 | else: 206 | z = 1.0 / (1 + exp(data)) 207 | result = np.append(result, z) 208 | 209 | return result 210 | 211 | 212 | def get_sigmoid_ration(data, alpha=2.0): 213 | """ 214 | Rationaly sigmoid. 215 | """ 216 | 217 | return data / (np.abs(data) + alpha) 218 | -------------------------------------------------------------------------------- /mas_tools/trade.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | import math 4 | import numpy as np 5 | import matplotlib.pyplot as plt 6 | 7 | from scipy.stats.stats import pearsonr 8 | from statsmodels.tsa.stattools import adfuller 9 | 10 | 11 | log = logging.getLogger(__name__) 12 | 13 | 14 | def calculate_stop_loss(data, direction: str, position=0, factor=3.0): 15 | """ 16 | Calculates the price of a stop order on an array of the latest prices. 17 | 18 | The minimum size of the history of the 12 last candles, 19 | including the bar for which the stop level is calculated, 20 | i.e. the size of the array must be more than 12. 21 | The method is described by Alexander Elder. 22 | 23 | Arguments 24 | data (list): an array of prices, the size of (n, 1) or (n, 4), 25 | where n is greater than or equal to 12. 26 | position (int): row index for calculate stop level. 27 | direction (str): direction, 'buy/sell', 'up/down'. 28 | Returns 29 | stop_price (float): The price of the breakdown in the opposite direction, 30 | at which it is necessary to close the position. 31 | """ 32 | 33 | buy = ['buy', 'up'] 34 | sell = ['sell', 'sale', 'down'] 35 | if direction.lower() not in buy and direction.lower() not in sell: 36 | raise ValueError('Illegal argument direction.') 37 | if len(data) < 12: 38 | # raise ValueError('To short') 39 | return 0.0 40 | 41 | data = np.array(data[position-12:position-1]) 42 | 43 | if len(data.shape) == 2: 44 | if data.shape[1] == 1: 45 | data = data[:, 0] 46 | elif data.shape[1] == 4: 47 | if direction.lower() in buy: 48 | data = data[:, 2] # low 49 | elif direction.lower() in sell: 50 | data = data[:, 1] # high 51 | else: 52 | raise ValueError('Array shape is not correct.') 53 | 54 | sum_bd = 0.0 55 | count_bd = 0 56 | bd_data = np.diff(data) 57 | if direction.lower() in buy: 58 | for item in bd_data: 59 | if item < 0: 60 | sum_bd += abs(item) 61 | count_bd += 1 62 | stop_price = data[position-2] - abs(bd_data[-1]) * factor 63 | if count_bd > 0: 64 | stop_price = data[position-2] - (sum_bd / count_bd) * factor 65 | elif direction.lower() in sell: 66 | for item in bd_data: 67 | if item > 0: 68 | sum_bd += item 69 | count_bd += 1 70 | stop_price = data[position-2] + abs(bd_data[-1]) * factor 71 | if count_bd > 0: 72 | stop_price = data[position-2] + (sum_bd / count_bd) * factor 73 | 74 | return stop_price 75 | 76 | 77 | def calculate_lot(one_lot_risk, balance_risk, min_lot, precision=2): 78 | """ 79 | Calculates the size of lot by the size of risk. 80 | Be careful, the risk values must be calculated in one currency. 81 | 82 | Arguments 83 | one_lot_risk (float): Risk when buying a single lot. It can be calculated 84 | as the absolute difference between the purchase price and the stop loss. 85 | balance_risk (float): Fixed risk for the current balance. 86 | min_lot (float): Minimum order size. 87 | """ 88 | 89 | if one_lot_risk * min_lot >= balance_risk or one_lot_risk <= 0: 90 | return min_lot 91 | elif min_lot == 1: 92 | return math.floor(balance_risk / one_lot_risk) 93 | else: 94 | return round(balance_risk / one_lot_risk - min_lot, precision) 95 | return -1.0 96 | 97 | 98 | def adjust_to_step(value, step, increase=False): 99 | """ 100 | Rounds any number to a multiple of the specified step. 101 | https://bablofil.ru 102 | 103 | Arguments 104 | increase (bool): if True - rounding will occur to a larger step value. 105 | """ 106 | 107 | return ((int(value * 100000000) - int(value * 100000000) % int( 108 | float(step) * 100000000)) / 100000000)+(float(step) if increase else 0) 109 | 110 | 111 | def calculate_cointegration_scores(x, y, log_info=True, plot_graph=False, 112 | save_graph_path='', raise_error=False): 113 | """ 114 | Write me, please 115 | 116 | Arguments 117 | x, y (array like): 118 | log_info (bool): 119 | plot_graph (bool): 120 | save_graph_path (str): 121 | 122 | Returns 123 | eps: 124 | mu: 125 | std: 126 | """ 127 | 128 | # correlation 129 | corr = pearsonr(x, y) 130 | if log_info: 131 | log.info('Pearson correlation coefficient: {}'.format(corr)) 132 | 133 | # stationary 134 | result_x, result_y = adfuller(x), adfuller(y) 135 | if log_info: 136 | log.info('P-values coefficient: x={} y={}'.format(result_x[1], result_y[1])) 137 | # stationary is <= 0.05 (p-value) 138 | if result_x[1] <= 0.05 or result_y[1] <= 0.05: 139 | info = 'Warning! One of the time series has stationarity.\nx p-value: {}; y p-value: {}'.format(result_x[1], result_y[1]) 140 | log.exception(info) 141 | if raise_error: 142 | raise ValueError(info) 143 | 144 | # Cointegration 145 | A = np.vstack([y, np.ones(len(y))]).T 146 | eps, mu = np.linalg.lstsq(A, x, rcond=None)[0] 147 | if log_info: 148 | log.info('Cointegration coef.: eps={} mu={}'.format(eps, mu)) 149 | if plot_graph: 150 | plt.plot(x, y, 'o', label='Original data', markersize=1) 151 | plt.plot(eps*y + mu, y, 'r', label='Fitted line') 152 | plt.legend() 153 | if len(save_graph_path) > 0: 154 | plt.savefig(save_graph_path + '_regression.png') 155 | plt.show() 156 | 157 | # difference graph - e 158 | z = x - eps*y - mu 159 | stat_z = adfuller(z)[1] 160 | if log_info: 161 | log.info('P-value coefficient of remainder: {}'.format(stat_z)) 162 | if stat_z > 0.05: 163 | info = 'Warning! Remainder is not stationary. p-value: {}'.format(stat_z) 164 | log.exception(info) 165 | if raise_error: 166 | raise ValueError(info) 167 | # if plot_graph: 168 | # plt.plot(z) 169 | # plt.title('Remainder') 170 | # plt.ylabel('Remainder') 171 | # plt.xlabel('bar') 172 | # if len(save_graph_path) > 0: 173 | # plt.savefig(save_graph_path + '_remainder.png') 174 | # plt.show() 175 | 176 | # z-score 177 | mean = np.mean(z) 178 | std = np.std(z) 179 | z_score1 = z / std 180 | # z_score2 = (z - mean) / std 181 | if plot_graph: 182 | plt.plot(z_score1) 183 | # plt.plot(z_score2) 184 | plt.title('Z-score') 185 | plt.ylabel('score') 186 | plt.xlabel('bar') 187 | plt.axhline(y=2., color='grey', linestyle='--') 188 | plt.axhline(y=0., color='grey', linestyle='--') 189 | plt.axhline(y=-2., color='grey', linestyle='--') 190 | if len(save_graph_path) > 0: 191 | plt.savefig(save_graph_path + '_z-score.png') 192 | plt.show() 193 | 194 | return eps, mu, std 195 | -------------------------------------------------------------------------------- /mas_cryptobot/train_1.3_ae_for_tickerbook.py: -------------------------------------------------------------------------------- 1 | import random 2 | import numpy as np 3 | 4 | # OpenCL GPU accelerating 5 | # http://github.com/plaidml 6 | import plaidml.keras 7 | plaidml.keras.install_backend() 8 | 9 | from mas_tools.models.autoencoders import deep_conv2d_vae 10 | from keras.callbacks import ModelCheckpoint, ReduceLROnPlateau 11 | 12 | from sklearn.model_selection import train_test_split 13 | from mas_tools.ml import plot_history 14 | import matplotlib.pyplot as plt 15 | 16 | # data 17 | dt_path = 'E:/Projects/market-analysis-system/data/tickers/' 18 | symbols = [''] 19 | tf = 240 # timeframe (period) 20 | window = 20 # size history window (bar count) 21 | 22 | # model 23 | action = 'train1' # train1, train2, predict 24 | wgt_path = 'E:/Projects/market-analysis-system/mas_vae/' 25 | code = 60 # latent tensor size 26 | filters = (3, 12) # convolution filters count 27 | dropout = 0.4 # inside dropout 28 | 29 | epochs = 10 30 | batch = 128 31 | img_width = window * 4 32 | img_size = img_width * img_width * 3 33 | 34 | 35 | #====== load data ====== 36 | print('Load data...') 37 | random.seed(666) 38 | symbols = random.sample(symbols, 6) 39 | filename = str(tf) + '_w' + str(window) + '.npz' 40 | # create form 41 | data = np.ones((img_size,), dtype=np.float) 42 | data = np.reshape(data, (1, img_width, img_width, 3)) 43 | # load 44 | for symbol in symbols: 45 | # compressed npz 46 | npz_file = np.load(dt_path + symbol + filename) 47 | new_data = npz_file.f.arr_0 48 | new_data = np.reshape(new_data, (len(new_data), img_width, img_width, 3)) 49 | data = np.vstack((data, new_data)) 50 | # clean first row 51 | data = data[1:] 52 | x_train, x_test = train_test_split(data, shuffle=True, test_size=0.1) 53 | # clear memory 54 | data = None 55 | # normalize imagess data 56 | x_train = x_train.astype('float32') / 255 57 | x_test = x_test.astype('float32') / 255 58 | 59 | print('New data shape: {}'.format(x_train.shape)) 60 | 61 | 62 | #====== Build VAE ====== 63 | print('Build autoencoder...') 64 | model_name = 'vae_img{}_flt{}-{}_code{}'.format(window, filters[0], filters[1], code) 65 | encoder, decoder, autoencoder, vae_loss = deep_conv2d_vae((img_width, img_width, 3), 66 | filters_count=filters, 67 | latent_dim=code, 68 | dropout=dropout) 69 | # print(type(vae_loss)) 70 | 71 | autoencoder.compile(optimizer='rmsprop', loss='mse', metrics=['acc']) 72 | 73 | if action in ['train2', 'predict']: 74 | autoencoder.load_weights(wgt_path + model_name + '.hdf5', by_name=True) 75 | if action in ['train1', 'train2']: 76 | # reduce_lr = ReduceLROnPlateau(factor=0.1, patience=3, min_lr=0.00001, verbose=1) 77 | 78 | history = autoencoder.fit(x_train, x_train, 79 | epochs=epochs, 80 | batch_size=batch, 81 | shuffle=True, 82 | # callbacks=[reduce_lr], 83 | validation_data=(x_test, x_test)) 84 | 85 | autoencoder.save_weights(wgt_path + model_name + '.hdf5') 86 | encoder.save_weights(wgt_path + model_name + '_enc.hdf5') 87 | # decoder.save_weights(wgt_path + model_name + '_dec.hdf5') 88 | plot_history(history, acc='acc') 89 | print('Weight saved.') 90 | 91 | 92 | #====== View ====== 93 | n = 10 94 | test_z_mean, test_z_log_var, test_z = encoder.predict(x_test, batch_size=batch) 95 | 96 | if action == 'predict': 97 | # display a histogram of the digit classes in the latent space 98 | filename = wgt_path + model_name + '_latent-distr.png' 99 | for idx in range(n*n): 100 | plt.hist(test_z[idx], bins=50) 101 | plt.xlabel('z size') 102 | plt.savefig(filename) 103 | plt.close() 104 | 105 | filename = wgt_path + model_name + '_latent-distr-mean.png' 106 | for idx in range(n*n): 107 | plt.hist(test_z_mean[idx], bins=50) 108 | plt.xlabel('z size') 109 | plt.savefig(filename) 110 | plt.close() 111 | 112 | filename = wgt_path + model_name + '_latent-distr--log.png' 113 | for idx in range(n*n): 114 | plt.hist(test_z_log_var[idx], bins=50) 115 | plt.xlabel('z size') 116 | plt.savefig(filename) 117 | plt.close() 118 | 119 | if action == 'train1' or action == 'train2': 120 | ## Origin test data 121 | figure = np.zeros((img_width * n, img_width * n, 3)) 122 | _test = x_test[:100] 123 | for idx in range(10): 124 | for jdx in range(10): 125 | digit = _test[idx*10+jdx].reshape(img_width, img_width, 3) 126 | figure[idx * img_width: (idx + 1) * img_width, 127 | jdx * img_width: (jdx + 1) * img_width] = digit 128 | 129 | filename = wgt_path + model_name + '_origin_pics.png' 130 | fig = plt.figure() 131 | img = plt.imshow(figure) 132 | plt.colorbar(img) 133 | plt.title('origin 100 examples') 134 | plt.savefig(filename) 135 | plt.close() 136 | 137 | ## Decoded test data 138 | figure = np.zeros((img_width * n, img_width * n, 3)) 139 | z_sample = test_z_mean[:100, :] 140 | x_decoded = decoder.predict(z_sample) 141 | for idx in range(10): 142 | for jdx in range(10): 143 | digit = x_decoded[idx*10+jdx].reshape(img_width, img_width, 3) 144 | figure[idx * img_width: (idx + 1) * img_width, 145 | jdx * img_width: (jdx + 1) * img_width] = digit * 100 146 | 147 | filename = wgt_path + model_name + '_restored_from_z__mean.png' 148 | fig = plt.figure() 149 | img = plt.imshow(figure) 150 | plt.colorbar(img) 151 | plt.title('z mean 100 examples') 152 | plt.savefig(filename) 153 | plt.close() 154 | 155 | figure = np.zeros((img_width * n, img_width * n, 3)) 156 | z_sample = test_z_log_var[:100, :] 157 | x_decoded = decoder.predict(z_sample) 158 | for idx in range(10): 159 | for jdx in range(10): 160 | digit = x_decoded[idx*10+jdx].reshape(img_width, img_width, 3) 161 | figure[idx * img_width: (idx + 1) * img_width, 162 | jdx * img_width: (jdx + 1) * img_width] = digit * 100 163 | 164 | filename = wgt_path + model_name + '_restored_from_z_log.png' 165 | fig = plt.figure() 166 | img = plt.imshow(figure) 167 | plt.colorbar(img) 168 | plt.title('z log 100 examples') 169 | plt.savefig(filename) 170 | plt.close() 171 | 172 | figure = np.zeros((img_width * n, img_width * n, 3)) 173 | z_sample = test_z[:100, :] 174 | x_decoded = decoder.predict(z_sample) 175 | for idx in range(10): 176 | for jdx in range(10): 177 | digit = x_decoded[idx*10+jdx].reshape(img_width, img_width, 3) 178 | figure[idx * img_width: (idx + 1) * img_width, 179 | jdx * img_width: (jdx + 1) * img_width] = digit * 100 180 | 181 | filename = wgt_path + model_name + '_restored_from_z.png' 182 | fig = plt.figure() 183 | img = plt.imshow(figure) 184 | plt.colorbar(img) 185 | plt.title('z 100 examples') 186 | plt.savefig(filename) 187 | plt.close() 188 | -------------------------------------------------------------------------------- /mas_tools/models/convolutions.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from keras.models import Model, Sequential 3 | 4 | from keras.layers import Input, concatenate, add 5 | from keras.layers import Dense, Activation 6 | from keras.layers import LSTM, GRU 7 | from keras.layers import BatchNormalization, Dropout 8 | from keras.layers import Flatten, Reshape 9 | 10 | from keras.layers import Conv1D, Conv2D 11 | from keras.layers import AveragePooling1D, MaxPooling1D 12 | from keras.layers import AveragePooling2D, MaxPooling2D 13 | from keras.layers import GlobalAveragePooling1D, GlobalMaxPooling1D 14 | 15 | from mas_tools.layers import Attention, AttentionWithContext, AttentionWeightedAverage 16 | 17 | from keras.activations import relu 18 | 19 | 20 | def cnn_model_2in(shape_a, shape_b, nb_output, activation='softmax'): 21 | """CNN for exchange bot. 22 | 23 | Arguments 24 | shape_a (): shape = (limit(timeseries or depth), features) 25 | shape_b (): shape = (limit(timeseries or depth), features) 26 | nb_output (int): Number of output classes. 27 | activation (string): activation function for model output. 28 | 29 | Returns 30 | model (keras.Model): Model of neural network.""" 31 | 32 | assert shape_a[0] == shape_b[0] 33 | 34 | conv_nb = (16, ) 35 | 36 | # Input A 37 | input_a = Input(shape=(1, shape_a[0], shape_a[1]), 38 | name='input_a') 39 | a = BatchNormalization()(input_a) 40 | a = Conv2D(filters=conv_nb[0], 41 | kernel_size=(3, 1), 42 | padding='same', # 'same' or 'causal' 43 | activation='relu', 44 | kernel_initializer='glorot_uniform', 45 | data_format='channels_first', 46 | )(a) 47 | a = Reshape((conv_nb[0], shape_a[0]*shape_a[1]))(a) 48 | a = LSTM(64, 49 | activation='relu', 50 | kernel_initializer='glorot_uniform', 51 | return_sequences=True 52 | )(a) 53 | 54 | # Input B 55 | input_b = Input(shape=(1, shape_b[0], shape_b[1]), 56 | name='input_b') 57 | b = BatchNormalization()(input_b) 58 | b = Conv2D(filters=conv_nb[0], 59 | kernel_size=(3, 1), 60 | padding='same', # 'same' or 'causal' 61 | activation='relu', 62 | kernel_initializer='glorot_uniform', 63 | data_format='channels_first', 64 | )(b) 65 | b = Reshape((conv_nb[0], shape_b[0]*shape_b[1]))(b) 66 | b = LSTM(64, 67 | activation='relu', 68 | kernel_initializer='glorot_uniform', 69 | return_sequences=True 70 | )(b) 71 | 72 | # Concat A and B 73 | x = concatenate([a, b]) 74 | 75 | x = LSTM(96, 76 | activation='relu', 77 | kernel_initializer='glorot_uniform', 78 | return_sequences=True 79 | )(x) 80 | x = LSTM(32, 81 | activation='relu', 82 | kernel_initializer='glorot_uniform' 83 | )(x) 84 | output = Dense(nb_output, activation=activation)(x) 85 | 86 | model = Model(inputs=[input_a, input_b], outputs=output) 87 | 88 | return model 89 | 90 | 91 | def cnn_model_2in_with_feedback(shape_a, shape_b, shape_fb, nb_output, activation='softmax'): 92 | """CNN for exchange bot. 93 | 94 | Arguments 95 | shape_a (tuple of int): shape = (limit(timeseries or depth), features) 96 | shape_b (tuple of int): shape = (limit(timeseries or depth), features) 97 | shape_fb (tuple of int or int): Shape of feedback data. 98 | nb_output (int): Number of output classes. 99 | activation (string): activation function for model output. 100 | 101 | Returns 102 | model (keras.Model): Model of neural network.""" 103 | 104 | assert shape_a[0] == shape_b[0] 105 | 106 | conv_nb = (16, 32, 64) 107 | dp = 0.2 108 | 109 | # Input A 110 | input_a = Input(shape=(1, shape_a[0], shape_a[1]), 111 | name='input_a') 112 | a = BatchNormalization()(input_a) 113 | a = Conv2D(filters=conv_nb[0], 114 | kernel_size=(5, 1), 115 | padding='same', # 'same' or 'causal' 116 | activation='relu', 117 | kernel_initializer='glorot_uniform', 118 | data_format='channels_first', 119 | )(a) 120 | a = Conv2D(filters=conv_nb[1], 121 | kernel_size=(3, 1), 122 | padding='same', # 'same' or 'causal' 123 | activation='relu', 124 | kernel_initializer='glorot_uniform', 125 | data_format='channels_first', 126 | )(a) 127 | a = MaxPooling2D(pool_size=(2,1), 128 | data_format='channels_first', 129 | )(a) 130 | a = Conv2D(filters=conv_nb[2], 131 | kernel_size=(3, 1), 132 | padding='same', 133 | activation='relu', 134 | kernel_initializer='glorot_uniform', 135 | data_format='channels_first', 136 | )(a) 137 | a = MaxPooling2D(pool_size=(2,1), 138 | data_format='channels_first', 139 | )(a) 140 | a = Reshape((conv_nb[2], int(shape_a[0]/4)*shape_a[1]))(a) 141 | a = Dense(64, 142 | activation='relu', 143 | kernel_initializer='glorot_uniform', 144 | # return_sequences=True 145 | )(a) 146 | a = Dropout(dp)(a) 147 | 148 | # Input B 149 | input_b = Input(shape=(1, shape_b[0], shape_b[1]), 150 | name='input_b') 151 | b = BatchNormalization()(input_b) 152 | b = Reshape((1, shape_b[0] * shape_b[1]))(b) 153 | b = LSTM(64, 154 | activation='relu', 155 | kernel_initializer='glorot_uniform', 156 | return_sequences=True 157 | )(b) 158 | b = Dense(64, 159 | activation='relu', 160 | kernel_initializer='glorot_uniform', 161 | )(b) 162 | b = Dropout(dp)(b) 163 | 164 | # Concat A and B 165 | x = concatenate([a, b], axis=1) 166 | 167 | x = LSTM(96, 168 | activation='relu', 169 | kernel_initializer='glorot_uniform', 170 | return_sequences=True 171 | )(x) 172 | x = LSTM(32, 173 | activation='relu', 174 | kernel_initializer='glorot_uniform' 175 | )(x) 176 | # x = Dense(32, activation='relu')(x) 177 | x = Dropout(dp)(x) 178 | 179 | # Input Feedback 180 | input_fb = Input(shape=(1, shape_fb), name='input_feedback') 181 | fb = BatchNormalization()(input_fb) 182 | # fb = Flatten()(fb) 183 | fb = LSTM(32, 184 | activation='relu', 185 | kernel_initializer='glorot_uniform', 186 | )(fb) 187 | fb = Dropout(dp)(fb) 188 | 189 | # Concat X and Feedback 190 | x = concatenate([x, fb]) 191 | 192 | x = Dense(10)(x) 193 | output = Dense(nb_output, activation=activation)(x) 194 | 195 | model = Model(inputs=[input_a, input_b, input_fb], outputs=output) 196 | 197 | return model 198 | 199 | 200 | if __name__ == "__main__": 201 | path = 'E:/Projects/market-analysis-system/' 202 | 203 | model = cnn_model_2in((50, 9), (50, 4), 3) 204 | save_model_arch(model, path+'cnn2in') 205 | model.summary() 206 | 207 | model = cnn_model_2in_with_feedback((50, 9), (50, 4), 8, 3) 208 | save_model_arch(model, path+'cnn2in_feedback') 209 | model.summary() 210 | -------------------------------------------------------------------------------- /mas_mt/v0.8_functional_models/simple_arch1.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import matplotlib.pyplot as plt 3 | import numpy as np 4 | import math 5 | 6 | from keras.backend import backend 7 | from keras.models import Model 8 | from keras.layers import Dense, GRU, Reshape, Dropout, Activation 9 | from keras.layers import Input, BatchNormalization, LeakyReLU, concatenate 10 | from keras.callbacks import ReduceLROnPlateau 11 | from keras.optimizers import RMSprop, Adam, SGD, Nadam 12 | from sklearn.metrics import mean_squared_error 13 | 14 | #=============================================================================# 15 | batch_size = 64 16 | fit_epoch = 100 17 | 18 | gru1 = 64 19 | gru2 = 32 20 | 21 | prefix = 'eurusd_w1_adam_' 22 | workfile = 'EURUSD.pro10080' 23 | path = 'C:/Program Files (x86)/STForex MetaTrader 4/MQL4/Files/ML-Assistant/' 24 | #=============================================================================# 25 | file_x = path + workfile + '_x.csv' 26 | file_y = path + workfile + '_y.csv' 27 | file_xx = path + workfile + '_xx.csv' 28 | file_yy = path + workfile + '_yy.csv' 29 | 30 | np.random.seed(7) 31 | 32 | print( 'Backend =', backend() ) 33 | print( 'Symbol =', workfile ) 34 | #=============================================================================# 35 | # L O A D D A T A # 36 | #=============================================================================# 37 | print( '\nPrepare Data...\n' ) 38 | 39 | data_x = np.genfromtxt( file_x, delimiter=';' ) 40 | data_yt = np.genfromtxt( file_y, delimiter=';' ) 41 | 42 | # batch_input_shape=( batch_size, timesteps, units ) 43 | data_x = np.reshape( data_x, (data_x.shape[0], 1, data_x.shape[1]) ) 44 | print( "data_x:", data_x.shape ) 45 | print( "data_y:", data_yt.shape ) 46 | 47 | # market, 4 ema 48 | data_x1, data_x2 = data_x[:,:,0:4], data_x[:,:,4:8] 49 | print( "data_x1 (mrkt):", data_x1.shape ) 50 | print( "data_x2 (4ema):", data_x2.shape ) 51 | 52 | data_y = np.array( [], ndmin=2 ) 53 | for item in data_yt: 54 | if item > 0: 55 | data_y = np.append( data_y, [abs(item), 0.0] ) 56 | if item < 0: 57 | data_y = np.append( data_y, [0.0, abs(item)] ) 58 | if item == 0: 59 | data_y = np.append( data_y, [0.0, 0.0] ) 60 | 61 | data_y = np.reshape( data_y, (data_yt.shape[0], 2) ) 62 | 63 | train_size = int( len(data_x) * 0.8 ) 64 | test_size = len(data_x) - train_size 65 | train_x, test_x = data_x[0:train_size,:], data_x[train_size:len(data_x),:] 66 | train_y, test_y = data_y[0:train_size,:], data_y[train_size:len(data_y),:] 67 | print( 'Train/Test :', len(train_y), '/', len(test_y) ) 68 | train_x1, test_x1 = train_x[:,:,0:4], test_x[:,:,0:4] 69 | train_x2, test_x2 = train_x[:,:,4:8], test_x[:,:,4:8] 70 | 71 | #=============================================================================# 72 | # P R E P A R E M O D E L # 73 | #=============================================================================# 74 | print( '\nCreating Model...\n' ) 75 | 76 | input1 = Input( shape=( 1, data_x1.shape[2] ), name='market' ) 77 | x1 = BatchNormalization()(input1) 78 | x1 = GRU( gru1 )(x1) 79 | #x1 = BatchNormalization()(x1) 80 | #x1 = Dropout( 0.2 )(x1) 81 | x1 = Reshape( ( 1, gru1 ) )(x1) 82 | x1 = GRU( gru2 )(x1) 83 | 84 | input2 = Input( shape=( 1, data_x1.shape[2] ), name='ema' ) 85 | x2 = BatchNormalization()(input2) 86 | x2 = GRU( gru1 )(x2) 87 | #x2 = BatchNormalization()(x2) 88 | #x2 = Dropout( 0.2 )(x2) 89 | x2 = Reshape( ( 1, gru1 ) )(x2) 90 | x2 = GRU( gru2 )(x2) 91 | 92 | x = concatenate([x1, x2]) 93 | #x = BatchNormalization()(x) 94 | x = Reshape( ( 1, gru2+gru2 ) )(x) 95 | x = GRU( gru2 )(x) 96 | output = Dense( 2, activation='softmax' )(x) 97 | 98 | model = Model( inputs=[input1,input2], outputs=output ) 99 | 100 | opt = Nadam(lr=0.001) 101 | reduce_lr = ReduceLROnPlateau( monitor='val_loss', factor=0.9, patience=5, min_lr=0.000001, verbose=1 ) 102 | 103 | # loss='mse', 'msle', 'categorical_crossentropy'(softmax) 104 | # optimizer='adam', 'rmsprop', 'sgd', opt 105 | model.compile( loss='categorical_crossentropy', optimizer=opt, metrics=['acc'] ) #'mae' 106 | 107 | 108 | #=============================================================================# 109 | # T R A I N I N G # 110 | #=============================================================================# 111 | print( '\nTraining...\n' ) 112 | 113 | history = model.fit( [train_x1, train_x2], train_y, #train_x, train_y, data_x, data_y, 114 | batch_size=batch_size, 115 | epochs=fit_epoch, 116 | validation_data=( [test_x1, test_x2], test_y ), 117 | callbacks=[reduce_lr] ) 118 | 119 | 120 | #=============================================================================# 121 | # P R E D I C T I N G # 122 | #=============================================================================# 123 | print( '\nPredicting...\n' ) 124 | 125 | # read data 126 | data_xx = np.genfromtxt( file_xx, delimiter=';' ) 127 | data_xx = np.reshape( data_xx, (data_xx.shape[0], 1, data_xx.shape[1]) ) 128 | print( "data_xx:", data_xx.shape ) 129 | 130 | # market, 4 ema 131 | data_xx1, data_xx2 = data_xx[:,:,0:4], data_xx[:,:,4:8] 132 | print( "data_xx1 (mrkt):", data_xx1.shape ) 133 | print( "data_xx2 (4ema):", data_xx2.shape ) 134 | 135 | predicted_output = model.predict( [data_xx1, data_xx2], batch_size=batch_size ) 136 | 137 | data_yy = np.array( [] ) 138 | for item in predicted_output: 139 | data_yy = np.append( data_yy, item[0] - item[1] ) 140 | 141 | np.savetxt( file_yy, data_yy, fmt='%.6f', delimiter=';' ) 142 | 143 | print( "Predict saved:", file_yy ) 144 | 145 | 146 | #=============================================================================# 147 | # Plot # 148 | #=============================================================================# 149 | # make predictions 150 | trainPredict = model.predict( [train_x1, train_x2] ) 151 | testPredict = model.predict( [test_x1, test_x2] ) 152 | # calculate root mean squared error 153 | trainScore = math.sqrt( mean_squared_error( train_y, trainPredict ) ) 154 | print( 'Train Score: %.6f RMSE' % ( trainScore ) ) 155 | testScore = math.sqrt( mean_squared_error( test_y, testPredict ) ) 156 | print( 'Test Score: %.6f RMSE' % ( testScore ) ) 157 | 158 | 159 | plt.plot( data_yy ) 160 | plt.title( 'Predicted' ) 161 | plt.ylabel( 'direction') 162 | plt.xlabel( 'bar') 163 | plt.show() 164 | 165 | 166 | plt.figure() 167 | plt.plot( history.history['loss'] ) 168 | plt.plot( history.history['val_loss'] ) 169 | plt.title( 'Model loss' ) 170 | plt.ylabel( 'loss' ) 171 | plt.xlabel( 'epoch' ) 172 | plt.legend( ['train', 'test'], loc='best' ) 173 | plt.show() 174 | 175 | plt.figure() 176 | plt.plot( history.history['acc'] ) 177 | plt.plot( history.history['val_acc'] ) 178 | plt.title( 'Model accuracy' ) 179 | plt.ylabel( 'acc') 180 | plt.xlabel( 'epoch') 181 | plt.legend( ['train', 'test'], loc='best' ) 182 | plt.show() 183 | # 184 | #plt.figure() 185 | #plt.plot( history.history['mean_absolute_error'] ) 186 | #plt.plot( history.history['val_mean_absolute_error'] ) 187 | #plt.title( 'Model mean absolute error' ) 188 | #plt.ylabel( 'mae') 189 | #plt.xlabel( 'epoch') 190 | #plt.legend( ['train', 'test'], loc='best' ) 191 | #plt.show() 192 | 193 | -------------------------------------------------------------------------------- /mas_cryptobot/vae/train_autoencoders.py: -------------------------------------------------------------------------------- 1 | import random 2 | import numpy as np 3 | 4 | # OpenCL GPU accelerating 5 | # http://github.com/plaidml 6 | import plaidml.keras 7 | plaidml.keras.install_backend() 8 | 9 | from mas_tools.models.autoencoders import deep_conv2d_vae 10 | from keras.callbacks import ModelCheckpoint, ReduceLROnPlateau 11 | 12 | from sklearn.model_selection import train_test_split 13 | from mas_tools.ml import plot_history 14 | import matplotlib.pyplot as plt 15 | 16 | # data 17 | dt_path = 'E:/Projects/market-analysis-system/data/pictured/' 18 | symbols = ['AUDJPY', 'AUDUSD', 'CHFJPY', 'EURAUD', 19 | 'EURCAD', 'EURCHF', 'EURGBP', 'EURJPY', 20 | 'EURRUB', 'EURUSD', 'GBPAUD', 'GBPCAD', 21 | 'GBPCHF', 'GBPJPY', 'GBPUSD', 'NZDJPY', 22 | 'NZDUSD', 'USDCAD', 'USDCHF', 'USDJPY', 23 | 'USDRUB', 'XAGUSD', 'XAUUSD'] 24 | tf = 240 # timeframe (period) 25 | window = 20 # size history window (bar count) 26 | 27 | # model 28 | action = 'train1' # train1, train2, predict 29 | wgt_path = 'E:/Projects/market-analysis-system/mas_cryptobot/vae/wgts/' 30 | code = 60 # latent tensor size 31 | filters = (3, 12) # convolution filters count 32 | dropout = 0.4 # inside dropout 33 | 34 | epochs = 10 35 | batch = 128 36 | img_width = window * 4 37 | img_size = img_width * img_width * 3 38 | 39 | 40 | #====== load data ====== 41 | print('Load data...') 42 | random.seed(666) 43 | symbols = random.sample(symbols, 6) 44 | filename = str(tf) + '_w' + str(window) + '.npz' 45 | # create form 46 | data = np.ones((img_size,), dtype=np.float) 47 | data = np.reshape(data, (1, img_width, img_width, 3)) 48 | # load 49 | for symbol in symbols: 50 | # compressed npz 51 | npz_file = np.load(dt_path + symbol + filename) 52 | new_data = npz_file.f.arr_0 53 | new_data = np.reshape(new_data, (len(new_data), img_width, img_width, 3)) 54 | data = np.vstack((data, new_data)) 55 | # clean first row 56 | data = data[1:] 57 | x_train, x_test = train_test_split(data, shuffle=True, test_size=0.1) 58 | # clear memory 59 | data = None 60 | # normalize imagess data 61 | x_train = x_train.astype('float32') / 255 62 | x_test = x_test.astype('float32') / 255 63 | 64 | print('New data shape: {}'.format(x_train.shape)) 65 | 66 | 67 | #====== Build VAE ====== 68 | print('Build autoencoder...') 69 | model_name = 'vae_img{}_flt{}-{}_code{}'.format(window, filters[0], filters[1], code) 70 | encoder, decoder, autoencoder, vae_loss = deep_conv2d_vae((img_width, img_width, 3), 71 | filters_count=filters, 72 | latent_dim=code, 73 | dropout=dropout) 74 | # print(type(vae_loss)) 75 | 76 | autoencoder.compile(optimizer='rmsprop', loss='mse', metrics=['acc']) 77 | 78 | if action in ['train2', 'predict']: 79 | autoencoder.load_weights(wgt_path + model_name + '.hdf5', by_name=True) 80 | if action in ['train1', 'train2']: 81 | # reduce_lr = ReduceLROnPlateau(factor=0.1, patience=3, min_lr=0.00001, verbose=1) 82 | 83 | history = autoencoder.fit(x_train, x_train, 84 | epochs=epochs, 85 | batch_size=batch, 86 | shuffle=True, 87 | # callbacks=[reduce_lr], 88 | validation_data=(x_test, x_test)) 89 | 90 | autoencoder.save_weights(wgt_path + model_name + '.hdf5') 91 | encoder.save_weights(wgt_path + model_name + '_enc.hdf5') 92 | # decoder.save_weights(wgt_path + model_name + '_dec.hdf5') 93 | plot_history(history, acc='acc') 94 | print('Weight saved.') 95 | 96 | 97 | #====== View ====== 98 | n = 10 99 | test_z_mean, test_z_log_var, test_z = encoder.predict(x_test, batch_size=batch) 100 | 101 | if action == 'predict': 102 | # display a histogram of the digit classes in the latent space 103 | filename = wgt_path + model_name + '_latent-distr.png' 104 | for idx in range(n*n): 105 | plt.hist(test_z[idx], bins=50) 106 | plt.xlabel('z size') 107 | plt.savefig(filename) 108 | plt.close() 109 | 110 | filename = wgt_path + model_name + '_latent-distr-mean.png' 111 | for idx in range(n*n): 112 | plt.hist(test_z_mean[idx], bins=50) 113 | plt.xlabel('z size') 114 | plt.savefig(filename) 115 | plt.close() 116 | 117 | filename = wgt_path + model_name + '_latent-distr--log.png' 118 | for idx in range(n*n): 119 | plt.hist(test_z_log_var[idx], bins=50) 120 | plt.xlabel('z size') 121 | plt.savefig(filename) 122 | plt.close() 123 | 124 | if action == 'train1' or action == 'train2': 125 | ## Origin test data 126 | figure = np.zeros((img_width * n, img_width * n, 3)) 127 | _test = x_test[:100] 128 | for idx in range(10): 129 | for jdx in range(10): 130 | digit = _test[idx*10+jdx].reshape(img_width, img_width, 3) 131 | figure[idx * img_width: (idx + 1) * img_width, 132 | jdx * img_width: (jdx + 1) * img_width] = digit 133 | 134 | filename = wgt_path + model_name + '_origin_pics.png' 135 | fig = plt.figure() 136 | img = plt.imshow(figure) 137 | plt.colorbar(img) 138 | plt.title('origin 100 examples') 139 | plt.savefig(filename) 140 | plt.close() 141 | 142 | ## Decoded test data 143 | figure = np.zeros((img_width * n, img_width * n, 3)) 144 | z_sample = test_z_mean[:100, :] 145 | x_decoded = decoder.predict(z_sample) 146 | for idx in range(10): 147 | for jdx in range(10): 148 | digit = x_decoded[idx*10+jdx].reshape(img_width, img_width, 3) 149 | figure[idx * img_width: (idx + 1) * img_width, 150 | jdx * img_width: (jdx + 1) * img_width] = digit * 100 151 | 152 | filename = wgt_path + model_name + '_restored_from_z__mean.png' 153 | fig = plt.figure() 154 | img = plt.imshow(figure) 155 | plt.colorbar(img) 156 | plt.title('z mean 100 examples') 157 | plt.savefig(filename) 158 | plt.close() 159 | 160 | figure = np.zeros((img_width * n, img_width * n, 3)) 161 | z_sample = test_z_log_var[:100, :] 162 | x_decoded = decoder.predict(z_sample) 163 | for idx in range(10): 164 | for jdx in range(10): 165 | digit = x_decoded[idx*10+jdx].reshape(img_width, img_width, 3) 166 | figure[idx * img_width: (idx + 1) * img_width, 167 | jdx * img_width: (jdx + 1) * img_width] = digit * 100 168 | 169 | filename = wgt_path + model_name + '_restored_from_z_log.png' 170 | fig = plt.figure() 171 | img = plt.imshow(figure) 172 | plt.colorbar(img) 173 | plt.title('z log 100 examples') 174 | plt.savefig(filename) 175 | plt.close() 176 | 177 | figure = np.zeros((img_width * n, img_width * n, 3)) 178 | z_sample = test_z[:100, :] 179 | x_decoded = decoder.predict(z_sample) 180 | for idx in range(10): 181 | for jdx in range(10): 182 | digit = x_decoded[idx*10+jdx].reshape(img_width, img_width, 3) 183 | figure[idx * img_width: (idx + 1) * img_width, 184 | jdx * img_width: (jdx + 1) * img_width] = digit * 100 185 | 186 | filename = wgt_path + model_name + '_restored_from_z.png' 187 | fig = plt.figure() 188 | img = plt.imshow(figure) 189 | plt.colorbar(img) 190 | plt.title('z 100 examples') 191 | plt.savefig(filename) 192 | plt.close() 193 | -------------------------------------------------------------------------------- /mas_tools/envs/envs.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import numpy as np 3 | 4 | from gym import Env, Space 5 | from gym.spaces import Discrete 6 | from gym.utils import seeding 7 | 8 | from mas_tools.markets import AbstractMarket 9 | 10 | 11 | class MarketEnv(Env): 12 | """ 13 | The class implements an Env class interface from the Gym (OpenAI) package, 14 | to communicate with real market data. 15 | """ 16 | 17 | reward_range = (-np.inf, np.inf) 18 | action_space = None 19 | observation_space = None 20 | actions = {'hold': 0, 'buy': 1, 'sell': 2} 21 | metadata = {'render.modes': ['human', 'rgb_array'], 22 | 'video.frames_per_second' : 15} 23 | viewer = None 24 | 25 | def __init__(self, market: AbstractMarket, 26 | use_deposit=False, use_last_action=False, 27 | **kwargs): 28 | """ 29 | MarketEnv constructor. 30 | 31 | Arguments 32 | market (AbstractMarket): Wrapper for access to the market through API, or other solutions. 33 | window (int): This is the size of the state of the environment (The number of time intervals). 34 | """ 35 | 36 | self.market = market 37 | self.use_deposit = use_deposit 38 | self.use_last_action = use_last_action 39 | 40 | self.last_action = dict() 41 | 42 | # TODO action space for multy symbols agent 43 | self.action_space = Discrete(3 * self.market.symbols_count) 44 | self.observation_space = Space(shape=self.market.shape, dtype=np.float) 45 | 46 | def step(self, action): 47 | """ 48 | Run one timestep of the environment's dynamics. When end of 49 | episode is reached, you are responsible for calling `reset()` 50 | to reset this environment's state. 51 | 52 | Accepts an action and returns a tuple (observation, reward, done, info). 53 | 54 | Arguments 55 | action (object): an action provided by the environment 56 | 57 | Returns 58 | observation (object): agent's observation of the current environment 59 | reward (float) : amount of reward returned after previous action 60 | done (boolean): whether the episode has ended, in which case further step() calls will return undefined results 61 | info (dict): contains auxiliary diagnostic information (helpful for debugging, and sometimes learning) 62 | """ 63 | # TODO multy action 64 | assert self.action_space.contains(action[0]), "%r (%s) invalid"%(action, type(action)) 65 | 66 | done = False 67 | reward = 0.0 68 | info = dict() 69 | 70 | observation = self.market.observation() 71 | feedback = [] 72 | 73 | # action is the max index from the model output (from three neurons) 74 | idx = 0 75 | for symbol in self.market.symbols: 76 | if action[idx] == self.actions['buy']: 77 | self.market.buy_order(symbol) 78 | elif action[idx] == self.actions['hold']: 79 | pass 80 | elif action[idx] == self.actions['sell']: 81 | self.market.sell_order(symbol) 82 | 83 | if self.use_deposit: 84 | feedback.append(self.market.deposit(symbol)) 85 | if self.use_last_action: 86 | feedback.append(self.last_action[symbol]) 87 | self.last_action[symbol] = action[idx] 88 | 89 | info[symbol] = { 90 | 'action': action[idx], 91 | 'reward': self.market.profit, 92 | 'deposit': self.market.deposit(symbol) 93 | } 94 | 95 | reward += self.market.profit 96 | idx += 1 97 | 98 | if self.use_deposit or self.use_last_action: 99 | observation.append(np.array(feedback)) 100 | if self.market.done or self.market.balance <= 0: 101 | done = True 102 | 103 | info['sum_reward'] = reward 104 | info['balance'] = self.market.balance 105 | 106 | return (observation, reward, done, info) 107 | 108 | def reset(self): 109 | """ 110 | Resets the state of the environment and returns an initial observation. 111 | 112 | Returns 113 | observation (object): the initial observation of the space. 114 | """ 115 | 116 | self.market.reset() 117 | observation = self.market.observation() 118 | 119 | feedback = [] 120 | for symbol in self.market.symbols: 121 | self.last_action[symbol] = 0 122 | if self.use_deposit: 123 | feedback.append(self.market.deposit(symbol)) 124 | if self.use_last_action: 125 | feedback.append(self.last_action[symbol]) 126 | 127 | if self.use_deposit or self.use_last_action: 128 | observation.append(np.array(feedback)) 129 | 130 | return observation 131 | 132 | @property 133 | def feedback_shape(self): 134 | """""" 135 | 136 | return ((len(self.market.symbols) if self.use_deposit else 0) + 137 | (len(self.market.symbols) if self.use_last_action else 0)) 138 | 139 | def render(self, mode='human', close=False): 140 | """ 141 | Renders the environment. 142 | The set of supported modes varies per environment. (And some 143 | environments do not support rendering at all.) By convention, 144 | if mode is: 145 | - human: render to the current display or terminal and 146 | return nothing. Usually for human consumption. 147 | - rgb_array: Return an numpy.ndarray with shape (x, y, 3), 148 | representing RGB values for an x-by-y pixel image, suitable 149 | for turning into a video. 150 | - ansi: Return a string (str) or StringIO.StringIO containing a 151 | terminal-style text representation. The text can include newlines 152 | and ANSI escape sequences (e.g. for colors). 153 | 154 | Note 155 | Make sure that your class's metadata 'render.modes' key includes 156 | the list of supported modes. It's recommended to call super() 157 | in implementations to use the functionality of this method. 158 | 159 | Arguments 160 | mode (str): The mode to render with. 161 | close (bool): Close all open renderings. 162 | """ 163 | 164 | if mode == 'rgb_array': 165 | # data = self.market.get_window(len=20) 166 | return np.array([]) # return RGB frame suitable for video 167 | elif mode is 'human': 168 | pass # pop up a window and render 169 | else: 170 | super(MarketEnv, self).render(mode=mode) # just raise an exception 171 | 172 | def configure(self, **kwargs): 173 | """ 174 | Provides runtime configuration to the environment. 175 | This configuration should consist of data that tells your 176 | environment how to run (such as an address of a remote server, 177 | or path to your ImageNet data). It should not affect the 178 | semantics of the environment. 179 | """ 180 | 181 | if kwargs['market']: 182 | del self.market 183 | self.market = kwargs['market'] 184 | 185 | self.reset() 186 | 187 | def close(self): 188 | """ 189 | Override _close in your subclass to perform any necessary cleanup. 190 | 191 | Environments will automatically close() themselves when 192 | garbage collected or when the program exits. 193 | """ 194 | if self.viewer: self.viewer.close() 195 | 196 | def seed(self, seed=None): 197 | """ 198 | Sets the seed for this env's random number generator(s). 199 | 200 | Returns 201 | list: Returns the list of seeds used in this env's random 202 | number generators. The first value in the list should be the 203 | "main" seed, or the value which a reproducer should pass to 204 | 'seed'. Often, the main seed equals the provided 'seed', but 205 | this won't be true if seed=None, for example. 206 | """ 207 | self.np_random, seed = seeding.np_random(seed) 208 | return [seed] 209 | 210 | def __del__(self): 211 | self.close() 212 | 213 | def __str__(self): 214 | return '<{} instance>'.format(type(self).__name__) 215 | -------------------------------------------------------------------------------- /mas_tools/models/autoencoders.py: -------------------------------------------------------------------------------- 1 | from keras.models import Model 2 | from keras.layers import Input, Dense, concatenate, Lambda 3 | from keras.layers import Flatten, Reshape, BatchNormalization, Dropout 4 | from keras.layers import Conv2D, MaxPooling2D, Conv2DTranspose, UpSampling2D 5 | from keras.losses import mse, mae, mape, binary_crossentropy 6 | from keras.regularizers import L1L2 7 | from keras import backend as K 8 | 9 | 10 | def dense_ae(input_shape, encoding_dim=64, output_activation='linear'): 11 | """ 12 | Example from https://habr.com/post/331382/ 13 | 14 | Arguments 15 | input_shape (tuple of int): 16 | encoding_dim (int): 17 | output_activation (str): 18 | 19 | Returns 20 | encoder: 21 | decoder: 22 | autoencoder: 23 | """ 24 | 25 | decoder_dim = 1 26 | for i in input_shape: 27 | decoder_dim *= i 28 | 29 | # Encoder 30 | input_tensor = Input(shape=input_shape) 31 | x = Flatten()(input_tensor) 32 | encoded = Dense(encoding_dim, 33 | activation='relu', 34 | kernel_initializer='glorot_uniform')(x) 35 | 36 | # Decoder 37 | input_encoded = Input(shape=(encoding_dim,)) 38 | y = Dense(decoder_dim, 39 | activation=output_activation, 40 | kernel_initializer='glorot_uniform')(input_encoded) 41 | decoded = Reshape(input_shape)(y) 42 | 43 | # Create models 44 | encoder = Model(input_tensor, encoded, name="encoder") 45 | decoder = Model(input_encoded, decoded, name="decoder") 46 | autoencoder = Model(input_tensor, decoder(encoder(input_tensor)), name="autoencoder") 47 | 48 | return encoder, decoder, autoencoder 49 | 50 | 51 | def deep_ae(input_shape, encoding_dim=64, 52 | output_activation='linear', kernel_activation='elu', 53 | lambda_l1=0.0): 54 | """ 55 | Example from https://habr.com/post/331382/ 56 | 57 | Arguments 58 | input_shape (tuple of int): 59 | encoding_dim (int): 60 | output_activation (str): 61 | kernel_activation (str): 62 | lambda_l1 (float): Regularisation value for sparse encoding. 63 | 64 | Returns 65 | encoder: 66 | decoder: 67 | autoencoder: 68 | """ 69 | 70 | decoder_dim = 1 71 | for i in input_shape: 72 | decoder_dim *= i 73 | 74 | # Encoder 75 | input_tensor = Input(shape=input_shape) 76 | x = Flatten()(input_tensor) 77 | # x = Dense(encoding_dim*4, activation=kernel_activation)(x) 78 | x = Dense(encoding_dim*3, activation=kernel_activation)(x) 79 | x = Dense(encoding_dim*2, activation=kernel_activation)(x) 80 | encoded = Dense(encoding_dim, activation='linear', 81 | activity_regularizer=L1L2(lambda_l1, 0))(x) 82 | 83 | # Decoder 84 | input_encoded = Input(shape=(encoding_dim,)) 85 | y = Dense(encoding_dim*2, activation=kernel_activation)(input_encoded) 86 | y = Dense(encoding_dim*3, activation=kernel_activation)(y) 87 | # y = Dense(encoding_dim*4, activation=kernel_activation)(y) 88 | y = Dense(decoder_dim, activation=output_activation)(y) 89 | decoded = Reshape(input_shape)(y) 90 | 91 | # Create models 92 | encoder = Model(input_tensor, encoded, name="encoder") 93 | decoder = Model(input_encoded, decoded, name="decoder") 94 | autoencoder = Model(input_tensor, decoder(encoder(input_tensor)), name="autoencoder") 95 | 96 | return encoder, decoder, autoencoder 97 | 98 | 99 | def deep_conv_ae(input_shape, latent_dim=32): 100 | """""" 101 | kernel_size = (1, 5) 102 | kernel_pooling = (1, 2) 103 | strides = (1, 1) 104 | 105 | # Encoder 106 | input_tensor = Input(shape=input_shape, name='encoder_input') 107 | x = Conv2D(filters=32, 108 | kernel_size=kernel_size, 109 | padding='same', 110 | activation='relu', 111 | strides=strides, 112 | input_shape=input_shape)(input_tensor) 113 | x = Conv2D(filters=64, 114 | kernel_size=kernel_size, 115 | padding='same', 116 | activation='relu', 117 | strides=strides)(x) 118 | # shape info needed to build decoder model 119 | shape = K.int_shape(x) 120 | # shape = enc.output_shape 121 | # generate latent vector Q(z|X) 122 | x = Flatten()(x) 123 | x = Dense(latent_dim, activation='relu', name='encoder_output')(x) 124 | 125 | # Decoder 126 | latent_inputs = Input(shape=(latent_dim,), name='latent_input') 127 | y = Dense(shape[1] * shape[2] * shape[3], activation='relu')(latent_inputs) 128 | y = Reshape((shape[1], shape[2], shape[3]))(y) 129 | y = Conv2DTranspose(filters=64, 130 | kernel_size=kernel_size, 131 | padding='same', 132 | activation='relu', 133 | strides=strides)(y) 134 | y = Conv2DTranspose(filters=32, 135 | kernel_size=kernel_size, 136 | padding='same', 137 | activation='relu', 138 | strides=strides)(y) 139 | y = Conv2DTranspose(filters=1, 140 | kernel_size=kernel_size, 141 | padding='same', 142 | activation='relu', 143 | strides=strides, 144 | name='decoder_output')(y) 145 | 146 | # Create models 147 | encoder = Model(input_tensor, x, name='encoder') 148 | decoder = Model(latent_inputs, y, name='decoder') 149 | autoencoder = Model(input_tensor, decoder(encoder(input_tensor)), name='ae') 150 | 151 | return encoder, decoder, autoencoder 152 | 153 | 154 | # reparameterization trick 155 | # instead of sampling from Q(z|X), sample eps = N(0,I) 156 | # then z = z_mean + sqrt(var)*eps 157 | def sampling(args): 158 | """Reparameterization trick by sampling fr an isotropic unit Gaussian. 159 | Arguments 160 | args (tensor): mean and log of variance of Q(z|X) 161 | Returns 162 | z (tensor): sampled latent vector 163 | """ 164 | 165 | z_mean, z_log_var = args 166 | batch = K.shape(z_mean)[0] 167 | dim = K.int_shape(z_mean)[1] 168 | # by default, random_normal has mean=0 and std=1.0 169 | epsilon = K.random_normal(shape=(batch, dim)) 170 | return z_mean + K.exp(0.5 * z_log_var) * epsilon 171 | 172 | 173 | def deep_conv2d_vae(input_shape, 174 | latent_dim=100, 175 | filters_count=(8, 24), 176 | dropout = 0.3): 177 | """""" 178 | kernel_size = (3, 3) 179 | kernel_pooling = (2, 2) 180 | strides = (1, 1) 181 | initializer = 'truncated_normal' 182 | 183 | # Encoder 184 | input_tensor = Input(shape=input_shape, name='encoder_input') 185 | if len(input_shape) == 2: 186 | x = Reshape((input_shape[0], input_shape[1], 1))(input_tensor) 187 | elif len(input_shape) == 3: 188 | x = input_tensor 189 | x = Conv2D(filters=filters_count[0], 190 | kernel_size=kernel_size, 191 | padding='same', 192 | activation='relu', 193 | kernel_initializer=initializer, 194 | strides=strides)(x) 195 | x = Dropout(dropout)(x) 196 | x = MaxPooling2D(kernel_pooling)(x) 197 | x = Conv2D(filters=filters_count[1], 198 | kernel_size=kernel_size, 199 | padding='same', 200 | activation='relu', 201 | kernel_initializer=initializer, 202 | strides=strides)(x) 203 | x = Dropout(dropout)(x) 204 | x = MaxPooling2D(kernel_pooling)(x) 205 | # shape info needed to build decoder model 206 | shape = K.int_shape(x) 207 | # shape = enc.output_shape 208 | # generate latent vector Q(z|X) 209 | x = Flatten()(x) 210 | x = Dense(latent_dim*3, activation='relu', name='encoder_output')(x) 211 | 212 | z_mean = Dense(latent_dim, name='z_mean')(x) 213 | z_log_var = Dense(latent_dim, name='z_log_var')(x) 214 | # use reparameterization trick to push the sampling out as input 215 | # note that "output_shape" isn't necessary with the TensorFlow backend 216 | z = Lambda(sampling, output_shape=(latent_dim,), name='z')([z_mean, z_log_var]) 217 | 218 | # Decoder 219 | latent_inputs = Input(shape=(latent_dim,), name='latent_input') 220 | y = Dense(shape[1] * shape[2] * shape[3], activation='relu')(latent_inputs) 221 | y = Reshape((shape[1], shape[2], shape[3]))(y) 222 | y = UpSampling2D(kernel_pooling)(y) 223 | y = Dropout(dropout)(y) 224 | y = Conv2DTranspose(filters=filters_count[1], 225 | kernel_size=kernel_size, 226 | padding='same', 227 | activation='relu', 228 | kernel_initializer=initializer, 229 | strides=strides)(y) 230 | y = UpSampling2D(kernel_pooling)(y) 231 | y = Dropout(dropout)(y) 232 | y = Conv2DTranspose(filters=filters_count[0], 233 | kernel_size=kernel_size, 234 | padding='same', 235 | activation='relu', 236 | kernel_initializer=initializer, 237 | strides=strides)(y) 238 | y = Conv2DTranspose(filters=(1 if len(input_shape) == 2 else 3), 239 | kernel_size=kernel_size, 240 | padding='same', 241 | activation='relu', 242 | kernel_initializer=initializer, 243 | strides=strides, 244 | name='decoder_output')(y) 245 | if len(input_shape) == 2: 246 | output = Reshape((input_shape[0], input_shape[1]))(y) 247 | elif len(input_shape) == 3: 248 | output = Reshape(input_shape)(y) 249 | 250 | # Create models 251 | encoder = Model(input_tensor, [z_mean, z_log_var, z], name='encoder') 252 | decoder = Model(latent_inputs, output, name='decoder') 253 | autoencoder = Model(input_tensor, decoder(encoder(input_tensor)[2]), name='ae') 254 | 255 | reconstruction_loss = mse(K.flatten(input_tensor), K.flatten(output)) 256 | if len(input_shape) == 2: 257 | reconstruction_loss *= (input_shape[0] * input_shape[1]) 258 | elif len(input_shape) == 3: 259 | reconstruction_loss *= (input_shape[0] * input_shape[1] * input_shape[2]) 260 | kl_loss = 1 + z_log_var - K.square(z_mean) - K.exp(z_log_var) 261 | kl_loss = K.sum(kl_loss, axis=-1) 262 | kl_loss *= -0.5 263 | vae_loss = K.mean(reconstruction_loss + kl_loss) 264 | # autoencoder.add_loss(vae_loss) 265 | # autoencoder.compile(optimizer='adam', metrics=['acc']) 266 | 267 | return encoder, decoder, autoencoder, vae_loss 268 | 269 | 270 | if __name__ == "__main__": 271 | _, _, ae = dense_ae((20, 4), 32) 272 | ae.summary() 273 | 274 | _, _, d_ae = deep_ae((1, 20, 4), 32) 275 | d_ae.summary() 276 | 277 | _, _, c_ae = deep_conv_ae((1, 20, 4), 32) 278 | c_ae.summary() 279 | 280 | _, _, vae, _ = deep_conv2d_vae((100, 100, 3) ) 281 | vae.summary() 282 | -------------------------------------------------------------------------------- /mas_mt/v0.4_simple_classifier/classifier_0.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import math 3 | import matplotlib.pyplot as plt 4 | import numpy as np 5 | 6 | from mas_tools.os import get_parameters 7 | from mas_tools.ml import plot_history 8 | from mas_tools.data import create_timeseries_matrix 9 | from mas_tools.data import get_delta, get_diff, get_log_diff 10 | from mas_tools.data import get_sigmoid_to_zero, get_sigmoid_ration 11 | from mas_tools.models import save_model, load_model 12 | from mas_tools.classes import signal_to_class, class_to_signal 13 | from sklearn.model_selection import train_test_split 14 | 15 | from keras.models import Sequential 16 | from keras.layers import BatchNormalization 17 | from keras.layers import Dense, Activation 18 | from keras.layers import LeakyReLU 19 | from keras.layers import Dropout, ActivityRegularization 20 | from keras import regularizers 21 | from keras.optimizers import RMSprop, SGD 22 | from keras.optimizers import Adam, Nadam, Adagrad, Adamax, Adadelta 23 | from keras.callbacks import ModelCheckpoint, ReduceLROnPlateau 24 | from keras.callbacks import CSVLogger, EarlyStopping 25 | 26 | from sklearn.metrics import mean_squared_error 27 | from sklearn.metrics import confusion_matrix 28 | from sklearn.metrics import classification_report 29 | from sklearn.metrics import matthews_corrcoef 30 | 31 | 32 | #=============================================================================# 33 | # P R E P A R E V A R I A B L E S # 34 | #=============================================================================# 35 | # params[symb+period, arg1, arg2, ..] 36 | params = ['EURUSD15', '-train', '100', '-graph'] 37 | # params = ['EURUSD15', '-predict'] 38 | limit = 8000 39 | batch_size = 256 40 | fit_epoch = 100 41 | train_test = 0.2 42 | ts_lookback = 12 43 | 44 | nclasses = 6 45 | normalize_class = True 46 | 47 | run_type = 0 48 | graph = False 49 | 50 | idx = 0 51 | for item in params: 52 | if idx > 0: 53 | if item == '-train': 54 | run_type = 0 55 | elif item == '-predict': 56 | run_type = 1 57 | elif item == '-graph': 58 | graph = True 59 | elif item == '-limit': 60 | pass 61 | elif int(item) > 0: 62 | if params[idx-1] == '-train': 63 | fit_epoch = int(item) 64 | elif params[idx-1] == '-limit': 65 | limit = int(item) 66 | idx += 1 67 | 68 | np.random.seed(7) 69 | 70 | 71 | path = 'C:/Users/Alexey/AppData/Roaming/MetaQuotes/Terminal/287469DEA9630EA94D0715D755974F1B/MQL4/Files/ML-Assistant/' 72 | workfile = params[0] 73 | file_x = path + workfile + '_x.csv' 74 | file_y = path + workfile + '_y.csv' 75 | file_xx = path + workfile + '_xx.csv' 76 | file_yy = path + workfile + '_yy.csv' 77 | prefix = 'wgts/classifier_0_' 78 | model = None 79 | data_x = np.array([]) 80 | data_y = np.array([]) 81 | train_x = np.array([]) 82 | train_y = np.array([]) 83 | test_x = np.array([]) 84 | test_y = np.array([]) 85 | history = None 86 | 87 | # print('Backend:', backend()) 88 | print('\nWork file:', workfile) 89 | 90 | 91 | #=============================================================================# 92 | # L O A D D A T A # 93 | #=============================================================================# 94 | def prepare_data(data): 95 | # for time(0, 6), market(7, 10), ema(11, 14), macd(15, 16) 96 | # for atr(17), cci(18), rsi(19), usdx(20), eurx(21) 97 | #---------------------------- 98 | # for market(0, 3), ema(4, 7), 99 | # for atr(8), cci(9), rsi(10) 100 | mrkt, ema = range(4), range(4, 8) 101 | delta = get_delta(data, mrkt[0], mrkt[3]) 102 | diff1 = get_diff(data[:, mrkt[1]]) 103 | diff2 = get_diff(data[:, mrkt[2]]) 104 | diff3 = get_diff(data[:, mrkt[3]]) 105 | # logdiff1 = get_log_diff(data[:, mrkt[1]]) 106 | # logdiff2 = get_log_diff(data[:, mrkt[2]]) 107 | # logdiff3 = get_log_diff(data[:, mrkt[3]]) 108 | detrend1 = get_delta(data, mrkt[3], ema[0]) # close - ema13 109 | detrend2 = get_delta(data, mrkt[3], ema[1]) # close - ema26 110 | # 111 | ediff1 = get_diff(data[:, ema[0]]) 112 | ediff2 = get_diff(data[:, ema[1]]) 113 | ediff3 = get_diff(data[:, ema[2]]) 114 | # elogdiff1 = get_log_diff(data[:, 11]) 115 | # elogdiff2 = get_log_diff(data[:, 12]) 116 | # elogdiff3 = get_log_diff(data[:, 13]) 117 | return np.array(np.column_stack(( 118 | # data[:, 5:6], # hours and minutes 119 | # data[:, 8:11], # prices (without open) 120 | delta, 121 | diff1, diff2, diff3, 122 | # logdiff1, logdiff2, logdiff3, 123 | detrend1, detrend2, 124 | ediff1, ediff2, ediff3, 125 | # elogdiff1, elogdiff2, elogdiff3, 126 | # data[:, 15:17], # macd 127 | data[:, 8:10], data[:, 10]-50, # atr, cci, rsi 128 | # data[:, 20:22], # usd and eur indexes 129 | )) 130 | ) 131 | 132 | if run_type == 0: 133 | print('Loading Data...') 134 | 135 | train_data = np.genfromtxt(file_x, delimiter=';') 136 | target_data = np.genfromtxt(file_y, delimiter=';') 137 | 138 | # train_data, target_data = train_data[-limit:,], target_data[-limit:] 139 | 140 | data_x = prepare_data(train_data) 141 | data_y = signal_to_class(target_data, n=nclasses, normalize=normalize_class) 142 | 143 | # For training validation 144 | train_x, test_x, train_y, test_y = train_test_split(data_x, data_y, test_size=train_test) 145 | 146 | print('Input data shape :', data_x.shape) 147 | print('Train/Test :', len(train_y), '/', len(test_y)) 148 | 149 | 150 | #=============================================================================# 151 | # P R E P A R E M O D E L # 152 | #=============================================================================# 153 | if run_type == 0: 154 | print('\nCreating Model...') 155 | 156 | batch_size = 256 157 | fa = 'elu' 158 | init = 'lecun_normal' #'lecun_uniform' #'random_uniform' 159 | init_b = 'random_uniform' 160 | reg = regularizers.l2 161 | rs = 0.01 162 | 163 | model = Sequential() 164 | model.add(BatchNormalization(batch_input_shape=(None, data_x.shape[1]))) 165 | model.add(Dense(32, 166 | # activation=fa, 167 | # kernel_initializer=init, 168 | # bias_initializer=init_b, 169 | # kernel_regularizer=reg(rs) 170 | )) 171 | model.add(LeakyReLU()) 172 | model.add(Dense(32, 173 | # activation=fa, 174 | # kernel_initializer=init, 175 | # bias_initializer=init_b, 176 | # kernel_regularizer=reg(rs) 177 | )) 178 | model.add(LeakyReLU()) 179 | # model.add(ActivityRegularization(l1=0.01, l2=0.01)) 180 | # model.add(Dropout(0.3)) 181 | model.add(Dense(32, 182 | # activation=fa, 183 | # kernel_initializer=init, 184 | # bias_initializer=init_b, 185 | # kernel_regularizer=reg(rs) 186 | )) 187 | model.add(LeakyReLU()) 188 | model.add(Dense(nclasses, 189 | activation='softmax', 190 | # kernel_initializer='lecun_uniform', 191 | # bias_initializer=init_b, 192 | # kernel_regularizer=regularizers.l2(rs) 193 | )) 194 | 195 | save_model(model, prefix + workfile + '.model') 196 | elif run_type == 1: 197 | model = load_model(prefix + workfile + '.model') 198 | 199 | # opt = SGD(lr=0.1, momentum=0.0, nesterov=True) 200 | # opt = RMSprop(lr=0.001) 201 | opt = Nadam(lr=0.002) 202 | model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['accuracy']) 203 | 204 | 205 | #=============================================================================# 206 | # T R A I N I N G # 207 | #=============================================================================# 208 | if run_type == 0: 209 | print('Training...') 210 | 211 | reduce_lr = ReduceLROnPlateau(factor=0.05, patience=5, min_lr=0.000001, verbose=1) 212 | # checkpointer = ModelCheckpoint(filepath=(prefix+workfile+"_{epoch:02d}-{val_loss:.2f}"+'.hdf5'), verbose=0, save_best_only=True) 213 | # es = EarlyStopping(patience=40, min_delta=0.0001) 214 | 215 | history = model.fit(train_x, train_y, 216 | batch_size=batch_size, 217 | epochs=fit_epoch, 218 | # callbacks=[reduce_lr], 219 | validation_data=(test_x, test_y) 220 | ) 221 | 222 | model.save_weights(prefix + workfile + '.hdf5') 223 | 224 | 225 | #=============================================================================# 226 | # P R E D I C T I N G # 227 | #=============================================================================# 228 | print('\nPredicting...') 229 | 230 | data_xx = prepare_data(np.genfromtxt(file_xx, delimiter=';')) 231 | print(data_xx.shape) 232 | 233 | if run_type == 1: 234 | model.load_weights(prefix + workfile + '.hdf5') 235 | 236 | # Prediction model 237 | data_yy = model.predict(data_xx, batch_size=batch_size) 238 | predicted = data_yy 239 | data_yy = class_to_signal(data_yy.reshape(data_xx.shape[0], nclasses), 240 | n=nclasses, 241 | normalized=normalize_class) 242 | 243 | np.savetxt(file_yy, data_yy, fmt='%.2f', delimiter=';') 244 | print("Predict saved:\n", file_yy) 245 | 246 | 247 | #=============================================================================# 248 | # P L O T # 249 | #=============================================================================# 250 | if graph: 251 | # test_y = class_to_signal(test_y, 252 | # n=nclasses, 253 | # normalized=normalize_class) 254 | # test_yy = class_to_signal(model.predict(test_x).reshape(test_x.shape[0], nclasses), 255 | # n=nclasses, 256 | # normalized=normalize_class) 257 | test_y_ = [np.argmax(x) for x in test_y] 258 | test_yy = [np.argmax(x) for x in model.predict(test_x).reshape(test_x.shape[0], nclasses)] 259 | 260 | print_classification_scores(test_y_, test_yy, nclasses) 261 | 262 | plt.plot(predicted) 263 | plt.title('Predict') 264 | plt.ylabel('class') 265 | plt.xlabel('bar') 266 | plt.legend(['buy', 'hold', 'sell']) 267 | plt.show() 268 | 269 | plt.plot(data_yy) 270 | plt.title('Saved predict') 271 | plt.ylabel('class') 272 | plt.xlabel('bar') 273 | plt.legend(['prediction']) 274 | plt.show() 275 | 276 | if run_type == 0: 277 | plot_history(history) 278 | 279 | -------------------------------------------------------------------------------- /mas_mt/v0.5_deep_regul/classifier_1.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import math 3 | import matplotlib.pyplot as plt 4 | import numpy as np 5 | 6 | from mas_tools.os import get_parameters 7 | from mas_tools.ml import plot_history 8 | from mas_tools.data import create_timeseries_matrix 9 | from mas_tools.data import get_delta, get_diff, get_log_diff 10 | from mas_tools.data import get_sigmoid_to_zero, get_sigmoid_ration 11 | from mas_tools.models import save_model, load_model 12 | from mas_tools.classes import signal_to_class, class_to_signal 13 | from sklearn.model_selection import train_test_split 14 | 15 | from keras.models import Sequential 16 | from keras.layers import BatchNormalization 17 | from keras.layers import Dense, Activation 18 | from keras.layers import LeakyReLU 19 | from keras.layers import Dropout, ActivityRegularization 20 | from keras import regularizers 21 | from keras.optimizers import RMSprop, SGD 22 | from keras.optimizers import Adam, Nadam, Adagrad, Adamax, Adadelta 23 | from keras.callbacks import ModelCheckpoint, ReduceLROnPlateau 24 | from keras.callbacks import CSVLogger, EarlyStopping 25 | 26 | from sklearn.metrics import mean_squared_error 27 | from sklearn.metrics import confusion_matrix 28 | from sklearn.metrics import classification_report 29 | from sklearn.metrics import matthews_corrcoef 30 | 31 | 32 | #=============================================================================# 33 | # P R E P A R E V A R I A B L E S # 34 | #=============================================================================# 35 | # params[symb+period, arg1, arg2, ..] 36 | params = ['EURUSD30', '-train', '100', '-graph'] 37 | # params = ['EURUSD15', '-predict'] 38 | limit = 8000 39 | batch_size = 256 40 | fit_epoch = 100 41 | train_test = 0.2 42 | ts_lookback = 6 43 | 44 | nclasses = 3 45 | normalize_class = True 46 | 47 | run_type = 0 48 | graph = False 49 | 50 | idx = 0 51 | for item in params: 52 | if idx > 0: 53 | if item == '-train': 54 | run_type = 0 55 | elif item == '-predict': 56 | run_type = 1 57 | elif item == '-graph': 58 | graph = True 59 | elif item == '-limit': 60 | pass 61 | elif int(item) > 0: 62 | if params[idx-1] == '-train': 63 | fit_epoch = int(item) 64 | elif params[idx-1] == '-limit': 65 | limit = int(item) 66 | idx += 1 67 | 68 | np.random.seed(7) 69 | 70 | 71 | path = 'C:/Users/Alexey/AppData/Roaming/MetaQuotes/Terminal/287469DEA9630EA94D0715D755974F1B/MQL4/Files/ML-Assistant/' 72 | workfile = params[0] 73 | file_x = path + workfile + '_x.csv' 74 | file_y = path + workfile + '_y.csv' 75 | file_xx = path + workfile + '_xx.csv' 76 | file_yy = path + workfile + '_yy.csv' 77 | prefix = 'wgts/classifier_0_' 78 | model = None 79 | data_x = np.array([]) 80 | data_y = np.array([]) 81 | train_x = np.array([]) 82 | train_y = np.array([]) 83 | test_x = np.array([]) 84 | test_y = np.array([]) 85 | history = None 86 | 87 | # print('Backend:', backend()) 88 | print('\nWork file:', workfile) 89 | 90 | 91 | #=============================================================================# 92 | # L O A D D A T A # 93 | #=============================================================================# 94 | def prepare_data(data): 95 | # for time(0, 6), market(7, 10), ema(11, 14), macd(15, 16) 96 | # for atr(17), cci(18), rsi(19), usdx(20), eurx(21) 97 | # 98 | # return data 99 | delta = get_delta(data, 0, 3) 100 | detrend1 = get_delta(data, 3, 4) # close - ema13 101 | detrend2 = get_delta(data, 3, 5) # close - ema26 102 | # diff1 = get_diff(data[:, 8]) 103 | # diff2 = get_diff(data[:, 9]) 104 | # diff3 = get_diff(data[:, 10]) 105 | # logdiff1 = get_log_diff(data[:, 8]) 106 | # logdiff2 = get_log_diff(data[:, 9]) 107 | # logdiff3 = get_log_diff(data[:, 10]) 108 | # 109 | # ediff1 = get_diff(data[:, 11]) 110 | # ediff2 = get_diff(data[:, 12]) 111 | # ediff3 = get_diff(data[:, 13]) 112 | # elogdiff1 = get_log_diff(data[:, 11]) 113 | # elogdiff2 = get_log_diff(data[:, 12]) 114 | # elogdiff3 = get_log_diff(data[:, 13]) 115 | # 116 | # xdiff1 = get_diff(data[:, 20]) 117 | # xdiff2 = get_diff(data[:, 21]) 118 | return np.array(np.column_stack(( 119 | # data[:, 5:6], # hours and minutes 120 | # data[:, 8:11], # prices (without open) 121 | delta, 122 | # diff1, diff2, diff3, 123 | # logdiff1, logdiff2, logdiff3, 124 | detrend1, detrend2, 125 | # ediff1, ediff2, ediff3, 126 | # elogdiff1, elogdiff2, elogdiff3, 127 | # data[:, 15:17], # macd 128 | data[:, 8:10], data[:, 10]-50, # atr, cci, rsi 129 | # data[:, 20:22], # usd and eur indexes 130 | # xdelta, 131 | # xdiff1, xdiff2, 132 | # xlogdiff1, xlogdiff2, 133 | )) 134 | ) 135 | 136 | if run_type == 0: 137 | print('Loading Data...') 138 | 139 | train_data = np.genfromtxt(file_x, delimiter=';') 140 | target_data = np.genfromtxt(file_y, delimiter=';') 141 | 142 | # train_data, target_data = train_data[-limit:,], target_data[-limit:] 143 | 144 | data_x = prepare_data(train_data) 145 | data_y = signal_to_class(target_data, n=nclasses, normalize=normalize_class) 146 | # data_x, data_y = create_timeseries_matrix(data_x, data_y, ts_lookback) 147 | 148 | # m_ind = sum(data_y[:, 1]) - max((sum(data_y[:, 0]), sum(data_y[:, 2]))) 149 | # col, ind = 0, 0 150 | # indexes = [] 151 | # while col < m_ind: 152 | # if data_y[ind, 1] == 1.0: 153 | # indexes.append(ind) 154 | # col += 1 155 | # ind += 1 156 | 157 | # data_y = np.delete(data_y, indexes, axis=0) 158 | # data_x = np.delete(data_x, indexes, axis=0) 159 | 160 | # For training validation 161 | train_x, test_x, train_y, test_y = train_test_split(data_x, data_y, test_size=train_test) 162 | 163 | print('Input data shape :', data_x.shape) 164 | print('Train/Test :', len(train_y), '/', len(test_y)) 165 | 166 | 167 | #=============================================================================# 168 | # P R E P A R E M O D E L # 169 | #=============================================================================# 170 | print('\nCreating Model...') 171 | 172 | batch_size = 256 173 | fa = 'elu' 174 | init = 'lecun_uniform' #'lecun_uniform' #'random_normal' 175 | init_b = 'lecun_uniform' 176 | reg = regularizers.l2 177 | rs = 0.001 178 | 179 | model = Sequential() 180 | model.add(BatchNormalization(batch_input_shape=(None, data_x.shape[1]))) 181 | model.add(Dense(32, 182 | # activation=fa, 183 | kernel_initializer=init, 184 | bias_initializer=init_b, 185 | kernel_regularizer=reg(rs) 186 | )) 187 | model.add(LeakyReLU()) 188 | model.add(Dense(32, 189 | # activation=fa, 190 | kernel_initializer=init, 191 | bias_initializer=init_b, 192 | kernel_regularizer=reg(rs) 193 | )) 194 | model.add(LeakyReLU()) 195 | # model.add(ActivityRegularization(l1=0.01, l2=0.01)) 196 | model.add(Dropout(0.3)) 197 | model.add(Dense(nclasses, 198 | activation='softmax', 199 | kernel_initializer='lecun_normal', 200 | bias_initializer=init_b, 201 | # kernel_regularizer=regularizers.l2(rs) 202 | )) 203 | 204 | save_model(model, prefix + workfile + '.model') 205 | elif run_type == 1: 206 | model = load_model(prefix + workfile + '.model') 207 | 208 | opt = Nadam(lr=0.002) 209 | model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['categorical_accuracy']) 210 | 211 | 212 | #=============================================================================# 213 | # T R A I N I N G # 214 | #=============================================================================# 215 | if run_type == 0: 216 | print('Training...') 217 | 218 | reduce_lr = ReduceLROnPlateau(factor=0.05, patience=5, min_lr=0.000001, verbose=1) 219 | # checkpointer = ModelCheckpoint(filepath=(prefix+workfile+"_{epoch:02d}-{val_loss:.2f}"+'.hdf5'), verbose=0, save_best_only=True) 220 | # es = EarlyStopping(patience=40, min_delta=0.0001) 221 | 222 | history = model.fit(train_x, train_y, 223 | batch_size=batch_size, 224 | epochs=fit_epoch, 225 | # callbacks=[reduce_lr], 226 | validation_data=(test_x, test_y) 227 | ) 228 | 229 | model.save_weights(prefix + workfile + '.hdf5') 230 | 231 | 232 | #=============================================================================# 233 | # P R E D I C T I N G # 234 | #=============================================================================# 235 | print('\nPredicting...') 236 | 237 | data_xx = prepare_data(np.genfromtxt(file_xx, delimiter=';')) 238 | # data_xx, empty = create_timeseries_matrix(data_xx, look_back=ts_lookback) 239 | print(data_xx.shape) 240 | 241 | if run_type == 1: 242 | model.load_weights(prefix + workfile + '.hdf5') 243 | 244 | # Prediction model 245 | data_yy = model.predict(data_xx) 246 | predicted = data_yy 247 | data_yy = class_to_signal(data_yy.reshape(data_xx.shape[0], nclasses), 248 | n=nclasses, 249 | normalized=normalize_class) 250 | 251 | np.savetxt(file_yy, data_yy, fmt='%.2f', delimiter=';') 252 | print("Predict saved:\n", file_yy) 253 | 254 | 255 | #=============================================================================# 256 | # P L O T # 257 | #=============================================================================# 258 | if graph: 259 | test_y = class_to_signal(test_y, 260 | n=nclasses, 261 | normalized=normalize_class) 262 | test_yy = class_to_signal(model.predict(test_x).reshape(test_x.shape[0], nclasses), 263 | n=nclasses, 264 | normalized=normalize_class) 265 | 266 | print_classification_scores(test_y, test_yy, nclasses) 267 | 268 | plt.plot(predicted) 269 | plt.title('Predict') 270 | plt.ylabel('class') 271 | plt.xlabel('bar') 272 | plt.legend(['buy', 'hold', 'sell']) 273 | plt.show() 274 | 275 | plt.plot(data_yy) 276 | plt.title('Saved predict') 277 | plt.ylabel('class') 278 | plt.xlabel('bar') 279 | plt.legend(['prediction']) 280 | plt.show() 281 | 282 | if run_type == 0: 283 | plot_history(history, acc='categorical_accuracy') 284 | 285 | -------------------------------------------------------------------------------- /mas_mt/v0.6_deep_recurrent/classifier_2.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import math 3 | import matplotlib.pyplot as plt 4 | import numpy as np 5 | 6 | from mas_tools.os import get_parameters 7 | from mas_tools.ml import plot_history 8 | from mas_tools.data import create_timeseries_matrix 9 | from mas_tools.data import get_delta, get_diff, get_log_diff 10 | from mas_tools.data import get_sigmoid_to_zero, get_sigmoid_ration 11 | from mas_tools.models import save_model, load_model 12 | from mas_tools.classes import signal_to_class, class_to_signal 13 | from sklearn.model_selection import train_test_split 14 | 15 | from keras.models import Sequential 16 | from keras.layers import BatchNormalization 17 | from keras.layers import Dense, Activation 18 | from keras.layers import LSTM, GRU 19 | from keras.layers import LeakyReLU 20 | from keras.layers import Dropout, ActivityRegularization 21 | from keras.layers.wrappers import Bidirectional 22 | from keras import regularizers 23 | from keras.optimizers import RMSprop, SGD 24 | from keras.optimizers import Adam, Nadam, Adagrad, Adamax, Adadelta 25 | from keras.callbacks import ModelCheckpoint, ReduceLROnPlateau 26 | from keras.callbacks import CSVLogger, EarlyStopping 27 | 28 | 29 | #=============================================================================# 30 | # P R E P A R E V A R I A B L E S # 31 | #=============================================================================# 32 | # params[symb+period, arg1, arg2, ..] 33 | params = ['EURUSD30', '-train', '100', '-graph'] 34 | # params = ['EURUSD15', '-predict'] 35 | limit = 8000 36 | batch_size = 256 37 | fit_epoch = 100 38 | train_test = 0.2 39 | ts_lookback = 5 40 | 41 | nclasses = 3 42 | normalize_class = True 43 | 44 | run_type = 0 45 | graph = False 46 | 47 | idx = 0 48 | for item in params: 49 | if idx > 0: 50 | if item == '-train': 51 | run_type = 0 52 | elif item == '-predict': 53 | run_type = 1 54 | elif item == '-graph': 55 | graph = True 56 | elif item == '-limit': 57 | pass 58 | elif int(item) > 0: 59 | if params[idx-1] == '-train': 60 | fit_epoch = int(item) 61 | elif params[idx-1] == '-limit': 62 | limit = int(item) 63 | idx += 1 64 | 65 | np.random.seed(7) 66 | 67 | 68 | path = 'C:/Users/Alexey/AppData/Roaming/MetaQuotes/Terminal/287469DEA9630EA94D0715D755974F1B/MQL4/Files/ML-Assistant/' 69 | workfile = params[0] 70 | file_x = path + workfile + '_x.csv' 71 | file_y = path + workfile + '_y.csv' 72 | file_xx = path + workfile + '_xx.csv' 73 | file_yy = path + workfile + '_yy.csv' 74 | prefix = 'wgts/classifier_0_' 75 | model = None 76 | data_x = np.array([]) 77 | data_y = np.array([]) 78 | train_x = np.array([]) 79 | train_y = np.array([]) 80 | test_x = np.array([]) 81 | test_y = np.array([]) 82 | history = None 83 | 84 | # print('Backend:', backend()) 85 | print('\nWork file:', workfile) 86 | 87 | 88 | #=============================================================================# 89 | # L O A D D A T A # 90 | #=============================================================================# 91 | def prepare_data(data): 92 | return data 93 | # for time(0, 6), market(7, 10), ema(11, 14), macd(15, 16) 94 | # for atr(17), cci(18), rsi(19), usdx(20), eurx(21) 95 | #---------------------------- 96 | # for market(0, 3), ema(4, 7), 97 | # for atr(8), cci(9), rsi(10) 98 | mrkt, ema = range(4), range(4, 8) 99 | # delta = get_delta(data, mrkt[0], mrkt[3]) 100 | diff1 = get_diff(data[:, mrkt[1]]) 101 | diff2 = get_diff(data[:, mrkt[2]]) 102 | diff3 = get_diff(data[:, mrkt[3]]) 103 | # logdiff1 = get_log_diff(data[:, mrkt[1]]) 104 | # logdiff2 = get_log_diff(data[:, mrkt[2]]) 105 | # logdiff3 = get_log_diff(data[:, mrkt[3]]) 106 | detrend1 = get_delta(data, mrkt[3], ema[0]) # close - ema13 107 | detrend2 = get_delta(data, mrkt[3], ema[1]) # close - ema26 108 | # 109 | ediff1 = get_diff(data[:, ema[0]]) 110 | ediff2 = get_diff(data[:, ema[1]]) 111 | ediff3 = get_diff(data[:, ema[2]]) 112 | # elogdiff1 = get_log_diff(data[:, 11]) 113 | # elogdiff2 = get_log_diff(data[:, 12]) 114 | # elogdiff3 = get_log_diff(data[:, 13]) 115 | return np.array(np.column_stack(( 116 | # data[:, 5:6], # hours and minutes 117 | # data[:, 8:11], # prices (without open) 118 | # delta, 119 | diff1, diff2, diff3, 120 | # logdiff1, logdiff2, logdiff3, 121 | detrend1, detrend2, 122 | ediff1, ediff2, ediff3, 123 | # elogdiff1, elogdiff2, elogdiff3, 124 | # data[:, 15:17], # macd 125 | # data[:, 17:19], data[:, 19]-50, # atr, cci, rsi 126 | # data[:, 20:22], # usd and eur indexes 127 | )) 128 | ) 129 | 130 | if run_type == 0: 131 | print('Loading Data...') 132 | 133 | train_data = np.genfromtxt(file_x, delimiter=';') 134 | target_data = np.genfromtxt(file_y, delimiter=';') 135 | 136 | # train_data, target_data = train_data[-limit:,], target_data[-limit:] 137 | 138 | data_x = prepare_data(train_data) 139 | shape_x = data_x.shape 140 | data_y = signal_to_class(target_data, n=nclasses, normalize=normalize_class) 141 | data_x, data_y = create_timeseries_matrix(data_x, data_y, ts_lookback) 142 | 143 | # batch_input_shape=(batch_size, timesteps, units) 144 | data_x = np.reshape(data_x, (data_x.shape[0], ts_lookback, shape_x[1])) 145 | 146 | # For training validation 147 | train_x, test_x, train_y, test_y = train_test_split(data_x, data_y, test_size=train_test) 148 | 149 | print('Input data shape :', data_x.shape) 150 | print('Train/Test :', len(train_y), '/', len(test_y)) 151 | 152 | 153 | #=============================================================================# 154 | # P R E P A R E M O D E L # 155 | #=============================================================================# 156 | print('\nCreating Model...') 157 | 158 | batch_size = 256 159 | fa = 'elu' 160 | init = 'lecun_uniform' #'lecun_uniform' #'random_normal' 161 | init_b = 'lecun_uniform' 162 | reg = regularizers.l2 163 | rs = 0.001 164 | Rcrnt = GRU 165 | 166 | model = Sequential() 167 | model.add(BatchNormalization(batch_input_shape=(None, ts_lookback, shape_x[1]))) 168 | model.add(Rcrnt(50, 169 | return_sequences=True, 170 | # activation=fa, 171 | kernel_initializer=init, 172 | bias_initializer=init_b, 173 | kernel_regularizer=reg(rs), 174 | recurrent_activation='elu', 175 | recurrent_regularizer=reg(rs) 176 | )) 177 | model.add(LeakyReLU()) 178 | # model.add(ActivityRegularization(l1=0.01, l2=0.01)) 179 | model.add(Rcrnt(32, 180 | return_sequences=True, 181 | # activation=fa, 182 | kernel_initializer=init, 183 | bias_initializer=init_b, 184 | kernel_regularizer=reg(rs), 185 | recurrent_activation='elu', 186 | recurrent_regularizer=reg(rs) 187 | )) 188 | model.add(LeakyReLU()) 189 | model.add(Dropout(0.3)) 190 | model.add(Rcrnt(32, 191 | # activation=fa, 192 | kernel_initializer=init, 193 | bias_initializer=init_b, 194 | kernel_regularizer=reg(rs), 195 | recurrent_activation='elu', 196 | recurrent_regularizer=reg(rs) 197 | )) 198 | model.add(LeakyReLU()) 199 | model.add(Dropout(0.3)) 200 | model.add(Dense(nclasses, 201 | activation='softmax', 202 | kernel_initializer=init, 203 | bias_initializer=init_b, 204 | # kernel_regularizer=regularizers.l2(rs) 205 | )) 206 | 207 | save_model(model, prefix + workfile + '.model') 208 | elif run_type == 1: 209 | model = load_model(prefix + workfile + '.model') 210 | 211 | opt = Nadam(lr=0.002) 212 | model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['categorical_accuracy']) 213 | 214 | 215 | #=============================================================================# 216 | # T R A I N I N G # 217 | #=============================================================================# 218 | if run_type == 0: 219 | print('Training...') 220 | 221 | reduce_lr = ReduceLROnPlateau(factor=0.1, patience=3, min_lr=0.00001, verbose=1) 222 | # checkpointer = ModelCheckpoint(filepath=(prefix+workfile+"_{epoch:02d}-{val_loss:.2f}"+'.hdf5'), verbose=0, save_best_only=True) 223 | es = EarlyStopping(patience=20, min_delta=0.001) 224 | 225 | history = model.fit(train_x, train_y, 226 | batch_size=batch_size, 227 | epochs=fit_epoch, 228 | callbacks=[reduce_lr, es], 229 | validation_data=(test_x, test_y) 230 | ) 231 | 232 | model.save_weights(prefix + workfile + '.hdf5') 233 | 234 | 235 | #=============================================================================# 236 | # P R E D I C T I N G # 237 | #=============================================================================# 238 | print('\nPredicting...') 239 | 240 | data_xx = prepare_data(np.genfromtxt(file_xx, delimiter=';')) 241 | shape_xx = data_xx.shape 242 | data_xx, empty = create_timeseries_matrix(data_xx, look_back=ts_lookback) 243 | print(data_xx.shape) 244 | data_xx = np.reshape(data_xx, (data_xx.shape[0], ts_lookback, shape_xx[1])) 245 | print(data_xx.shape) 246 | 247 | if run_type == 1: 248 | model.load_weights(prefix + workfile + '.hdf5') 249 | 250 | # Prediction model 251 | data_yy = model.predict(data_xx) 252 | predicted = data_yy 253 | data_yy = class_to_signal(data_yy.reshape(data_xx.shape[0], nclasses), 254 | n=nclasses, 255 | normalized=normalize_class) 256 | 257 | np.savetxt(file_yy, data_yy, fmt='%.2f', delimiter=';') 258 | print("Predict saved:\n", file_yy) 259 | 260 | 261 | #=============================================================================# 262 | # P L O T # 263 | #=============================================================================# 264 | if graph: 265 | test_y_ = [np.argmax(x) for x in test_y] 266 | test_yy = [np.argmax(x) for x in model.predict(test_x).reshape(test_x.shape[0], nclasses)] 267 | 268 | print_classification_scores(test_y_, test_yy, nclasses) 269 | 270 | plt.plot(predicted) 271 | plt.title('Predict') 272 | plt.ylabel('class') 273 | plt.xlabel('bar') 274 | plt.legend(['buy', 'hold', 'sell']) 275 | plt.show() 276 | 277 | plt.plot(data_yy) 278 | plt.title('Saved predict') 279 | plt.ylabel('class') 280 | plt.xlabel('bar') 281 | plt.legend(['prediction']) 282 | plt.show() 283 | 284 | if run_type == 0: 285 | plot_history(history, acc='categorical_accuracy') 286 | 287 | -------------------------------------------------------------------------------- /mas_mt/v0.7_conv+rnn/classifier_3.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import math 3 | import matplotlib.pyplot as plt 4 | import numpy as np 5 | 6 | from mas_tools.os import get_parameters 7 | from mas_tools.ml import plot_history 8 | from mas_tools.data import create_timeseries_matrix 9 | from mas_tools.data import get_delta, get_diff, get_log_diff 10 | from mas_tools.data import get_sigmoid_to_zero, get_sigmoid_ration 11 | from mas_tools.models import save_model, load_model 12 | from mas_tools.classes import signal_to_class, class_to_signal 13 | from sklearn.model_selection import train_test_split 14 | 15 | from keras.models import Sequential 16 | from keras.layers import BatchNormalization 17 | from keras.layers import Conv1D, MaxPooling1D 18 | from keras.layers import AveragePooling1D, GlobalMaxPooling1D, GlobalAveragePooling1D 19 | from keras.layers import Dense, Activation 20 | from keras.layers import LSTM, GRU 21 | from keras.layers import LeakyReLU, Flatten 22 | from keras.layers import Dropout, ActivityRegularization 23 | from keras.layers.wrappers import Bidirectional 24 | from keras import regularizers 25 | from keras.optimizers import RMSprop, SGD 26 | from keras.optimizers import Adam, Nadam, Adagrad, Adamax, Adadelta 27 | from keras.callbacks import ModelCheckpoint, ReduceLROnPlateau 28 | from keras.callbacks import CSVLogger, EarlyStopping 29 | 30 | from sklearn.metrics import mean_squared_error 31 | from sklearn.metrics import confusion_matrix 32 | from sklearn.metrics import classification_report 33 | from sklearn.metrics import matthews_corrcoef 34 | 35 | 36 | #=============================================================================# 37 | # P R E P A R E V A R I A B L E S # 38 | #=============================================================================# 39 | # params[symb+period, arg1, arg2, ..] 40 | params = ['EURUSD30', '-train', '100', '-graph'] 41 | # params = ['EURUSD15', '-predict'] 42 | limit = 8000 43 | batch_size = 256 44 | fit_epoch = 100 45 | train_test = 0.2 46 | ts_lookback = 20 47 | 48 | nclasses = 3 49 | normalize_class = True 50 | 51 | run_type = 0 52 | graph = False 53 | 54 | idx = 0 55 | for item in params: 56 | if idx > 0: 57 | if item == '-train': 58 | run_type = 0 59 | elif item == '-predict': 60 | run_type = 1 61 | elif item == '-graph': 62 | graph = True 63 | elif item == '-limit': 64 | pass 65 | elif int(item) > 0: 66 | if params[idx-1] == '-train': 67 | fit_epoch = int(item) 68 | elif params[idx-1] == '-limit': 69 | limit = int(item) 70 | idx += 1 71 | 72 | np.random.seed(7) 73 | 74 | 75 | path = 'C:/Users/Alexey/AppData/Roaming/MetaQuotes/Terminal/287469DEA9630EA94D0715D755974F1B/MQL4/Files/ML-Assistant/' 76 | workfile = params[0] 77 | file_x = path + workfile + '_x.csv' 78 | file_y = path + workfile + '_y.csv' 79 | file_xx = path + workfile + '_xx.csv' 80 | file_yy = path + workfile + '_yy.csv' 81 | prefix = 'wgts/classifier_0_' 82 | model = None 83 | data_x = np.array([]) 84 | data_y = np.array([]) 85 | train_x = np.array([]) 86 | train_y = np.array([]) 87 | test_x = np.array([]) 88 | test_y = np.array([]) 89 | history = None 90 | 91 | # print('Backend:', backend()) 92 | print('\nWork file:', workfile) 93 | 94 | 95 | #=============================================================================# 96 | # L O A D D A T A # 97 | #=============================================================================# 98 | def prepare_data(data): 99 | return data 100 | # for time(0, 6), market(7, 10), ema(11, 14), macd(15, 16) 101 | # for atr(17), cci(18), rsi(19), usdx(20), eurx(21) 102 | #---------------------------- 103 | # for market(0, 3), ema(4, 7), 104 | # for atr(8), cci(9), rsi(10) 105 | mrkt, ema = range(4), range(4, 8) 106 | delta = get_delta(data, mrkt[0], mrkt[3]) 107 | # diff1 = get_diff(data[:, mrkt[1]]) 108 | # diff2 = get_diff(data[:, mrkt[2]]) 109 | # diff3 = get_diff(data[:, mrkt[3]]) 110 | # logdiff1 = get_log_diff(data[:, mrkt[1]]) 111 | # logdiff2 = get_log_diff(data[:, mrkt[2]]) 112 | # logdiff3 = get_log_diff(data[:, mrkt[3]]) 113 | detrend1 = get_delta(data, mrkt[3], ema[0]) # close - ema13 114 | detrend2 = get_delta(data, mrkt[3], ema[1]) # close - ema26 115 | # 116 | # ediff1 = get_diff(data[:, ema[0]]) 117 | # ediff2 = get_diff(data[:, ema[1]]) 118 | # ediff3 = get_diff(data[:, ema[2]]) 119 | # elogdiff1 = get_log_diff(data[:, 11]) 120 | # elogdiff2 = get_log_diff(data[:, 12]) 121 | # elogdiff3 = get_log_diff(data[:, 13]) 122 | return np.column_stack(( 123 | # data[:, 5:6], # hours and minutes 124 | # data[:, 8:11], # prices (without open) 125 | delta, 126 | # diff1, diff2, diff3, 127 | # logdiff1, logdiff2, logdiff3, 128 | detrend1, detrend2, 129 | # ediff1, ediff2, ediff3, 130 | # elogdiff1, elogdiff2, elogdiff3, 131 | # data[:, 15:17], # macd 132 | data[:, 8:10], data[:, 10]-50, # atr, cci, rsi 133 | # data[:, 20:22], # usd and eur indexes 134 | )) 135 | 136 | 137 | if run_type == 0: 138 | print('Loading Data...') 139 | 140 | train_data = np.genfromtxt(file_x, delimiter=';') 141 | target_data = np.genfromtxt(file_y, delimiter=';') 142 | 143 | # train_data, target_data = train_data[-limit:,], target_data[-limit:] 144 | 145 | data_x = prepare_data(train_data) 146 | shape_x = data_x.shape 147 | data_y = signal_to_class(target_data, n=nclasses, normalize=normalize_class) 148 | data_x, data_y = create_timeseries_matrix(data_x, data_y, ts_lookback) 149 | 150 | # batch_input_shape=(batch_size, timesteps, units) 151 | data_x = np.reshape(data_x, (data_x.shape[0], ts_lookback, shape_x[1])) 152 | 153 | # data_x, data_y = shuffle_xy(data_x, data_y) 154 | 155 | # For training validation 156 | train_x, test_x, train_y, test_y = train_test_split(data_x, data_y, test_size=train_test) 157 | train_x, train_y = shuffle_xy(train_x, train_y) 158 | 159 | print('Input data shape :', data_x.shape) 160 | print('Train/Test :', len(train_y), '/', len(test_y)) 161 | 162 | 163 | #=============================================================================# 164 | # P R E P A R E M O D E L # 165 | #=============================================================================# 166 | print('\nCreating Model...') 167 | 168 | batch_size = 512 169 | fa = 'tanh' 170 | init = 'lecun_normal' #'lecun_uniform' #'random_uniform', 'glorot_uniform' 171 | init_b = 'lecun_normal' 172 | reg = regularizers.l2 173 | rs = 0.001 174 | Pooling = MaxPooling1D #MaxPooling1D #GlobalMaxPooling1D #GlobalAveragePooling1D 175 | Rcrnt = LSTM 176 | 177 | model = Sequential() 178 | model.add(BatchNormalization(batch_input_shape=(None, ts_lookback, shape_x[1]))) 179 | model.add(Conv1D(#input_shape = (None, ts_lookback, shape_x[1]), 180 | filters=16, 181 | kernel_size=4, 182 | padding='valid')) 183 | # model.add(Pooling(2)) 184 | # model.add(LeakyReLU()) 185 | model.add(Conv1D(filters=8, 186 | kernel_size=3, 187 | padding='valid')) 188 | model.add(Pooling(2)) 189 | model.add(LeakyReLU()) 190 | # model.add(Flatten()) 191 | model.add(Rcrnt(300, 192 | return_sequences=True, 193 | # activation=fa, 194 | kernel_initializer=init, 195 | bias_initializer=init_b, 196 | # kernel_regularizer=reg(rs) 197 | )) 198 | model.add(LeakyReLU()) 199 | model.add(Dropout(0.3)) 200 | model.add(Rcrnt(64, 201 | # activation=fa, 202 | kernel_initializer=init, 203 | bias_initializer=init_b, 204 | # kernel_regularizer=reg(rs) 205 | )) 206 | model.add(LeakyReLU()) 207 | # model.add(ActivityRegularization(l1=0.01, l2=0.01)) 208 | model.add(Dropout(0.3)) 209 | model.add(Dense(nclasses, 210 | activation='softmax', 211 | # kernel_initializer='lecun_uniform', 212 | # bias_initializer=init_b, 213 | # kernel_regularizer=regularizers.l2(rs) 214 | )) 215 | 216 | save_model(model, prefix + workfile + '.model') 217 | elif run_type == 1: 218 | model = load_model(prefix + workfile + '.model') 219 | 220 | opt = Nadam(lr=0.002) 221 | model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['categorical_accuracy']) 222 | 223 | 224 | #=============================================================================# 225 | # T R A I N I N G # 226 | #=============================================================================# 227 | if run_type == 0: 228 | print('Training...') 229 | 230 | reduce_lr = ReduceLROnPlateau(factor=0.1, patience=3, min_lr=0.00001, verbose=1) 231 | # checkpointer = ModelCheckpoint(filepath=(prefix+workfile+"_{epoch:02d}-{val_loss:.2f}"+'.hdf5'), verbose=0, save_best_only=True) 232 | es = EarlyStopping(patience=20, min_delta=0.0001) 233 | 234 | history = model.fit(train_x, train_y, 235 | batch_size=batch_size, 236 | epochs=fit_epoch, 237 | callbacks=[reduce_lr, es], 238 | validation_data=(test_x, test_y) 239 | ) 240 | 241 | model.save_weights(prefix + workfile + '.hdf5') 242 | 243 | 244 | #=============================================================================# 245 | # P R E D I C T I N G # 246 | #=============================================================================# 247 | print('\nPredicting...') 248 | 249 | data_xx = prepare_data(np.genfromtxt(file_xx, delimiter=';')) 250 | shape_xx = data_xx.shape 251 | data_xx, empty = create_timeseries_matrix(data_xx, look_back=ts_lookback) 252 | print(data_xx.shape) 253 | data_xx = np.reshape(data_xx, (data_xx.shape[0], ts_lookback, shape_xx[1])) 254 | print(data_xx.shape) 255 | 256 | if run_type == 1: 257 | model.load_weights(prefix + workfile + '.hdf5') 258 | 259 | # Prediction model 260 | data_yy = model.predict(data_xx) 261 | predicted = data_yy 262 | data_yy = class_to_signal(data_yy.reshape(data_xx.shape[0], nclasses), 263 | n=nclasses, 264 | normalized=normalize_class) 265 | 266 | np.savetxt(file_yy, data_yy, fmt='%.2f', delimiter=';') 267 | print("Predict saved:\n", file_yy) 268 | 269 | 270 | #=============================================================================# 271 | # P L O T # 272 | #=============================================================================# 273 | if graph: 274 | test_y = [np.argmax(x) for x in test_y] 275 | test_yy = [np.argmax(x) for x in model.predict(test_x).reshape(test_x.shape[0], nclasses)] 276 | 277 | print_classification_scores(test_y, test_yy, nclasses) 278 | 279 | plt.plot(predicted) 280 | plt.title('Predict') 281 | plt.ylabel('class') 282 | plt.xlabel('bar') 283 | plt.legend(['buy', 'hold', 'sell']) 284 | plt.show() 285 | 286 | plt.plot(data_yy) 287 | plt.title('Saved predict') 288 | plt.ylabel('class') 289 | plt.xlabel('bar') 290 | plt.legend(['prediction']) 291 | plt.show() 292 | 293 | if run_type == 0: 294 | plot_history(history, acc='categorical_accuracy') 295 | 296 | -------------------------------------------------------------------------------- /mas_mt/v0.4_simple_classifier/classifier_0_activation.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import math 3 | import matplotlib.pyplot as plt 4 | import numpy as np 5 | 6 | from mas_tools.os import get_parameters 7 | from mas_tools.ml import plot_history 8 | from mas_tools.data import create_timeseries_matrix 9 | from mas_tools.data import get_delta, get_diff, get_log_diff 10 | from mas_tools.data import get_sigmoid_to_zero, get_sigmoid_ration 11 | from mas_tools.models import save_model, load_model 12 | from mas_tools.classes import signal_to_class, class_to_signal 13 | from sklearn.model_selection import train_test_split 14 | 15 | from keras.models import Sequential 16 | from keras.layers import BatchNormalization 17 | from keras.layers import Dense, Activation 18 | from keras.layers import LeakyReLU 19 | from keras.layers import Dropout, ActivityRegularization 20 | from keras import regularizers 21 | from keras.optimizers import RMSprop, SGD 22 | from keras.optimizers import Adam, Nadam, Adagrad, Adamax, Adadelta 23 | from keras.callbacks import ModelCheckpoint, ReduceLROnPlateau 24 | from keras.callbacks import CSVLogger, EarlyStopping 25 | 26 | from sklearn.metrics import mean_squared_error 27 | from sklearn.metrics import confusion_matrix 28 | from sklearn.metrics import classification_report 29 | from sklearn.metrics import matthews_corrcoef 30 | 31 | 32 | #=============================================================================# 33 | # P R E P A R E V A R I A B L E S # 34 | #=============================================================================# 35 | # params[symb+period, arg1, arg2, ..] 36 | params = ['EURUSD15', '-train', '100', '-graph'] 37 | # params = ['EURUSD15', '-predict'] 38 | limit = 8000 39 | batch_size = 256 40 | fit_epoch = 100 41 | train_test = 0.2 42 | ts_lookback = 12 43 | 44 | nclasses = 3 45 | normalize_class = True 46 | 47 | run_type = 0 48 | graph = False 49 | 50 | idx = 0 51 | for item in params: 52 | if idx > 0: 53 | if item == '-train': 54 | run_type = 0 55 | elif item == '-predict': 56 | run_type = 1 57 | elif item == '-graph': 58 | graph = True 59 | elif item == '-limit': 60 | pass 61 | elif int(item) > 0: 62 | if params[idx-1] == '-train': 63 | fit_epoch = int(item) 64 | elif params[idx-1] == '-limit': 65 | limit = int(item) 66 | idx += 1 67 | 68 | np.random.seed(7) 69 | 70 | 71 | path = 'C:/Users/Alexey/AppData/Roaming/MetaQuotes/Terminal/287469DEA9630EA94D0715D755974F1B/MQL4/Files/ML-Assistant/' 72 | workfile = params[0] 73 | file_x = path + workfile + '_x.csv' 74 | file_y = path + workfile + '_y.csv' 75 | file_xx = path + workfile + '_xx.csv' 76 | file_yy = path + workfile + '_yy.csv' 77 | prefix = 'wgts/classifier_0_' 78 | model = None 79 | data_x = np.array([]) 80 | data_y = np.array([]) 81 | train_x = np.array([]) 82 | train_y = np.array([]) 83 | test_x = np.array([]) 84 | test_y = np.array([]) 85 | history = None 86 | 87 | # print('Backend:', backend()) 88 | print('\nWork file:', workfile) 89 | 90 | 91 | #=============================================================================# 92 | # L O A D D A T A # 93 | #=============================================================================# 94 | def prepare_data(data): 95 | # for time(0, 6), market(7, 10), ema(11, 14), macd(15, 16) 96 | # for atr(17), cci(18), rsi(19), usdx(20), eurx(21) 97 | # 98 | delta = get_delta(data, 7, 10) 99 | # diff1 = get_diff(data[:, 8]) 100 | # diff2 = get_diff(data[:, 9]) 101 | # diff3 = get_diff(data[:, 10]) 102 | logdiff1 = get_log_diff(data[:, 8]) 103 | logdiff2 = get_log_diff(data[:, 9]) 104 | logdiff3 = get_log_diff(data[:, 10]) 105 | detrend1 = get_delta(data, 10, 11) # close - ema13 106 | detrend2 = get_delta(data, 10, 12) # close - ema26 107 | # 108 | # ediff1 = get_diff(data[:, 11]) 109 | # ediff2 = get_diff(data[:, 12]) 110 | # ediff3 = get_diff(data[:, 13]) 111 | elogdiff1 = get_log_diff(data[:, 11]) 112 | elogdiff2 = get_log_diff(data[:, 12]) 113 | elogdiff3 = get_log_diff(data[:, 13]) 114 | # 115 | # xdiff1 = get_diff(data[:, 20]) 116 | # xdiff2 = get_diff(data[:, 21]) 117 | return np.array(np.column_stack(( 118 | # data[:, 5:6], # hours and minutes 119 | # data[:, 8:11], # prices (without open) 120 | delta, 121 | # diff1, diff2, diff3, 122 | # logdiff1, logdiff2, logdiff3, 123 | detrend1, detrend2, 124 | # ediff1, ediff2, ediff3, 125 | # elogdiff1, elogdiff2, elogdiff3, 126 | # data[:, 15:17], # macd 127 | data[:, 17:19], data[:, 19]-50, # atr, cci, rsi 128 | # data[:, 20:22], # usd and eur indexes 129 | # xdelta, 130 | # xdiff1, xdiff2, 131 | # xlogdiff1, xlogdiff2, 132 | )) 133 | ) 134 | 135 | if run_type == 0: 136 | print('Loading Data...') 137 | 138 | train_data = np.genfromtxt(file_x, delimiter=';') 139 | target_data = np.genfromtxt(file_y, delimiter=';') 140 | 141 | # train_data, target_data = train_data[-limit:,], target_data[-limit:] 142 | 143 | data_x = prepare_data(train_data) 144 | shape_x = data_x.shape 145 | data_y = signal_to_class(target_data, n=nclasses, normalize=normalize_class) 146 | # data_x, data_y = create_timeseries_matrix(train_data, data_y, ts_lookback) 147 | 148 | # batch_input_shape=(batch_size, timesteps, units) 149 | # data_x = np.reshape(data_x, (shape_x[0], ts_lookback, shape_x[1])) 150 | 151 | # For training validation 152 | train_x, test_x, train_y, test_y = train_test_split(data_x, data_y, test_size=train_test) 153 | 154 | print('Input data shape :', data_x.shape) 155 | print('Train/Test :', len(train_y), '/', len(test_y)) 156 | 157 | 158 | #=============================================================================# 159 | # P R E P A R E M O D E L # 160 | #=============================================================================# 161 | if run_type == 0: 162 | print('\nCreating Model...') 163 | 164 | batch_size = 256 165 | fa = 'elu' 166 | init = 'lecun_normal' #'lecun_uniform' #'random_uniform' 167 | init_b = 'random_uniform' 168 | reg = regularizers.l2 169 | rs = 0.01 170 | 171 | model = Sequential() 172 | model.add(BatchNormalization(batch_input_shape=(None, data_x.shape[1]))) 173 | model.add(Dense(32, 174 | # activation=fa, 175 | # kernel_initializer=init, 176 | # bias_initializer=init_b, 177 | # kernel_regularizer=reg(rs) 178 | )) 179 | model.add(LeakyReLU()) 180 | model.add(Dense(32, 181 | # activation=fa, 182 | # kernel_initializer=init, 183 | # bias_initializer=init_b, 184 | # kernel_regularizer=reg(rs) 185 | )) 186 | model.add(LeakyReLU()) 187 | # model.add(ActivityRegularization(l1=0.01, l2=0.01)) 188 | # model.add(Dropout(0.3)) 189 | model.add(Dense(32, 190 | # activation=fa, 191 | # kernel_initializer=init, 192 | # bias_initializer=init_b, 193 | # kernel_regularizer=reg(rs) 194 | )) 195 | model.add(LeakyReLU()) 196 | model.add(Dense(nclasses, 197 | activation='softmax', 198 | # kernel_initializer='lecun_uniform', 199 | # bias_initializer=init_b, 200 | # kernel_regularizer=regularizers.l2(rs) 201 | )) 202 | 203 | save_model(model, prefix + workfile + '.model') 204 | elif run_type == 1: 205 | model = load_model(prefix + workfile + '.model') 206 | 207 | # opt = SGD(lr=0.1, momentum=0.0, nesterov=True) 208 | # opt = RMSprop(lr=0.001) 209 | opt = Nadam(lr=0.002) 210 | model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['accuracy']) 211 | 212 | 213 | #=============================================================================# 214 | # T R A I N I N G # 215 | #=============================================================================# 216 | if run_type == 0: 217 | print('Training...') 218 | 219 | reduce_lr = ReduceLROnPlateau(factor=0.05, patience=5, min_lr=0.000001, verbose=1) 220 | # checkpointer = ModelCheckpoint(filepath=(prefix+workfile+"_{epoch:02d}-{val_loss:.2f}"+'.hdf5'), verbose=0, save_best_only=True) 221 | # es = EarlyStopping(patience=40, min_delta=0.0001) 222 | 223 | history = model.fit(train_x, train_y, 224 | batch_size=batch_size, 225 | epochs=fit_epoch, 226 | # callbacks=[reduce_lr], 227 | validation_data=(test_x, test_y) 228 | ) 229 | 230 | model.save_weights(prefix + workfile + '.hdf5') 231 | 232 | 233 | #=============================================================================# 234 | # P R E D I C T I N G # 235 | #=============================================================================# 236 | print('\nPredicting...') 237 | 238 | data_xx = prepare_data(np.genfromtxt(file_xx, delimiter=';')) 239 | shape_xx = data_xx.shape 240 | print(data_xx.shape) 241 | # data_xx, empty = create_timeseries_matrix(data_xx, look_back=ts_lookback) 242 | # print(data_xx.shape) 243 | # data_xx = np.reshape(data_xx, (shape_xx[0], ts_lookback, shape_xx[1])) 244 | # print(data_xx.shape) 245 | 246 | if run_type == 1: 247 | model.load_weights(prefix + workfile + '.hdf5') 248 | 249 | # Prediction model 250 | data_yy = model.predict(data_xx, batch_size=batch_size) 251 | predicted = data_yy 252 | data_yy = class_to_signal(data_yy.reshape(data_xx.shape[0], nclasses), 253 | n=nclasses, 254 | normalized=normalize_class) 255 | 256 | np.savetxt(file_yy, data_yy, fmt='%.2f', delimiter=';') 257 | print("Predict saved:\n", file_yy) 258 | 259 | 260 | #=============================================================================# 261 | # P L O T # 262 | #=============================================================================# 263 | if graph: 264 | test_y = class_to_signal(test_y, 265 | n=nclasses, 266 | normalized=normalize_class) 267 | test_yy = class_to_signal(model.predict(test_x).reshape(test_x.shape[0], nclasses), 268 | n=nclasses, 269 | normalized=normalize_class) 270 | 271 | print('-' * 20) 272 | print('\nMATTHEWS CORRELATION') 273 | print(matthews_corrcoef(test_y, test_yy)) 274 | CM = confusion_matrix(test_y, test_yy, labels=[1, 0, -1]) 275 | print('\nCONFUSION MATRIX') 276 | print(CM / CM.astype(np.float).sum(axis=1)) 277 | print('\nCLASSIFICATION REPORT') 278 | print(classification_report(test_y, test_yy, labels=[1, 0, -1], target_names=['buy', 'hold', 'sell'])) 279 | print('-' * 20) 280 | 281 | plt.plot(predicted) 282 | plt.title('Predict') 283 | plt.ylabel('class') 284 | plt.xlabel('bar') 285 | plt.legend(['buy', 'hold', 'sell']) 286 | plt.show() 287 | 288 | plt.plot(data_yy) 289 | plt.title('Saved predict') 290 | plt.ylabel('class') 291 | plt.xlabel('bar') 292 | plt.legend(['prediction']) 293 | plt.show() 294 | 295 | if run_type == 0: 296 | plot_history(history) 297 | 298 | -------------------------------------------------------------------------------- /mas_mt/v0.4_simple_classifier/classifier_0_feature_selection.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import math 3 | import matplotlib.pyplot as plt 4 | import numpy as np 5 | 6 | from mas_tools.os import get_parameters 7 | from mas_tools.ml import plot_history 8 | from mas_tools.data import create_timeseries_matrix 9 | from mas_tools.data import get_delta, get_diff, get_log_diff 10 | from mas_tools.data import get_sigmoid_to_zero, get_sigmoid_ration 11 | from mas_tools.models import save_model, load_model 12 | from mas_tools.classes import signal_to_class, class_to_signal 13 | from sklearn.model_selection import train_test_split 14 | 15 | from keras.models import Sequential 16 | from keras.layers import BatchNormalization 17 | from keras.layers import Dense, Activation 18 | from keras.layers import LeakyReLU 19 | from keras.layers import Dropout, ActivityRegularization 20 | from keras import regularizers 21 | from keras.optimizers import RMSprop, SGD 22 | from keras.optimizers import Adam, Nadam, Adagrad, Adamax, Adadelta 23 | from keras.callbacks import ModelCheckpoint, ReduceLROnPlateau 24 | from keras.callbacks import CSVLogger, EarlyStopping 25 | 26 | from sklearn.metrics import mean_squared_error 27 | from sklearn.metrics import confusion_matrix 28 | from sklearn.metrics import classification_report 29 | from sklearn.metrics import matthews_corrcoef 30 | 31 | 32 | #=============================================================================# 33 | # P R E P A R E V A R I A B L E S # 34 | #=============================================================================# 35 | # params[symb+period, arg1, arg2, ..] 36 | params = ['EURUSD15', '-train', '100', '-graph'] 37 | # params = ['EURUSD15', '-predict'] 38 | limit = 8000 39 | batch_size = 256 40 | fit_epoch = 100 41 | train_test = 0.2 42 | ts_lookback = 12 43 | 44 | nclasses = 3 45 | normalize_class = True 46 | 47 | run_type = 0 48 | graph = False 49 | 50 | idx = 0 51 | for item in params: 52 | if idx > 0: 53 | if item == '-train': 54 | run_type = 0 55 | elif item == '-predict': 56 | run_type = 1 57 | elif item == '-graph': 58 | graph = True 59 | elif item == '-limit': 60 | pass 61 | elif int(item) > 0: 62 | if params[idx-1] == '-train': 63 | fit_epoch = int(item) 64 | elif params[idx-1] == '-limit': 65 | limit = int(item) 66 | idx += 1 67 | 68 | np.random.seed(7) 69 | 70 | 71 | path = 'C:/Users/Alexey/AppData/Roaming/MetaQuotes/Terminal/287469DEA9630EA94D0715D755974F1B/MQL4/Files/ML-Assistant/' 72 | workfile = params[0] 73 | file_x = path + workfile + '_x.csv' 74 | file_y = path + workfile + '_y.csv' 75 | file_xx = path + workfile + '_xx.csv' 76 | file_yy = path + workfile + '_yy.csv' 77 | prefix = 'wgts/classifier_0_' 78 | model = None 79 | data_x = np.array([]) 80 | data_y = np.array([]) 81 | train_x = np.array([]) 82 | train_y = np.array([]) 83 | test_x = np.array([]) 84 | test_y = np.array([]) 85 | history = None 86 | 87 | # print('Backend:', backend()) 88 | print('\nWork file:', workfile) 89 | 90 | 91 | #=============================================================================# 92 | # L O A D D A T A # 93 | #=============================================================================# 94 | def prepare_data(data): 95 | # for time(0, 6), market(7, 10), ema(11, 14), macd(15, 16) 96 | # for atr(17), cci(18), rsi(19), usdx(20), eurx(21) 97 | # 98 | delta = get_delta(data, 7, 10) 99 | # diff1 = get_diff(data[:, 8]) 100 | # diff2 = get_diff(data[:, 9]) 101 | # diff3 = get_diff(data[:, 10]) 102 | logdiff1 = get_log_diff(data[:, 8]) 103 | logdiff2 = get_log_diff(data[:, 9]) 104 | logdiff3 = get_log_diff(data[:, 10]) 105 | detrend1 = get_delta(data, 10, 11) # close - ema13 106 | detrend2 = get_delta(data, 10, 12) # close - ema26 107 | # 108 | # ediff1 = get_diff(data[:, 11]) 109 | # ediff2 = get_diff(data[:, 12]) 110 | # ediff3 = get_diff(data[:, 13]) 111 | elogdiff1 = get_log_diff(data[:, 11]) 112 | elogdiff2 = get_log_diff(data[:, 12]) 113 | elogdiff3 = get_log_diff(data[:, 13]) 114 | # 115 | # xdiff1 = get_diff(data[:, 20]) 116 | # xdiff2 = get_diff(data[:, 21]) 117 | return np.array(np.column_stack(( 118 | # data[:, 5:6], # hours and minutes 119 | # data[:, 8:11], # prices (without open) 120 | delta, 121 | # diff1, diff2, diff3, 122 | # logdiff1, logdiff2, logdiff3, 123 | detrend1, detrend2, 124 | # ediff1, ediff2, ediff3, 125 | # elogdiff1, elogdiff2, elogdiff3, 126 | # data[:, 15:17], # macd 127 | data[:, 17:19], data[:, 19]-50, # atr, cci, rsi 128 | # data[:, 20:22], # usd and eur indexes 129 | # xdelta, 130 | # xdiff1, xdiff2, 131 | # xlogdiff1, xlogdiff2, 132 | )) 133 | ) 134 | 135 | if run_type == 0: 136 | print('Loading Data...') 137 | 138 | train_data = np.genfromtxt(file_x, delimiter=';') 139 | target_data = np.genfromtxt(file_y, delimiter=';') 140 | 141 | # train_data, target_data = train_data[-limit:,], target_data[-limit:] 142 | 143 | data_x = prepare_data(train_data) 144 | shape_x = data_x.shape 145 | data_y = signal_to_class(target_data, n=nclasses, normalize=normalize_class) 146 | # data_x, data_y = create_timeseries_matrix(train_data, data_y, ts_lookback) 147 | 148 | # batch_input_shape=(batch_size, timesteps, units) 149 | # data_x = np.reshape(data_x, (shape_x[0], ts_lookback, shape_x[1])) 150 | 151 | # For training validation 152 | train_x, test_x, train_y, test_y = train_test_split(data_x, data_y, test_size=train_test) 153 | 154 | print('Input data shape :', data_x.shape) 155 | print('Train/Test :', len(train_y), '/', len(test_y)) 156 | 157 | 158 | #=============================================================================# 159 | # P R E P A R E M O D E L # 160 | #=============================================================================# 161 | if run_type == 0: 162 | print('\nCreating Model...') 163 | 164 | batch_size = 256 165 | fa = 'tanh' 166 | init = 'lecun_normal' #'lecun_uniform' #'random_uniform' 167 | init_b = 'random_uniform' 168 | reg = regularizers.l2 169 | rs = 0.01 170 | 171 | model = Sequential() 172 | model.add(BatchNormalization(batch_input_shape=(None, data_x.shape[1]))) 173 | model.add(Dense(32, 174 | # activation=fa, 175 | # kernel_initializer=init, 176 | # bias_initializer=init_b, 177 | # kernel_regularizer=reg(rs) 178 | )) 179 | model.add(LeakyReLU()) 180 | model.add(Dense(32, 181 | # activation=fa, 182 | # kernel_initializer=init, 183 | # bias_initializer=init_b, 184 | # kernel_regularizer=reg(rs) 185 | )) 186 | model.add(LeakyReLU()) 187 | # model.add(ActivityRegularization(l1=0.01, l2=0.01)) 188 | # model.add(Dropout(0.3)) 189 | model.add(Dense(32, 190 | # activation=fa, 191 | # kernel_initializer=init, 192 | # bias_initializer=init_b, 193 | # kernel_regularizer=reg(rs) 194 | )) 195 | model.add(LeakyReLU()) 196 | model.add(Dense(nclasses, 197 | activation='softmax', 198 | # kernel_initializer='lecun_uniform', 199 | # bias_initializer=init_b, 200 | # kernel_regularizer=regularizers.l2(rs) 201 | )) 202 | 203 | save_model(model, prefix + workfile + '.model') 204 | elif run_type == 1: 205 | model = load_model(prefix + workfile + '.model') 206 | 207 | # opt = SGD(lr=0.1, momentum=0.0, nesterov=True) 208 | # opt = RMSprop(lr=0.001) 209 | opt = Nadam(lr=0.002) 210 | model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['accuracy']) 211 | 212 | 213 | #=============================================================================# 214 | # T R A I N I N G # 215 | #=============================================================================# 216 | if run_type == 0: 217 | print('Training...') 218 | 219 | reduce_lr = ReduceLROnPlateau(factor=0.05, patience=5, min_lr=0.000001, verbose=1) 220 | # checkpointer = ModelCheckpoint(filepath=(prefix+workfile+"_{epoch:02d}-{val_loss:.2f}"+'.hdf5'), verbose=0, save_best_only=True) 221 | # es = EarlyStopping(patience=40, min_delta=0.0001) 222 | 223 | history = model.fit(train_x, train_y, 224 | batch_size=batch_size, 225 | epochs=fit_epoch, 226 | # callbacks=[reduce_lr], 227 | validation_data=(test_x, test_y) 228 | ) 229 | 230 | model.save_weights(prefix + workfile + '.hdf5') 231 | 232 | 233 | #=============================================================================# 234 | # P R E D I C T I N G # 235 | #=============================================================================# 236 | print('\nPredicting...') 237 | 238 | data_xx = prepare_data(np.genfromtxt(file_xx, delimiter=';')) 239 | shape_xx = data_xx.shape 240 | print(data_xx.shape) 241 | # data_xx, empty = create_timeseries_matrix(data_xx, look_back=ts_lookback) 242 | # print(data_xx.shape) 243 | # data_xx = np.reshape(data_xx, (shape_xx[0], ts_lookback, shape_xx[1])) 244 | # print(data_xx.shape) 245 | 246 | if run_type == 1: 247 | model.load_weights(prefix + workfile + '.hdf5') 248 | 249 | # Prediction model 250 | data_yy = model.predict(data_xx, batch_size=batch_size) 251 | predicted = data_yy 252 | data_yy = class_to_signal(data_yy.reshape(data_xx.shape[0], nclasses), 253 | n=nclasses, 254 | normalized=normalize_class) 255 | 256 | np.savetxt(file_yy, data_yy, fmt='%.2f', delimiter=';') 257 | print("Predict saved:\n", file_yy) 258 | 259 | 260 | #=============================================================================# 261 | # P L O T # 262 | #=============================================================================# 263 | if graph: 264 | test_y = class_to_signal(test_y, 265 | n=nclasses, 266 | normalized=normalize_class) 267 | test_yy = class_to_signal(model.predict(test_x).reshape(test_x.shape[0], nclasses), 268 | n=nclasses, 269 | normalized=normalize_class) 270 | 271 | print('-' * 20) 272 | print('\nMATTHEWS CORRELATION') 273 | print(matthews_corrcoef(test_y, test_yy)) 274 | CM = confusion_matrix(test_y, test_yy, labels=[1, 0, -1]) 275 | print('\nCONFUSION MATRIX') 276 | print(CM / CM.astype(np.float).sum(axis=1)) 277 | print('\nCLASSIFICATION REPORT') 278 | print(classification_report(test_y, test_yy, labels=[1, 0, -1], target_names=['buy', 'hold', 'sell'])) 279 | print('-' * 20) 280 | 281 | plt.plot(predicted) 282 | plt.title('Predict') 283 | plt.ylabel('class') 284 | plt.xlabel('bar') 285 | plt.legend(['buy', 'hold', 'sell']) 286 | plt.show() 287 | 288 | plt.plot(data_yy) 289 | plt.title('Saved predict') 290 | plt.ylabel('class') 291 | plt.xlabel('bar') 292 | plt.legend(['prediction']) 293 | plt.show() 294 | 295 | if run_type == 0: 296 | plot_history(history) 297 | 298 | --------------------------------------------------------------------------------