├── test ├── __init__.py ├── test_mt5.py └── test_correlation.py ├── mt5_correlation ├── __init__.py ├── gui │ ├── __init__.py │ ├── mdi_child_log.py │ ├── mdi_child_help.py │ ├── mdi_child_diverged_symbols.py │ ├── mdi_child_status.py │ ├── mdi_child_divergedgraph.py │ ├── mdi_child_correlationgraph.py │ └── mdi.py ├── mt5.py └── correlation.py ├── .gitignore ├── definitions.py ├── .idea ├── vcs.xml ├── misc.xml ├── inspectionProfiles │ └── profiles_settings.xml ├── other.xml ├── modules.xml └── mt5-correlation.iml ├── requirements.txt ├── LICENSE ├── mt5_correlation.py ├── config.yaml ├── README.md └── configmeta.yaml /test/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /mt5_correlation/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /venv/ 2 | /.idea/workspace.xml 3 | /*.log 4 | /*.log.? 5 | -------------------------------------------------------------------------------- /mt5_correlation/gui/__init__.py: -------------------------------------------------------------------------------- 1 | from mt5_correlation.gui.mdi import CorrelationMDIFrame 2 | -------------------------------------------------------------------------------- /definitions.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) 4 | HELP_FILE = fr"{ROOT_DIR}\README.md" 5 | LOG_FILE = fr"{ROOT_DIR}\debug.log" 6 | -------------------------------------------------------------------------------- /.idea/vcs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /.idea/misc.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | pandas==1.2.1 2 | matplotlib==3.4.2 3 | MetaTrader5==5.0.34 4 | pytz==2021.1 5 | scipy==1.6.0 6 | logging==0.4.9.6 7 | markdown==3.3.4 8 | wxpython==4.1.1 9 | mock==4.0.3 10 | numpy==1.20.0 11 | wxconfig==1.2 -------------------------------------------------------------------------------- /.idea/inspectionProfiles/profiles_settings.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 6 | -------------------------------------------------------------------------------- /.idea/other.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 7 | -------------------------------------------------------------------------------- /.idea/modules.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /.idea/mt5-correlation.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 16 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Jamie Cash 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /mt5_correlation.py: -------------------------------------------------------------------------------- 1 | """ 2 | Application to monitor previously correlated symbol pairs for correlation divergence. 3 | """ 4 | import definitions 5 | import logging.config 6 | from mt5_correlation.gui import CorrelationMDIFrame 7 | import wxconfig as cfg 8 | import wx 9 | import wx.lib.mixins.inspection as wit 10 | 11 | 12 | class InspectionApp(wx.App, wit.InspectionMixin): 13 | # Override app to use inspection. 14 | def OnInit(self): 15 | self.Init() # initialize the inspection tool 16 | return True 17 | 18 | 19 | if __name__ == "__main__": 20 | # Load the config 21 | cfg.Config().load(fr"{definitions.ROOT_DIR}\config.yaml", meta=fr"{definitions.ROOT_DIR}\configmeta.yaml") 22 | 23 | # Get logging config and configure the logger 24 | log_config = cfg.Config().get('logging') 25 | logging.config.dictConfig(log_config) 26 | 27 | # Do we have inspection turned on. Create correct version of app 28 | inspection = cfg.Config().get('developer.inspection') 29 | if inspection: 30 | app = InspectionApp() 31 | else: 32 | app = wx.App(False) 33 | 34 | # Start the app 35 | frame = CorrelationMDIFrame() 36 | frame.Show() 37 | app.MainLoop() 38 | -------------------------------------------------------------------------------- /mt5_correlation/gui/mdi_child_log.py: -------------------------------------------------------------------------------- 1 | import definitions 2 | import wx 3 | import wx.html 4 | 5 | import mt5_correlation.gui.mdi as mdi 6 | 7 | 8 | class MDIChildLog(mdi.CorrelationMDIChild): 9 | """ 10 | Shows the debug.log file 11 | """ 12 | 13 | __log_window = None # Widget to display log file in 14 | 15 | def __init__(self, parent): 16 | # Super 17 | mdi.CorrelationMDIChild.__init__(self, parent=parent, id=wx.ID_ANY, pos=wx.DefaultPosition, title="Log", 18 | size=wx.Size(width=800, height=200), 19 | style=wx.DEFAULT_FRAME_STYLE) 20 | 21 | # Panel and sizer for help file 22 | panel = wx.Panel(self, wx.ID_ANY) 23 | sizer = wx.BoxSizer() 24 | panel.SetSizer(sizer) 25 | 26 | # Log file window 27 | self.__log_window = wx.TextCtrl(parent=panel, id=wx.ID_ANY, style=wx.HSCROLL | wx.TE_MULTILINE | wx.TE_READONLY) 28 | sizer.Add(self.__log_window, 1, wx.ALL | wx.EXPAND) 29 | 30 | # Refresh to populate 31 | self.refresh() 32 | 33 | def refresh(self): 34 | """ 35 | Refresh the log file 36 | :return: 37 | """ 38 | # Load the help file 39 | self.__log_window.LoadFile(definitions.LOG_FILE) 40 | 41 | # Scroll to bottom 42 | self.__log_window.SetInsertionPoint(-1) 43 | 44 | -------------------------------------------------------------------------------- /mt5_correlation/gui/mdi_child_help.py: -------------------------------------------------------------------------------- 1 | import definitions 2 | import markdown 3 | import wx 4 | import wx.html 5 | 6 | import mt5_correlation.gui.mdi as mdi 7 | 8 | 9 | class MDIChildHelp(mdi.CorrelationMDIChild): 10 | """ 11 | Shows the README.md file 12 | """ 13 | 14 | def __init__(self, parent): 15 | # Super 16 | mdi.CorrelationMDIChild.__init__(self, parent=parent, id=wx.ID_ANY, pos=wx.DefaultPosition, title="Help", 17 | size=wx.Size(width=800, height=-1), 18 | style=wx.DEFAULT_FRAME_STYLE) 19 | 20 | # Panel and sizer for help file 21 | panel = wx.Panel(self, wx.ID_ANY) 22 | sizer = wx.BoxSizer() 23 | panel.SetSizer(sizer) 24 | 25 | # HtmlWindow 26 | html_widget = wx.html.HtmlWindow(parent=panel, id=wx.ID_ANY, style=wx.html.HW_SCROLLBAR_AUTO | wx.html.HW_NO_SELECTION) 27 | sizer.Add(html_widget, 1, wx.ALL | wx.EXPAND) 28 | 29 | # Load the help file, convert markdown to HTML and display. The markdown library doesnt understand the shell 30 | # format so we will remove. 31 | markdown_text = open(definitions.HELP_FILE).read() 32 | markdown_text = markdown_text.replace("```shell", "```") 33 | html = markdown.markdown(markdown_text) 34 | html_widget.SetPage(html) 35 | 36 | def refresh(self): 37 | """ 38 | Nothing to do on refresh. Help file doesnt change during outside of development. 39 | :return: 40 | """ 41 | pass 42 | -------------------------------------------------------------------------------- /config.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | calculate: 3 | from: 4 | days: 10 5 | timeframe: 15 6 | min_prices: 400 7 | max_set_size_diff_pct: 90 8 | overlap_pct: 90 9 | max_p_value: 0.05 10 | monitor: 11 | interval: 10 12 | calculations: 13 | long: 14 | from: 30 15 | min_prices: 300 16 | max_set_size_diff_pct: 50 17 | overlap_pct: 50 18 | max_p_value: 0.05 19 | medium: 20 | from: 10 21 | min_prices: 100 22 | max_set_size_diff_pct: 50 23 | overlap_pct: 50 24 | max_p_value: 0.05 25 | short: 26 | from: 2 27 | min_prices: 30 28 | max_set_size_diff_pct: 50 29 | overlap_pct: 50 30 | max_p_value: 0.05 31 | monitoring_threshold: 0.9 32 | divergence_threshold: 0.8 33 | monitor_inverse: true 34 | tick_cache_time: 10 35 | autosave: true 36 | logging: 37 | version: 1 38 | disable_existing_loggers: false 39 | formatters: 40 | brief: 41 | format: '%(asctime)s - %(name)s - %(levelname)s - %(message)s' 42 | datefmt: '%H:%M:%S' 43 | precice: 44 | format: '%(asctime)s - %(threadName)s - %(name)s - %(levelname)s - %(message)s' 45 | datefmt: '%Y-%m-%d %H:%M:%S' 46 | handlers: 47 | console: 48 | level: INFO 49 | class: logging.StreamHandler 50 | formatter: brief 51 | stream: ext://sys.stdout 52 | file: 53 | level: DEBUG 54 | class: logging.handlers.RotatingFileHandler 55 | formatter: precice 56 | filename: debug.log 57 | mode: a 58 | maxBytes: 2560000 59 | backupCount: 1 60 | root: 61 | level: DEBUG 62 | handlers: 63 | - console 64 | - file 65 | loggers: 66 | mt5-correlation: 67 | level: DEBUG 68 | handlers: 69 | - console 70 | - file 71 | propagate: 0 72 | charts: 73 | colormap: Dark2 74 | developer: 75 | inspection: true 76 | window: 77 | x: -4 78 | y: 1 79 | width: 1626 80 | height: 1043 81 | style: 541072960 82 | settings_window: 83 | x: 354 84 | y: 299 85 | width: 624 86 | height: 328 87 | style: 524352 88 | ... -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # mt5-correlation 2 | Calculates correlation coefficient between all symbols in MetaTrader5 Market Watch and monitors previously correlated symbol pairs for divergence 3 | 4 | # Setup 5 | 1) Set up your MetaTrader 5 environment ensuring that all symbols that you would like to assess for correlation are shown in your Market Watch window; 6 | 2) Set up your python environment; and 7 | 3) Install the required libraries. 8 | 9 | ```shell 10 | pip install -r mt5-correlation/requirements.txt 11 | ``` 12 | 13 | # Usage 14 | If you set up a virtual environment in the Setup step, ensure this is activated. Then run the script. 15 | 16 | ```shell 17 | python -m mt5_correlations/mt5_correlations.py 18 | ``` 19 | 20 | This will open a GUI. 21 | 22 | ## Calculating Baseline Coefficients 23 | On the first time that you run, you will want to calculate the initial set of correlations. 24 | 1) Open settings and review the settings under the 'Calculate' tab. Hover over the individual settings for help. 25 | 2) Set the threshold for the correlations to monitor. This can be set under the Settings 'Monitoring' tab and is named 'Monitoring Threshold'. Only settings with a coefficient over this threshold will be displayed and monitored. 26 | 3) Calculate the coefficients by selecting File/Calculate. All symbol pairs that have a correlation coefficient greater than the monitoring threshold will be displayed. A graph showing the price candle data used to calculate the coefficient for the pair will be displayed when you select the row. 27 | 4) You may want to save. Select File/Save and choose a file name. This file will contain all the calculated coefficients, and the price data used to calculate them. This file can be loaded to avoid having to recalculate the baseline coefficients every time you use the application. 28 | 29 | ## Monitoring for Divergence 30 | Once you have calculated or loaded the baseline coefficients, they can be monitored for divergence. 31 | 1) Open settings and review the settings under the 'Monitor' tab. Hover over the individual settings for help. 32 | 2) Switch the monitoring toggle to on. The application will continuously monitor for divergence. The data frame will be updated with the last time that the correlation was checked, and the last coefficient. The chart frame will contain 3 charts which will be updated after every monitoring event: 33 | - One showing the price history data used to calculate the baseline coefficient for both symbols in the correlated pair; 34 | - One showing the tick data used to calculate the latest coefficient for both symbols in the correlated pair. 35 | - One showing every correlation coefficient calculated for the symbol pair. -------------------------------------------------------------------------------- /test/test_mt5.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from unittest.mock import patch 3 | import mt5_correlation.mt5 as mt5 4 | import pandas as pd 5 | from datetime import datetime 6 | 7 | 8 | class Symbol: 9 | """ A Mock symbol class""" 10 | name = None 11 | visible = None 12 | 13 | def __init__(self, name, visible): 14 | self.name = name 15 | self.visible = visible 16 | 17 | 18 | class TestMT5(unittest.TestCase): 19 | """ 20 | Unit test for MT5. Uses mock to mock MetaTrader5 connection. 21 | """ 22 | 23 | # Mock symbols. 5 Symbols, 4 visible. 24 | mock_symbols = [Symbol(name='SYMBOL1', visible=True), 25 | Symbol(name='SYMBOL2', visible=True), 26 | Symbol(name='SYMBOL3', visible=False), 27 | Symbol(name='SYMBOL4', visible=True), 28 | Symbol(name='SYMBOL5', visible=True)] 29 | 30 | # Mock prices for symbol 1 31 | mock_prices = pd.DataFrame(columns=['time', 'close'], 32 | data=[[datetime(2021, 1, 1, 1, 5, 0), 123.123], 33 | [datetime(2021, 1, 1, 1, 10, 0), 123.124], 34 | [datetime(2021, 1, 1, 1, 15, 0), 123.125], 35 | [datetime(2021, 1, 1, 1, 20, 0), 125.126], 36 | [datetime(2021, 1, 1, 1, 25, 0), 123.127], 37 | [datetime(2021, 1, 1, 1, 30, 0), 123.128]]) 38 | 39 | @patch('mt5_correlation.mt5.MetaTrader5') 40 | def test_get_symbols(self, mock): 41 | # Mock return value 42 | mock.symbols_get.return_value = self.mock_symbols 43 | 44 | # Call get_symbols 45 | symbols = mt5.MT5().get_symbols() 46 | 47 | # There should be four, as one is set as not visible 48 | self.assertTrue(len(symbols) == 4, "There should be 5 symbols returned from MT5.") 49 | 50 | @patch('mt5_correlation.mt5.MetaTrader5') 51 | def test_get_prices(self, mock): 52 | # Mock return value 53 | mock.copy_rates_range.return_value = self.mock_prices 54 | 55 | # Call get prices 56 | prices = mt5.MT5().get_prices(symbol='SYMBOL1', from_date='01-JAN-2021 01:00:00', 57 | to_date='01-JAN-2021 01:10:25', timeframe=mt5.TIMEFRAME_M5) 58 | 59 | # There should be 6 60 | self.assertTrue(len(prices.index) == 6, "There should be 6 prices.") 61 | 62 | @patch('mt5_correlation.mt5.MetaTrader5') 63 | def test_get_ticks(self, mock): 64 | # Mock return value 65 | mock.copy_ticks_range.return_value = self.mock_prices 66 | 67 | # Call get ticks 68 | ticks = mt5.MT5().get_ticks(symbol='SYMBOL1', from_date='01-JAN-2021 01:00:00', to_date='01-JAN-2021 01:10:25') 69 | 70 | # There should be 6 71 | self.assertTrue(len(ticks.index) == 6, "There should be 6 prices.") 72 | 73 | 74 | if __name__ == '__main__': 75 | unittest.main() 76 | 77 | 78 | -------------------------------------------------------------------------------- /mt5_correlation/mt5.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | import MetaTrader5 3 | import logging 4 | 5 | # Timeframes 6 | TIMEFRAME_M1 = MetaTrader5.TIMEFRAME_M1 7 | TIMEFRAME_M2 = MetaTrader5.TIMEFRAME_M2 8 | TIMEFRAME_M3 = MetaTrader5.TIMEFRAME_M3 9 | TIMEFRAME_M4 = MetaTrader5.TIMEFRAME_M4 10 | TIMEFRAME_M5 = MetaTrader5.TIMEFRAME_M5 11 | TIMEFRAME_M6 = MetaTrader5.TIMEFRAME_M6 12 | TIMEFRAME_M10 = MetaTrader5.TIMEFRAME_M10 13 | TIMEFRAME_M12 = MetaTrader5.TIMEFRAME_M12 14 | TIMEFRAME_M15 = MetaTrader5.TIMEFRAME_M15 15 | TIMEFRAME_M20 = MetaTrader5.TIMEFRAME_M20 16 | TIMEFRAME_M30 = MetaTrader5.TIMEFRAME_M30 17 | TIMEFRAME_H1 = MetaTrader5.TIMEFRAME_H1 18 | TIMEFRAME_H2 = MetaTrader5.TIMEFRAME_H2 19 | TIMEFRAME_H3 = MetaTrader5.TIMEFRAME_H3 20 | TIMEFRAME_H4 = MetaTrader5.TIMEFRAME_H4 21 | TIMEFRAME_H6 = MetaTrader5.TIMEFRAME_H6 22 | TIMEFRAME_H8 = MetaTrader5.TIMEFRAME_H8 23 | TIMEFRAME_H12 = MetaTrader5.TIMEFRAME_H12 24 | TIMEFRAME_D1 = MetaTrader5.TIMEFRAME_D1 25 | TIMEFRAME_W1 = MetaTrader5.TIMEFRAME_W1 26 | TIMEFRAME_MN1 = MetaTrader5.TIMEFRAME_MN1 27 | 28 | 29 | class MT5: 30 | """ 31 | A class to connect to and interface with MetaTrader 5 32 | """ 33 | 34 | def __init__(self): 35 | # Connect to MetaTrader5. Opens if not already open. 36 | 37 | # Logger 38 | self.__log = logging.getLogger(__name__) 39 | 40 | # Open MT5 and log error if it could not open 41 | if not MetaTrader5.initialize(): 42 | self.__log.error("initialize() failed") 43 | MetaTrader5.shutdown() 44 | 45 | # Print connection status 46 | self.__log.debug(MetaTrader5.terminal_info()) 47 | 48 | # Print data on MetaTrader 5 version 49 | self.__log.debug(MetaTrader5.version()) 50 | 51 | def __del__(self): 52 | # shut down connection to the MetaTrader 5 terminal 53 | MetaTrader5.shutdown() 54 | 55 | def get_symbols(self): 56 | """ 57 | Gets list of symbols open in MT5 market watch. 58 | :return: list of symbol names 59 | """ 60 | # Iterate symbols and get those in market watch. 61 | symbols = MetaTrader5.symbols_get() 62 | selected_symbols = [] 63 | for symbol in symbols: 64 | if symbol.visible: 65 | selected_symbols.append(symbol.name) 66 | 67 | # Log symbol counts 68 | total_symbols = MetaTrader5.symbols_total() 69 | num_selected_symbols = len(selected_symbols) 70 | self.__log.debug(f"{num_selected_symbols} of {total_symbols} available symbols in Market Watch.") 71 | 72 | return selected_symbols 73 | 74 | def get_prices(self, symbol, from_date, to_date, timeframe): 75 | """ 76 | Gets OHLC price data for the specified symbol. 77 | :param symbol: The name of the symbol to get the price data for. 78 | :param from_date: Date from when to retrieve data 79 | :param to_date: Date where to receive data to 80 | :param timeframe: The timeframe for the candes. Possible values are: 81 | TIMEFRAME_M1: 1 minute 82 | TIMEFRAME_M2: 2 minutes 83 | TIMEFRAME_M3: 3 minutes 84 | TIMEFRAME_M4: 4 minutes 85 | TIMEFRAME_M5: 5 minutes 86 | TIMEFRAME_M6: 6 minutes 87 | TIMEFRAME_M10: 10 minutes 88 | TIMEFRAME_M12: 12 minutes 89 | TIMEFRAME_M15: 15 minutes 90 | TIMEFRAME_M20: 20 minutes 91 | TIMEFRAME_M30: 30 minutes 92 | TIMEFRAME_H1: 1 hour 93 | TIMEFRAME_H2: 2 hours 94 | TIMEFRAME_H3: 3 hours 95 | TIMEFRAME_H4: 4 hours 96 | TIMEFRAME_H6: 6 hours 97 | TIMEFRAME_H8: 8 hours 98 | TIMEFRAME_H12: 12 hours 99 | TIMEFRAME_D1: 1 day 100 | TIMEFRAME_W1: 1 week 101 | TIMEFRAME_MN1: 1 month 102 | :return: Price data for symbol as dataframe 103 | """ 104 | 105 | prices_dataframe = None 106 | 107 | # Get prices from MT5 108 | prices = MetaTrader5.copy_rates_range(symbol, timeframe, from_date, to_date) 109 | if prices is not None: 110 | self.__log.debug(f"{len(prices)} prices retrieved for {symbol}.") 111 | 112 | # Create dataframe from data and convert time in seconds to datetime format 113 | prices_dataframe = pd.DataFrame(prices) 114 | prices_dataframe['time'] = pd.to_datetime(prices_dataframe['time'], unit='s') 115 | 116 | return prices_dataframe 117 | 118 | def get_ticks(self, symbol, from_date, to_date): 119 | """ 120 | Gets OHLC price data for the specified symbol. 121 | :param symbol: The name of the symbol to get the price data for. 122 | :param from_date: Date from when to retrieve data 123 | :param to_date: Date where to receive data to 124 | :return: Tick data for symbol as dataframe 125 | """ 126 | 127 | ticks_dataframe = None 128 | 129 | # Get ticks from MT5 130 | ticks = MetaTrader5.copy_ticks_range(symbol, from_date, to_date, MetaTrader5.COPY_TICKS_ALL) 131 | 132 | # If ticks is None, there was an error 133 | if ticks is None: 134 | error = MetaTrader5.last_error() 135 | self.__log.error(f"Error retrieving ticks for {symbol}: {error}") 136 | else: 137 | self.__log.debug(f"{len(ticks)} ticks retrieved for {symbol}.") 138 | 139 | # Create dataframe from data and convert time in seconds to datetime format 140 | try: 141 | ticks_dataframe = pd.DataFrame(ticks) 142 | ticks_dataframe['time'] = pd.to_datetime(ticks_dataframe['time'], unit='s') 143 | except RecursionError: 144 | self.__log.warning("Error converting ticks to dataframe.") 145 | 146 | return ticks_dataframe 147 | -------------------------------------------------------------------------------- /mt5_correlation/gui/mdi_child_diverged_symbols.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import pandas as pd 3 | import wx 4 | import wx.grid 5 | 6 | import mt5_correlation.gui.mdi as mdi 7 | 8 | # Columns for diverged symbols table 9 | COLUMN_INDEX = 0 10 | COLUMN_SYMBOL = 1 11 | COLUMN_NUM_DIVERGENCES = 2 12 | 13 | 14 | class MDIChildDivergedSymbols(mdi.CorrelationMDIChild): 15 | """ 16 | Shows the status of all correlations that are within the monitoring threshold 17 | """ 18 | 19 | # The table and grid containing the symbols that form part of the diverged correlations. Defined at instance level 20 | # to enable refresh. 21 | __table = None 22 | __grid = None 23 | 24 | # Number of rows. Required for and updated by refresh method 25 | __rows = 0 26 | 27 | __log = None # The logger 28 | 29 | def __init__(self, parent): 30 | # Super 31 | wx.MDIChildFrame.__init__(self, parent=parent, id=wx.ID_ANY, title="Diverged Symbols", 32 | size=wx.Size(width=240, height=-1), style=wx.DEFAULT_FRAME_STYLE) 33 | 34 | # Create logger 35 | self.__log = logging.getLogger(__name__) 36 | 37 | # Panel and sizer for table 38 | panel = wx.Panel(self, wx.ID_ANY) 39 | sizer = wx.BoxSizer(wx.VERTICAL) 40 | panel.SetSizer(sizer) 41 | 42 | # Create the grid. This is a data table using pandas dataframe for underlying data. Add the 43 | # grid to the sizer. 44 | self.__table = _DataTable(columns=self.GetMDIParent().cor.diverged_symbols.columns) 45 | self.__grid = wx.grid.Grid(panel, wx.ID_ANY) 46 | self.__grid.SetTable(self.__table, takeOwnership=True) 47 | self.__grid.EnableEditing(False) 48 | self.__grid.EnableDragRowSize(False) 49 | self.__grid.EnableDragColSize(True) 50 | self.__grid.EnableDragGridSize(True) 51 | self.__grid.SetSelectionMode(wx.grid.Grid.SelectRows) 52 | self.__grid.SetRowLabelSize(0) 53 | self.__grid.SetColSize(COLUMN_INDEX, 0) # Index. Hide 54 | self.__grid.SetColSize(COLUMN_SYMBOL, 100) # Symbol 55 | self.__grid.SetColSize(COLUMN_NUM_DIVERGENCES, 100) # Num divergences 56 | self.__grid.SetMinSize((220, 100)) 57 | sizer.Add(self.__grid, 1, wx.ALL | wx.EXPAND) 58 | 59 | # Bind row doubleclick 60 | self.Bind(wx.grid.EVT_GRID_CELL_LEFT_DCLICK, self.__on_doubleckick_row, self.__grid) 61 | 62 | # Refresh to populate 63 | self.refresh() 64 | 65 | def refresh(self): 66 | """ 67 | Refreshes grid. Notifies if rows have been added or deleted. 68 | :return: 69 | """ 70 | self.__log.debug(f"Refreshing grid.") 71 | 72 | # Update data 73 | self.__table.data = self.GetMDIParent().cor.diverged_symbols.copy() 74 | 75 | # Start refresh 76 | self.__grid.BeginBatch() 77 | 78 | # Check if num rows in dataframe has changed, and send appropriate APPEND or DELETE messages 79 | cur_rows = len(self.GetMDIParent().cor.diverged_symbols.index) 80 | if cur_rows < self.__rows: 81 | # Data has been deleted. Send message 82 | msg = wx.grid.GridTableMessage(self.__table, wx.grid.GRIDTABLE_NOTIFY_ROWS_DELETED, 83 | self.__rows - cur_rows, self.__rows - cur_rows) 84 | self.__grid.ProcessTableMessage(msg) 85 | elif cur_rows > self.__rows: 86 | # Data has been added. Send message 87 | msg = wx.grid.GridTableMessage(self.__table, wx.grid.GRIDTABLE_NOTIFY_ROWS_APPENDED, 88 | cur_rows - self.__rows) # how many 89 | self.__grid.ProcessTableMessage(msg) 90 | 91 | self.__grid.EndBatch() 92 | 93 | # Send updated message 94 | msg = wx.grid.GridTableMessage(self.__table, wx.grid.GRIDTABLE_REQUEST_VIEW_GET_VALUES) 95 | self.__grid.ProcessTableMessage(msg) 96 | 97 | # Update row count 98 | self.__rows = cur_rows 99 | 100 | def __on_doubleckick_row(self, evt): 101 | """ 102 | Open the graphs when a row is doubleclicked. 103 | :param evt: 104 | :return: 105 | """ 106 | row = evt.GetRow() 107 | symbol = self.__grid.GetCellValue(row, COLUMN_SYMBOL) 108 | 109 | mdi.FrameManager.open_frame(parent=self.GetMDIParent(), 110 | frame_module='mt5_correlation.gui.mdi_child_divergedgraph', 111 | frame_class='MDIChildDivergedGraph', 112 | raise_if_open=True, 113 | symbol=symbol) 114 | 115 | 116 | class _DataTable(wx.grid.GridTableBase): 117 | """ 118 | A data table that holds data in a pandas dataframe. 119 | """ 120 | data = None # The data for this table. A Pandas DataFrame 121 | 122 | def __init__(self, columns): 123 | wx.grid.GridTableBase.__init__(self) 124 | self.headerRows = 1 125 | self.data = pd.DataFrame(columns=columns) 126 | 127 | def GetNumberRows(self): 128 | return len(self.data) 129 | 130 | def GetNumberCols(self): 131 | return len(self.data.columns) + 1 132 | 133 | def GetValue(self, row, col): 134 | if row < self.RowsCount and col < self.ColsCount: 135 | return self.data.index[row] if col == 0 else self.data.iloc[row, col - 1] 136 | else: 137 | raise Exception(f"Trying to access row {row} and col {col} which does not exist.") 138 | 139 | def SetValue(self, row, col, value): 140 | self.data.iloc[row, col - 1] = value 141 | 142 | def GetColLabelValue(self, col): 143 | if col == 0: 144 | if self.data.index.name is None: 145 | return 'Index' 146 | else: 147 | return self.data.index.name 148 | return str(self.data.columns[col - 1]) 149 | 150 | def GetTypeName(self, row, col): 151 | return wx.grid.GRID_VALUE_STRING 152 | 153 | def GetAttr(self, row, col, prop): 154 | attr = wx.grid.GridCellAttr() 155 | 156 | return attr 157 | -------------------------------------------------------------------------------- /mt5_correlation/gui/mdi_child_status.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import pandas as pd 3 | import wx 4 | import wx.grid 5 | 6 | from mt5_correlation import correlation as cor 7 | import mt5_correlation.gui.mdi as mdi 8 | 9 | # Columns for coefficient table 10 | COLUMN_INDEX = 0 11 | COLUMN_SYMBOL1 = 1 12 | COLUMN_SYMBOL2 = 2 13 | COLUMN_BASE_COEFFICIENT = 3 14 | COLUMN_DATE_FROM = 4 15 | COLUMN_DATE_TO = 5 16 | COLUMN_TIMEFRAME = 6 17 | COLUMN_LAST_CALCULATION = 7 18 | COLUMN_STATUS = 8 19 | 20 | 21 | class MDIChildStatus(mdi.CorrelationMDIChild): 22 | """ 23 | Shows the status of all correlations that are within the monitoring threshold 24 | """ 25 | 26 | # The table and grid containing the status of correlations. Defined at instance level to enable refresh. 27 | __table = None 28 | __grid = None 29 | 30 | # Number of rows. Required for and updated by refresh method 31 | __rows = 0 32 | 33 | __log = None # The logger 34 | 35 | def __init__(self, parent): 36 | # Super 37 | wx.MDIChildFrame.__init__(self, parent=parent, id=wx.ID_ANY, title="Correlation Status", 38 | size=wx.Size(width=440, height=-1), style=wx.DEFAULT_FRAME_STYLE) 39 | 40 | # Create logger 41 | self.__log = logging.getLogger(__name__) 42 | 43 | # Panel and sizer for table 44 | panel = wx.Panel(self, wx.ID_ANY) 45 | sizer = wx.BoxSizer(wx.VERTICAL) 46 | panel.SetSizer(sizer) 47 | 48 | # Create the correlations grid. This is a data table using pandas dataframe for underlying data. Add the 49 | # correlations_grid to the correlations sizer. 50 | self.__table = _DataTable(columns=self.GetMDIParent().cor.filtered_coefficient_data.columns) 51 | self.__grid = wx.grid.Grid(panel, wx.ID_ANY) 52 | self.__grid.SetTable(self.__table, takeOwnership=True) 53 | self.__grid.EnableEditing(False) 54 | self.__grid.EnableDragRowSize(False) 55 | self.__grid.EnableDragColSize(True) 56 | self.__grid.EnableDragGridSize(True) 57 | self.__grid.SetSelectionMode(wx.grid.Grid.SelectRows) 58 | self.__grid.SetRowLabelSize(0) 59 | self.__grid.SetColSize(COLUMN_INDEX, 0) # Index. Hide 60 | self.__grid.SetColSize(COLUMN_SYMBOL1, 100) # Symbol 1 61 | self.__grid.SetColSize(COLUMN_SYMBOL2, 100) # Symbol 2 62 | self.__grid.SetColSize(COLUMN_BASE_COEFFICIENT, 100) # Base Coefficient 63 | self.__grid.SetColSize(COLUMN_DATE_FROM, 0) # UTC Date From. Hide 64 | self.__grid.SetColSize(COLUMN_DATE_TO, 0) # UTC Date To. Hide 65 | self.__grid.SetColSize(COLUMN_TIMEFRAME, 0) # Timeframe. Hide. 66 | self.__grid.SetColSize(COLUMN_LAST_CALCULATION, 0) # Last Calculation. Hide 67 | self.__grid.SetColSize(COLUMN_STATUS, 100) # Status 68 | self.__grid.SetMinSize((420, 500)) 69 | sizer.Add(self.__grid, 1, wx.ALL | wx.EXPAND) 70 | 71 | # Bind row doubleclick 72 | self.Bind(wx.grid.EVT_GRID_CELL_LEFT_DCLICK, self.__on_doubleckick_row, self.__grid) 73 | 74 | # Refresh to populate 75 | self.refresh() 76 | 77 | def refresh(self): 78 | """ 79 | Refreshes grid. Notifies if rows have been added or deleted. 80 | :return: 81 | """ 82 | self.__log.debug(f"Refreshing grid.") 83 | 84 | # Update data 85 | self.__table.data = self.GetMDIParent().cor.filtered_coefficient_data.copy() 86 | 87 | # Format 88 | self.__table.data.loc[:, 'Base Coefficient'] = self.__table.data['Base Coefficient'].map('{:.5f}'.format) 89 | self.__table.data.loc[:, 'Last Calculation'] = pd.to_datetime(self.__table.data['Last Calculation'], utc=True) 90 | self.__table.data.loc[:, 'Last Calculation'] = \ 91 | self.__table.data['Last Calculation'].dt.strftime('%d-%m-%y %H:%M:%S') 92 | 93 | # Start refresh 94 | self.__grid.BeginBatch() 95 | 96 | # Check if num rows in dataframe has changed, and send appropriate APPEND or DELETE messages 97 | cur_rows = len(self.GetMDIParent().cor.filtered_coefficient_data.index) 98 | if cur_rows < self.__rows: 99 | # Data has been deleted. Send message 100 | msg = wx.grid.GridTableMessage(self.__table, wx.grid.GRIDTABLE_NOTIFY_ROWS_DELETED, 101 | self.__rows - cur_rows, self.__rows - cur_rows) 102 | self.__grid.ProcessTableMessage(msg) 103 | elif cur_rows > self.__rows: 104 | # Data has been added. Send message 105 | msg = wx.grid.GridTableMessage(self.__table, wx.grid.GRIDTABLE_NOTIFY_ROWS_APPENDED, 106 | cur_rows - self.__rows) # how many 107 | self.__grid.ProcessTableMessage(msg) 108 | 109 | self.__grid.EndBatch() 110 | 111 | # Send updated message 112 | msg = wx.grid.GridTableMessage(self.__table, wx.grid.GRIDTABLE_REQUEST_VIEW_GET_VALUES) 113 | self.__grid.ProcessTableMessage(msg) 114 | 115 | # Update row count 116 | self.__rows = cur_rows 117 | 118 | def __on_doubleckick_row(self, evt): 119 | """ 120 | Open the graphs when a row is doubleclicked. 121 | :param evt: 122 | :return: 123 | """ 124 | row = evt.GetRow() 125 | symbol1 = self.__grid.GetCellValue(row, COLUMN_SYMBOL1) 126 | symbol2 = self.__grid.GetCellValue(row, COLUMN_SYMBOL2) 127 | 128 | mdi.FrameManager.open_frame(parent=self.GetMDIParent(), 129 | frame_module='mt5_correlation.gui.mdi_child_correlationgraph', 130 | frame_class='MDIChildCorrelationGraph', 131 | raise_if_open=True, 132 | symbols=[symbol1, symbol2]) 133 | 134 | 135 | class _DataTable(wx.grid.GridTableBase): 136 | """ 137 | A data table that holds data in a pandas dataframe. Contains highlighting rules for status. 138 | """ 139 | data = None # The data for this table. A Pandas DataFrame 140 | 141 | def __init__(self, columns): 142 | wx.grid.GridTableBase.__init__(self) 143 | self.headerRows = 1 144 | self.data = pd.DataFrame(columns=columns) 145 | 146 | def GetNumberRows(self): 147 | return len(self.data) 148 | 149 | def GetNumberCols(self): 150 | return len(self.data.columns) + 1 151 | 152 | def GetValue(self, row, col): 153 | if row < self.RowsCount and col < self.ColsCount: 154 | return self.data.index[row] if col == 0 else self.data.iloc[row, col - 1] 155 | else: 156 | raise Exception(f"Trying to access row {row} and col {col} which does not exist.") 157 | 158 | def SetValue(self, row, col, value): 159 | self.data.iloc[row, col - 1] = value 160 | 161 | def GetColLabelValue(self, col): 162 | if col == 0: 163 | if self.data.index.name is None: 164 | return 'Index' 165 | else: 166 | return self.data.index.name 167 | return str(self.data.columns[col - 1]) 168 | 169 | def GetTypeName(self, row, col): 170 | return wx.grid.GRID_VALUE_STRING 171 | 172 | def GetAttr(self, row, col, prop): 173 | attr = wx.grid.GridCellAttr() 174 | 175 | # Check that we are not out of bounds 176 | if row < self.RowsCount: 177 | # If column is status, check and highlight if diverging or converging. 178 | if col in [COLUMN_STATUS]: 179 | # Is status one of interest 180 | value = self.GetValue(row, col) 181 | if value != "": 182 | if value in [cor.STATUS_DIVERGING]: 183 | attr.SetBackgroundColour(wx.RED) 184 | elif value in [cor.STATUS_CONVERGING]: 185 | attr.SetBackgroundColour(wx.GREEN) 186 | else: 187 | attr.SetBackgroundColour(wx.WHITE) 188 | 189 | return attr 190 | -------------------------------------------------------------------------------- /mt5_correlation/gui/mdi_child_divergedgraph.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import matplotlib.dates 3 | import matplotlib.pyplot as plt 4 | import matplotlib.ticker as mticker 5 | import wx 6 | import wx.lib.scrolledpanel as scrolled 7 | import wxconfig as cfg 8 | 9 | from matplotlib.backends.backend_wxagg import FigureCanvasWxAgg as FigureCanvas 10 | from mpl_toolkits.axes_grid1 import host_subplot 11 | from mpl_toolkits import axisartist 12 | 13 | import mt5_correlation.gui.mdi as mdi 14 | from mt5_correlation import correlation as cor 15 | 16 | 17 | class MDIChildDivergedGraph(mdi.CorrelationMDIChild): 18 | """ 19 | Shows the graphs for the specified correlation 20 | """ 21 | 22 | symbol = None # Symbol to chart divergence for. Public as we use to check if window for the symbol is already open. 23 | 24 | # Logger 25 | __log = None 26 | 27 | # Date formats for graphs 28 | __tick_fmt_date = matplotlib.dates.DateFormatter('%d-%b') 29 | __tick_fmt_time = matplotlib.dates.DateFormatter('%H:%M:%S') 30 | 31 | # Graph Canvas 32 | __canvas = None 33 | 34 | # Colors for graph lines 35 | __colours = matplotlib.cm.get_cmap(cfg.Config().get("charts.colormap")).colors 36 | 37 | # We will store the other symbols last plotted. This will save us rebuilding teh figure if teh symbols haven't 38 | # changed. 39 | __other_symbols = None 40 | 41 | def __init__(self, parent, **kwargs): 42 | # Super 43 | wx.MDIChildFrame.__init__(self, parent=parent, id=wx.ID_ANY, 44 | title=f"Divergence Graph for {kwargs['symbol']}") 45 | 46 | # Create logger 47 | self.__log = logging.getLogger(__name__) 48 | 49 | # Store the symbol 50 | self.symbol = kwargs['symbol'] 51 | 52 | # Create panel and sizer 53 | panel = scrolled.ScrolledPanel(self, wx.ID_ANY) 54 | sizer = wx.BoxSizer() 55 | panel.SetSizer(sizer) 56 | 57 | # Create figure and canvas. Add canvas to sizer 58 | self.__canvas = FigureCanvas(panel, wx.ID_ANY, plt.figure()) 59 | panel.GetSizer().Add(self.__canvas, 1, wx.ALL | wx.EXPAND) 60 | panel.SetupScrolling() 61 | 62 | # Refresh to show content 63 | self.refresh() 64 | 65 | def refresh(self): 66 | """ 67 | Refresh the graph 68 | :return: 69 | """ 70 | 71 | # Get tick data for base symbol 72 | symbol_tick_data = self.GetMDIParent().cor.get_ticks(self.symbol, cache_only=True) 73 | 74 | # Get the other symbols and their tick data 75 | other_symbols_data = self.__get_other_symbols_data() 76 | 77 | # Delete all axes from the figure. They will need to be recreated as the symbols may be different 78 | for axes in self.__canvas.figure.axes: 79 | axes.remove() 80 | 81 | # Plot for all other symbols 82 | num_subplots = 1 if len(other_symbols_data.keys()) == 0 else len(other_symbols_data.keys()) 83 | plotnum = 1 84 | axs = [] # Store the axes for sharing x axis 85 | for other_symbol in other_symbols_data: 86 | axs.append(self.__canvas.figure.add_subplot(num_subplots, 1, plotnum)) 87 | self.__plot(axes=axs[-1], base_symbol=self.symbol, other_symbol=other_symbol, 88 | base_symbol_data=symbol_tick_data, other_symbol_data=other_symbols_data[other_symbol]) 89 | 90 | # Next plot 91 | plotnum += 1 92 | 93 | # Share x axis of the last axes with all the others 94 | self.__share_xaxis(axs) 95 | 96 | # Redraw canvas 97 | self.__canvas.figure.tight_layout(pad=0.5) 98 | self.__canvas.draw() 99 | 100 | def __get_other_symbols_data(self): 101 | """ 102 | Gets the data required for the graphs 103 | :param self: 104 | :return: dict of other symbols their tick data 105 | """ 106 | # Get the symbols that this one has diverged against. 107 | data = self.GetMDIParent().cor.filtered_coefficient_data 108 | filtered_data = data.loc[( 109 | (data['Status'] == cor.STATUS_DIVERGED) | 110 | (data['Status'] == cor.STATUS_DIVERGING) | 111 | (data['Status'] == cor.STATUS_CONVERGING) 112 | ) & 113 | ( 114 | (data['Symbol 1'] == self.symbol) | 115 | (data['Symbol 2'] == self.symbol) 116 | )] 117 | 118 | # Get all symbols and remove the base one. We will need to ensure that this is first in the list 119 | other_symbols = list(filtered_data['Symbol 1'].append(filtered_data['Symbol 2']).drop_duplicates()) 120 | if self.symbol in other_symbols: 121 | other_symbols.remove(self.symbol) 122 | 123 | # Get the tick data other symbols and add to dict 124 | other_tick_data = {} 125 | for symbol in other_symbols: 126 | tick_data = self.GetMDIParent().cor.get_ticks(symbol, cache_only=True) 127 | other_tick_data[symbol] = tick_data 128 | 129 | return other_tick_data 130 | 131 | def __plot(self, axes, base_symbol, other_symbol, base_symbol_data, other_symbol_data): 132 | """ 133 | Plots the data on the axes 134 | :param axes: The subplot to plot onto 135 | :param base_symbol 136 | :param other_symbol: 137 | """ 138 | # Create the other axes. Will need an axes for the base symbol data and another for the other symbol data 139 | other_axes = axes.twinx() 140 | 141 | # Set plot title and axis labels 142 | axes.set_title(f"Tick Data for {base_symbol}:{other_symbol}") 143 | axes.set_ylabel(base_symbol, color=self.__colours[0], labelpad=10) 144 | other_axes.set_ylabel(other_symbol, color=self.__colours[1], labelpad=10) 145 | 146 | # Set the tick and axis colors 147 | self.__set_axes_color(axes, self.__colours[0], 'left') 148 | self.__set_axes_color(other_axes, self.__colours[1], 'right') 149 | 150 | # Plot both lines 151 | axes.plot(base_symbol_data['time'], base_symbol_data['ask'], color=self.__colours[0]) 152 | other_axes.plot(other_symbol_data['time'], other_symbol_data['ask'], color=self.__colours[1]) 153 | 154 | @staticmethod 155 | def __set_axes_color(axes, color, axis_loc='right'): 156 | """ 157 | Set the color for the axes, including axis line, ticks and tick labels 158 | :param axes: The axes to set color for. 159 | :param color: The color to set to 160 | :param axis_loc: The location of the axis, label and ticks. Either left for base symbol or right for others 161 | :return: 162 | """ 163 | # Change the color for the axis line, ticks and tick labels 164 | axes.spines[axis_loc].set_color(color) 165 | axes.tick_params(axis='y', colors=color) 166 | 167 | def __share_xaxis(self, axs): 168 | """ 169 | Share the xaxis of the last axes with all other axes. Remove axis tick labels for all but the last. Format axis 170 | tick labels for the last. 171 | :param axs: 172 | :return: 173 | """ 174 | if len(axs) > 0: 175 | last_ax = axs[-1] 176 | for ax in axs: 177 | # If we are not on the last one, share and hide tick labels. If we are on the last one, format tick 178 | # labels. 179 | if ax != last_ax: 180 | ax.sharex(last_ax) 181 | plt.setp(ax.xaxis.get_majorticklabels(), visible=False) 182 | else: 183 | # Ticks, labels and formats. Fixing xticks with FixedLocator but also using MaxNLocator to avoid 184 | # cramped x-labels 185 | ax.xaxis.set_major_locator(mticker.MaxNLocator(10)) 186 | ticks_loc = ax.get_xticks().tolist() 187 | ax.xaxis.set_major_locator(mticker.FixedLocator(ticks_loc)) 188 | ax.set_xticklabels(ticks_loc) 189 | ax.xaxis.set_major_formatter(self.__tick_fmt_time) 190 | plt.setp(ax.xaxis.get_majorticklabels(), rotation=45) 191 | 192 | def __del__(self): 193 | # Close all plots 194 | plt.close('all') 195 | -------------------------------------------------------------------------------- /configmeta.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | calculate: 3 | from: 4 | days: 5 | __label: Calculate from (days) 6 | __helptext: The number of days of data to be used to calculate the coefficient. 7 | timeframe: 8 | __label: Timeframe 9 | __helptext: The timeframe for price candles to use for the calculation. Possible values are 1=1 Minute candles; 2=2 Minute candles; 3=3 Minute candles; 4=4 Minute candles; 5=5 Minute candles; 6=6 Minute candles; 10=10 Minute candles; 15=15 Minute candles; 20=20 Minute candles; 30=30 Minute candles; 16385=1 Hour candles; 16386=2 Hour candles; 16387=3 Hour candles; 16388=4 Hour candles; 16390=6 Hour candles; 16392=8 Hour candles; 16396=12 Hour candles; 16408=1 Day candles; 32769=1 Week candles; or 49153=1 Month candles. 10 | min_prices: 11 | __label: Min Prices 12 | __helptext: The minimum number of candles required to calculate a coefficient from. If any of the symbols do not have at least this number of candles then the coefficient won't be calculated. 13 | max_set_size_diff_pct: 14 | __label: Min Set Size Difference % 15 | __helptext: For a meaningful coefficient calculation, the two sets of data should be of a similar size. This setting specifies the % difference allowed for a correlation to be calculated. The smallest set of candle data must be at least this values % of the largest set. 16 | overlap_pct: 17 | __label: Overlap % 18 | __helptext: The dates and times in the two sets of data must match. The coefficient will only be calculated against the dates that overlap. Any non overlapping dates will be discarded. This setting specifies the minimum size of the overlapping data when compared to the smallest set as a %. A coefficient will not be calculated if this threshold is not met. 19 | max_p_value: 20 | __label: Max Pearsonr P Value 21 | __helptext: The maximum P value for the coefficient to be considered valid. A full explanation on the correlation coefficient P value is available in the scipy pearsonr documentation. 22 | monitor: 23 | interval: 24 | __label: Monitoring Interval 25 | __helptext: The number of seconds between monitoring events. 26 | calculations: 27 | long: 28 | __label: Long 29 | __helptext: The settings for the correlation calculation using the longest timeframe. 30 | from: 31 | __label: From (Minutes) 32 | __helptext: The number of minutes of data to be used to calculate the long coefficient. 33 | min_prices: 34 | __label: Min Prices 35 | __helptext: Tick data will be converted to 1 second candles prior to calculation. This will enable data to be matched between symbols. This setting specifies the minimum number of price candles required to calculate a coefficient from. If any of the symbols do not have at least number of candles then the coefficient won't be calculated. 36 | max_set_size_diff_pct: 37 | __label: Max Set Size Difference % 38 | __helptext: For a meaningful coefficient calculation, the two sets of data should be of a similar size. This setting specifies the % difference allowed for a correlation to be calculated. The smallest set of price candles must be at least this values % of the largest set. 39 | overlap_pct: 40 | __label: Overlap % 41 | __helptext: The dates and times in the two sets of data must match. The ticks will be converted to 1 second price candles before calculation. The coefficient will only be calculated against the times from the candles that overlap. Any non overlapping times will be discarded. This setting specifies the minimum size of the overlapping data when compared to the smallest set as a %. A coefficient will not be calculated if this threshold is not met. 42 | max_p_value: 43 | __label: Max Pearsonr P Value 44 | __helptext: The maximum P value for the coefficient to be considered valid. A full explanation on the correlation coefficient P value is available in the scipy pearsonr documentation. 45 | medium: 46 | __label: Medium 47 | __helptext: The settings for the correlation calculation using a medium timeframe. 48 | from: 49 | __label: From (Minutes) 50 | __helptext: The number of minutes of data to be used to calculate the medium coefficient. 51 | min_prices: 52 | __label: Min Prices 53 | __helptext: Tick data will be converted to 1 second candles prior to calculation. This will enable data to be matched between symbols. This setting specifies the minimum number of price candles required to calculate a coefficient from. If any of the symbols do not have at least number of candles then the coefficient won't be calculated. 54 | max_set_size_diff_pct: 55 | __label: Max Set Size Difference % 56 | __helptext: For a meaningful coefficient calculation, the two sets of data should be of a similar size. This setting specifies the % difference allowed for a correlation to be calculated. The smallest set of price candles must be at least this values % of the largest set. 57 | overlap_pct: 58 | __label: Overlap % 59 | __helptext: The dates and times in the two sets of data must match. The ticks will be converted to 1 second price candles before calculation. The coefficient will only be calculated against the times from the candles that overlap. Any non overlapping times will be discarded. This setting specifies the minimum size of the overlapping data when compared to the smallest set as a %. A coefficient will not be calculated if this threshold is not met. 60 | max_p_value: 61 | __label: Max Pearsonr P Value 62 | __helptext: The maximum P value for the coefficient to be considered valid. A full explanation on the correlation coefficient P value is available in the scipy pearsonr documentation. 63 | short: 64 | __label: Short 65 | __helptext: The settings for the correlation calculation using the shortest timeframe. 66 | from: 67 | __label: From (Minutes) 68 | __helptext: The number of minutes of data to be used to calculate the short coefficient. 69 | min_prices: 70 | __label: Min Prices 71 | __helptext: Tick data will be converted to 1 second candles prior to calculation. This will enable data to be matched between symbols. This setting specifies the minimum number of price candles required to calculate a coefficient from. If any of the symbols do not have at least number of candles then the coefficient won't be calculated. 72 | max_set_size_diff_pct: 73 | __label: Max Set Size Difference % 74 | __helptext: For a meaningful coefficient calculation, the two sets of data should be of a similar size. This setting specifies the % difference allowed for a correlation to be calculated. The smallest set of price candles must be at least this values % of the largest set. 75 | overlap_pct: 76 | __label: Overlap % 77 | __helptext: The dates and times in the two sets of data must match. The ticks will be converted to 1 second price candles before calculation. The coefficient will only be calculated against the times from the candles that overlap. Any non overlapping times will be discarded. This setting specifies the minimum size of the overlapping data when compared to the smallest set as a %. A coefficient will not be calculated if this threshold is not met. 78 | max_p_value: 79 | __label: Max Pearsonr P Value 80 | __helptext: The maximum P value for the coefficient to be considered valid. A full explanation on the correlation coefficient P value is available in the scipy pearsonr documentation. 81 | monitoring_threshold: 82 | __label: Monitoring Threshold 83 | __helptext: Only pairs with a coefficient over this threshold will be displayed and monitored. 84 | divergence_threshold: 85 | __label: Divergence Threshold 86 | __helptext: The application will consider a pair to have diverged if the correlation coefficient for all timeframes (long, medium and short) falls below this threshold. 87 | monitor_inverse: 88 | __label: Monitor Inverse 89 | __helptext: Monitor Inverse Correlations (uses negative scale with -1 being fully inversly correlated) 90 | tick_cache_time: 91 | __label: Tick Cache Time 92 | __helptext: Every calculation requires tick data for both symbols. Tick data will be cached for this number of seconds before being retrieved from MetaTrader. Some caching is recommended as a single monitoring run will request the same data for symbols that form multiple correlated pairs. 93 | autosave: 94 | __label: Auto Save 95 | __helptext: Whether to auto save after every monitoring event. If a file was opened or has been saved, then the data will be saved to this file, otherwise the data will be saved to a file named autosave.cpd. 96 | charts: 97 | colormap: 98 | __label: Color Map 99 | __helptext: The matplotlib color pallet to use for plotting graphs. A list of pallets is available at https://matplotlib.org/stable/tutorials/colors/colormaps.html 100 | developer: 101 | inspection: 102 | __label: Inspection 103 | __helptext: Provide GUI Inspection guidelines for developers modifying the GUI. 104 | ... -------------------------------------------------------------------------------- /mt5_correlation/gui/mdi_child_correlationgraph.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import matplotlib.dates 3 | import matplotlib.pyplot as plt 4 | import matplotlib.ticker as mticker 5 | import wx 6 | import wxconfig as cfg 7 | import wx.lib.scrolledpanel as scrolled 8 | 9 | from matplotlib.backends.backend_wxagg import FigureCanvasWxAgg as FigureCanvas 10 | 11 | import mt5_correlation.gui.mdi as mdi 12 | 13 | 14 | class MDIChildCorrelationGraph(mdi.CorrelationMDIChild): 15 | """ 16 | Shows the graphs for the specified correlation 17 | """ 18 | 19 | symbols = None # Symbols for correlation. Public as we use to check if window for the symbol pair is already open. 20 | 21 | # Date formats for graphs 22 | __tick_fmt_date = matplotlib.dates.DateFormatter('%d-%b') 23 | __tick_fmt_time = matplotlib.dates.DateFormatter('%H:%M:%S') 24 | 25 | # Colors for graph lines for symbol1 and symbol2. Will use first 2 colours in colormap 26 | __colours = matplotlib.cm.get_cmap(cfg.Config().get("charts.colormap")).colors 27 | 28 | # Fig, axes and canvas 29 | __fig = None 30 | __axs = None 31 | __canvas = None 32 | 33 | def __init__(self, parent, **kwargs): 34 | # Super 35 | wx.MDIChildFrame.__init__(self, parent=parent, id=wx.ID_ANY, 36 | title=f"Correlation Status for {kwargs['symbols'][0]}:{kwargs['symbols'][1]}") 37 | 38 | # Create logger 39 | self.__log = logging.getLogger(__name__) 40 | 41 | # Store the symbols 42 | self.symbols = kwargs['symbols'] 43 | 44 | # We will freeze this frame and thaw once constructed to avoid flicker. 45 | self.Freeze() 46 | 47 | # Draw the empty graphs. We will populate with data in refresh. We will have 3 charts: 48 | # 1) Data used to calculate base coefficient for both symbols (2 lines on chart); 49 | # 2) Data used to calculate latest coefficient for both symbols (2 lines on chart); and 50 | # 3) Coefficient history and the divergence threshold lines 51 | 52 | # Create fig and 3 axes. 53 | self.__fig, self.__axs = plt.subplots(3) 54 | 55 | # Create additional axis for second line on charts 1 & 2 56 | self.__s2axs = [self.__axs[0].twinx(), self.__axs[1].twinx()] 57 | 58 | # Set titles 59 | self.__axs[0].set_title(f"Base Coefficient Price Data for {self.symbols[0]}:{self.symbols[1]}") 60 | self.__axs[1].set_title(f"Coefficient Tick Data for {self.symbols[0]}:{self.symbols[1]}") 61 | self.__axs[2].set_title(f"Coefficient History for {self.symbols[0]}:{self.symbols[1]}") 62 | 63 | # Set Y Labels and tick colours for charts 1 & 2. Left for symbol1, right for symbol2 64 | for i in range(0, 2): 65 | self.__axs[i].set_ylabel(f"{self.symbols[0]}", color=self.__colours[0], labelpad=10) 66 | self.__axs[i].tick_params(axis='y', labelcolor=self.__colours[0]) 67 | self.__s2axs[i].set_ylabel(f"{self.symbols[1]}", color=self.__colours[1], labelpad=10) 68 | self.__s2axs[i].tick_params(axis='y', labelcolor=self.__colours[1]) 69 | 70 | # Set Y label and limits for 3rd chart. Limits will be coefficients range from -1 to 1 71 | self.__axs[2].set_ylabel('Coefficient', labelpad=10) 72 | self.__axs[2].set_ylim([-1, 1]) 73 | 74 | # Set X labels to ''. Workaround as matplotlib is not leaving space for ticks 75 | for ax in self.__axs: 76 | ax.set_xlabel(" ", labelpad=10) 77 | 78 | # Layout with padding between charts 79 | self.__fig.tight_layout(pad=0.5) 80 | 81 | # Create panel and sizer. This will provide scrollbar 82 | panel = scrolled.ScrolledPanel(self, wx.ID_ANY) 83 | sizer = wx.BoxSizer() 84 | panel.SetSizer(sizer) 85 | 86 | # Add fig to canvas and canvas to sizer. Thaw window to update 87 | self.__canvas = FigureCanvas(panel, wx.ID_ANY, self.__fig) 88 | sizer.Add(self.__canvas, 1, wx.ALL | wx.EXPAND) 89 | self.Thaw() 90 | 91 | # Setup scrolling 92 | panel.SetupScrolling() 93 | 94 | # Refresh to show content 95 | self.refresh() 96 | 97 | def refresh(self): 98 | """ 99 | Refresh the graph 100 | :return: 101 | """ 102 | # Get the price data for the base coefficient calculation, tick data that was used to calculate last 103 | # coefficient and and the coefficient history data 104 | price_data = [self.GetMDIParent().cor.get_price_data(self.symbols[0]), 105 | self.GetMDIParent().cor.get_price_data(self.symbols[1])] 106 | 107 | tick_data = [self.GetMDIParent().cor.get_ticks(self.symbols[0], cache_only=True), 108 | self.GetMDIParent().cor.get_ticks(self.symbols[1], cache_only=True)] 109 | 110 | history_data = [] 111 | for timeframe in cfg.Config().get('monitor.calculations'): 112 | frm = cfg.Config().get(f'monitor.calculations.{timeframe}.from') 113 | history_data.append(self.GetMDIParent().cor.get_coefficient_history( 114 | {'Symbol 1': self.symbols[0], 'Symbol 2': self.symbols[1], 'Timeframe': frm})) 115 | 116 | # Check what data we have available 117 | price_data_available = price_data is not None and len(price_data) == 2 and price_data[0] is not None and \ 118 | price_data[1] is not None and len(price_data[0]) > 0 and len(price_data[1]) > 0 119 | 120 | tick_data_available = tick_data is not None and len(tick_data) == 2 and tick_data[0] is not None and \ 121 | tick_data[1] is not None and len(tick_data[0]) > 0 and len(tick_data[1]) > 0 122 | 123 | history_data_available = history_data is not None and len(history_data) > 0 124 | 125 | # Get all plots for coefficient history. History can contain multiple plots for different timeframes. They 126 | # will all be plotted on the same chart. 127 | times = [] 128 | coefficients = [] 129 | if history_data_available: 130 | for hist in history_data: 131 | times.append(hist['Date To']) 132 | coefficients.append(hist['Coefficient']) 133 | 134 | # Update graphs where we have data available 135 | if price_data_available: 136 | # Update range and ticks 137 | xrange = [min(min(price_data[0]['time']), min(price_data[1]['time'])), 138 | max(max(price_data[0]['time']), max(price_data[1]['time']))] 139 | self.__axs[0].set_xlim(xrange) 140 | 141 | # Plot both lines 142 | self.__axs[0].plot(price_data[0]['time'], price_data[0]['close'], 143 | color=self.__colours[0]) 144 | self.__s2axs[0].plot(price_data[1]['time'], price_data[1]['close'], 145 | color=self.__colours[1]) 146 | 147 | # Ticks, labels and formats. Fixing xticks with FixedLocator but also using MaxNLocator to avoid 148 | # cramped x-labels 149 | if len(price_data[0]['time']) > 0: 150 | self.__axs[0].xaxis.set_major_locator(mticker.MaxNLocator(10)) 151 | ticks_loc = self.__axs[0].get_xticks().tolist() 152 | self.__axs[0].xaxis.set_major_locator(mticker.FixedLocator(ticks_loc)) 153 | self.__axs[0].set_xticklabels(ticks_loc) 154 | self.__axs[0].xaxis.set_major_formatter(self.__tick_fmt_date) 155 | plt.setp(self.__axs[0].xaxis.get_majorticklabels(), rotation=45) 156 | 157 | if tick_data_available: 158 | # Update range and ticks 159 | xrange = [min(min(tick_data[0]['time']), min(tick_data[1]['time'])), 160 | max(max(tick_data[0]['time']), max(tick_data[1]['time']))] 161 | self.__axs[1].set_xlim(xrange) 162 | 163 | # Plot both lines 164 | self.__axs[1].plot(tick_data[0]['time'], tick_data[0]['ask'], 165 | color=self.__colours[0]) 166 | self.__s2axs[1].plot(tick_data[1]['time'], tick_data[1]['ask'], 167 | color=self.__colours[1]) 168 | 169 | if len(tick_data[0]['time']) > 0: 170 | self.__axs[1].xaxis.set_major_locator(mticker.MaxNLocator(10)) 171 | ticks_loc = self.__axs[1].get_xticks().tolist() 172 | self.__axs[1].xaxis.set_major_locator(mticker.FixedLocator(ticks_loc)) 173 | self.__axs[1].set_xticklabels(ticks_loc) 174 | self.__axs[1].xaxis.set_major_formatter(self.__tick_fmt_time) 175 | plt.setp(self.__axs[1].xaxis.get_majorticklabels(), rotation=45) 176 | 177 | if history_data_available: 178 | # Plot. There may be more than one set of data for chart. One for each coefficient date range. Convert 179 | # single data to list, then loop to plot 180 | xdata = times if isinstance(times, list) else [times, ] 181 | ydata = coefficients if isinstance(coefficients, list) else [coefficients, ] 182 | 183 | for i in range(0, len(xdata)): 184 | self.__axs[2].scatter(xdata[i], ydata[i], s=1) 185 | 186 | # Ticks, labels and formats. Fixing xticks with FixedLocator but also using MaxNLocator to avoid 187 | # cramped x-labels 188 | if len(times[0].array) > 0: 189 | self.__axs[2].xaxis.set_major_locator(mticker.MaxNLocator(10)) 190 | ticks_loc = self.__axs[2].get_xticks().tolist() 191 | self.__axs[2].xaxis.set_major_locator(mticker.FixedLocator(ticks_loc)) 192 | self.__axs[2].set_xticklabels(ticks_loc) 193 | self.__axs[2].xaxis.set_major_formatter(self.__tick_fmt_time) 194 | plt.setp(self.__axs[2].xaxis.get_majorticklabels(), rotation=45) 195 | 196 | # Legend 197 | self.__axs[2].legend([f"{cfg.Config().get('monitor.calculations.long.from')} Minutes", 198 | f"{cfg.Config().get('monitor.calculations.medium.from')} Minutes", 199 | f"{cfg.Config().get('monitor.calculations.short.from')} Minutes"]) 200 | 201 | # Lines showing divergence threshold. 2 if we are monitoring inverse correlations. 202 | divergence_threshold = self.GetMDIParent().cor.divergence_threshold 203 | monitor_inverse = self.GetMDIParent().cor.monitor_inverse 204 | 205 | if divergence_threshold is not None: 206 | self.__axs[2].axhline(y=divergence_threshold, color="red", label='_nolegend_', linewidth=1) 207 | if monitor_inverse: 208 | self.__axs[2].axhline(y=divergence_threshold * -1, color="red", label='_nolegend_', linewidth=1) 209 | 210 | # Redraw canvas 211 | self.__canvas.draw() 212 | 213 | def __del__(self): 214 | # Close all plots 215 | plt.close('all') 216 | -------------------------------------------------------------------------------- /mt5_correlation/gui/mdi.py: -------------------------------------------------------------------------------- 1 | import abc 2 | import importlib 3 | import logging 4 | 5 | import pytz 6 | import wx 7 | import wx.lib.inspection as ins 8 | import wxconfig 9 | import wxconfig as cfg 10 | 11 | from datetime import datetime, timedelta 12 | 13 | from mt5_correlation import correlation as cor 14 | 15 | 16 | class CorrelationMDIFrame(wx.MDIParentFrame): 17 | """ 18 | The MDI Frame window for the correlation monitoring application 19 | """ 20 | # The correlation instance that calculates coefficients and monitors for divergence. Needs to be accessible to 21 | # child frames. 22 | cor = None 23 | 24 | __opened_filename = None # So we can save to same file as we opened 25 | __log = None # The logger 26 | __menu_item_monitor = None # We need to store this menu item so that we can check if it is checked or not. 27 | 28 | def __init__(self): 29 | # Super 30 | wx.MDIParentFrame.__init__(self, parent=None, id=wx.ID_ANY, title="Divergence Monitor", 31 | pos=wx.Point(x=cfg.Config().get('window.x'), y=cfg.Config().get('window.y')), 32 | size=wx.Size(width=cfg.Config().get('window.width'), 33 | height=cfg.Config().get('window.height')), 34 | style=cfg.Config().get('window.style')) 35 | 36 | # Create logger 37 | self.__log = logging.getLogger(__name__) 38 | 39 | # Create correlation instance to maintain state of calculated coefficients. Set params from config 40 | self.cor = cor.Correlation(monitoring_threshold=cfg.Config().get("monitor.monitoring_threshold"), 41 | divergence_threshold=cfg.Config().get("monitor.divergence_threshold"), 42 | monitor_inverse=cfg.Config().get("monitor.monitor_inverse")) 43 | 44 | # Status bar. 2 fields, one for monitoring status and one for general status. On open, monitoring status is not 45 | # monitoring. SetBackgroundColour will change colour of both. Couldn't find a way to set on single field only. 46 | self.__statusbar = self.CreateStatusBar(2) 47 | self.__statusbar.SetStatusWidths([100, -1]) 48 | self.SetStatusText("Not Monitoring", 0) 49 | 50 | # Create menu bar and bind menu items to methods 51 | self.menubar = wx.MenuBar() 52 | 53 | # File menu and items 54 | menu_file = wx.Menu() 55 | self.Bind(wx.EVT_MENU, self.__on_open_file, menu_file.Append(wx.ID_ANY, "&Open", "Open correlations file.")) 56 | self.Bind(wx.EVT_MENU, self.__on_save_file, menu_file.Append(wx.ID_ANY, "Save", "Save correlations file.")) 57 | self.Bind(wx.EVT_MENU, self.__on_save_file_as, 58 | menu_file.Append(wx.ID_ANY, "Save As", "Save correlations file.")) 59 | menu_file.AppendSeparator() 60 | self.Bind(wx.EVT_MENU, self.__on_open_settings, 61 | menu_file.Append(wx.ID_ANY, "Settings", "Change application settings.")) 62 | menu_file.AppendSeparator() 63 | self.Bind(wx.EVT_MENU, self.__on_exit, menu_file.Append(wx.ID_ANY, "Exit", "Close the application")) 64 | self.menubar.Append(menu_file, "&File") 65 | 66 | # Coefficient menu and items 67 | menu_coef = wx.Menu() 68 | self.Bind(wx.EVT_MENU, self.__on_calculate, 69 | menu_coef.Append(wx.ID_ANY, "Calculate", "Calculate base coefficients.")) 70 | self.__menu_item_monitor = menu_coef.Append(wx.ID_ANY, "Monitor", 71 | "Monitor correlated pairs for changes to coefficient.", 72 | kind=wx.ITEM_CHECK) 73 | self.Bind(wx.EVT_MENU, self.__on_monitor, self.__menu_item_monitor) 74 | menu_coef.AppendSeparator() 75 | self.Bind(wx.EVT_MENU, self.__on_clear, 76 | menu_coef.Append(wx.ID_ANY, "Clear", "Clear coefficient and price history.")) 77 | self.menubar.Append(menu_coef, "Coefficient") 78 | 79 | # View menu and items 80 | menu_view = wx.Menu() 81 | self.Bind(wx.EVT_MENU, self.__on_view_status, menu_view.Append(wx.ID_ANY, "Status", 82 | "View status of correlations.")) 83 | self.Bind(wx.EVT_MENU, self.__on_view_diverged, menu_view.Append(wx.ID_ANY, "Diverged Symbols", 84 | "View diverged symbols.")) 85 | self.menubar.Append(menu_view, "&View") 86 | 87 | # Help menu and items 88 | help_menu = wx.Menu() 89 | self.Bind(wx.EVT_MENU, self.__on_view_log, help_menu.Append(wx.ID_ANY, "View Log", "Show application log.")) 90 | self.Bind(wx.EVT_MENU, self.__on_view_help, help_menu.Append(wx.ID_ANY, "Help", 91 | "Show application usage instructions.")) 92 | 93 | self.menubar.Append(help_menu, "&Help") 94 | 95 | # Set menu bar 96 | self.SetMenuBar(self.menubar) 97 | 98 | # Set up timer to refresh 99 | self.timer = wx.Timer(self) 100 | self.Bind(wx.EVT_TIMER, self.__on_timer, self.timer) 101 | 102 | # Bind window close event 103 | self.Bind(wx.EVT_CLOSE, self.__on_close, self) 104 | 105 | def __on_close(self, event): 106 | """ 107 | Window closing. Save coefficients and stop monitoring. 108 | :param event: 109 | :return: 110 | """ 111 | # Save pos and size 112 | x, y = self.GetPosition() 113 | width, height = self.GetSize() 114 | cfg.Config().set('window.x', x) 115 | cfg.Config().set('window.y', y) 116 | cfg.Config().set('window.width', width) 117 | cfg.Config().set('window.height', height) 118 | 119 | # Style 120 | style = self.GetWindowStyle() 121 | cfg.Config().set('window.style', style) 122 | 123 | cfg.Config().save() 124 | 125 | # Stop monitoring 126 | self.cor.stop_monitor() 127 | 128 | # End 129 | event.Skip() 130 | 131 | def __on_open_file(self, evt): 132 | with wx.FileDialog(self, "Open Coefficients file", wildcard="cpd (*.cpd)|*.cpd", 133 | style=wx.FD_OPEN | wx.FD_FILE_MUST_EXIST) as fileDialog: 134 | if fileDialog.ShowModal() == wx.ID_CANCEL: 135 | return # the user changed their mind 136 | 137 | # Load the file chosen by the user. 138 | self.__opened_filename = fileDialog.GetPath() 139 | 140 | self.SetStatusText(f"Loading file {self.__opened_filename}.", 1) 141 | self.cor.load(self.__opened_filename) 142 | 143 | # Show calculated data and refresh all opened frames 144 | self.__on_view_status(evt) 145 | self.__refresh() 146 | 147 | self.SetStatusText(f"File {self.__opened_filename} loaded.", 1) 148 | 149 | def __on_save_file(self, evt): 150 | self.SetStatusText(f"Saving file as {self.__opened_filename}", 1) 151 | 152 | if self.__opened_filename is None: 153 | self.__on_save_file_as(evt) 154 | else: 155 | self.cor.save(self.__opened_filename) 156 | 157 | self.SetStatusText(f"File saved as {self.__opened_filename}", 1) 158 | 159 | def __on_save_file_as(self, evt): 160 | with wx.FileDialog(self, "Save Coefficients file", wildcard="cpd (*.cpd)|*.cpd", 161 | style=wx.FD_SAVE) as fileDialog: 162 | if fileDialog.ShowModal() == wx.ID_CANCEL: 163 | return # the user changed their mind 164 | 165 | # Save the file and price data file, changing opened filename so next save writes to new file 166 | self.SetStatusText(f"Saving file as {self.__opened_filename}", 1) 167 | 168 | self.__opened_filename = fileDialog.GetPath() 169 | self.cor.save(self.__opened_filename) 170 | 171 | self.SetStatusText(f"File saved as {self.__opened_filename}", 1) 172 | 173 | def __on_open_settings(self, evt): 174 | settings_dialog = cfg.SettingsDialog(parent=self, exclude=['window']) 175 | res = settings_dialog.ShowModal() 176 | if res == wx.ID_OK: 177 | # Stop the monitor 178 | self.cor.stop_monitor() 179 | 180 | # Build calculation params and restart the monitor 181 | calculation_params = [cfg.Config().get('monitor.calculations.long'), 182 | cfg.Config().get('monitor.calculations.medium'), 183 | cfg.Config().get('monitor.calculations.short')] 184 | 185 | self.cor.start_monitor(interval=cfg.Config().get('monitor.interval'), 186 | calculation_params=calculation_params, 187 | cache_time=cfg.Config().get('monitor.tick_cache_time'), 188 | autosave=cfg.Config().get('monitor.autosave'), 189 | filename=self.__opened_filename) 190 | 191 | # Refresh all open child frames 192 | self.__refresh() 193 | 194 | def __on_exit(self, evt): 195 | # Close 196 | self.Close() 197 | 198 | def __on_calculate(self, evt): 199 | # set time zone to UTC to avoid local offset issues, and get from and to dates (a week ago to today) 200 | timezone = pytz.timezone("Etc/UTC") 201 | utc_to = datetime.now(tz=timezone) 202 | utc_from = utc_to - timedelta(days=cfg.Config().get('calculate.from.days')) 203 | 204 | # Calculate 205 | self.SetStatusText("Calculating coefficients.", 1) 206 | self.cor.calculate(date_from=utc_from, date_to=utc_to, 207 | timeframe=cfg.Config().get('calculate.timeframe'), 208 | min_prices=cfg.Config().get('calculate.min_prices'), 209 | max_set_size_diff_pct=cfg.Config().get('calculate.max_set_size_diff_pct'), 210 | overlap_pct=cfg.Config().get('calculate.overlap_pct'), 211 | max_p_value=cfg.Config().get('calculate.max_p_value')) 212 | self.SetStatusText("", 1) 213 | 214 | # Show calculated data and refresh frames 215 | self.__on_view_status(evt) 216 | self.__refresh() 217 | 218 | def __on_monitor(self, evt): 219 | # Check state of toggle menu. If on, then start monitoring, else stop 220 | if self.__menu_item_monitor.IsChecked(): 221 | self.__log.info("Starting monitoring for changes to coefficients.") 222 | self.SetStatusText("Monitoring", 0) 223 | self.__statusbar.SetBackgroundColour('green') 224 | self.__statusbar.Refresh() 225 | 226 | self.timer.Start(cfg.Config().get('monitor.interval') * 1000) 227 | 228 | # Autosave filename 229 | filename = self.__opened_filename if self.__opened_filename is not None else 'autosave.cpd' 230 | 231 | # Build calculation params and start monitor 232 | calculation_params = [cfg.Config().get('monitor.calculations.long'), 233 | cfg.Config().get('monitor.calculations.medium'), 234 | cfg.Config().get('monitor.calculations.short')] 235 | 236 | self.cor.start_monitor(interval=cfg.Config().get('monitor.interval'), 237 | calculation_params=calculation_params, 238 | cache_time=cfg.Config().get('monitor.tick_cache_time'), 239 | autosave=cfg.Config().get('monitor.autosave'), 240 | filename=filename) 241 | else: 242 | self.__log.info("Stopping monitoring.") 243 | self.SetStatusText("Not Monitoring", 0) 244 | self.__statusbar.SetBackgroundColour('lightgray') 245 | self.__statusbar.Refresh() 246 | self.timer.Stop() 247 | self.cor.stop_monitor() 248 | 249 | def __on_clear(self, evt): 250 | # Clear the history 251 | self.cor.clear_coefficient_history() 252 | 253 | # Refresh opened child frames 254 | self.__refresh() 255 | 256 | def __on_timer(self, evt): 257 | # Refresh opened child frames 258 | self.__refresh() 259 | 260 | # Set status message 261 | self.SetStatusText(f"Status updated at {self.cor.get_last_calculation():%d-%b %H:%M:%S}.", 1) 262 | 263 | def __on_view_status(self, evt): 264 | FrameManager.open_frame(parent=self, frame_module='mt5_correlation.gui.mdi_child_status', 265 | frame_class='MDIChildStatus', 266 | raise_if_open=True) 267 | 268 | def __on_view_diverged(self, evt): 269 | FrameManager.open_frame(parent=self, frame_module='mt5_correlation.gui.mdi_child_diverged_symbols', 270 | frame_class='MDIChildDivergedSymbols', 271 | raise_if_open=True) 272 | 273 | def __on_view_log(self, evt): 274 | FrameManager.open_frame(parent=self, frame_module='mt5_correlation.gui.mdi_child_log', 275 | frame_class='MDIChildLog', 276 | raise_if_open=True) 277 | 278 | def __on_view_help(self, evt): 279 | FrameManager.open_frame(parent=self, frame_module='mt5_correlation.gui.mdi_child_help', 280 | frame_class='MDIChildHelp', 281 | raise_if_open=True) 282 | 283 | def __refresh(self): 284 | """ 285 | Refresh all open child frames 286 | :return: 287 | """ 288 | children = self.GetChildren() 289 | 290 | for child in children: 291 | if isinstance(child, CorrelationMDIChild): 292 | child.refresh() 293 | elif isinstance(child, wx.StatusBar) or isinstance(child, ins.InspectionFrame) or \ 294 | isinstance(child, wxconfig.SettingsDialog): 295 | # Ignore 296 | pass 297 | else: 298 | raise Exception(f"MDI Child for application must implement CorrelationMDIChild. MDI Child is " 299 | f"{type(child)}.") 300 | 301 | 302 | class CorrelationMDIChild(wx.MDIChildFrame): 303 | """ 304 | Interface for all MDI Children supported by the MDIParent 305 | """ 306 | 307 | @abc.abstractmethod 308 | def refresh(self): 309 | """ 310 | Must be implemented. Refreshes the content. Called by MDIParents __refresh method 311 | :return: 312 | """ 313 | raise NotImplementedError 314 | 315 | 316 | class FrameManager: 317 | """ 318 | Manages the opening and raising of MDIChild frames 319 | """ 320 | @staticmethod 321 | def open_frame(parent, frame_module, frame_class, raise_if_open=True, **kwargs): 322 | """ 323 | Opens the frame specified by the frame class 324 | :param parent: The MDIParentFrame to open the child frame into 325 | :param frame_module: A string specifying the module containing the frame class to open or raise 326 | :param frame_class: A string specifying the frame class to open or raise 327 | :param raise_if_open: Whether the frame should raise rather than open if an instance is already open. 328 | :param kwargs: A dict of parameters to pass to frame constructor. These will also be checked in raise_if_open 329 | to determine uniqueness (i.e. If a frame of the same class is already open but its params are different, 330 | then the frame will be opened again with the new params instead of being raised.) 331 | :return: 332 | """ 333 | 334 | # Load the module and class 335 | module = importlib.import_module(frame_module) 336 | clazz = getattr(module, frame_class) 337 | 338 | # Do we have an opened instance 339 | opened_instance = None 340 | for child in parent.GetChildren(): 341 | if isinstance(child, clazz): 342 | # do the args match 343 | match = True 344 | for key in kwargs: 345 | if kwargs[key] != getattr(child, key): 346 | match = False 347 | 348 | # Only open existing instance if args matched 349 | if match: 350 | opened_instance = child 351 | 352 | # If we dont have an opened instance or raise_on_open is False then open new frame, otherwise raise it 353 | if opened_instance is None or raise_if_open is False: 354 | if len(kwargs) == 0: 355 | clazz(parent=parent).Show(True) 356 | else: 357 | clazz(parent=parent, **kwargs).Show(True) 358 | else: 359 | opened_instance.Raise() 360 | -------------------------------------------------------------------------------- /test/test_correlation.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from unittest.mock import patch, PropertyMock 3 | import time 4 | import mt5_correlation.correlation as correlation 5 | import pandas as pd 6 | from datetime import datetime, timedelta 7 | from test_mt5 import Symbol 8 | import random 9 | import os 10 | 11 | 12 | class TestCorrelation(unittest.TestCase): 13 | # Mock symbols. 4 Symbols, 3 visible. 14 | mock_symbols = [Symbol(name='SYMBOL1', visible=True), 15 | Symbol(name='SYMBOL2', visible=True), 16 | Symbol(name='SYMBOL3', visible=False), 17 | Symbol(name='SYMBOL4', visible=True), 18 | Symbol(name='SYMBOL5', visible=True)] 19 | 20 | # Start and end date for price data and mock prices: base; correlated; and uncorrelated. 21 | start_date = None 22 | end_date = None 23 | price_columns = None 24 | mock_base_prices = None 25 | mock_correlated_prices = None 26 | mock_uncorrelated_prices = None 27 | 28 | def setUp(self): 29 | """ 30 | Creates some price data fro use in tests 31 | :return: 32 | """ 33 | # Start and end date for price data and mock price dataframes. One for: base; correlated; uncorrelated and 34 | # different dates. 35 | self.start_date = datetime(2021, 1, 1, 1, 5, 0) 36 | self.end_date = datetime(2021, 1, 1, 11, 30, 0) 37 | self.price_columns = ['time', 'close'] 38 | self.mock_base_prices = pd.DataFrame(columns=self.price_columns) 39 | self.mock_correlated_prices = pd.DataFrame(columns=self.price_columns) 40 | self.mock_uncorrelated_prices = pd.DataFrame(columns=self.price_columns) 41 | self.mock_correlated_different_dates = pd.DataFrame(columns=self.price_columns) 42 | self.mock_inverse_correlated_prices = pd.DataFrame(columns=self.price_columns) 43 | 44 | # Build the price data for the test. One price every 5 minutes for 500 rows. Base will use min for price, 45 | # correlated will use min + 5 and uncorrelated will use random 46 | for date in (self.start_date + timedelta(minutes=m) for m in range(0, 500*5, 5)): 47 | self.mock_base_prices = self.mock_base_prices.append(pd.DataFrame(columns=self.price_columns, 48 | data=[[date, date.minute]])) 49 | self.mock_correlated_prices = \ 50 | self.mock_correlated_prices.append(pd.DataFrame(columns=self.price_columns, 51 | data=[[date, date.minute + 5]])) 52 | self.mock_uncorrelated_prices = \ 53 | self.mock_uncorrelated_prices.append(pd.DataFrame(columns=self.price_columns, 54 | data=[[date, random.randint(0, 1000000)]])) 55 | 56 | self.mock_correlated_different_dates = \ 57 | self.mock_correlated_different_dates.append(pd.DataFrame(columns=self.price_columns, 58 | data=[[date + timedelta(minutes=100), 59 | date.minute + 5]])) 60 | self.mock_inverse_correlated_prices = \ 61 | self.mock_inverse_correlated_prices.append(pd.DataFrame(columns=self.price_columns, 62 | data=[[date, (date.minute + 5) * -1]])) 63 | 64 | @patch('mt5_correlation.mt5.MetaTrader5') 65 | def test_calculate(self, mock): 66 | """ 67 | Test the calculate method. Uses mock for MT5 symbols and prices. 68 | :param mock: 69 | :return: 70 | """ 71 | # Mock symbol return values 72 | mock.symbols_get.return_value = self.mock_symbols 73 | 74 | # Correlation class 75 | cor = correlation.Correlation(monitoring_threshold=1, monitor_inverse=True) 76 | 77 | # Calculate for price data. We should have 100% matching dates in sets. Get prices should be called 4 times. 78 | # We don't have a SYMBOL3 as this is set as not visible. Correlations should be as follows: 79 | # SYMBOL1:SYMBOL2 should be fully correlated (1) 80 | # SYMBOL1:SYMBOL4 should be uncorrelated (0) 81 | # SYMBOL1:SYMBOL5 should be negatively correlated 82 | # SYMBOL2:SYMBOL5 should be negatively correlated 83 | # We will not use p_value as the last set uses random numbers so p value will not be useful. 84 | mock.copy_rates_range.side_effect = [self.mock_base_prices, self.mock_correlated_prices, 85 | self.mock_uncorrelated_prices, self.mock_inverse_correlated_prices] 86 | cor.calculate(date_from=self.start_date, date_to=self.end_date, timeframe=5, min_prices=100, 87 | max_set_size_diff_pct=100, overlap_pct=100, max_p_value=1) 88 | 89 | # Test the output. We should have 6 rows. S1:S2 c=1, S1:S4 c<1, S1:S5 c=-1, S2:S5 c=-1. We are not checking 90 | # S2:S4 or S4:S5 91 | self.assertEqual(len(cor.coefficient_data.index), 6, "There should be six correlations rows calculated.") 92 | self.assertEqual(cor.get_base_coefficient('SYMBOL1', 'SYMBOL2'), 1, 93 | "The correlation for SYMBOL1:SYMBOL2 should be 1.") 94 | self.assertTrue(cor.get_base_coefficient('SYMBOL1', 'SYMBOL4') < 1, 95 | "The correlation for SYMBOL1:SYMBOL4 should be <1.") 96 | self.assertEqual(cor.get_base_coefficient('SYMBOL1', 'SYMBOL5'), -1, 97 | "The correlation for SYMBOL1:SYMBOL5 should be -1.") 98 | self.assertEqual(cor.get_base_coefficient('SYMBOL2', 'SYMBOL5'), -1, 99 | "The correlation for SYMBOL2:SYMBOL5 should be -1.") 100 | 101 | # Monitoring threshold is 1 and we are monitoring inverse. Get filtered correlations. There should be 3 (S1:S2, 102 | # S1:S5 and S2:S5) 103 | self.assertEqual(len(cor.filtered_coefficient_data.index), 3, 104 | "There should be 3 rows in filtered coefficient data when we are monitoring inverse " 105 | "correlations.") 106 | 107 | # Now aren't monitoring inverse correlations. There should only be one correlation when filtered 108 | cor.monitor_inverse = False 109 | self.assertEqual(len(cor.filtered_coefficient_data.index), 1, 110 | "There should be only 1 rows in filtered coefficient data when we are not monitoring inverse " 111 | "correlations.") 112 | 113 | # Now were going to recalculate, but this time SYMBOL1:SYMBOL2 will have non overlapping dates and coefficient 114 | # should be None. There shouldn't be a row. We should have correlations for S1:S4, S1:S5 and S4:S5 115 | mock.copy_rates_range.side_effect = [self.mock_base_prices, self.mock_correlated_different_dates, 116 | self.mock_correlated_prices, self.mock_correlated_prices] 117 | cor.calculate(date_from=self.start_date, date_to=self.end_date, timeframe=5, min_prices=100, 118 | max_set_size_diff_pct=100, overlap_pct=100, max_p_value=1) 119 | self.assertEqual(len(cor.coefficient_data.index), 3, "There should be three correlations rows calculated.") 120 | self.assertEqual(cor.coefficient_data.iloc[0, 2], 1, "The correlation for SYMBOL1:SYMBOL4 should be 1.") 121 | self.assertEqual(cor.coefficient_data.iloc[1, 2], 1, "The correlation for SYMBOL1:SYMBOL5 should be 1.") 122 | self.assertEqual(cor.coefficient_data.iloc[2, 2], 1, "The correlation for SYMBOL4:SYMBOL5 should be 1.") 123 | 124 | # Get the price data used to calculate the coefficients for symbol 1. It should match mock_base_prices. 125 | price_data = cor.get_price_data('SYMBOL1') 126 | self.assertTrue(price_data.equals(self.mock_base_prices), "Price data returned post calculation should match " 127 | "mock price data.") 128 | 129 | def test_calculate_coefficient(self): 130 | """ 131 | Tests the coefficient calculation. 132 | :return: 133 | """ 134 | # Correlation class 135 | cor = correlation.Correlation() 136 | 137 | # Test 2 correlated sets 138 | coefficient = cor.calculate_coefficient(self.mock_base_prices, self.mock_correlated_prices) 139 | self.assertEqual(coefficient, 1, "Coefficient should be 1.") 140 | 141 | # Test 2 uncorrelated sets. Set p value to 1 to force correlation to be returned. 142 | coefficient = cor.calculate_coefficient(self.mock_base_prices, self.mock_uncorrelated_prices, max_p_value=1) 143 | self.assertTrue(coefficient < 1, "Coefficient should be < 1.") 144 | 145 | # Test 2 sets where prices dont overlap 146 | coefficient = cor.calculate_coefficient(self.mock_base_prices, self.mock_correlated_different_dates) 147 | self.assertTrue(coefficient < 1, "Coefficient should be None.") 148 | 149 | # Test 2 inversely correlated sets 150 | coefficient = cor.calculate_coefficient(self.mock_base_prices, self.mock_inverse_correlated_prices) 151 | self.assertEqual(coefficient, -1, "Coefficient should be -1.") 152 | 153 | @patch('mt5_correlation.mt5.MetaTrader5') 154 | def test_get_ticks(self, mock): 155 | """ 156 | Test that caching works. For the purpose of this test, we can use price data rather than tick data. 157 | Mock 2 different sets of prices. Get three times. Base, One within cache threshold and one outside. Set 1 158 | should match set 2 but differ from set 3. 159 | :param mock: 160 | :return: 161 | """ 162 | 163 | # Correlation class to test 164 | cor = correlation.Correlation() 165 | 166 | # Mock the tick data to contain 2 different sets. Then get twice. They should match as the data was cached. 167 | mock.copy_ticks_range.side_effect = [self.mock_base_prices, self.mock_correlated_prices] 168 | 169 | # We need to start and stop the monitor as this will set the cache time 170 | cor.start_monitor(interval=10, calculation_params={'from': 10, 'min_prices': 0, 'max_set_size_diff_pct': 0, 171 | 'overlap_pct': 0, 'max_p_value': 1}, cache_time=3) 172 | cor.stop_monitor() 173 | 174 | # Get the ticks within cache time and check that they match 175 | base_ticks = cor.get_ticks('SYMBOL1', None, None) 176 | cached_ticks = cor.get_ticks('SYMBOL1', None, None) 177 | self.assertTrue(base_ticks.equals(cached_ticks), 178 | "Both sets of tick data should match as set 2 came from cache.") 179 | 180 | # Wait 3 seconds 181 | time.sleep(3) 182 | 183 | # Retrieve again. This one should be different as the cache has expired. 184 | non_cached_ticks = cor.get_ticks('SYMBOL1', None, None) 185 | self.assertTrue(not base_ticks.equals(non_cached_ticks), 186 | "Both sets of tick data should differ as cached data had expired.") 187 | 188 | @patch('mt5_correlation.mt5.MetaTrader5') 189 | def test_start_monitor(self, mock): 190 | """ 191 | Test that starting the monitor and running for 2 seconds produces two sets of coefficient history when using an 192 | interval of 1 second. 193 | :param mock: 194 | :return: 195 | """ 196 | # Mock symbol return values 197 | mock.symbols_get.return_value = self.mock_symbols 198 | 199 | # Create correlation class. We will set a divergence threshold so that we can test status. 200 | cor = correlation.Correlation(divergence_threshold=0.8, monitor_inverse=True) 201 | 202 | # Calculate for price data. We should have 100% matching dates in sets. Get prices should be called 4 times. 203 | # We dont have a SYMBOL2 as this is set as not visible. All pairs should be correlated for the purpose of this 204 | # test. 205 | mock.copy_rates_range.side_effect = [self.mock_base_prices, self.mock_correlated_prices, 206 | self.mock_correlated_prices, self.mock_inverse_correlated_prices] 207 | 208 | cor.calculate(date_from=self.start_date, date_to=self.end_date, timeframe=5, min_prices=100, 209 | max_set_size_diff_pct=100, overlap_pct=100, max_p_value=1) 210 | 211 | # We will build some tick data for each symbol and patch it in. Tick data will be from 10 seconds ago to now. 212 | # We only need to patch in one set of tick data for each symbol as it will be cached. 213 | columns = ['time', 'ask'] 214 | starttime = datetime.now() - timedelta(seconds=10) 215 | tick_data_s1 = pd.DataFrame(columns=columns) 216 | tick_data_s2 = pd.DataFrame(columns=columns) 217 | tick_data_s4 = pd.DataFrame(columns=columns) 218 | tick_data_s5 = pd.DataFrame(columns=columns) 219 | 220 | now = datetime.now() 221 | price_base = 1 222 | while starttime < now: 223 | tick_data_s1 = tick_data_s1.append(pd.DataFrame(columns=columns, data=[[starttime, price_base * 0.5]])) 224 | tick_data_s2 = tick_data_s1.append(pd.DataFrame(columns=columns, data=[[starttime, price_base * 0.1]])) 225 | tick_data_s4 = tick_data_s1.append(pd.DataFrame(columns=columns, data=[[starttime, price_base * 0.25]])) 226 | tick_data_s5 = tick_data_s1.append(pd.DataFrame(columns=columns, data=[[starttime, price_base * -0.25]])) 227 | starttime = starttime + timedelta(milliseconds=10*random.randint(0, 100)) 228 | price_base += 1 229 | 230 | # Patch it in 231 | mock.copy_ticks_range.side_effect = [tick_data_s1, tick_data_s2, tick_data_s4, tick_data_s5] 232 | 233 | # Start the monitor. Run every second. Use ~10 and ~5 seconds of data. Were not testing the overlap and price 234 | # data quality metrics here as that is set elsewhere so these can be set to not take effect. Set cache level 235 | # high and don't use autosave. Timer runs in a separate thread so test can continue after it has started. 236 | cor.start_monitor(interval=1, calculation_params=[{'from': 0.66, 'min_prices': 0, 237 | 'max_set_size_diff_pct': 0, 'overlap_pct': 0, 238 | 'max_p_value': 1}, 239 | {'from': 0.33, 'min_prices': 0, 240 | 'max_set_size_diff_pct': 0, 'overlap_pct': 0, 241 | 'max_p_value': 1}], cache_time=100, autosave=False) 242 | 243 | # Wait 2 seconds so timer runs twice 244 | time.sleep(2) 245 | 246 | # Stop the monitor 247 | cor.stop_monitor() 248 | 249 | # We should have 2 coefficients calculated for each symbol pair (6), for each date_from value (2), 250 | # for each run (2) so 24 in total. 251 | self.assertEqual(len(cor.coefficient_history.index), 24) 252 | 253 | # We should have 2 coefficients calculated for a single symbol pair and timeframe 254 | self.assertEqual(len(cor.get_coefficient_history({'Symbol 1': 'SYMBOL1', 'Symbol 2': 'SYMBOL2', 255 | 'Timeframe': 0.66})), 256 | 2, "We should have 2 history records for SYMBOL1:SYMBOL2 using the 0.66 min timeframe.") 257 | 258 | # The status should be DIVERGED for SYMBOL1:SYMBOL2 and CORRELATED for SYMBOL1:SYMBOL4 and SYMBOL2:SYMBOL4. 259 | self.assertTrue(cor.get_last_status('SYMBOL1', 'SYMBOL2') == correlation.STATUS_DIVERGED) 260 | self.assertTrue(cor.get_last_status('SYMBOL1', 'SYMBOL4') == correlation.STATUS_CORRELATED) 261 | self.assertTrue(cor.get_last_status('SYMBOL2', 'SYMBOL4') == correlation.STATUS_CORRELATED) 262 | 263 | # We are monitoring inverse correlations, status for SYMBOL1:SYMBOL5 should be DIVERGED 264 | self.assertTrue(cor.get_last_status('SYMBOL2', 'SYMBOL5') == correlation.STATUS_DIVERGED) 265 | 266 | @patch('mt5_correlation.mt5.MetaTrader5') 267 | def test_load_and_save(self, mock): 268 | """Calculate and run monitor for a few seconds. Store the data. Save it, load it then compare against stored 269 | data.""" 270 | 271 | # Correlation class 272 | cor = correlation.Correlation() 273 | 274 | # Patch symbol and price data, then calculate 275 | mock.symbols_get.return_value = self.mock_symbols 276 | mock.copy_rates_range.side_effect = [self.mock_base_prices, self.mock_correlated_prices, 277 | self.mock_correlated_prices, self.mock_inverse_correlated_prices] 278 | cor.calculate(date_from=self.start_date, date_to=self.end_date, timeframe=5, min_prices=100, 279 | max_set_size_diff_pct=100, overlap_pct=100, max_p_value=1) 280 | 281 | # Patch the tick data 282 | columns = ['time', 'ask'] 283 | starttime = datetime.now() - timedelta(seconds=10) 284 | tick_data_s1 = pd.DataFrame(columns=columns) 285 | tick_data_s3 = pd.DataFrame(columns=columns) 286 | tick_data_s4 = pd.DataFrame(columns=columns) 287 | now = datetime.now() 288 | price_base = 1 289 | while starttime < now: 290 | tick_data_s1 = tick_data_s1.append(pd.DataFrame(columns=columns, data=[[starttime, price_base * 0.5]])) 291 | tick_data_s3 = tick_data_s1.append(pd.DataFrame(columns=columns, data=[[starttime, price_base * 0.1]])) 292 | tick_data_s4 = tick_data_s1.append(pd.DataFrame(columns=columns, data=[[starttime, price_base * 0.25]])) 293 | starttime = starttime + timedelta(milliseconds=10 * random.randint(0, 100)) 294 | price_base += 1 295 | mock.copy_ticks_range.side_effect = [tick_data_s1, tick_data_s3, tick_data_s4] 296 | 297 | # Start monitor and run for a seconds with a 1 second interval to produce some coefficient history. Then stop 298 | # the monitor 299 | cor.start_monitor(interval=1, calculation_params={'from': 0.66, 'min_prices': 0, 'max_set_size_diff_pct': 0, 300 | 'overlap_pct': 0, 'max_p_value': 1}, 301 | cache_time=100, autosave=False) 302 | time.sleep(2) 303 | cor.stop_monitor() 304 | 305 | # Get copies of data that will be saved. 306 | cd_copy = cor.coefficient_data 307 | pd_copy = cor.get_price_data('SYMBOL1') 308 | mtd_copy = cor.get_ticks('SYMBOL1', cache_only=True) 309 | ch_copy = cor.coefficient_history 310 | 311 | # Save, reset data, then reload 312 | cor.save("unittest.cpd") 313 | cor.load("unittest.cpd") 314 | 315 | # Test that the reloaded data matches the original 316 | self.assertTrue(cd_copy.equals(cor.coefficient_data), 317 | "Saved and reloaded coefficient data should match original.") 318 | self.assertTrue(pd_copy.equals(cor.get_price_data('SYMBOL1')), 319 | "Saved and reloaded price data should match original.") 320 | self.assertTrue(mtd_copy.equals(cor.get_ticks('SYMBOL1', cache_only=True)), 321 | "Saved and reloaded tick data should match original.") 322 | self.assertTrue(ch_copy.equals(cor.coefficient_history), 323 | "Saved and reloaded coefficient history should match original.") 324 | 325 | # Cleanup. delete the file 326 | os.remove("unittest.cpd") 327 | 328 | @patch('mt5_correlation.correlation.Correlation.coefficient_data', new_callable=PropertyMock) 329 | def test_diverged_symbols(self, mock): 330 | """ 331 | Test that diverged_symbols property correctly groups symbols and counts. 332 | :param mock: 333 | :return: 334 | """ 335 | # Correlation class 336 | cor = correlation.Correlation() 337 | 338 | # Mock the correlation data. Symbol 1 has diverged 3 times; symbols 2 has diverged twice; symbol 3 has 339 | # diverged once; symbols 4 has diverged twice and symbol 5 has not diverged at all. Use all diverged status' 340 | # (diverged, diverging & converging). Also add a row for a non diverged pair. 341 | mock.return_value = pd.DataFrame(columns=['Symbol 1', 'Symbol 2', 'Status'], data=[ 342 | ['SYMBOL1', 'SYMBOL2', correlation.STATUS_DIVERGED], 343 | ['SYMBOL1', 'SYMBOL3', correlation.STATUS_DIVERGING], 344 | ['SYMBOL1', 'SYMBOL4', correlation.STATUS_CONVERGING], 345 | ['SYMBOL2', 'SYMBOL4', correlation.STATUS_DIVERGED], 346 | ['SYMBOL2', 'SYMBOL3', correlation.STATUS_CORRELATED]]) 347 | 348 | # Get the diverged_symbols data and check the counts 349 | diverged_symbols = cor.diverged_symbols 350 | self.assertEqual(diverged_symbols.loc[(diverged_symbols['Symbol'] == 'SYMBOL1')]['Count'].iloc[0], 3, 351 | "Symbol 1 has diverged three times.") 352 | self.assertEqual(diverged_symbols.loc[(diverged_symbols['Symbol'] == 'SYMBOL2')]['Count'].iloc[0], 2, 353 | "Symbol 2 has diverged twice.") 354 | self.assertEqual(diverged_symbols.loc[(diverged_symbols['Symbol'] == 'SYMBOL3')]['Count'].iloc[0], 1, 355 | "Symbol 3 has diverged once.") 356 | self.assertEqual(diverged_symbols.loc[(diverged_symbols['Symbol'] == 'SYMBOL4')]['Count'].iloc[0], 2, 357 | "Symbol 4 has diverged twice.") 358 | self.assertFalse('SYMBOL5' in diverged_symbols['Symbol']) 359 | 360 | 361 | if __name__ == '__main__': 362 | unittest.main() 363 | -------------------------------------------------------------------------------- /mt5_correlation/correlation.py: -------------------------------------------------------------------------------- 1 | import math 2 | import logging 3 | import pandas as pd 4 | from datetime import datetime, timedelta 5 | import time 6 | import sched 7 | import threading 8 | import pytz 9 | from scipy.stats.stats import pearsonr 10 | import pickle 11 | import inspect 12 | import sys 13 | 14 | from mt5_correlation.mt5 import MT5 15 | 16 | 17 | class CorrelationStatus: 18 | """ 19 | The status of the monitoring event for a symbol pair. 20 | """ 21 | 22 | val = None 23 | text = None 24 | long_text = None 25 | 26 | def __init__(self, status_val, status_text, status_long_text=None): 27 | """ 28 | Creates a status. 29 | :param status_val: 30 | :param status_text: 31 | :param status_long_text 32 | :return: 33 | """ 34 | self.val = status_val 35 | self.text = status_text 36 | self.long_text = status_long_text 37 | 38 | def __eq__(self, other): 39 | """ 40 | Compare the status val. We can compare against other CorrelationStatus instances or against int. 41 | :param other: 42 | :return: 43 | """ 44 | if isinstance(other, self.__class__): 45 | return self.val == other.val 46 | elif isinstance(other, int): 47 | return self.val == other 48 | else: 49 | return False 50 | 51 | def __str__(self): 52 | """ 53 | str is the text for the status. 54 | :return: 55 | """ 56 | return self.text 57 | 58 | 59 | # All status's for symbol pair from monitoring. Status set from assessing coefficient for all timeframes from last run. 60 | STATUS_NOT_CALCULATED = CorrelationStatus(-1, 'NOT CALC', 'Coefficient could not be calculated') 61 | STATUS_CORRELATED = CorrelationStatus(1, 'CORRELATED', 'Coefficients for all timeframes are equal to or above the ' 62 | 'divergence threshold') 63 | STATUS_DIVERGED = CorrelationStatus(2, 'DIVERGED', 'Coefficients for all timeframes are below the divergence threshold') 64 | STATUS_INCONSISTENT = CorrelationStatus(3, 'INCONSISTENT', 'Coefficients not consistently above or below divergence ' 65 | 'threshold and are neither trending towards divergence or ' 66 | 'convergence') 67 | STATUS_DIVERGING = CorrelationStatus(4, 'DIVERGING', 'Coefficients, when ordered by timeframe, are trending ' 68 | 'towards convergence. The shortest timeframe is below the ' 69 | 'divergence threshold and the longest timeframe is above the ' 70 | 'divergence threshold') 71 | STATUS_CONVERGING = CorrelationStatus(5, 'CONVERGING', 'Coefficients, when ordered by timeframe, are trending ' 72 | 'towards divergence. The shortest timeframe is above the ' 73 | 'divergence threshold and the longest timeframe is below the ' 74 | 'divergence threshold') 75 | 76 | 77 | class Correlation: 78 | """ 79 | A class to maintain the state of the calculated correlation coefficients. 80 | """ 81 | 82 | # Connection to MetaTrader5 83 | __mt5 = None 84 | 85 | # Minimum base coefficient for monitoring. Symbol pairs with a lower correlation 86 | # coefficient than ths won't be monitored. 87 | monitoring_threshold = 0.9 88 | 89 | # Threshold for divergence. Correlation coefficients that were previously above the monitoring_threshold and fall 90 | # below this threshold will be considered as having diverged 91 | divergence_threshold = 0.8 92 | 93 | # Flag to determine we monitor and report on inverse correlations 94 | monitor_inverse = False 95 | 96 | # Toggle on whether we are monitoring or not. Set through start_monitor and stop_monitor 97 | __monitoring = False 98 | 99 | # Monitoring calculation params, interval, cache_time, autosave and filename. Passed to start_monitor 100 | __monitoring_params = [] 101 | __interval = None 102 | __cache_time = None 103 | __autosave = None 104 | __filename = None 105 | 106 | # First run of scheduler 107 | __first_run = True 108 | 109 | # The price data used to calculate the correlations 110 | __price_data = None 111 | 112 | # Coefficient data and history. Will be created in init call to __reset_coefficient_data 113 | coefficient_data = None 114 | coefficient_history = None 115 | 116 | # Stores tick data used to calculate coefficient during Monitor. 117 | # Dict: {Symbol: [retrieved datetime, ticks dataframe]} 118 | __monitor_tick_data = {} 119 | 120 | def __init__(self, monitoring_threshold=0.9, divergence_threshold=0.8, monitor_inverse=False): 121 | """ 122 | Initialises the Correlation class. 123 | :param monitoring_threshold: Only correlations that are greater than or equal to this threshold will be 124 | monitored. 125 | :param divergence_threshold: Correlations that are being monitored and fall below this threshold are considered 126 | to have diverged. 127 | :param monitor_inverse: Whether we will monitor and report on negative / inverse correlations. 128 | """ 129 | # Logger 130 | self.__log = logging.getLogger(__name__) 131 | 132 | # Connection to MetaTrader5 133 | self.__mt5 = MT5() 134 | 135 | # Create dataframe for coefficient data 136 | self.__reset_coefficient_data() 137 | 138 | # Create timer for continuous monitoring 139 | self.__scheduler = sched.scheduler(time.time, time.sleep) 140 | 141 | # Set thresholds and flags 142 | self.monitoring_threshold = monitoring_threshold 143 | self.divergence_threshold = divergence_threshold 144 | self.monitor_inverse = monitor_inverse 145 | 146 | @property 147 | def filtered_coefficient_data(self): 148 | """ 149 | :return: Coefficient data filtered so that all base coefficients >= monitoring_threshold 150 | """ 151 | filtered_data = None 152 | if self.coefficient_data is not None: 153 | if self.monitor_inverse: 154 | filtered_data = self.coefficient_data \ 155 | .loc[(self.coefficient_data['Base Coefficient'] >= self.monitoring_threshold) | 156 | (self.coefficient_data['Base Coefficient'] <= self.monitoring_threshold * -1)] 157 | 158 | else: 159 | filtered_data = self.coefficient_data.loc[self.coefficient_data['Base Coefficient'] >= 160 | self.monitoring_threshold] 161 | 162 | return filtered_data 163 | 164 | @property 165 | def diverged_symbols(self): 166 | """ 167 | :return: dataframe containing all diverged, diverging or converging symbols and count of number of 168 | divergences for those symbols. 169 | """ 170 | filtered_data = None 171 | 172 | if self.coefficient_data is not None: 173 | # Only rows where we have a divergence 174 | filtered_data = self.coefficient_data \ 175 | .loc[(self.coefficient_data['Status'] == STATUS_DIVERGED) | 176 | (self.coefficient_data['Status'] == STATUS_DIVERGING) | 177 | (self.coefficient_data['Status'] == STATUS_CONVERGING)] 178 | 179 | # We only need the symbols 180 | all_symbols = pd.DataFrame(columns=['Symbol', 'Count'], 181 | data={'Symbol': filtered_data['Symbol 1'].append(filtered_data['Symbol 2']), 182 | 'Count': 1}) 183 | 184 | # Group and count. Reset index so that we have named SYMBOL column. 185 | filtered_data = all_symbols.groupby(by='Symbol').count().reset_index() 186 | 187 | # Sort 188 | filtered_data = filtered_data.sort_values('Count', ascending=False) 189 | 190 | return filtered_data 191 | 192 | def load(self, filename): 193 | """ 194 | Loads calculated coefficients, price data used to calculate them and tick data used during monitoring. 195 | coefficients 196 | :param filename: The filename for the coefficient data to load. 197 | :return: 198 | """ 199 | # Load data 200 | with open(filename, 'rb') as file: 201 | loaded_dict = pickle.load(file) 202 | 203 | # Get data from loaded dict and save 204 | self.coefficient_data = loaded_dict["coefficient_data"] 205 | self.__price_data = loaded_dict["price_data"] 206 | self.__monitor_tick_data = loaded_dict["monitor_tick_data"] 207 | self.coefficient_history = loaded_dict["coefficient_history"] 208 | 209 | def save(self, filename): 210 | """ 211 | Saves the calculated coefficients, the price data used to calculate and the tick data for monitoring to a file. 212 | :param filename: The filename to save the data to. 213 | :return: 214 | """ 215 | # Add data to dict then use pickle to save 216 | save_dict = {"coefficient_data": self.coefficient_data, "price_data": self.__price_data, 217 | "monitor_tick_data": self.__monitor_tick_data, "coefficient_history": self.coefficient_history} 218 | with open(filename, 'wb') as file: 219 | pickle.dump(save_dict, file, protocol=pickle.HIGHEST_PROTOCOL) 220 | 221 | def calculate(self, date_from, date_to, timeframe, min_prices=100, max_set_size_diff_pct=90, overlap_pct=90, 222 | max_p_value=0.05): 223 | """ 224 | Calculates correlation coefficient between all symbols in MetaTrader5 Market Watch. Updates coefficient data. 225 | 226 | :param date_from: From date for price data from which to calculate correlation coefficients 227 | :param date_to: To date for price data from which to calculate correlation coefficients 228 | :param timeframe: Timeframe for price data from which to calculate correlation coefficients 229 | :param min_prices: The minimum number of prices that should be used to calculate coefficient. If this threshold 230 | is not met then returned coefficient will be None 231 | :param max_set_size_diff_pct: Correlations will only be calculated if the sizes of the two price data sets are 232 | within this pct of each other 233 | :param overlap_pct:The dates and times in the two sets of data must match. The coefficient will only be 234 | calculated against the dates that overlap. Any non overlapping dates will be discarded. This setting 235 | specifies the minimum size of the overlapping data when compared to the smallest set as a %. A coefficient 236 | will not be calculated if this threshold is not met. 237 | :param max_p_value: The maximum p value for the correlation to be meaningful 238 | 239 | :return: 240 | """ 241 | 242 | coefficient = None 243 | 244 | # If we are monitoring, stop. We will need to restart later 245 | was_monitoring = self.__monitoring 246 | if self.__monitoring: 247 | self.stop_monitor() 248 | 249 | # Clear the existing correlations 250 | self.__reset_coefficient_data() 251 | 252 | # Get all visible symbols 253 | symbols = self.__mt5.get_symbols() 254 | 255 | # Get price data for selected symbols. 1 week of 15 min OHLC data for each symbol. Add to dict. 256 | self.__price_data = {} 257 | for symbol in symbols: 258 | self.__price_data[symbol] = self.__mt5.get_prices(symbol=symbol, from_date=date_from, to_date=date_to, 259 | timeframe=timeframe) 260 | 261 | # Loop through all symbol pair combinations and calculate coefficient. Make sure you don't double count pairs 262 | # eg. (USD/GBP AUD/USD vs AUD/USD USD/GBP). Use grid of all symbols with i and j axis. j starts at i + 1 to 263 | # avoid duplicating. We will store all coefficients in a dataframe. 264 | index = 0 265 | # There will be (x^2 - x) / 2 pairs where x is number of symbols 266 | num_pair_combinations = int((len(symbols) ** 2 - len(symbols)) / 2) 267 | 268 | for i in range(0, len(symbols)): 269 | symbol1 = symbols[i] 270 | 271 | for j in range(i + 1, len(symbols)): 272 | symbol2 = symbols[j] 273 | index += 1 274 | 275 | # Get price data for both symbols 276 | symbol1_price_data = self.__price_data[symbol1] 277 | symbol2_price_data = self.__price_data[symbol2] 278 | 279 | # Get coefficient 280 | if symbol1_price_data is not None and symbol2_price_data is not None: 281 | coefficient = self.calculate_coefficient(symbol1_prices=symbol1_price_data, 282 | symbol2_prices=symbol2_price_data, 283 | min_prices=min_prices, 284 | max_set_size_diff_pct=max_set_size_diff_pct, 285 | overlap_pct=overlap_pct, max_p_value=max_p_value) 286 | 287 | # Store if valid 288 | if coefficient is not None: 289 | 290 | self.coefficient_data = \ 291 | self.coefficient_data.append({'Symbol 1': symbol1, 'Symbol 2': symbol2, 292 | 'Base Coefficient': coefficient, 'UTC Date From': date_from, 293 | 'UTC Date To': date_to, 'Timeframe': timeframe, 'Status': ''}, 294 | ignore_index=True) 295 | 296 | self.__log.debug(f"Pair {index} of {num_pair_combinations}: {symbol1}:{symbol2} has a " 297 | f"coefficient of {coefficient}.") 298 | else: 299 | self.__log.debug(f"Coefficient for pair {index} of {num_pair_combinations}: {symbol1}:" 300 | f"{symbol2} could no be calculated.") 301 | 302 | # Sort, highest correlated first 303 | self.coefficient_data = self.coefficient_data.sort_values('Base Coefficient', ascending=False) 304 | 305 | # If we were monitoring, we stopped, so start again. 306 | if was_monitoring: 307 | self.start_monitor(interval=self.__interval, calculation_params=self.__monitoring_params, 308 | cache_time=self.__cache_time, autosave=self.__autosave, filename=self.__filename) 309 | 310 | def get_price_data(self, symbol): 311 | """ 312 | Returns the price data used to calculate the base coefficients for the specified symbol 313 | :param symbol: Symbol to get price data for. 314 | :return: price data 315 | """ 316 | price_data = None 317 | if symbol in self.__price_data: 318 | price_data = self.__price_data[symbol] 319 | 320 | return price_data 321 | 322 | def start_monitor(self, interval, calculation_params, cache_time=10, autosave=False, filename='autosave.cpd'): 323 | """ 324 | Starts monitor to continuously update the coefficient for all symbol pairs in that meet the min_coefficient 325 | threshold. 326 | 327 | :param interval: How often to check in seconds 328 | :param calculation_params: A single dict or list of dicts containing the parameters for the coefficient 329 | calculations. On every iteration, a coefficient will be calculated for every set of params in list. Params 330 | contain the following values: 331 | from: The number of minutes of tick data to use for calculation. This can be a single value or 332 | a list. If a list, then calculations will be performed for every from date in list. 333 | min_prices: The minimum number of prices that should be used to calculate coefficient. If this threshold 334 | is not met then returned coefficient will be None 335 | max_set_size_diff_pct: Correlations will only be calculated if the sizes of the two price data sets are 336 | within this pct of each other 337 | overlap_pct: The dates and times in the two sets of data must match. The coefficient will only be 338 | calculated against the dates that overlap. Any non overlapping dates will be discarded. This 339 | setting specifies the minimum size of the overlapping data when compared to the smallest set as a %. 340 | A coefficient will not be calculated if this threshold is not met. 341 | max_p_value: The maximum p value for the correlation to be meaningful 342 | :param cache_time: Tick data is cached so that we can check coefficients for multiple symbol pairs and reuse 343 | the tick data. Number of seconds to cache tick data for before it becomes stale. 344 | :param autosave: Whether to autosave after every monitor run. If there is no filename specified then will 345 | create one named autosave.cpd 346 | :param filename: Filename for autosave. Default is autosave.cpd. 347 | 348 | :return: correlation coefficient, or None if coefficient could not be calculated. 349 | """ 350 | 351 | if self.__monitoring: 352 | self.__log.debug(f"Request to start monitor when monitor is already running. Monitor will be stopped and" 353 | f"restarted with new parameters.") 354 | self.stop_monitor() 355 | 356 | self.__log.debug(f"Starting monitor.") 357 | self.__monitoring = True 358 | 359 | # Store the calculation params. If it isn't a list, convert to list of one to make code simpler later on. 360 | self.__monitoring_params = calculation_params if isinstance(calculation_params, list) \ 361 | else [calculation_params, ] 362 | 363 | # Store the other params. We will need these later if monitor is stopped and needs to be restarted. This 364 | # happens in calculate. 365 | self.__interval = interval 366 | self.__cache_time = cache_time 367 | self.__autosave = autosave 368 | self.__filename = filename 369 | 370 | # Create thread to run monitoring This will call private __monitor method that will run the calculation and 371 | # keep scheduling itself while self.monitoring is True. 372 | thread = threading.Thread(target=self.__monitor) 373 | thread.start() 374 | 375 | def stop_monitor(self): 376 | """ 377 | Stops monitoring symbol pairs for correlation. 378 | :return: 379 | """ 380 | if self.__monitoring: 381 | self.__log.debug(f"Stopping monitor.") 382 | self.__monitoring = False 383 | else: 384 | self.__log.debug(f"Request to stop monitor when it is not running. No action taken.") 385 | 386 | def calculate_coefficient(self, symbol1_prices, symbol2_prices, min_prices: int = 100, 387 | max_set_size_diff_pct: int = 90, overlap_pct: int = 90, 388 | max_p_value: float = 0.05): 389 | """ 390 | Calculates the correlation coefficient between two sets of price data. Uses close price. 391 | 392 | :param symbol1_prices: Pandas dataframe containing prices for symbol 1 393 | :param symbol2_prices: Pandas dataframe containing prices for symbol 2 394 | :param min_prices: The minimum number of prices that should be used to calculate coefficient. If this threshold 395 | is not met then returned coefficient will be None 396 | :param max_set_size_diff_pct: Correlations will only be calculated if the sizes of the two price data sets are 397 | within this pct of each other 398 | :param overlap_pct: 399 | :param max_p_value: The maximum p value for the correlation to be meaningful 400 | 401 | :return: correlation coefficient, or None if coefficient could not be calculated. 402 | :rtype: float or None 403 | """ 404 | 405 | assert symbol1_prices is not None and symbol2_prices is not None 406 | 407 | # Calculate size of intersection and determine if prices for symbols have enough overlapping timestamps for 408 | # correlation coefficient calculation to be meaningful. Is the smallest set at least max_set_size_diff_pct % of 409 | # the size of the largest set and is the overlap set size at least overlap_pct % the size of the smallest set? 410 | coefficient = None 411 | 412 | intersect_dates = (set(symbol1_prices['time']) & set(symbol2_prices['time'])) 413 | len_smallest_set = int(min([len(symbol1_prices.index), len(symbol2_prices.index)])) 414 | len_largest_set = int(max([len(symbol1_prices.index), len(symbol2_prices.index)])) 415 | similar_size = len_largest_set * (max_set_size_diff_pct / 100) <= len_smallest_set 416 | enough_overlap = len(intersect_dates) >= len_smallest_set * (overlap_pct / 100) 417 | enough_prices = len_smallest_set >= min_prices 418 | suitable = similar_size and enough_overlap and enough_prices 419 | 420 | if suitable: 421 | # Calculate coefficient on close prices 422 | 423 | # First filter prices to only include those that intersect 424 | symbol1_prices_filtered = symbol1_prices[symbol1_prices['time'].isin(intersect_dates)] 425 | symbol2_prices_filtered = symbol2_prices[symbol2_prices['time'].isin(intersect_dates)] 426 | 427 | # Calculate coefficient. Only use if p value is < max_p_value (highly likely that coefficient is valid 428 | # and null hypothesis is false). 429 | coefficient_with_p_value = pearsonr(symbol1_prices_filtered['close'], symbol2_prices_filtered['close']) 430 | coefficient = None if coefficient_with_p_value[1] > max_p_value else coefficient_with_p_value[0] 431 | 432 | # If NaN, change to None 433 | if coefficient is not None and math.isnan(coefficient): 434 | coefficient = None 435 | 436 | self.__log.debug(f"Calculate coefficient returning {coefficient}. " 437 | f"Symbol 1 Prices: {len(symbol1_prices)} Symbol 2 Prices: {len(symbol2_prices)} " 438 | f"Overlap Prices: {len(intersect_dates)} Similar size: {similar_size} " 439 | f"Enough overlap: {enough_overlap} Enough prices: {enough_prices} Suitable: {suitable}.") 440 | 441 | return coefficient 442 | 443 | def get_coefficient_history(self, filters=None): 444 | """ 445 | Returns the coefficient history that matches the supplied filter. 446 | :param filters: Dict of all filters to apply. Possible values in dict are: 447 | Symbol 1 448 | Symbol 2 449 | Coefficient 450 | Timeframe 451 | Date From 452 | Date To 453 | 454 | If filter is not supplied, then all history is returned. 455 | 456 | :return: dataframe containing history of coefficient data. 457 | """ 458 | history = self.coefficient_history 459 | 460 | # Apply filters 461 | if filters is not None: 462 | for key in filters: 463 | if key in history.columns: 464 | history = history[history[key] == filters[key]] 465 | else: 466 | self.__log.warning(f"Invalid column name provided for filter. Filter column: {key} " 467 | f"Valid columns: {history.columns}") 468 | 469 | return history 470 | 471 | def clear_coefficient_history(self): 472 | """ 473 | Clears the coefficient history for all symbol pairs 474 | :return: 475 | """ 476 | # Create dataframes for coefficient history. 477 | coefficient_history_columns = ['Symbol 1', 'Symbol 2', 'Coefficient', 'Timeframe', 'Date To'] 478 | self.coefficient_history = pd.DataFrame(columns=coefficient_history_columns) 479 | 480 | # Clear tick data 481 | self.__monitor_tick_data = {} 482 | 483 | # Clear status from coefficient data 484 | self.coefficient_data['Status'] = '' 485 | 486 | def get_ticks(self, symbol, date_from=None, date_to=None, cache_only=False): 487 | """ 488 | Returns the ticks for the specified symbol. Get's from cache if available and not older than cache_timeframe. 489 | 490 | :param symbol: Name of symbol to get ticks for. 491 | :param date_from: Date to get ticks from. Can only be None if getting from cache (cache_only=True) 492 | :param date_to:Date to get ticks to. Can only be None if getting from cache (cache_only=True) 493 | :param cache_only: Only retrieve from cache. cache_time is ignored. Returns None if symbol is not available in 494 | cache. 495 | 496 | :return: 497 | """ 498 | 499 | timezone = pytz.timezone("Etc/UTC") 500 | utc_now = datetime.now(tz=timezone) 501 | 502 | ticks = None 503 | 504 | # Cache only 505 | if cache_only: 506 | if symbol in self.__monitor_tick_data: 507 | ticks = self.__monitor_tick_data[symbol][1] 508 | # Check if we have a cache time defined, if we already have the tick data and it is not stale 509 | elif self.__cache_time is not None and symbol in self.__monitor_tick_data and utc_now < \ 510 | self.__monitor_tick_data[symbol][0] + timedelta(seconds=self.__cache_time): 511 | # Cached ticks are not stale. Get them 512 | ticks = self.__monitor_tick_data[symbol][1] 513 | self.__log.debug(f"Ticks for {symbol} retrieved from cache.") 514 | else: 515 | # Data does not exist in cache or cached data is stale. Retrieve from source and cache. 516 | ticks = self.__mt5.get_ticks(symbol=symbol, from_date=date_from, to_date=date_to) 517 | self.__monitor_tick_data[symbol] = [utc_now, ticks] 518 | self.__log.debug(f"Ticks for {symbol} retrieved from source and cached.") 519 | return ticks 520 | 521 | def get_last_status(self, symbol1, symbol2): 522 | """ 523 | Get the last status for the specified symbol pair. 524 | :param symbol1 525 | :param symbol2 526 | 527 | :return: CorrelationStatus instance for symbol pair 528 | """ 529 | status_col = self.coefficient_data.loc[(self.coefficient_data['Symbol 1'] == symbol1) & 530 | (self.coefficient_data['Symbol 2'] == symbol2), 'Status'] 531 | status = status_col.values[0] 532 | return status 533 | 534 | def get_last_calculation(self, symbol1=None, symbol2=None): 535 | """ 536 | Get the last calculation time the specified symbol pair. If no symbols are specified, then gets the last 537 | calculation time across all pairs 538 | :param symbol1 539 | :param symbol2 540 | 541 | :return: last calculation time 542 | """ 543 | last_calc = None 544 | if self.coefficient_data is not None and len(self.coefficient_data.index) > 0: 545 | data = self.coefficient_data.copy() 546 | 547 | # Filter by symbols if specified 548 | data = data.loc[data['Symbol 1'] == symbol1] if symbol1 is not None else data 549 | data = data.loc[data['Symbol 2'] == symbol2] if symbol2 is not None else data 550 | 551 | # Filter to remove blank dates 552 | data = data.dropna(subset=['Last Calculation']) 553 | 554 | # Get the column 555 | col = data['Last Calculation'] 556 | 557 | # Get max date from column 558 | if col is not None and len(col) > 0: 559 | last_calc = max(col.values) 560 | 561 | return last_calc 562 | 563 | def get_base_coefficient(self, symbol1, symbol2): 564 | """ 565 | Returns the base coefficient for the specified symbol pair 566 | :param symbol1: 567 | :param symbol2: 568 | :return: 569 | """ 570 | base_coefficient = None 571 | if self.coefficient_data is not None: 572 | row = self.coefficient_data[(self.coefficient_data['Symbol 1'] == symbol1) & 573 | (self.coefficient_data['Symbol 2'] == symbol2)] 574 | 575 | if row is not None and len(row) == 1: 576 | base_coefficient = row.iloc[0]['Base Coefficient'] 577 | 578 | return base_coefficient 579 | 580 | def __monitor(self): 581 | """ 582 | The actual monitor method. Private. This should not be called outside of this class. Use start_monitoring and 583 | stop_monitoring. 584 | 585 | :return: correlation coefficient, or None if coefficient could not be calculated. 586 | """ 587 | self.__log.debug(f"In monitor event. Monitoring: {self.__monitoring}.") 588 | 589 | # Only run if monitor is not stopped 590 | if self.__monitoring: 591 | # Update all coefficients 592 | self.__update_all_coefficients() 593 | 594 | # Autosave 595 | if self.__autosave: 596 | self.save(filename=self.__filename) 597 | 598 | # Schedule the timer to run again 599 | self.__scheduler.enter(delay=self.__interval, priority=1, action=self.__monitor) 600 | 601 | # Log the stack. Debug stack overflow 602 | self.__log.debug(f"Current stack size: {len(inspect.stack())} Recursion limit: {sys.getrecursionlimit()}") 603 | 604 | # Run 605 | if self.__first_run: 606 | self.__first_run = False 607 | self.__scheduler.run() 608 | 609 | def __update_coefficients(self, symbol1, symbol2): 610 | """ 611 | Updates the coefficients for the specified symbol pair 612 | :param symbol1: Name of symbol to calculate coefficient for. 613 | :param symbol2: Name of symbol to calculate coefficient for. 614 | :return: correlation coefficient, or None if coefficient could not be calculated. 615 | """ 616 | 617 | # Get the largest value of from in monitoring_params. This will be used to retrieve the data. We will only 618 | # retrieve once and use for every set of params by getting subset of the data. 619 | max_from = None 620 | for params in self.__monitoring_params: 621 | if max_from is None: 622 | max_from = params['from'] 623 | else: 624 | max_from = max(max_from, params['from']) 625 | 626 | # Date range for data 627 | timezone = pytz.timezone("Etc/UTC") 628 | date_to = datetime.now(tz=timezone) 629 | date_from = date_to - timedelta(minutes=max_from) 630 | 631 | # Get the tick data for the longest timeframe calculation. 632 | symbol1ticks = self.get_ticks(symbol=symbol1, date_from=date_from, date_to=date_to) 633 | symbol2ticks = self.get_ticks(symbol=symbol2, date_from=date_from, date_to=date_to) 634 | 635 | # Resample to 1 sec OHLC, this will help with coefficient calculation ensuring that we dont have more than 636 | # one tick per second and ensuring that times can match. We will need to set the index to time for the 637 | # resample then revert back to a 'time' column. We will then need to remove rows with nan in 'close' price 638 | s1_prices = None 639 | s2_prices = None 640 | if symbol1ticks is not None and symbol2ticks is not None and len(symbol1ticks.index) > 0 and \ 641 | len(symbol2ticks.index) > 0: 642 | 643 | try: 644 | symbol1ticks = symbol1ticks.set_index('time') 645 | symbol2ticks = symbol2ticks.set_index('time') 646 | s1_prices = symbol1ticks['ask'].resample('1S').ohlc() 647 | s2_prices = symbol2ticks['ask'].resample('1S').ohlc() 648 | except RecursionError: 649 | self.__log.warning(f"Coefficient could not be calculated for {symbol1}:{symbol2}. prices could not " 650 | f"be resampled.") 651 | else: 652 | s1_prices.reset_index(inplace=True) 653 | s2_prices.reset_index(inplace=True) 654 | s1_prices = s1_prices[s1_prices['close'].notna()] 655 | s2_prices = s2_prices[s2_prices['close'].notna()] 656 | 657 | # Calculate for all sets of monitoring_params 658 | if s1_prices is not None and s2_prices is not None: 659 | coefficients = {} 660 | for params in self.__monitoring_params: 661 | # Get the from date as a datetime64 662 | date_from_subset = pd.Timestamp(date_to - timedelta(minutes=params['from'])).to_datetime64() 663 | 664 | # Get subset of the price data 665 | s1_prices_subset = s1_prices[(s1_prices['time'] >= date_from_subset)] 666 | s2_prices_subset = s2_prices[(s2_prices['time'] >= date_from_subset)] 667 | 668 | # Calculate the coefficient 669 | coefficient = \ 670 | self.calculate_coefficient(symbol1_prices=s1_prices_subset, symbol2_prices=s2_prices_subset, 671 | min_prices=params['min_prices'], 672 | max_set_size_diff_pct=params['max_set_size_diff_pct'], 673 | overlap_pct=params['overlap_pct'], max_p_value=params['max_p_value']) 674 | 675 | self.__log.debug(f"Symbol pair {symbol1}:{symbol2} has a coefficient of {coefficient} for last " 676 | f"{params['from']} minutes.") 677 | 678 | # Add the coefficient to a dict {timeframe: coefficient}. We will update together for all for 679 | # symbol pair and time 680 | coefficients[params['from']] = coefficient 681 | 682 | # Update coefficient data for all coefficients for all timeframes for this run and symbol pair. 683 | self.__update_coefficient_data(symbol1=symbol1, symbol2=symbol2, coefficients=coefficients, 684 | date_to=date_to) 685 | 686 | def __update_all_coefficients(self): 687 | """ 688 | Updates the coefficient for all symbol pairs in that meet the min_coefficient threshold. Symbol pairs that meet 689 | the threshold can be accessed through the filtered_coefficient_data property. 690 | """ 691 | # Update latest coefficient for every pair 692 | for index, row in self.filtered_coefficient_data.iterrows(): 693 | symbol1 = row['Symbol 1'] 694 | symbol2 = row['Symbol 2'] 695 | self.__update_coefficients(symbol1=symbol1, symbol2=symbol2) 696 | 697 | def __reset_coefficient_data(self): 698 | """ 699 | Clears coefficient data and history. 700 | :return: 701 | """ 702 | # Create dataframes for coefficient data. 703 | coefficient_data_columns = ['Symbol 1', 'Symbol 2', 'Base Coefficient', 'UTC Date From', 'UTC Date To', 704 | 'Timeframe', 'Last Calculation', 'Status'] 705 | self.coefficient_data = pd.DataFrame(columns=coefficient_data_columns) 706 | 707 | # Clear coefficient history 708 | self.clear_coefficient_history() 709 | 710 | # Clear price data 711 | self.__price_data = None 712 | 713 | def __update_coefficient_data(self, symbol1, symbol2, coefficients, date_to): 714 | """ 715 | Updates the coefficient data with the latest coefficient and adds to coefficient history. 716 | :param symbol1: 717 | :param symbol2: 718 | :param coefficients: Dict of all coefficients calculated for this run and symbol pair. {timeframe: coefficient} 719 | :param date_to: The date from for which the coefficient was calculated 720 | :return: 721 | """ 722 | 723 | timezone = pytz.timezone("Etc/UTC") 724 | now = datetime.now(tz=timezone) 725 | 726 | # Update data if we have a coefficient and add to history 727 | if coefficients is not None: 728 | # Update the coefficient data table with the Last Calculation time. 729 | self.coefficient_data.loc[(self.coefficient_data['Symbol 1'] == symbol1) & 730 | (self.coefficient_data['Symbol 2'] == symbol2), 731 | 'Last Calculation'] = now 732 | 733 | # Are we an inverse correlation 734 | inverse = self.get_base_coefficient(symbol1, symbol2) <= self.monitoring_threshold * -1 735 | 736 | # Calculate status and update 737 | status = self.__calculate_status(coefficients=coefficients, inverse=inverse) 738 | self.coefficient_data.loc[(self.coefficient_data['Symbol 1'] == symbol1) & 739 | (self.coefficient_data['Symbol 2'] == symbol2), 740 | 'Status'] = status 741 | 742 | # Update history data 743 | for key in coefficients: 744 | row = pd.DataFrame(columns=self.coefficient_history.columns, 745 | data=[[symbol1, symbol2, coefficients[key], key, date_to]]) 746 | self.coefficient_history = self.coefficient_history.append(row) 747 | 748 | def __calculate_status(self, coefficients, inverse): 749 | """ 750 | Calculates the status from the supplied set of coefficients 751 | :param coefficients: Dict of timeframes and coefficients {timeframe: coefficient} to calculate status from 752 | :param: Whether we are calculating status based on normal or inverse correlation 753 | :return: status 754 | """ 755 | status = STATUS_NOT_CALCULATED 756 | 757 | # Only continue if we have calculated all coefficients, otherwise we will return STATUS_NOT_CALCULATED 758 | if None not in coefficients.values(): 759 | # Get the values ordered by timeframe descending 760 | ordered_values = [] 761 | for key in sorted(coefficients, reverse=True): 762 | ordered_values.append(coefficients[key]) 763 | 764 | if self.monitor_inverse and inverse: 765 | # Calculation for inverse calculations 766 | if all(i <= self.divergence_threshold * -1 for i in ordered_values): 767 | status = STATUS_CORRELATED 768 | elif all(i > self.divergence_threshold * -1 for i in ordered_values): 769 | status = STATUS_DIVERGED 770 | elif all(ordered_values[i] <= ordered_values[i+1] for i in range(0, len(ordered_values)-1, 1)): 771 | status = STATUS_CONVERGING 772 | elif all(ordered_values[i] > ordered_values[i+1] for i in range(0, len(ordered_values)-1, 1)): 773 | status = STATUS_DIVERGING 774 | else: 775 | status = STATUS_INCONSISTENT 776 | else: 777 | # Calculation for standard correlations 778 | if all(i >= self.divergence_threshold for i in ordered_values): 779 | status = STATUS_CORRELATED 780 | elif all(i < self.divergence_threshold for i in ordered_values): 781 | status = STATUS_DIVERGED 782 | elif all(ordered_values[i] <= ordered_values[i+1] for i in range(0, len(ordered_values)-1, 1)): 783 | status = STATUS_DIVERGING 784 | elif all(ordered_values[i] > ordered_values[i+1] for i in range(0, len(ordered_values)-1, 1)): 785 | status = STATUS_CONVERGING 786 | else: 787 | status = STATUS_INCONSISTENT 788 | 789 | return status 790 | --------------------------------------------------------------------------------