├── web ├── static │ └── css │ │ └── dashboard.css ├── templates │ └── dashboard.html └── app.py ├── for_setup_airlfow ├── airflowignore.template └── docker-compose.yaml ├── .gitignore ├── airflow_requirements.txt ├── requirements.txt ├── tests ├── reports │ └── financial_reports │ │ ├── resources │ │ ├── mock_financial_report_fail_example.html │ │ └── mock_financial_report_success_example.html │ │ └── test_financial_report_agent.py ├── toolbox │ └── test_date_tool.py └── db_bridges │ └── mongo_bridge │ └── test_collection_structure.py ├── reports ├── financial_reports │ ├── balance_sheet.py │ ├── comprehensive_income_sheet.py │ ├── sheet.py │ └── financial_report_agent.py └── monthly_revenue │ └── monthly_revenue.py ├── dags ├── common │ └── setup_project.py ├── parse_monthly_revenue │ └── parse_monthly_revenue.py ├── three_days_strategy │ └── three_days_strategy.py ├── crawl_daily_transaction │ ├── crawl_daily_transaction.py │ └── crawl_yahoo_daily_transaction.py └── parse_financial_report │ └── parse_financial_report.py ├── strategies └── three_days_strategy.py ├── toolbox └── date_tool.py ├── db_bridges └── mongo_bridge │ └── collection_structure.py ├── daily_transaction ├── yahoo_daily_transaction.py └── daily_transaction.py ├── README.md └── lab └── experiment1 ├── .ipynb_checkpoints └── three_days_strategy-checkpoint.ipynb └── three_days_strategy.ipynb /web/static/css/dashboard.css: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /for_setup_airlfow/airflowignore.template: -------------------------------------------------------------------------------- 1 | venv/ 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | venv/ 2 | __pycache__ 3 | *.swp 4 | .idea 5 | .DS_Store 6 | -------------------------------------------------------------------------------- /airflow_requirements.txt: -------------------------------------------------------------------------------- 1 | beautifulsoup4 2 | requests 3 | apache-airflow-providers-mongo==2.0.0 -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | apache-airflow==2.1.3 2 | autopep8 3 | beautifulsoup4 4 | requests 5 | apache-airflow-providers-mongo==2.0.0 6 | plotly==5.2.1 7 | pandas==1.2.4 8 | pymongo==3.12.0 9 | Flask==1.1.4 10 | pytest==6.2.5 -------------------------------------------------------------------------------- /tests/reports/financial_reports/resources/mock_financial_report_fail_example.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/YanHaoChen/tw-financial-report-analysis/HEAD/tests/reports/financial_reports/resources/mock_financial_report_fail_example.html -------------------------------------------------------------------------------- /tests/reports/financial_reports/resources/mock_financial_report_success_example.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/YanHaoChen/tw-financial-report-analysis/HEAD/tests/reports/financial_reports/resources/mock_financial_report_success_example.html -------------------------------------------------------------------------------- /reports/financial_reports/balance_sheet.py: -------------------------------------------------------------------------------- 1 | from reports.financial_reports.sheet import Sheet 2 | from bs4 import BeautifulSoup 3 | 4 | 5 | class BalanceSheet(Sheet): 6 | def __init__(self, sheet: BeautifulSoup): 7 | self.magic_id = '' 8 | self.sheet = sheet 9 | self.dollar_unit = 0 10 | 11 | def set_magic_id(self): 12 | self.magic_id = 'BalanceSheet' 13 | -------------------------------------------------------------------------------- /reports/financial_reports/comprehensive_income_sheet.py: -------------------------------------------------------------------------------- 1 | from reports.financial_reports.sheet import Sheet 2 | from bs4 import BeautifulSoup 3 | 4 | 5 | class ComprehensiveIncomeSheet(Sheet): 6 | def __init__(self, sheet: BeautifulSoup): 7 | self.magic_id = '' 8 | self.sheet = sheet 9 | self.dollar_unit = 0 10 | 11 | def set_magic_id(self): 12 | self.magic_id = 'StatementOfComprehensiveIncome' 13 | -------------------------------------------------------------------------------- /web/templates/dashboard.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 |

tw-financial-report-analysis

8 |

ROA(Per Season)

9 |
10 |

ROE(Per Season)

11 |
12 | 13 | 14 | 21 | -------------------------------------------------------------------------------- /tests/toolbox/test_date_tool.py: -------------------------------------------------------------------------------- 1 | from toolbox.date_tool import DateTool 2 | import unittest 3 | 4 | 5 | class DateToolTestCase(unittest.TestCase): 6 | 7 | def test_date_to_ex_year_and_season_successfully(self): 8 | ex_year, ex_season = DateTool.date_to_ex_year_and_season(2019, 10) 9 | assert (ex_year == 2019 and ex_season == 3) is True 10 | 11 | ex_year, ex_season = DateTool.date_to_ex_year_and_season(2020, 1) 12 | assert (ex_year == 2019 and ex_season == 4) is True 13 | 14 | def test_season_to_ex_year_and_season_int_successfully(self): 15 | ex_year_and_season_int = DateTool.season_to_ex_year_and_season_int(20201) 16 | assert ex_year_and_season_int == 20194 17 | 18 | def test_to_tw_year_successfully(self): 19 | tw_year = DateTool.to_tw_year(2020) 20 | assert tw_year == 109 21 | 22 | def test_tw_year_to_year_successfully(self): 23 | year = DateTool.tw_year_to_year(109) 24 | assert year == 2020 25 | 26 | 27 | if __name__ == '__main__': 28 | unittest.main() 29 | -------------------------------------------------------------------------------- /dags/common/setup_project.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | from functools import wraps 4 | 5 | from airflow.utils.dates import days_ago 6 | from airflow.settings import AIRFLOW_HOME 7 | from airflow import DAG 8 | from airflow.operators.bash import BashOperator 9 | 10 | 11 | class EnvSetting(object): 12 | PROJECT_HOME = f'{AIRFLOW_HOME}/dags/tw-financial-report-analysis' 13 | 14 | @staticmethod 15 | def append_project_to_path(f): 16 | @wraps(f) 17 | def insert_path(*args, **kwds): 18 | sys.path.insert(0, EnvSetting.PROJECT_HOME) 19 | return f(*args, **kwds) 20 | 21 | return insert_path 22 | 23 | 24 | args = { 25 | 'owner': 'sean', 26 | } 27 | 28 | dag = DAG( 29 | dag_id='setup_tw_financial_report_analysis', 30 | default_args=args, 31 | schedule_interval='@daily', 32 | start_date=days_ago(1), 33 | ) 34 | 35 | check_requirements = BashOperator( 36 | task_id='check_requirements', 37 | bash_command=f'pip3 install -r {EnvSetting.PROJECT_HOME}/airflow_requirements.txt', 38 | depends_on_past=True, 39 | dag=dag 40 | ) 41 | 42 | check_requirements 43 | -------------------------------------------------------------------------------- /web/app.py: -------------------------------------------------------------------------------- 1 | import json 2 | import logging 3 | import plotly 4 | import plotly.express as px 5 | from flask import Flask, render_template 6 | import pandas as pd 7 | from pymongo import MongoClient 8 | from urllib.parse import quote_plus 9 | 10 | uri = "mongodb://%s:%s@%s:%s/%s" % ( 11 | quote_plus('airflower'), 12 | quote_plus('money'), 13 | '127.0.0.1', 14 | '27017', 15 | 'stock' 16 | ) 17 | stock_db = MongoClient(uri).stock 18 | 19 | app = Flask(__name__, template_folder='./templates') 20 | 21 | @app.route("/") 22 | def dashboard(): 23 | financial_report_data = pd.DataFrame(stock_db.financialReports.find({})) 24 | financial_report_data['stockCode'] = financial_report_data['stockCode'].astype(str) 25 | roa_fig = px.bar(financial_report_data, x="yearAndSeason", y="roa", color="stockCode", barmode="group") 26 | roa_fig.update_layout(xaxis_type='category') 27 | roa_fig.update_xaxes(categoryorder='category ascending') 28 | roa_json = json.dumps(roa_fig, cls=plotly.utils.PlotlyJSONEncoder) 29 | 30 | roe_fig = px.bar(financial_report_data, x="yearAndSeason", y="roe", color="stockCode", barmode="group") 31 | roe_fig.update_layout(xaxis_type='category') 32 | roe_fig.update_xaxes(categoryorder='category ascending') 33 | roe_json = json.dumps(roe_fig, cls=plotly.utils.PlotlyJSONEncoder) 34 | return render_template('dashboard.html', roaJson=roa_json, roeJson=roe_json) 35 | 36 | if __name__ == '__main__': 37 | app.run() -------------------------------------------------------------------------------- /strategies/three_days_strategy.py: -------------------------------------------------------------------------------- 1 | from db_bridges.mongo_bridge.collection_structure import CollectionStructure 2 | from toolbox.date_tool import DateTool 3 | 4 | 5 | class ThreeDaysStrategyDataCollection(CollectionStructure): 6 | def set_name(self): 7 | return 'threeDaysStrategyData' 8 | 9 | def set_unique_fields(self): 10 | return { 11 | 'stockCode': str, 12 | 'year': int, 13 | 'month': int, 14 | 'day': int, 15 | 'intDate': int, 16 | } 17 | 18 | def set_data_fields(self): 19 | return { 20 | 'd1AgoTradeVolume': 0, 21 | 'd1AgoAmount': 0, 22 | 'd1AgoOpeningPrice': 0.0, 23 | 'd1AgoHighestPrice': 0.0, 24 | 'd1AgoLowestPrice': 0.0, 25 | 'd1AgoClosingPrice': 0.0, 26 | 'd1AgoClosingDiff': 0.0, 27 | 'd1AgoTransaction': 0, 28 | 'd1AgoMaxDiff': 0.0, 29 | 'd2AgoTradeVolume': 0, 30 | 'd2AgoAmount': 0, 31 | 'd2AgoOpeningPrice': 0.0, 32 | 'd2AgoHighestPrice': 0.0, 33 | 'd2AgoLowestPrice': 0.0, 34 | 'd2AgoClosingPrice': 0.0, 35 | 'd2AgoClosingDiff': 0.0, 36 | 'd2AgoTransaction': 0, 37 | 'd2AgoMaxDiff': 0.0, 38 | 'd3AgoTradeVolume': 0, 39 | 'd3AgoAmount': 0, 40 | 'd3AgoOpeningPrice': 0.0, 41 | 'd3AgoHighestPrice': 0.0, 42 | 'd3AgoLowestPrice': 0.0, 43 | 'd3AgoClosingPrice': 0.0, 44 | 'd3AgoClosingDiff': 0.0, 45 | 'd3AgoTransaction': 0, 46 | 'd1AgoMaxDiff': 0.0, 47 | 'dDayPriseRise': 0, 48 | 'dDayClosingDiff': 0, 49 | } 50 | -------------------------------------------------------------------------------- /tests/db_bridges/mongo_bridge/test_collection_structure.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from db_bridges.mongo_bridge.collection_structure import CollectionStructure 3 | 4 | 5 | class FakeCollection(CollectionStructure): 6 | 7 | def set_name(self): 8 | return 'fake' 9 | 10 | def set_unique_fields(self): 11 | return {'a': int, 'b': str} 12 | 13 | def set_data_fields(self): 14 | 15 | return {'c': 1, 'd': 'hi'} 16 | 17 | 18 | class IncompleteFakeCollection(CollectionStructure): 19 | 20 | def set_name(self): 21 | return 'fake' 22 | 23 | def set_unique_fields(self): 24 | return {'a': int, 'b': str} 25 | 26 | # without this 27 | # def set_data_fields(self): 28 | # return {'c': 1, 'd': 'hi'} 29 | 30 | 31 | class TestCollectionStructure(unittest.TestCase): 32 | 33 | def test_create_collection_structure_successfully(self): 34 | try: 35 | fake = FakeCollection() 36 | new_document = fake.document_operator(a=123, b='str', c=2) 37 | 38 | self.assertEqual(fake.name, 'fake') 39 | self.assertEqual(fake.unique_fields, {'a': int, 'b': str}) 40 | self.assertEqual(fake.data_fields, {'c': 1, 'd': 'hi'}) 41 | self.assertEqual(new_document, { 42 | 'a': 123, 43 | 'b': 'str', 44 | 'c': 2, 45 | 'd': 'hi' 46 | }) 47 | 48 | except Exception as e: 49 | self.fail(f'Raise {e}. So The testing is fail.') 50 | 51 | def test_create_collection_structure_unsuccessfully(self): 52 | try: 53 | _ = IncompleteFakeCollection() 54 | self.fail(f'Except create fail') 55 | except TypeError as e: 56 | self.assertEqual( 57 | str(e), 58 | "Can't instantiate abstract class IncompleteFakeCollection with abstract methods set_data_fields" 59 | ) 60 | pass 61 | -------------------------------------------------------------------------------- /toolbox/date_tool.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | import time 3 | 4 | 5 | class DateTool: 6 | 7 | @staticmethod 8 | def date_to_timestamp(year: int, month: int, day: int, end=False): 9 | if end: 10 | this_date = datetime( 11 | year=year, 12 | month=month, 13 | day=day, 14 | hour=23, 15 | minute=59, 16 | second=59 17 | ) 18 | else: 19 | this_date = datetime( 20 | year=year, 21 | month=month, 22 | day=day 23 | ) 24 | 25 | return int(time.mktime(this_date.timetuple())) 26 | 27 | @staticmethod 28 | def date_to_ex_year_and_season(year, month): 29 | ex_season_mapping = { 30 | 1: 4, 31 | 2: 4, 32 | 3: 4, 33 | 4: 1, 34 | 5: 1, 35 | 6: 1, 36 | 7: 2, 37 | 8: 2, 38 | 9: 2, 39 | 10: 3, 40 | 11: 3, 41 | 12: 3 42 | } 43 | season = ex_season_mapping[month] 44 | season_year = year if season != 4 else year - 1 45 | 46 | return season_year, season 47 | 48 | @staticmethod 49 | def season_to_ex_year_and_season_int(now_year_and_season: int): 50 | now_season = now_year_and_season % 10 51 | now_year = now_year_and_season // 10 52 | ex_season = now_season - 1 53 | ex_year = now_year 54 | if ex_season == 0: 55 | ex_season = 4 56 | ex_year -= 1 57 | return ex_year * 10 + ex_season 58 | 59 | @staticmethod 60 | def to_tw_year(year: int) -> int: 61 | return year - 1911 62 | 63 | @staticmethod 64 | def tw_year_to_year(year: int) -> int: 65 | return year + 1911 66 | 67 | @staticmethod 68 | def to_next_year_month(year: int, month: int) -> (int, int): 69 | month += 1 70 | if month == 13: 71 | year += 1 72 | month = 1 73 | return year, month 74 | 75 | -------------------------------------------------------------------------------- /db_bridges/mongo_bridge/collection_structure.py: -------------------------------------------------------------------------------- 1 | import abc 2 | import logging 3 | 4 | 5 | class CollectionStructure(abc.ABC): 6 | def __init__(self): 7 | self.name = self.set_name() 8 | self.__unique_fields = self.set_unique_fields() 9 | self.__data_fields = self.set_data_fields() 10 | 11 | @abc.abstractmethod 12 | def set_name(self): 13 | # For Example: return 'collectionA' 14 | raise NotImplementedError 15 | 16 | @abc.abstractmethod 17 | def set_unique_fields(self): 18 | # {'key': 'type'} For Example: return {'a': int(), 'b': str} 19 | raise NotImplementedError 20 | 21 | @property 22 | def unique_fields(self): 23 | return self.__unique_fields 24 | 25 | @abc.abstractmethod 26 | def set_data_fields(self): 27 | # {key: default_value} For Example: return {'c': 123} 28 | raise NotImplementedError 29 | 30 | @property 31 | def data_fields(self): 32 | return self.__data_fields 33 | 34 | def get_unique_fields(self, document): 35 | unique_fields = {} 36 | for key in self.unique_fields.keys(): 37 | unique_fields[key] = document[key] 38 | 39 | return unique_fields 40 | 41 | def bind_db(self, db): 42 | return db[self.name] 43 | 44 | def document_operator(self, **kwargs): 45 | document_dict = {} 46 | document_dict.update(self.__data_fields) 47 | try: 48 | for field in self.__unique_fields.keys(): 49 | document_dict[field] = kwargs[field] 50 | except KeyError as e: 51 | logging.error(f'The unique field, {e}:{self.__unique_fields[e]}, is not included in kwargs.') 52 | return None 53 | 54 | for field in self.__data_fields.keys(): 55 | try: 56 | document_dict[field] = kwargs[field] 57 | except KeyError as e: 58 | logging.warning((f'The data field, {e}, is not included in kwargs. ' 59 | f'So use the default value for this.')) 60 | 61 | return document_dict 62 | 63 | 64 | class TryThis(CollectionStructure): 65 | def set_name(self): 66 | return 'monthly' 67 | 68 | def set_unique_fields(self): 69 | return ['a', 'b'] 70 | 71 | def set_data_fields(self): 72 | return ['c', 'd'] 73 | 74 | 75 | if __name__ == '__main__': 76 | a = TryThis() 77 | print(a.name) 78 | print(a.key_fields) 79 | 80 | print(a.data_fields) -------------------------------------------------------------------------------- /reports/financial_reports/sheet.py: -------------------------------------------------------------------------------- 1 | import abc 2 | import re 3 | 4 | from bs4 import BeautifulSoup 5 | 6 | 7 | class Sheet(abc.ABC): 8 | def __init__(self): 9 | self.magic_id = "" 10 | self.dollar_unit = 0 11 | self.sheet = None 12 | 13 | def set_dollar_unit(self, dollar_unit): 14 | self.dollar_unit = dollar_unit 15 | 16 | @abc.abstractmethod 17 | def set_magic_id(self): 18 | raise NotImplementedError 19 | 20 | def get_magic_id(self): 21 | return self.magic_id 22 | 23 | def parse_items_to_dict(self, item_set, columns={1: ''}): 24 | result = self.search_key_string_set(self.sheet, item_set, columns) 25 | self.sheet_str_to_number(result) 26 | return result 27 | 28 | @staticmethod 29 | def search_key_string_set(sheet: BeautifulSoup, search_set: set, columns): 30 | result = {} 31 | 32 | td = sheet.find_next('td') 33 | while search_set and td: 34 | zh_td_span = td.find('span', class_='zh') 35 | en_td_span = td.find('span', class_='en') 36 | for td_span in [zh_td_span, en_td_span]: 37 | if td_span: 38 | span_item = td_span.string.lstrip() 39 | if span_item in search_set: 40 | span_item = re.split(' \\(|（', span_item)[0] 41 | str_list = re.split(' |-', span_item.lower()) 42 | formatted_item_name = f'{str_list[0]}' 43 | for this_str in str_list[1:]: 44 | formatted_item_name += f'{this_str[0].upper()}{this_str[1:]}' 45 | 46 | siblings_tds = td.find_next_siblings('td') 47 | for idx, prefix in columns.items(): 48 | td_value = siblings_tds[idx - 1].find('ix:nonfraction').string 49 | if prefix != '': 50 | new_formatted = f'{formatted_item_name[0].upper()}{formatted_item_name[1:]}' 51 | result[f'{prefix}{new_formatted}'] = td_value 52 | else: 53 | result[formatted_item_name] = td_value 54 | search_set.remove(span_item) 55 | break 56 | 57 | td = td.find_next('td') 58 | 59 | return result 60 | 61 | @staticmethod 62 | def sheet_str_to_number(result_dict: dict): 63 | for key, acc_number in result_dict.items(): 64 | number = acc_number.replace(",", "") 65 | if '(' in number: 66 | result_dict[key] = float(number[1:-1]) * -1.0 67 | else: 68 | result_dict[key] = float(number) 69 | -------------------------------------------------------------------------------- /daily_transaction/yahoo_daily_transaction.py: -------------------------------------------------------------------------------- 1 | import requests 2 | 3 | from db_bridges.mongo_bridge.collection_structure import CollectionStructure 4 | from toolbox.date_tool import DateTool 5 | 6 | 7 | class YahooDailyTransactionsCollection(CollectionStructure): 8 | def set_name(self): 9 | return 'yahooDailyTransactions' 10 | 11 | def set_unique_fields(self): 12 | return { 13 | 'stockCode': str, 14 | 'year': int, 15 | 'month': int, 16 | 'day': int, 17 | 'intDate': int, 18 | } 19 | 20 | def set_data_fields(self): 21 | return { 22 | 'high': 0.0, 23 | 'low': 0.0, 24 | 'volume': 0, 25 | 'close': 0.0, 26 | 'open': 0.0, 27 | } 28 | 29 | 30 | class YahooDailyTransaction(object): 31 | 32 | def __init__(self, stock: str, year: int, month: int, day: int): 33 | self.document = None 34 | start_period = DateTool.date_to_timestamp( 35 | year=year, 36 | month=month, 37 | day=day 38 | ) 39 | end_period = DateTool.date_to_timestamp( 40 | year=year, 41 | month=month, 42 | day=day, 43 | end=True 44 | ) 45 | headers = { 46 | 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) ' 47 | 'Chrome/39.0.2171.95 Safari/537.36' 48 | } 49 | 50 | resp = requests.get( 51 | url=f'https://query1.finance.yahoo.com/v8/finance/chart/{stock}', 52 | params={ 53 | 'period1': start_period, 54 | 'period2': end_period, 55 | 'interval': '1d', 56 | 'events': 'history', 57 | }, 58 | headers=headers 59 | ) 60 | resp.encoding = 'utf8' 61 | json_resp = resp.json() 62 | this_day_data = json_resp['chart']['result'][0]['indicators']['quote'][0] 63 | if this_day_data: 64 | self.document = YahooDailyTransactionsCollection().document_operator( 65 | stockCode=stock, 66 | year=year, 67 | month=month, 68 | day=day, 69 | intDate=year*10000+month*100+day, 70 | high=this_day_data['high'][0], 71 | low=this_day_data['low'][0], 72 | volume=this_day_data['volume'][0], 73 | close=this_day_data['close'][0], 74 | open=this_day_data['open'][0] 75 | ) 76 | 77 | 78 | def main(): 79 | # ^GSPC 80 | new_stock = YahooDailyTransaction('^GSPC', 2022, 1, 2) 81 | print(new_stock.document) 82 | pass 83 | 84 | 85 | if __name__ == '__main__': 86 | main() -------------------------------------------------------------------------------- /dags/parse_monthly_revenue/parse_monthly_revenue.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import sys 3 | from datetime import datetime 4 | from datetime import timedelta 5 | from functools import wraps 6 | 7 | from airflow.settings import AIRFLOW_HOME 8 | from airflow import DAG 9 | from airflow.operators.python import PythonOperator 10 | from functools import wraps 11 | 12 | 13 | class EnvSetting(object): 14 | PROJECT_HOME = f'{AIRFLOW_HOME}/dags/tw-financial-report-analysis' 15 | 16 | @staticmethod 17 | def append_project_to_path(f): 18 | @wraps(f) 19 | def insert_path(*args, **kwds): 20 | sys.path.insert(0, EnvSetting.PROJECT_HOME) 21 | return f(*args, **kwds) 22 | 23 | return insert_path 24 | 25 | 26 | # 2633 A 27 | 28 | def init_dag(start_date, schedule_interval='30 12 12 * *'): 29 | args = { 30 | 'owner': 'sean', 31 | } 32 | dag = DAG( 33 | dag_id='parse_monthly_revenue', 34 | default_args=args, 35 | max_active_runs=2, 36 | concurrency=2, 37 | schedule_interval=schedule_interval, 38 | start_date=start_date, 39 | ) 40 | 41 | @EnvSetting.append_project_to_path 42 | def parse_monthly_revenue(**context): 43 | from airflow.providers.mongo.hooks.mongo import MongoHook 44 | 45 | from reports.monthly_revenue.monthly_revenue import MonthlyRevenue 46 | from reports.monthly_revenue.monthly_revenue import MonthlyRevenueCollection 47 | from toolbox.date_tool import DateTool 48 | 49 | mongo_hook = MongoHook(conn_id='stock_mongo') 50 | stock_db = mongo_hook.get_conn().stock 51 | mr_collection_struct = MonthlyRevenueCollection() 52 | monthly_revenue_collection = mr_collection_struct.bind_db(stock_db) 53 | 54 | next_execution_date = context['next_ds'] 55 | target_year, target_month, _ = map(int, next_execution_date.split('-')) 56 | logging.info(f'parsing {target_year}/{target_month} monthly revenue.') 57 | 58 | mr_results = MonthlyRevenue(tw_year=DateTool.to_tw_year(target_year), month=target_month).monthly_revenues 59 | 60 | for result in mr_results: 61 | monthly_revenue_collection.update_one( 62 | filter=mr_collection_struct.get_unique_fields(result), 63 | update={'$set': result}, 64 | upsert=True 65 | ) 66 | 67 | logging.info(f'{target_year}/{target_month} monthly revenue is finished!') 68 | 69 | parse_monthly_revenue_task = PythonOperator( 70 | task_id='parse_monthly_revenue', 71 | provide_context=True, 72 | python_callable=parse_monthly_revenue, 73 | depends_on_past=True, 74 | retries=3, 75 | retry_delay=timedelta(seconds=30), 76 | dag=dag 77 | ) 78 | 79 | parse_monthly_revenue_task 80 | 81 | return dag 82 | 83 | 84 | parse_monthly_revenue_dag = init_dag(start_date=datetime(year=2018, month=12, day=1)) 85 | -------------------------------------------------------------------------------- /reports/monthly_revenue/monthly_revenue.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import requests 3 | 4 | from db_bridges.mongo_bridge.collection_structure import CollectionStructure 5 | from toolbox.date_tool import DateTool 6 | 7 | class MonthlyRevenueCollection(CollectionStructure): 8 | def set_name(self): 9 | return 'monthlyRevenues' 10 | 11 | def set_unique_fields(self): 12 | return { 13 | 'stockCode': str, 14 | 'year': int, 15 | 'month': int, 16 | } 17 | 18 | def set_data_fields(self): 19 | return { 20 | 'company': 'unknown', 21 | 'industrialClass': 'unknown', 22 | 'theMonthlyRevenue': 0.0, 23 | 'exMonthlyRevenue': 0.0, 24 | 'exYearTheMonthlyRevenue': 0.0, 25 | 'mom': 0.0, 26 | 'yoy': 0.0, 27 | 'cumTheMonthlyRevenue': 0.0, 28 | 'cumExYearTheMonthlyRevenue': 0.0, 29 | 'cmocm': 0.0, 30 | 'note': '-' 31 | } 32 | 33 | 34 | class MonthlyRevenue(object): 35 | def __init__(self, tw_year, month): 36 | resp = requests.post( 37 | url='https://mops.twse.com.tw/server-java/FileDownLoad', 38 | data={ 39 | 'step': '9', 40 | 'functionName': 'show_file', 41 | 'filePath': '/home/html/nas/t21/sii/', 42 | 'fileName': f't21sc03_{tw_year}_{month}.csv' 43 | } 44 | ) 45 | resp.encoding = 'utf8' 46 | self.content = resp.text 47 | 48 | self.monthly_revenues = list() 49 | 50 | def str_to_float(this_str: str): 51 | try: 52 | return float(this_str) 53 | except ValueError: 54 | return 0.0 55 | 56 | for company in resp.text.split('\r\n')[1:-1]: 57 | items = company.replace('"', '').split(',') 58 | tw_year, month = map(int, items[1].split('/')) 59 | 60 | self.monthly_revenues.append( 61 | MonthlyRevenueCollection().document_operator( 62 | stockCode=items[2], 63 | year=DateTool.tw_year_to_year(tw_year), 64 | month=month, 65 | company=items[3], 66 | industrialClass=items[4], 67 | theMonthlyRevenue=str_to_float(items[5]), 68 | exMonthlyRevenue=str_to_float(items[6]), 69 | exYearTheMonthlyRevenue=str_to_float(items[7]), 70 | mom=str_to_float(items[8]), 71 | yoy=str_to_float(items[9]), 72 | cumTheMonthlyRevenue=str_to_float(items[10]), 73 | cumExYearTheMonthlyRevenue=str_to_float(items[11]), 74 | cmocm=str_to_float(items[12]), 75 | note=items[13] 76 | ) 77 | ) 78 | 79 | 80 | if __name__ == '__main__': 81 | for document in MonthlyRevenue(110, 8).monthly_revenues: 82 | print(document) -------------------------------------------------------------------------------- /dags/three_days_strategy/three_days_strategy.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import sys 3 | from datetime import datetime 4 | from datetime import timedelta 5 | from functools import wraps 6 | 7 | from airflow.settings import AIRFLOW_HOME 8 | from airflow import DAG 9 | from airflow.operators.python import PythonOperator 10 | from functools import wraps 11 | 12 | 13 | class EnvSetting(object): 14 | PROJECT_HOME = f'{AIRFLOW_HOME}/dags/tw-financial-report-analysis' 15 | 16 | @staticmethod 17 | def append_project_to_path(f): 18 | @wraps(f) 19 | def insert_path(*args, **kwds): 20 | sys.path.insert(0, EnvSetting.PROJECT_HOME) 21 | return f(*args, **kwds) 22 | 23 | return insert_path 24 | 25 | 26 | # 2633 A 27 | 28 | def init_dag(stock_code, start_date, schedule_interval='30 12 1 * *'): 29 | args = { 30 | 'owner': 'sean', 31 | } 32 | dag = DAG( 33 | dag_id=f'crawl_{stock_code}_daily_transaction', 34 | default_args=args, 35 | max_active_runs=2, 36 | concurrency=2, 37 | schedule_interval=schedule_interval, 38 | start_date=start_date, 39 | tags=['stock'], 40 | ) 41 | 42 | @EnvSetting.append_project_to_path 43 | def crawl_daily_transaction(**context): 44 | from time import sleep 45 | 46 | from airflow.providers.mongo.hooks.mongo import MongoHook 47 | 48 | from daily_transaction.daily_transaction import DailyTransactionsInMonth 49 | from daily_transaction.daily_transaction import DailyTransactionsCollection 50 | 51 | mongo_hook = MongoHook(conn_id='stock_mongo') 52 | stock_db = mongo_hook.get_conn().stock 53 | dt_collection_struct = DailyTransactionsCollection() 54 | daily_transaction_collection = dt_collection_struct.bind_db(stock_db) 55 | 56 | execution_date = context['ds'] 57 | target_year, target_month, _ = map(int, execution_date.split('-')) 58 | logging.info(f'parsing daily stock of {target_year}/{target_month}.') 59 | 60 | dt_results = DailyTransactionsInMonth(stock=stock_code, year=target_year, month=target_month).documents 61 | 62 | for result in dt_results: 63 | daily_transaction_collection.update_one( 64 | filter=dt_collection_struct.get_unique_fields(result), 65 | update={'$set': result}, 66 | upsert=True 67 | ) 68 | 69 | logging.info(f'sleep 5 second.') 70 | sleep(5) 71 | logging.info(f'{target_year}/{target_month} daily transaction is finished!') 72 | 73 | parse_daily_transaction_task = PythonOperator( 74 | task_id='crawl_daily_transaction', 75 | provide_context=True, 76 | python_callable=crawl_daily_transaction, 77 | depends_on_past=True, 78 | retries=3, 79 | retry_delay=timedelta(seconds=30), 80 | dag=dag 81 | ) 82 | 83 | parse_daily_transaction_task 84 | 85 | return dag 86 | 87 | 88 | parse_0050_transaction_dag = init_dag(stock_code='0050', start_date=datetime(year=2018, month=12, day=1)) 89 | -------------------------------------------------------------------------------- /dags/crawl_daily_transaction/crawl_daily_transaction.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import sys 3 | from datetime import datetime 4 | from datetime import timedelta 5 | from functools import wraps 6 | 7 | from airflow.settings import AIRFLOW_HOME 8 | from airflow import DAG 9 | from airflow.operators.python import PythonOperator 10 | from functools import wraps 11 | 12 | 13 | class EnvSetting(object): 14 | PROJECT_HOME = f'{AIRFLOW_HOME}/dags/tw-financial-report-analysis' 15 | 16 | @staticmethod 17 | def append_project_to_path(f): 18 | @wraps(f) 19 | def insert_path(*args, **kwds): 20 | sys.path.insert(0, EnvSetting.PROJECT_HOME) 21 | return f(*args, **kwds) 22 | 23 | return insert_path 24 | 25 | 26 | # 2633 A 27 | 28 | def init_dag(stock_code, start_date, schedule_interval='30 12 * * *'): 29 | args = { 30 | 'owner': 'sean', 31 | } 32 | dag = DAG( 33 | dag_id=f'crawl_{stock_code}_daily_transaction', 34 | default_args=args, 35 | max_active_runs=2, 36 | concurrency=2, 37 | schedule_interval=schedule_interval, 38 | start_date=start_date, 39 | tags=['stock'], 40 | ) 41 | 42 | @EnvSetting.append_project_to_path 43 | def crawl_daily_transaction(**context): 44 | from time import sleep 45 | 46 | from airflow.providers.mongo.hooks.mongo import MongoHook 47 | 48 | from daily_transaction.daily_transaction import DailyTransactionsInMonth 49 | from daily_transaction.daily_transaction import DailyTransactionsCollection 50 | 51 | mongo_hook = MongoHook(conn_id='stock_mongo') 52 | stock_db = mongo_hook.get_conn().stock 53 | dt_collection_struct = DailyTransactionsCollection() 54 | daily_transaction_collection = dt_collection_struct.bind_db(stock_db) 55 | 56 | execution_date = context['ds'] 57 | target_year, target_month, _ = map(int, execution_date.split('-')) 58 | logging.info(f'parsing daily stock of {target_year}/{target_month}.') 59 | 60 | dt_results = DailyTransactionsInMonth(stock=stock_code, year=target_year, month=target_month).documents 61 | 62 | for result in dt_results: 63 | daily_transaction_collection.update_one( 64 | filter=dt_collection_struct.get_unique_fields(result), 65 | update={'$set': result}, 66 | upsert=True 67 | ) 68 | 69 | logging.info(f'sleep 5 second.') 70 | sleep(5) 71 | logging.info(f'{target_year}/{target_month} daily transaction is finished!') 72 | 73 | parse_daily_transaction_task = PythonOperator( 74 | task_id='crawl_daily_transaction', 75 | provide_context=True, 76 | python_callable=crawl_daily_transaction, 77 | depends_on_past=True, 78 | retries=3, 79 | retry_delay=timedelta(seconds=30), 80 | dag=dag 81 | ) 82 | 83 | parse_daily_transaction_task 84 | 85 | return dag 86 | 87 | 88 | parse_0050_transaction_dag = init_dag(stock_code='0050', start_date=datetime(year=2018, month=12, day=1)) 89 | -------------------------------------------------------------------------------- /daily_transaction/daily_transaction.py: -------------------------------------------------------------------------------- 1 | import requests 2 | 3 | from db_bridges.mongo_bridge.collection_structure import CollectionStructure 4 | from toolbox.date_tool import DateTool 5 | 6 | 7 | class DailyTransactionsCollection(CollectionStructure): 8 | def set_name(self): 9 | return 'dailyTransactions' 10 | 11 | def set_unique_fields(self): 12 | return { 13 | 'stockCode': str, 14 | 'year': int, 15 | 'month': int, 16 | 'day': int, 17 | 'intDate': int, 18 | } 19 | 20 | def set_data_fields(self): 21 | return { 22 | 'tradeVolume': 0, 23 | 'amount': 0, 24 | 'openingPrice': 0.0, 25 | 'highestPrice': 0.0, 26 | 'lowestPrice': 0.0, 27 | 'closingPrice': 0.0, 28 | 'closingDiff': 0.0, 29 | 'transaction': 0, 30 | } 31 | 32 | 33 | class DailyTransactionsInMonth(object): 34 | @staticmethod 35 | def remove_comma_to_int(with_comma: str) -> int: 36 | return int(with_comma.replace(',', '')) 37 | 38 | @staticmethod 39 | def remove_comma_to_float(with_comma: str) -> float: 40 | return float(with_comma.replace(',', '')) 41 | 42 | @staticmethod 43 | def distinguish_sign_to_float(with_sign: str) -> float: 44 | if with_sign[0] == '-': 45 | return -1.0 * float(with_sign[1:]) 46 | else: 47 | return float(with_sign[1:]) 48 | 49 | def __init__(self, stock: str, year: int, month: int): 50 | self.documents = list() 51 | 52 | resp = requests.get( 53 | url='https://www.twse.com.tw/exchangeReport/STOCK_DAY', 54 | params={ 55 | 'response': 'json', 56 | 'date': f'{year}{month:0>2}01', 57 | 'stockNo': stock, 58 | } 59 | ) 60 | resp.encoding = 'utf8' 61 | json_resp = resp.json() 62 | 63 | for daily_info in json_resp['data']: 64 | got_tw_year, got_month, got_day = map(int, daily_info[0].split('/')) 65 | got_year = DateTool.tw_year_to_year(got_tw_year) 66 | self.documents.append( 67 | DailyTransactionsCollection().document_operator( 68 | stockCode=stock, 69 | year=got_year, 70 | month=got_month, 71 | day=got_day, 72 | intDate=got_year * 10000 + got_month * 100 + got_day, 73 | tradeVolume=self.__class__.remove_comma_to_int(daily_info[1]), 74 | amount=self.__class__.remove_comma_to_int(daily_info[2]), 75 | openingPrice=self.__class__.remove_comma_to_float(daily_info[3]), 76 | highestPrice=self.__class__.remove_comma_to_float(daily_info[4]), 77 | lowestPrice=self.__class__.remove_comma_to_float(daily_info[5]), 78 | closingPrice=self.__class__.remove_comma_to_float(daily_info[6]), 79 | closingDiff=self.__class__.distinguish_sign_to_float(daily_info[7]), 80 | transaction=self.__class__.remove_comma_to_int(daily_info[8]), 81 | ) 82 | ) 83 | 84 | 85 | def main(): 86 | new_stock = DailyTransactionsInMonth('0050', 2021, 1) 87 | print(new_stock.documents) 88 | pass 89 | 90 | 91 | if __name__ == '__main__': 92 | main() -------------------------------------------------------------------------------- /reports/financial_reports/financial_report_agent.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import logging 3 | 4 | from bs4 import BeautifulSoup 5 | 6 | from reports.financial_reports.sheet import Sheet 7 | from reports.financial_reports.balance_sheet import BalanceSheet 8 | from reports.financial_reports.comprehensive_income_sheet import ComprehensiveIncomeSheet 9 | 10 | 11 | class FinancialReportAgent(object): 12 | def __new__( 13 | cls, 14 | company_id: str, 15 | year: int, 16 | season: int, 17 | report_type: str 18 | ): 19 | try: 20 | resp = requests.get( 21 | f'https://mops.twse.com.tw/server-java/t164sb01?step=1&' 22 | f'CO_ID={company_id}&' 23 | f'SYEAR={year}&' 24 | f'SSEASON={season}&' 25 | f'REPORT_ID={report_type}') 26 | resp.encoding = 'big5' 27 | soup = BeautifulSoup(resp.text, 'html.parser') 28 | 29 | check_status = soup.find('h4') 30 | # print(type(check_status)) 31 | if check_status and check_status.string == '檔案不存在!': 32 | return None 33 | else: 34 | instance = super(FinancialReportAgent, cls).__new__(cls) 35 | instance.__dict__['soup'] = soup 36 | return instance 37 | except Exception as e: 38 | logging.exception(e) 39 | 40 | def __init__( 41 | self, 42 | company_id: str, 43 | year: int, 44 | season: int, 45 | report_type: str, 46 | ): 47 | self.company_id = company_id 48 | self.year = year 49 | self.season = season 50 | self.report_type = report_type 51 | self.balance_sheet = BalanceSheet(self.soup.find('table')) 52 | self.parse_sheet_unit(self.balance_sheet, self.soup) 53 | self.comprehensive_income_sheet = ComprehensiveIncomeSheet(self.balance_sheet.sheet.find_next_sibling('table')) 54 | self.parse_sheet_unit(self.comprehensive_income_sheet, self.soup) 55 | 56 | @staticmethod 57 | def parse_sheet_unit(sheet: Sheet, report_html: BeautifulSoup): 58 | unit_string = report_html.find( 59 | 'div', id=sheet.get_magic_id() 60 | ).find_next( 61 | 'div', 'rptidx' 62 | ).find( 63 | 'span', 'en' 64 | ).string 65 | 66 | if 'thousands' in unit_string: 67 | sheet.set_dollar_unit(1000) 68 | else: 69 | logging.warning(f'Unkown unit: {unit_string}') 70 | 71 | return sheet 72 | 73 | 74 | if __name__ == '__main__': 75 | # search_comprehensive_income_set = { 76 | # 'Total operating revenue', 77 | # 'Total operating costs', 78 | # 'Total basic earnings per share' 79 | # } 80 | fn_report_agent = FinancialReportAgent("2605", 2020, 3, "C") 81 | search_balance_sheet_set = { 82 | 'Total assets', 83 | 'Total current assets', 84 | 'Total non-current assets', 85 | 'Total liabilities', 86 | 'Total current liabilities', 87 | 'Total non-current liabilities', 88 | 'Total equity' 89 | } 90 | balance_sheet_res = fn_report_agent.balance_sheet.parse_items_to_dict(search_balance_sheet_set) 91 | print(balance_sheet_res) 92 | search_comprehensive_income_sheet_set = { 93 | 'Total operating revenue', 94 | 'Total operating costs', 95 | 'Total basic earnings per share' 96 | } 97 | income_res = fn_report_agent.comprehensive_income_sheet.parse_items_to_dict(search_comprehensive_income_sheet_set) 98 | print(income_res) 99 | -------------------------------------------------------------------------------- /dags/crawl_daily_transaction/crawl_yahoo_daily_transaction.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import sys 3 | from datetime import datetime 4 | from datetime import timedelta 5 | from functools import wraps 6 | 7 | from airflow.settings import AIRFLOW_HOME 8 | from airflow import DAG 9 | from airflow.operators.python import BranchPythonOperator 10 | from airflow.operators.dummy import DummyOperator 11 | 12 | 13 | class EnvSetting(object): 14 | PROJECT_HOME = f'{AIRFLOW_HOME}/dags/tw-financial-report-analysis' 15 | 16 | @staticmethod 17 | def append_project_to_path(f): 18 | @wraps(f) 19 | def insert_path(*args, **kwds): 20 | sys.path.insert(0, EnvSetting.PROJECT_HOME) 21 | return f(*args, **kwds) 22 | 23 | return insert_path 24 | 25 | 26 | def init_dag(stock_code, start_date, schedule_interval='30 12 * * *'): 27 | args = { 28 | 'owner': 'sean', 29 | } 30 | dag = DAG( 31 | dag_id=f'crawl_{stock_code}_yahoo_daily_transaction', 32 | default_args=args, 33 | max_active_runs=2, 34 | concurrency=2, 35 | schedule_interval=schedule_interval, 36 | start_date=start_date, 37 | tags=['stock'], 38 | ) 39 | 40 | @EnvSetting.append_project_to_path 41 | def crawl_daily_transaction(**context): 42 | from time import sleep 43 | 44 | from airflow.providers.mongo.hooks.mongo import MongoHook 45 | 46 | from daily_transaction.yahoo_daily_transaction import YahooDailyTransaction 47 | from daily_transaction.yahoo_daily_transaction import YahooDailyTransactionsCollection 48 | 49 | mongo_hook = MongoHook(conn_id='stock_mongo') 50 | stock_db = mongo_hook.get_conn().stock 51 | ydt_collection_struct = YahooDailyTransactionsCollection() 52 | yahoo_daily_transaction_collection = ydt_collection_struct.bind_db(stock_db) 53 | 54 | execution_date = context['ds'] 55 | target_year, target_month, target_day = map(int, execution_date.split('-')) 56 | logging.info(f'parsing daily stock of {target_year}/{target_month}/{target_day}.') 57 | 58 | ydt_result = YahooDailyTransaction( 59 | stock=stock_code, 60 | year=target_year, 61 | month=target_month, 62 | day=target_day 63 | ).document 64 | logging.info(f'data: {ydt_result}') 65 | 66 | if ydt_result: 67 | yahoo_daily_transaction_collection.update_one( 68 | filter=ydt_collection_struct.get_unique_fields(ydt_result), 69 | update={'$set': ydt_result}, 70 | upsert=True 71 | ) 72 | logging.info(f'sleep 10 second.') 73 | sleep(10) 74 | logging.info(f'{target_year}/{target_month}/{target_day} daily transaction is finished!') 75 | return 'done' 76 | else: 77 | return 'no_data' 78 | 79 | parse_daily_transaction_task = BranchPythonOperator( 80 | task_id='crawl_daily_transaction', 81 | provide_context=True, 82 | python_callable=crawl_daily_transaction, 83 | depends_on_past=True, 84 | retries=3, 85 | retry_delay=timedelta(seconds=30), 86 | dag=dag 87 | ) 88 | 89 | no_data_task = DummyOperator( 90 | task_id='no_data', 91 | dag=dag 92 | ) 93 | 94 | done_task = DummyOperator( 95 | task_id='done', 96 | dag=dag 97 | ) 98 | 99 | parse_daily_transaction_task >> [no_data_task, done_task] 100 | 101 | return dag 102 | 103 | 104 | parse_0050_transaction_dag = init_dag(stock_code='0050.TW', start_date=datetime(year=2018, month=12, day=1)) 105 | -------------------------------------------------------------------------------- /tests/reports/financial_reports/test_financial_report_agent.py: -------------------------------------------------------------------------------- 1 | from reports.financial_reports.financial_report_agent import FinancialReportAgent 2 | import unittest 3 | from unittest import mock 4 | 5 | FINANCIAL_REPORT_RESOURCE_PATH = './tests/reports/financial_reports/resources' 6 | 7 | 8 | class FinancialReportAgentTestCase(unittest.TestCase): 9 | 10 | def mocked_requests_get(*args, **kwargs): 11 | class MockResponse(object): 12 | def __init__(self, text): 13 | self.encoding = 'big5' 14 | self.text = text 15 | 16 | if args[0] == (f'https://mops.twse.com.tw/server-java/t164sb01?step=1&' 17 | f'CO_ID=6666&' 18 | f'SYEAR=2021&' 19 | f'SSEASON=4&' 20 | f'REPORT_ID=C'): 21 | 22 | with open( 23 | f'{FINANCIAL_REPORT_RESOURCE_PATH}/mock_financial_report_success_example.html', 24 | 'r', 25 | encoding='big5' 26 | ) as f: 27 | success_html = f.read() 28 | return MockResponse(text=success_html) 29 | elif args[0] == (f'https://mops.twse.com.tw/server-java/t164sb01?step=1&' 30 | f'CO_ID=6666&' 31 | f'SYEAR=2021&' 32 | f'SSEASON=5&' 33 | f'REPORT_ID=C'): 34 | 35 | with open( 36 | f'{FINANCIAL_REPORT_RESOURCE_PATH}/mock_financial_report_fail_example.html', 37 | 'r', 38 | encoding='big5' 39 | ) as f: 40 | fail_html = f.read() 41 | return MockResponse(text=fail_html) 42 | 43 | @mock.patch('requests.get', side_effect=mocked_requests_get) 44 | def test_get_financial_report_successfully(self, mock_get): 45 | agent = FinancialReportAgent(company_id='6666', 46 | year=2021, 47 | season=4, 48 | report_type='C') 49 | 50 | search_balance_sheet_set_us = {'Total assets'} 51 | search_balance_sheet_set_tw = {'資產總計'} 52 | us_result = agent.balance_sheet.parse_items_to_dict(search_balance_sheet_set_us, {2: ''}) 53 | tw_result = agent.balance_sheet.parse_items_to_dict(search_balance_sheet_set_tw, {2: ''}) 54 | 55 | self.assertIsNotNone(agent) 56 | self.assertIn(mock.call(f'https://mops.twse.com.tw/server-java/t164sb01?step=1&' 57 | f'CO_ID=6666&' 58 | f'SYEAR=2021&' 59 | f'SSEASON=4&' 60 | f'REPORT_ID=C'), mock_get.call_args_list) 61 | 62 | # test parsing unit 63 | self.assertEqual(agent.balance_sheet.dollar_unit, 1000) 64 | # test parsing column 65 | self.assertEqual(us_result['totalAssets'], tw_result['資產總計']) 66 | # test parsing result 67 | self.assertEqual(us_result['totalAssets'], 5564612) 68 | 69 | @mock.patch('requests.get', side_effect=mocked_requests_get) 70 | def test_get_financial_report_when_file_is_not_exist_unsuccessfully(self, mock_get): 71 | agent = FinancialReportAgent(company_id='6666', 72 | year=2021, 73 | season=5, 74 | report_type='C') 75 | self.assertIsNone(agent) 76 | self.assertIn(mock.call(f'https://mops.twse.com.tw/server-java/t164sb01?step=1&' 77 | f'CO_ID=6666&' 78 | f'SYEAR=2021&' 79 | f'SSEASON=5&' 80 | f'REPORT_ID=C'), mock_get.call_args_list) 81 | 82 | 83 | if __name__ == '__main__': 84 | unittest.main() 85 | -------------------------------------------------------------------------------- /for_setup_airlfow/docker-compose.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | # 18 | 19 | # Basic Airflow cluster configuration for CeleryExecutor with Redis and PostgreSQL. 20 | # 21 | # WARNING: This configuration is for local development. Do not use it in a production deployment. 22 | # 23 | # This configuration supports basic configuration using environment variables or an .env file 24 | # The following variables are supported: 25 | # 26 | # AIRFLOW_IMAGE_NAME - Docker image name used to run Airflow. 27 | # Default: apache/airflow:master-python3.8 28 | # AIRFLOW_UID - User ID in Airflow containers 29 | # Default: 50000 30 | # AIRFLOW_GID - Group ID in Airflow containers 31 | # Default: 50000 32 | # _AIRFLOW_WWW_USER_USERNAME - Username for the administrator account. 33 | # Default: airflow 34 | # _AIRFLOW_WWW_USER_PASSWORD - Password for the administrator account. 35 | # Default: airflow 36 | # 37 | # Feel free to modify this file to suit your needs. 38 | --- 39 | version: '3' 40 | x-airflow-common: 41 | &airflow-common 42 | image: ${AIRFLOW_IMAGE_NAME:-apache/airflow:2.1.3-python3.8} 43 | environment: 44 | &airflow-common-env 45 | AIRFLOW__CORE__EXECUTOR: CeleryExecutor 46 | AIRFLOW__CORE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@postgres/airflow 47 | AIRFLOW__CELERY__RESULT_BACKEND: db+postgresql://airflow:airflow@postgres/airflow 48 | AIRFLOW__CELERY__BROKER_URL: redis://:@redis:6379/0 49 | AIRFLOW__CORE__FERNET_KEY: '' 50 | AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: 'true' 51 | AIRFLOW__CORE__LOAD_EXAMPLES: 'false' 52 | volumes: 53 | - ./dags:/opt/airflow/dags 54 | - ./logs:/opt/airflow/logs 55 | - ./plugins:/opt/airflow/plugins 56 | user: "${AIRFLOW_UID:-50000}:${AIRFLOW_GID:-50000}" 57 | depends_on: 58 | redis: 59 | condition: service_healthy 60 | postgres: 61 | condition: service_healthy 62 | 63 | services: 64 | mongo: 65 | image: mongo 66 | hostname: stock_mongo 67 | restart: always 68 | ports: 69 | - 27017:27017 70 | environment: 71 | MONGO_INITDB_ROOT_USERNAME: root 72 | MONGO_INITDB_ROOT_PASSWORD: example 73 | 74 | postgres: 75 | image: postgres:13 76 | environment: 77 | POSTGRES_USER: airflow 78 | POSTGRES_PASSWORD: airflow 79 | POSTGRES_DB: airflow 80 | volumes: 81 | - postgres-db-volume:/var/lib/postgresql/data 82 | healthcheck: 83 | test: ["CMD", "pg_isready", "-U", "airflow"] 84 | interval: 5s 85 | retries: 5 86 | restart: always 87 | 88 | redis: 89 | image: redis:latest 90 | ports: 91 | - 6379:6379 92 | healthcheck: 93 | test: ["CMD", "redis-cli", "ping"] 94 | interval: 5s 95 | timeout: 30s 96 | retries: 50 97 | restart: always 98 | 99 | airflow-webserver: 100 | <<: *airflow-common 101 | command: webserver 102 | ports: 103 | - 8080:8080 104 | healthcheck: 105 | test: ["CMD", "curl", "--fail", "http://localhost:8080/health"] 106 | interval: 10s 107 | timeout: 10s 108 | retries: 5 109 | restart: always 110 | 111 | airflow-scheduler: 112 | <<: *airflow-common 113 | command: scheduler 114 | restart: always 115 | 116 | airflow-worker: 117 | <<: *airflow-common 118 | command: celery worker 119 | restart: always 120 | 121 | airflow-init: 122 | <<: *airflow-common 123 | command: version 124 | environment: 125 | <<: *airflow-common-env 126 | _AIRFLOW_DB_UPGRADE: 'true' 127 | _AIRFLOW_WWW_USER_CREATE: 'true' 128 | _AIRFLOW_WWW_USER_USERNAME: ${_AIRFLOW_WWW_USER_USERNAME:-airflow} 129 | _AIRFLOW_WWW_USER_PASSWORD: aLrfIow 130 | 131 | flower: 132 | <<: *airflow-common 133 | command: celery flower 134 | ports: 135 | - 5555:5555 136 | healthcheck: 137 | test: ["CMD", "curl", "--fail", "http://localhost:5555/"] 138 | interval: 10s 139 | timeout: 10s 140 | retries: 5 141 | restart: always 142 | 143 | volumes: 144 | postgres-db-volume: 145 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # tw-financial-report-analysis 2 | 3 | 遲早都要分析財報，怎麼不一開始就自己分析？ 4 | 5 | 這個專案希望能提供一個可以快速上手的財報分析系統樣板。此系統樣板可以分成三個主要單元： 6 | 7 | * `dags(Airflow)`：數據收集，排程分析。 8 | * `reports` Modules：財報解析。 9 | * `web`：視覺化數據。 10 | 11 | ## 已完成項目 12 | * 財報定期抓取 13 | * 資產負載表 14 | * 綜合損益表 15 | * 每月營業收入 16 | * 指標 17 | * ROE 18 | * ROA 19 | * 每股淨值（bookValuePerShare） 20 | * EPS（totalBasicEarningsPerShare） 21 | * 財報公佈日期（publishedDate） 22 | * 視覺化 23 | * ROA and ROE 24 | 25 | ## Get Started! 26 | 27 | ### 環境安裝 28 | 29 | 在家目錄下安裝： 30 | ```bash 31 | $ cd ~ 32 | $ mkdir airflow 33 | $ cd airflow 34 | $ curl -LfO 'https://github.com/YanHaoChen/tw-financial-report-analysis/raw/main/for_setup_airlfow/docker-compose.yaml' 35 | $ mkdir ./dags ./logs ./plugins 36 | $ echo -e "AIRFLOW_UID=$(id -u)\nAIRFLOW_GID=0" > .env 37 | $ docker-compose up airflow-init 38 | $ docker-compose up -d 39 | ``` 40 | 41 | ### 建立 mongo 使用者 42 | 43 | ```bash 44 | $ docker exec -it airflow-mongo-1 bash 45 | $ mongosh -u root -p example 46 | 47 | > use stock 48 | > db.createUser( 49 | { 50 | user:"airflower", 51 | pwd: "money", 52 | roles:[ 53 | { role: "readWrite", db: "stock" } 54 | ] 55 | } 56 | ) 57 | 58 | > exit 59 | $ exit 60 | ``` 61 | ### 建立 Mongo Connection 62 | 在建立之前，先讓 Airflow Webserver 新增 Mongo 的 Connection Type。 63 | ``` 64 | $ docker exec -it airflow-airflow-webserver-1 pip3 install apache-airflow-providers-mongo==2.0.0 65 | 66 | # 刷新 Airflow Web 67 | $ docker-compose restart 68 | ``` 69 | 接下來，連進 Airflow Webserver `127.0.0.1:8080`。點選 Admin -> Connections -> 點選加號(Add a new record)。再照以下項目填寫： 70 | 71 | * Conn Id: stock_mongo 72 | * Conn Type: MongoDB 73 | * Host: stock_mongo 74 | * Schema: stock 75 | * Login: airflower 76 | * Password: money 77 | * Port: 27017 78 | 79 | > Airflow Webserver 的帳號密碼如下： 80 | > account: airflow 81 | > password: aLrfIow 82 | 83 | ### 把檔案導入 `dags` 84 | 在把專案放進 `dags` 前，先把 `airflowignore.template` 放進去。 85 | 86 | ```bash 87 | $ cd ./dags 88 | $ curl -LfO 'https://github.com/YanHaoChen/tw-financial-report-analysis/raw/main/for_setup_airlfow/airflowignore.template' 89 | $ mv airflowignore.template .airflowignore 90 | ``` 91 | 現在把可以把專案拉下來囉！ 92 | ```bash 93 | $ git clone https://github.com/YanHaoChen/tw-financial-report-analysis.git 94 | ``` 95 | 現在回到 Airflow Webserver， 等 3~5 分鐘就可以看到這個專案裡的拉報表的 DAG 囉！ 96 | 97 | ### 初始化專案 98 | 99 | 在 Airflow Webserver 上，開啟 `setup_parse_financial_report` DAG。把所需 package 在 Worker 上安裝好。 100 | 101 | ### 開始拉報表囉！ 102 | 103 | 在 Airflow Webserver 上，開啟 `stock_2633`。檢查是否有資料導入 Mongo： 104 | ```bash 105 | $ docker exec -it airflow_mongo_1 bash 106 | $ mongosh -u airflower -p money stock 107 | stock> db.financialReports.find({}) 108 | [ 109 | { 110 | _id: ObjectId("612a16f0163eec9d5d34d670"), 111 | stockCode: 2633, 112 | year: 2021, 113 | season: 2, 114 | yearAndSeason: 20212, 115 | publishedDate: ISODate("2021-08-05T08:24:03.000Z"), 116 | balanceSheetUnit: 1000, 117 | bookValuePerShare: 11.7343, 118 | comprehensiveIncomeSheetUnit: 1000, 119 | netWorth: 66044317, 120 | ordinaryShare: 56282930, 121 | roa: 0.12, 122 | roe: 0.72, 123 | shares: 5628293, 124 | totalAssets: 425359870, 125 | totalBasicEarningsPerShare: 0.09, 126 | totalComprehensiveIncome: 493284, 127 | totalCurrentAssets: 42588522, 128 | totalCurrentLiabilities: 18949504, 129 | totalEquity: 66044317, 130 | totalLiabilities: 359315553, 131 | totalNonCurrentAssets: 382771348, 132 | totalNonCurrentLiabilities: 340366049, 133 | totalOperatingCosts: 6459969, 134 | totalOperatingRevenue: 6101721 135 | } 136 | ] 137 | ``` 138 | 139 | ### 新增其他公司財報 140 | 1. 先到以下網址，查詢欲新增的公司。 141 | 142 | [公開資訊觀測站-資產負債表查詢](https://mops.twse.com.tw/mops/web/t164sb03) 143 | 144 | 2. 取得欲新增公司的財報類型。 145 | 146 | 點選 「投資人若需了解更詳細資訊可至XBRL資訊平台或電子書查詢」中的`XBRL資訊平台`。透過 URL 就可以知道財報的種類。 147 | ``` 148 | # 以 2207 為例，財報類型為: REPORT_ID=C 149 | https://mops.twse.com.tw/server-java/t164sb01?step=1&CO_ID=2207&SYEAR=2021&SSEASON=2&REPORT_ID=C 150 | ``` 151 | 3. 新增對應 DAG。 152 | 至檔案 `dags/parse_financial_report/parse_financial_report.py` 的最下方，加入以下程式碼即可： 153 | ```python 154 | ... 155 | stock_2633 = init_dag( 156 | 'stock_2633', 157 | stock_code=2633, 158 | report_type='A', 159 | start_date=datetime(year=2019, month=4, day=1), 160 | schedule_interval='1 0 * * *', 161 | ) 162 | 163 | stock_5283 = init_dag( 164 | 'stock_5283', 165 | stock_code=5283, 166 | report_type='C', 167 | start_date=datetime(year=2019, month=4, day=1), 168 | schedule_interval='2 0 * * *', 169 | ) 170 | 171 | stock_2207 = init_dag( 172 | 'stock_2207', 173 | stock_code=2207, 174 | report_type='C', 175 | start_date=datetime(year=2019, month=4, day=1), 176 | schedule_interval='3 0 * * *', 177 | ) 178 | 179 | ``` 180 | > 目前規劃抓取 2019 年第一季以後的非金融財報格式，所以 `start_date` 需大於 2019-04-01 才能正確抓取，也暫時無法抓取銀行財報。 181 | 182 | ### 看到圖表比較有感覺嗎? 那你可以試試這個 183 | 184 | 在 `web` 資料夾中，放了一個簡易 flask + plotly 的範例，來呈現 ROA 和 ROE。其執行方式，如下： 185 | 186 | ```bash 187 | # 至專案目錄 188 | $ cd ~/airflow/dags/tw-financial-report-analysis/ 189 | 190 | # 安裝相關套件 191 | $ pip3 install -r requirements.txt 192 | 193 | # 至 web 目錄下，並執行以下指令 194 | $ cd web 195 | $ flask run [±main ●] 196 | * Environment: production 197 | WARNING: This is a development server. Do not use it in a production deployment. 198 | Use a production WSGI server instead. 199 | * Debug mode: off 200 | * Running on http://127.0.0.1:5000/ (Press CTRL+C to quit) 201 | ``` 202 | 203 | 現在，你可以透過 [127.0.0.1:5000](http://127.0.0.1:5000/) 看到 ROA 及 ROE 的圖表囉！ 204 | 205 | roa_with_flask_and_plotly 206 | 207 | ## 財報 208 | 209 | ### 欄位對應 210 | 211 | 同 XBRL 資訊平台上的英文版欄位名稱。例如： 212 | 213 | | XBRL 資訊平台上的英文版欄位 | Mongo 中對欄位名稱 | 214 | | --- | --- | 215 | | Total assets | totalAssets | 216 | 217 | 某些欄位最會加上前綴 `cumulative` 表示是從當年年初`累計`至當季的意思。 218 | 219 | | XBRL 資訊平台上的英文版欄位 | Mongo 中對欄位名稱 | 220 | | --- | --- | 221 | | Total Comprehensive Income | cumulativeTotalComprehensiveIncome | 222 | 223 | ## 每月營收 224 | 225 | ### 欄位對應 226 | | 欄位名稱 | 敘述 | 227 | | --- | --- | 228 | | stockCode | 公司代號 | 229 | | year | 報表年份 | 230 | | month | 報表月份 | 231 | | company | 公司代號 | 232 | | industrialClass | 產業別 | 233 | | theMonthlyRevenue | 營業收入-當月營收 | 234 | | exMonthlyRevenue | 營業收入-上月營收 | 235 | | exYearTheMonthlyRevenue | 營業收入-去年當月營收 | 236 | | mom |營業收入-上月比較增減(%) | 237 | | yoy | 營業收入-去年同月增減(%) | 238 | | cumTheMonthlyRevenue | 累計營業收入-當月累計營收 | 239 | | cumExYearTheMonthlyRevenue | 累計營業收入-去年累計營收 | 240 | | cmocm | 累計營業收入-前期比較增減(%) | 241 | | note | 備註 | 242 | -------------------------------------------------------------------------------- /dags/parse_financial_report/parse_financial_report.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from datetime import datetime 3 | import logging 4 | from functools import wraps 5 | 6 | from airflow.settings import AIRFLOW_HOME 7 | from airflow import DAG 8 | from airflow.operators.python import BranchPythonOperator 9 | from airflow.operators.dummy import DummyOperator 10 | from functools import wraps 11 | 12 | 13 | class EnvSetting(object): 14 | PROJECT_HOME = f'{AIRFLOW_HOME}/dags/tw-financial-report-analysis' 15 | 16 | @staticmethod 17 | def append_project_to_path(f): 18 | @wraps(f) 19 | def insert_path(*args, **kwds): 20 | sys.path.insert(0, EnvSetting.PROJECT_HOME) 21 | return f(*args, **kwds) 22 | 23 | return insert_path 24 | 25 | 26 | # 2633 A 27 | 28 | def init_dag(dag_id, stock_code, report_type, start_date, schedule_interval='0 0 27 * *'): 29 | args = { 30 | 'owner': 'sean', 31 | } 32 | dag = DAG( 33 | dag_id=dag_id, 34 | default_args=args, 35 | max_active_runs=1, 36 | concurrency=1, 37 | schedule_interval=schedule_interval, 38 | start_date=start_date, 39 | ) 40 | 41 | @EnvSetting.append_project_to_path 42 | def check_report_in_mongo(code, **context): 43 | from airflow.providers.mongo.hooks.mongo import MongoHook 44 | 45 | from toolbox.date_tool import DateTool 46 | 47 | mongo_hook = MongoHook(conn_id='stock_mongo') 48 | stock_db = mongo_hook.get_conn().stock 49 | execution_date = context['ds'] 50 | year, month, day = map(int, execution_date.split('-')) 51 | season_year, season = DateTool.date_to_ex_year_and_season(year, month) 52 | upload_key = { 53 | 'stockCode': code, 54 | 'year': season_year, 55 | 'season': season, 56 | 'yearAndSeason': season_year * 10 + season 57 | } 58 | has_report = stock_db.financialReports.find_one(upload_key) 59 | if has_report: 60 | return 'report_already_in_mongo' 61 | else: 62 | return 'check_report_released' 63 | 64 | check_report_in_mongo_task = BranchPythonOperator( 65 | task_id='check_report_in_mongo', 66 | python_callable=check_report_in_mongo, 67 | op_kwargs={ 68 | 'code': stock_code, 69 | }, 70 | provide_context=True, 71 | dag=dag, 72 | ) 73 | 74 | report_already_in_mongo_task = DummyOperator( 75 | task_id='report_already_in_mongo', 76 | dag=dag 77 | ) 78 | 79 | @EnvSetting.append_project_to_path 80 | def check_report_released(code, **context): 81 | import requests 82 | from time import sleep 83 | from datetime import datetime 84 | import pytz 85 | 86 | from airflow.providers.mongo.hooks.mongo import MongoHook 87 | from bs4 import BeautifulSoup 88 | 89 | from toolbox.date_tool import DateTool 90 | 91 | mongo_hook = MongoHook(conn_id='stock_mongo') 92 | stock_db = mongo_hook.get_conn().stock 93 | execution_date = context['ds'] 94 | year, month, day = map(int, execution_date.split('-')) 95 | season_year, season = DateTool.date_to_ex_year_and_season(year, month) 96 | tw_year = DateTool.to_tw_year(season_year) 97 | 98 | resp = requests.get( 99 | f'https://doc.twse.com.tw/server-java/t57sb01?step=1&colorchg=1&' 100 | f'co_id={code}&' 101 | f'year={tw_year}&' 102 | f'seamon={season}&' 103 | f'mtype=A') 104 | resp.encoding = 'big5' 105 | soup = BeautifulSoup(resp.text, 'html.parser') 106 | 107 | logging.info(f'sleep 5 seconds.') 108 | sleep(5) 109 | 110 | if soup.h4 and soup.h4.text == '查無所需資料': 111 | return 'the_report_is_not_exist' 112 | 113 | published_date_str = soup.center.table.table.find(attrs={'align': 'cetern'}).text 114 | published_tw_year, *others = published_date_str.split('/') 115 | published_year = DateTool.tw_year_to_year(int(published_tw_year)) 116 | tw_date_str = '/'.join([str(published_year)] + others) 117 | published_date = datetime.strptime(tw_date_str, '%Y/%m/%d %H:%M:%S') 118 | timezone = pytz.timezone('Asia/Taipei') 119 | published_date = timezone.localize(published_date) 120 | upload_data = { 121 | 'stockCode': code, 122 | 'year': season_year, 123 | 'season': season, 124 | 'yearAndSeason': season_year * 10 + season, 125 | 'publishedDate': published_date 126 | } 127 | stock_db.financialReports.insert(upload_data) 128 | 129 | return 'load_report_to_mongo' 130 | 131 | check_report_released_task = BranchPythonOperator( 132 | task_id='check_report_released', 133 | python_callable=check_report_released, 134 | op_kwargs={ 135 | 'code': stock_code, 136 | }, 137 | provide_context=True, 138 | trigger_rule='none_failed_or_skipped', 139 | dag=dag, 140 | ) 141 | 142 | the_report_is_not_exist_task = DummyOperator( 143 | task_id='the_report_is_not_exist', 144 | dag=dag 145 | ) 146 | 147 | @EnvSetting.append_project_to_path 148 | def load_report_to_mongo_by_stock_code(code, r_type, **context): 149 | from time import sleep 150 | 151 | from reports.financial_reports.financial_report_agent import FinancialReportAgent 152 | from toolbox.date_tool import DateTool 153 | from airflow.providers.mongo.hooks.mongo import MongoHook 154 | mongo_hook = MongoHook(conn_id='stock_mongo') 155 | stock_db = mongo_hook.get_conn().stock 156 | execution_date = context['ds'] 157 | year, month, day = map(int, execution_date.split('-')) 158 | season_year, season = DateTool.date_to_ex_year_and_season(year, month) 159 | fn_report_agent = FinancialReportAgent(code, season_year, season, r_type) 160 | upload_key = { 161 | 'stockCode': code, 162 | 'year': season_year, 163 | 'season': season, 164 | 'yearAndSeason': season_year * 10 + season, 165 | } 166 | upload_data = {} 167 | 168 | logging.info(f'sleep 5 seconds.') 169 | sleep(5) 170 | 171 | if not fn_report_agent: 172 | return 'cant_get_the_report' 173 | 174 | ''' Balance Sheet ''' 175 | search_balance_sheet_set = { 176 | 'Total assets', 177 | 'Total current assets', 178 | 'Total non-current assets', 179 | 'Total liabilities', 180 | 'Total current liabilities', 181 | 'Total non-current liabilities', 182 | 'Total equity', 183 | 'Ordinary share' 184 | } 185 | balance_sheet_res = fn_report_agent.balance_sheet.parse_items_to_dict(search_balance_sheet_set) 186 | upload_data.update(balance_sheet_res) 187 | # record the unit of Balance Sheet 188 | upload_data.update( 189 | { 190 | 'balanceSheetUnit': fn_report_agent.balance_sheet.dollar_unit 191 | } 192 | ) 193 | 194 | ''' Comprehensive Income Sheet ''' 195 | search_comprehensive_income_sheet_set = { 196 | 'Total operating revenue', 197 | 'Total operating costs', 198 | 'Total comprehensive income', 199 | 'Total basic earnings per share', 200 | } 201 | 202 | if season == 3: 203 | income_parse_columns = { 204 | 1: '', 205 | 3: 'cumulative' 206 | } 207 | elif season == 4: 208 | income_parse_columns = { 209 | 1: 'cumulative', 210 | } 211 | else: 212 | income_parse_columns = { 213 | 1: '', 214 | } 215 | 216 | income_res = fn_report_agent.comprehensive_income_sheet.parse_items_to_dict( 217 | item_set=search_comprehensive_income_sheet_set, 218 | columns=income_parse_columns 219 | ) 220 | upload_data.update(income_res) 221 | 222 | # Record the unit of Comprehensive Income Sheet 223 | upload_data.update( 224 | { 225 | 'comprehensiveIncomeSheetUnit': fn_report_agent.comprehensive_income_sheet.dollar_unit 226 | } 227 | ) 228 | 229 | ex_season_report = stock_db.financialReports.find_one({ 230 | 'stockCode': code, 231 | 'yearAndSeason': DateTool.season_to_ex_year_and_season_int(season_year * 10 + season), 232 | }) 233 | 234 | if ex_season_report: 235 | if season == 4: 236 | print(ex_season_report['yearAndSeason']) 237 | upload_data['totalComprehensiveIncome'] = ( 238 | upload_data.get('cumulativeTotalComprehensiveIncome') - 239 | ex_season_report.get('cumulativeTotalComprehensiveIncome')) 240 | print(upload_data['totalComprehensiveIncome']) 241 | 242 | avg_total_asset = (ex_season_report.get('totalAssets', 1) + upload_data.get('totalAssets', 1)) / 2 243 | avg_total_equity = (ex_season_report.get('totalEquity', 1) + upload_data.get('totalEquity', 1)) / 2 244 | 245 | else: 246 | avg_total_asset = upload_data.get('totalAssets', 1) 247 | avg_total_equity = upload_data.get('totalEquity', 1) 248 | 249 | single_season_net_income = upload_data.get('totalComprehensiveIncome', 0) 250 | 251 | ''' compute ROA, ROE and Book Value Per Share ''' 252 | roa = round(single_season_net_income / avg_total_asset * 100, 2) 253 | roe = round(single_season_net_income / avg_total_equity * 100, 2) 254 | assets = upload_data.get('totalAssets', 0) 255 | liabilities = upload_data.get('totalLiabilities', 0) 256 | net_worth = assets - liabilities 257 | shares = (upload_data.get('ordinaryShare', 0) / 10) 258 | book_value_per_share = (net_worth / shares) if shares > 0 else 0 259 | 260 | upload_data.update( 261 | { 262 | 'roa': roa, 263 | 'roe': roe, 264 | 'netWorth': net_worth, 265 | 'shares': shares, 266 | 'bookValuePerShare': round(book_value_per_share, 4), 267 | } 268 | ) 269 | 270 | stock_db.financialReports.update( 271 | upload_key, 272 | { 273 | "$set": upload_data 274 | }, 275 | upsert=True 276 | ) 277 | 278 | return 'done' 279 | 280 | load_report_to_mongo_task = BranchPythonOperator( 281 | task_id='load_report_to_mongo', 282 | python_callable=load_report_to_mongo_by_stock_code, 283 | op_kwargs={ 284 | 'code': stock_code, 285 | 'r_type': report_type 286 | }, 287 | trigger_rule='none_failed_or_skipped', 288 | provide_context=True, 289 | dag=dag, 290 | ) 291 | 292 | cant_get_the_report_task = DummyOperator( 293 | task_id='cant_get_the_report', 294 | dag=dag 295 | ) 296 | 297 | done_task = DummyOperator( 298 | task_id='done', 299 | dag=dag 300 | ) 301 | check_report_in_mongo_task >> [report_already_in_mongo_task, check_report_released_task] 302 | check_report_released_task >> [the_report_is_not_exist_task, load_report_to_mongo_task] 303 | load_report_to_mongo_task >> [cant_get_the_report_task, done_task] 304 | 305 | return dag 306 | 307 | 308 | stock_2633 = init_dag( 309 | 'stock_2633', 310 | stock_code=2633, 311 | report_type='A', 312 | start_date=datetime(year=2019, month=4, day=1), 313 | schedule_interval='1 0 * * *', 314 | ) 315 | 316 | stock_5283 = init_dag( 317 | 'stock_5283', 318 | stock_code=5283, 319 | report_type='C', 320 | start_date=datetime(year=2019, month=4, day=1), 321 | schedule_interval='2 0 * * *', 322 | ) 323 | 324 | stock_2207 = init_dag( 325 | 'stock_2207', 326 | stock_code=2207, 327 | report_type='C', 328 | start_date=datetime(year=2019, month=4, day=1), 329 | schedule_interval='3 0 * * *', 330 | ) 331 | -------------------------------------------------------------------------------- /lab/experiment1/.ipynb_checkpoints/three_days_strategy-checkpoint.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 2, 6 | "id": "10c819c0", 7 | "metadata": {}, 8 | "outputs": [], 9 | "source": [ 10 | "from pymongo import MongoClient\n", 11 | "from pymongo import DESCENDING\n", 12 | "\n", 13 | "from urllib.parse import quote_plus" 14 | ] 15 | }, 16 | { 17 | "cell_type": "code", 18 | "execution_count": 3, 19 | "id": "a06c39d0", 20 | "metadata": {}, 21 | "outputs": [], 22 | "source": [ 23 | "uri = \"mongodb://%s:%s@%s/?authSource=stock\" % (\n", 24 | " quote_plus('airflower'), quote_plus('money'), 'localhost')\n", 25 | "client = MongoClient(uri)" 26 | ] 27 | }, 28 | { 29 | "cell_type": "code", 30 | "execution_count": 4, 31 | "id": "47d1ac75", 32 | "metadata": {}, 33 | "outputs": [], 34 | "source": [ 35 | "results = list(client.stock.dailyTransactions.find({'stockCode': '0050'}).sort('intDate', DESCENDING))" 36 | ] 37 | }, 38 | { 39 | "cell_type": "code", 40 | "execution_count": 5, 41 | "id": "06d99ba7", 42 | "metadata": {}, 43 | "outputs": [ 44 | { 45 | "name": "stdout", 46 | "output_type": "stream", 47 | "text": [ 48 | "{'_id': ObjectId('6188f926bc1c0ea656d5b1b5'), 'day': 29, 'intDate': 20211029, 'month': 10, 'stockCode': '0050', 'year': 2021, 'amount': 699434063, 'closingDiff': -1.35, 'closingPrice': 135.8, 'highestPrice': 137.05, 'lowestPrice': 135.35, 'openingPrice': 137.05, 'tradeVolume': 5145743, 'transaction': 6484} {'_id': ObjectId('6188f926bc1c0ea656d5b1b3'), 'day': 28, 'intDate': 20211028, 'month': 10, 'stockCode': '0050', 'year': 2021, 'amount': 630055512, 'closingDiff': -0.4, 'closingPrice': 137.15, 'highestPrice': 137.65, 'lowestPrice': 136.6, 'openingPrice': 137.45, 'tradeVolume': 4597622, 'transaction': 5081}\n" 49 | ] 50 | } 51 | ], 52 | "source": [ 53 | "print(results[0], results[1])" 54 | ] 55 | }, 56 | { 57 | "cell_type": "code", 58 | "execution_count": 23, 59 | "id": "fdca3755", 60 | "metadata": {}, 61 | "outputs": [], 62 | "source": [ 63 | "def total_rise(days):\n", 64 | " count = 0\n", 65 | " for day in days:\n", 66 | " count += 1 if day['closingDiff'] > 0 else 0\n", 67 | " \n", 68 | " return count\n", 69 | "\n", 70 | "data = [ \n", 71 | " {\n", 72 | " # target\n", 73 | " 'dDayPriseRise': 1 if results[i]['closingDiff'] > 0 else 0,\n", 74 | " 'dDayClosingDiff': results[i]['closingDiff'],\n", 75 | " # summary\n", 76 | " # 'totalRise': total_rise(results[i+1:i+3]),\n", 77 | " # day1 columns\n", 78 | " 'd1TradeVolume': results[i+1]['tradeVolume'],\n", 79 | " 'd1Amount': results[i+1]['amount'],\n", 80 | " 'd1OpeningPrice': results[i+1]['openingPrice'],\n", 81 | " 'd1AgoHighestPrice': results[i+1]['highestPrice'],\n", 82 | " 'd1AgoLowestPrice': results[i+1]['lowestPrice'],\n", 83 | " 'd1AgoClosingPrice': results[i+1]['closingPrice'],\n", 84 | " 'd1AgoClosingDiff': results[i+1]['closingDiff'],\n", 85 | " 'd1AgoTransaction': results[i+1]['transaction'],\n", 86 | " 'd1AgoMaxDiff': round(results[i+1]['highestPrice'] - results[i+1]['lowestPrice'], 2),\n", 87 | " 'd1DayPriseRise': 1 if results[i+1]['closingDiff'] > 0 else 0,\n", 88 | " }\n", 89 | " for i in range(0, len(results)-3)\n", 90 | "]" 91 | ] 92 | }, 93 | { 94 | "cell_type": "code", 95 | "execution_count": 24, 96 | "id": "aac0f0c0", 97 | "metadata": {}, 98 | "outputs": [], 99 | "source": [ 100 | "import pandas as pd" 101 | ] 102 | }, 103 | { 104 | "cell_type": "code", 105 | "execution_count": 25, 106 | "id": "28d2f155", 107 | "metadata": {}, 108 | "outputs": [], 109 | "source": [ 110 | "df = pd.DataFrame(data)" 111 | ] 112 | }, 113 | { 114 | "cell_type": "code", 115 | "execution_count": 19, 116 | "id": "ccd6bc37", 117 | "metadata": {}, 118 | "outputs": [ 119 | { 120 | "data": { 121 | "text/html": [ 122 | "
\n", 123 | "\n", 136 | "\n", 137 | " \n", 138 | " \n", 139 | " \n", 140 | " \n", 141 | " \n", 142 | " \n", 143 | " \n", 144 | " \n", 145 | " \n", 146 | " \n", 147 | " \n", 148 | " \n", 149 | " \n", 150 | " \n", 151 | " \n", 152 | " \n", 153 | " \n", 154 | " \n", 155 | " \n", 156 | " \n", 157 | " \n", 158 | " \n", 159 | " \n", 160 | " \n", 161 | " \n", 162 | " \n", 163 | " \n", 164 | " \n", 165 | " \n", 166 | " \n", 167 | " \n", 168 | " \n", 169 | " \n", 170 | " \n", 171 | " \n", 172 | " \n", 173 | " \n", 174 | " \n", 175 | " \n", 176 | " \n", 177 | " \n", 178 | " \n", 179 | " \n", 180 | " \n", 181 | " \n", 182 | " \n", 183 | " \n", 184 | " \n", 185 | " \n", 186 | " \n", 187 | " \n", 188 | " \n", 189 | " \n", 190 | " \n", 191 | " \n", 192 | " \n", 193 | " \n", 194 | " \n", 195 | " \n", 196 | " \n", 197 | " \n", 198 | " \n", 199 | " \n", 200 | " \n", 201 | " \n", 202 | " \n", 203 | " \n", 204 | " \n", 205 | " \n", 206 | " \n", 207 | " \n", 208 | " \n", 209 | " \n", 210 | " \n", 211 | " \n", 212 | " \n", 213 | " \n", 214 | " \n", 215 | " \n", 216 | " \n", 217 | " \n", 218 | " \n", 219 | " \n", 220 | " \n", 221 | " \n", 222 | " \n", 223 | " \n", 224 | " \n", 225 | " \n", 226 | " \n", 227 | " \n", 228 | " \n", 229 | " \n", 230 | " \n", 231 | " \n", 232 | " \n", 233 | " \n", 234 | " \n", 235 | " \n", 236 | " \n", 237 | " \n", 238 | " \n", 239 | " \n", 240 | " \n", 241 | " \n", 242 | " \n", 243 | " \n", 244 | " \n", 245 | " \n", 246 | " \n", 247 | " \n", 248 | " \n", 249 | " \n", 250 | " \n", 251 | " \n", 252 | " \n", 253 | " \n", 254 | " \n", 255 | " \n", 256 | " \n", 257 | " \n", 258 | " \n", 259 | " \n", 260 | " \n", 261 | " \n", 262 | " \n", 263 | " \n", 264 | " \n", 265 | " \n", 266 | " \n", 267 | " \n", 268 | " \n", 269 | " \n", 270 | " \n", 271 | " \n", 272 | " \n", 273 | " \n", 274 | " \n", 275 | " \n", 276 | " \n", 277 | " \n", 278 | " \n", 279 | " \n", 280 | " \n", 281 | " \n", 282 | " \n", 283 | " \n", 284 | " \n", 285 | " \n", 286 | " \n", 287 | " \n", 288 | " \n", 289 | " \n", 290 | " \n", 291 | " \n", 292 | " \n", 293 | " \n", 294 | " \n", 295 | " \n", 296 | " \n", 297 | " \n", 298 | " \n", 299 | " \n", 300 | " \n", 301 | " \n", 302 | " \n", 303 | " \n", 304 | " \n", 305 | " \n", 306 | " \n", 307 | " \n", 308 | " \n", 309 | " \n", 310 | " \n", 311 | " \n", 312 | " \n", 313 | " \n", 314 | " \n", 315 | " \n", 316 | " \n", 317 | " \n", 318 | " \n", 319 | " \n", 320 | " \n", 321 | " \n", 322 | " \n", 323 | " \n", 324 | " \n", 325 | " \n", 326 | " \n", 327 | " \n", 328 | " \n", 329 | " \n", 330 | " \n", 331 | " \n", 332 | " \n", 333 | "
dDayPriseRisedDayClosingDifftotalRised1TradeVolumed1Amountd1OpeningPriced1AgoHighestPriced1AgoLowestPriced1AgoClosingPriced1AgoClosingDiffd1AgoTransactiond1AgoMaxDiffd1DayPriseRise
00-1.3504597622630055512137.45137.65136.60137.15-0.4050811.050
10-0.40175277891031896075136.95137.55136.70137.550.0058760.850
200.001107977041483480452136.65137.75136.60137.551.1597891.151
311.15175223251022937288136.00136.70135.20136.40-0.1071241.500
40-0.1015321361725081014136.25136.75135.65136.500.4046571.101
..........................................
70011.151531622839765790174.5075.1074.4575.050.6028080.651
70110.6011301328497056767774.5074.9074.4574.45-1.0058100.450
7020-1.001678100851228850275.6075.7575.3075.450.3546230.451
70310.35020347421152996527075.8575.8574.8575.10-1.55105191.000
7040-1.550963636674054455077.1077.2076.6576.65-1.6036020.550
\n", 334 | "

705 rows × 13 columns

\n", 335 | "
" 336 | ], 337 | "text/plain": [ 338 | " dDayPriseRise dDayClosingDiff totalRise d1TradeVolume d1Amount \\\n", 339 | "0 0 -1.35 0 4597622 630055512 \n", 340 | "1 0 -0.40 1 7527789 1031896075 \n", 341 | "2 0 0.00 1 10797704 1483480452 \n", 342 | "3 1 1.15 1 7522325 1022937288 \n", 343 | "4 0 -0.10 1 5321361 725081014 \n", 344 | ".. ... ... ... ... ... \n", 345 | "700 1 1.15 1 5316228 397657901 \n", 346 | "701 1 0.60 1 13013284 970567677 \n", 347 | "702 0 -1.00 1 6781008 512288502 \n", 348 | "703 1 0.35 0 20347421 1529965270 \n", 349 | "704 0 -1.55 0 9636366 740544550 \n", 350 | "\n", 351 | " d1OpeningPrice d1AgoHighestPrice d1AgoLowestPrice d1AgoClosingPrice \\\n", 352 | "0 137.45 137.65 136.60 137.15 \n", 353 | "1 136.95 137.55 136.70 137.55 \n", 354 | "2 136.65 137.75 136.60 137.55 \n", 355 | "3 136.00 136.70 135.20 136.40 \n", 356 | "4 136.25 136.75 135.65 136.50 \n", 357 | ".. ... ... ... ... \n", 358 | "700 74.50 75.10 74.45 75.05 \n", 359 | "701 74.50 74.90 74.45 74.45 \n", 360 | "702 75.60 75.75 75.30 75.45 \n", 361 | "703 75.85 75.85 74.85 75.10 \n", 362 | "704 77.10 77.20 76.65 76.65 \n", 363 | "\n", 364 | " d1AgoClosingDiff d1AgoTransaction d1AgoMaxDiff d1DayPriseRise \n", 365 | "0 -0.40 5081 1.05 0 \n", 366 | "1 0.00 5876 0.85 0 \n", 367 | "2 1.15 9789 1.15 1 \n", 368 | "3 -0.10 7124 1.50 0 \n", 369 | "4 0.40 4657 1.10 1 \n", 370 | ".. ... ... ... ... \n", 371 | "700 0.60 2808 0.65 1 \n", 372 | "701 -1.00 5810 0.45 0 \n", 373 | "702 0.35 4623 0.45 1 \n", 374 | "703 -1.55 10519 1.00 0 \n", 375 | "704 -1.60 3602 0.55 0 \n", 376 | "\n", 377 | "[705 rows x 13 columns]" 378 | ] 379 | }, 380 | "execution_count": 19, 381 | "metadata": {}, 382 | "output_type": "execute_result" 383 | } 384 | ], 385 | "source": [ 386 | "df" 387 | ] 388 | }, 389 | { 390 | "cell_type": "code", 391 | "execution_count": 22, 392 | "id": "c452c275", 393 | "metadata": {}, 394 | "outputs": [ 395 | { 396 | "data": { 397 | "text/html": [ 398 | "
\n", 399 | "\n", 412 | "\n", 413 | " \n", 414 | " \n", 415 | " \n", 416 | " \n", 417 | " \n", 418 | " \n", 419 | " \n", 420 | " \n", 421 | " \n", 422 | " \n", 423 | " \n", 424 | " \n", 425 | " \n", 426 | " \n", 427 | " \n", 428 | " \n", 429 | " \n", 430 | " \n", 431 | " \n", 432 | " \n", 433 | " \n", 434 | " \n", 435 | " \n", 436 | " \n", 437 | " \n", 438 | " \n", 439 | " \n", 440 | " \n", 441 | " \n", 442 | " \n", 443 | " \n", 444 | " \n", 445 | " \n", 446 | " \n", 447 | " \n", 448 | " \n", 449 | " \n", 450 | " \n", 451 | " \n", 452 | " \n", 453 | " \n", 454 | " \n", 455 | " \n", 456 | " \n", 457 | " \n", 458 | " \n", 459 | " \n", 460 | " \n", 461 | " \n", 462 | " \n", 463 | " \n", 464 | " \n", 465 | " \n", 466 | " \n", 467 | " \n", 468 | " \n", 469 | " \n", 470 | " \n", 471 | " \n", 472 | " \n", 473 | " \n", 474 | " \n", 475 | " \n", 476 | " \n", 477 | " \n", 478 | " \n", 479 | " \n", 480 | " \n", 481 | " \n", 482 | " \n", 483 | " \n", 484 | " \n", 485 | " \n", 486 | " \n", 487 | " \n", 488 | " \n", 489 | " \n", 490 | " \n", 491 | " \n", 492 | " \n", 493 | " \n", 494 | " \n", 495 | " \n", 496 | " \n", 497 | " \n", 498 | " \n", 499 | " \n", 500 | " \n", 501 | " \n", 502 | " \n", 503 | " \n", 504 | " \n", 505 | " \n", 506 | " \n", 507 | " \n", 508 | " \n", 509 | " \n", 510 | " \n", 511 | " \n", 512 | " \n", 513 | " \n", 514 | " \n", 515 | " \n", 516 | " \n", 517 | " \n", 518 | " \n", 519 | " \n", 520 | " \n", 521 | " \n", 522 | " \n", 523 | " \n", 524 | " \n", 525 | " \n", 526 | " \n", 527 | " \n", 528 | " \n", 529 | " \n", 530 | " \n", 531 | " \n", 532 | " \n", 533 | " \n", 534 | " \n", 535 | " \n", 536 | " \n", 537 | " \n", 538 | " \n", 539 | " \n", 540 | " \n", 541 | " \n", 542 | " \n", 543 | " \n", 544 | " \n", 545 | " \n", 546 | " \n", 547 | " \n", 548 | " \n", 549 | " \n", 550 | " \n", 551 | " \n", 552 | " \n", 553 | " \n", 554 | " \n", 555 | " \n", 556 | " \n", 557 | " \n", 558 | " \n", 559 | " \n", 560 | " \n", 561 | " \n", 562 | " \n", 563 | " \n", 564 | " \n", 565 | " \n", 566 | " \n", 567 | " \n", 568 | " \n", 569 | " \n", 570 | " \n", 571 | " \n", 572 | " \n", 573 | " \n", 574 | " \n", 575 | " \n", 576 | " \n", 577 | " \n", 578 | " \n", 579 | " \n", 580 | " \n", 581 | " \n", 582 | " \n", 583 | " \n", 584 | " \n", 585 | " \n", 586 | " \n", 587 | " \n", 588 | " \n", 589 | " \n", 590 | " \n", 591 | " \n", 592 | " \n", 593 | " \n", 594 | " \n", 595 | " \n", 596 | " \n", 597 | " \n", 598 | " \n", 599 | " \n", 600 | " \n", 601 | " \n", 602 | " \n", 603 | " \n", 604 | " \n", 605 | " \n", 606 | " \n", 607 | " \n", 608 | " \n", 609 | " \n", 610 | " \n", 611 | " \n", 612 | " \n", 613 | " \n", 614 | " \n", 615 | " \n", 616 | " \n", 617 | " \n", 618 | " \n", 619 | " \n", 620 | " \n", 621 | " \n", 622 | " \n", 623 | " \n", 624 | " \n", 625 | " \n", 626 | " \n", 627 | " \n", 628 | " \n", 629 | " \n", 630 | " \n", 631 | " \n", 632 | " \n", 633 | " \n", 634 | " \n", 635 | " \n", 636 | " \n", 637 | " \n", 638 | " \n", 639 | " \n", 640 | " \n", 641 | "
dDayPriseRisedDayClosingDifftotalRised1TradeVolumed1Amountd1OpeningPriced1AgoHighestPriced1AgoLowestPriced1AgoClosingPriced1AgoClosingDiffd1AgoTransactiond1AgoMaxDiffd1DayPriseRise
dDayPriseRise1.0000000.6900770.017607-0.055300-0.056692-0.051408-0.050721-0.049532-0.0500180.020438-0.029423-0.0538260.021118
dDayClosingDiff0.6900771.0000000.021934-0.045731-0.033772-0.008455-0.007423-0.007124-0.0080500.012131-0.004265-0.0112690.022485
totalRise0.0176070.0219341.000000-0.118961-0.108564-0.024623-0.019629-0.013800-0.0110270.498864-0.197942-0.1665560.714534
d1TradeVolume-0.055300-0.045731-0.1189611.0000000.916421-0.052051-0.045398-0.064629-0.055689-0.1871880.7239700.502675-0.075356
d1Amount-0.056692-0.033772-0.1085640.9164211.0000000.2754170.2817880.2625720.271583-0.1799600.8908790.641260-0.076290
d1OpeningPrice-0.051408-0.008455-0.024623-0.0520510.2754171.0000000.9997100.9995100.9993250.0207810.4230860.430105-0.028490
d1AgoHighestPrice-0.050721-0.007423-0.019629-0.0453980.2817880.9997101.0000000.9994520.9996980.0339490.4262830.439682-0.019473
d1AgoLowestPrice-0.049532-0.007124-0.013800-0.0646290.2625720.9995100.9994521.0000000.9996260.0383520.4082500.409719-0.016922
d1AgoClosingPrice-0.050018-0.008050-0.011027-0.0556890.2715830.9993250.9996980.9996261.0000000.0480750.4153080.426655-0.010142
d1AgoClosingDiff0.0204380.0121310.498864-0.187188-0.1799600.0207810.0339490.0383520.0480751.000000-0.309428-0.1050860.690124
d1AgoTransaction-0.029423-0.004265-0.1979420.7239700.8908790.4230860.4262830.4082500.415308-0.3094281.0000000.670530-0.177660
d1AgoMaxDiff-0.053826-0.011269-0.1665560.5026750.6412600.4301050.4396820.4097190.426655-0.1050860.6705301.000000-0.077514
d1DayPriseRise0.0211180.0224850.714534-0.075356-0.076290-0.028490-0.019473-0.016922-0.0101420.690124-0.177660-0.0775141.000000
\n", 642 | "
" 643 | ], 644 | "text/plain": [ 645 | " dDayPriseRise dDayClosingDiff totalRise d1TradeVolume \\\n", 646 | "dDayPriseRise 1.000000 0.690077 0.017607 -0.055300 \n", 647 | "dDayClosingDiff 0.690077 1.000000 0.021934 -0.045731 \n", 648 | "totalRise 0.017607 0.021934 1.000000 -0.118961 \n", 649 | "d1TradeVolume -0.055300 -0.045731 -0.118961 1.000000 \n", 650 | "d1Amount -0.056692 -0.033772 -0.108564 0.916421 \n", 651 | "d1OpeningPrice -0.051408 -0.008455 -0.024623 -0.052051 \n", 652 | "d1AgoHighestPrice -0.050721 -0.007423 -0.019629 -0.045398 \n", 653 | "d1AgoLowestPrice -0.049532 -0.007124 -0.013800 -0.064629 \n", 654 | "d1AgoClosingPrice -0.050018 -0.008050 -0.011027 -0.055689 \n", 655 | "d1AgoClosingDiff 0.020438 0.012131 0.498864 -0.187188 \n", 656 | "d1AgoTransaction -0.029423 -0.004265 -0.197942 0.723970 \n", 657 | "d1AgoMaxDiff -0.053826 -0.011269 -0.166556 0.502675 \n", 658 | "d1DayPriseRise 0.021118 0.022485 0.714534 -0.075356 \n", 659 | "\n", 660 | " d1Amount d1OpeningPrice d1AgoHighestPrice \\\n", 661 | "dDayPriseRise -0.056692 -0.051408 -0.050721 \n", 662 | "dDayClosingDiff -0.033772 -0.008455 -0.007423 \n", 663 | "totalRise -0.108564 -0.024623 -0.019629 \n", 664 | "d1TradeVolume 0.916421 -0.052051 -0.045398 \n", 665 | "d1Amount 1.000000 0.275417 0.281788 \n", 666 | "d1OpeningPrice 0.275417 1.000000 0.999710 \n", 667 | "d1AgoHighestPrice 0.281788 0.999710 1.000000 \n", 668 | "d1AgoLowestPrice 0.262572 0.999510 0.999452 \n", 669 | "d1AgoClosingPrice 0.271583 0.999325 0.999698 \n", 670 | "d1AgoClosingDiff -0.179960 0.020781 0.033949 \n", 671 | "d1AgoTransaction 0.890879 0.423086 0.426283 \n", 672 | "d1AgoMaxDiff 0.641260 0.430105 0.439682 \n", 673 | "d1DayPriseRise -0.076290 -0.028490 -0.019473 \n", 674 | "\n", 675 | " d1AgoLowestPrice d1AgoClosingPrice d1AgoClosingDiff \\\n", 676 | "dDayPriseRise -0.049532 -0.050018 0.020438 \n", 677 | "dDayClosingDiff -0.007124 -0.008050 0.012131 \n", 678 | "totalRise -0.013800 -0.011027 0.498864 \n", 679 | "d1TradeVolume -0.064629 -0.055689 -0.187188 \n", 680 | "d1Amount 0.262572 0.271583 -0.179960 \n", 681 | "d1OpeningPrice 0.999510 0.999325 0.020781 \n", 682 | "d1AgoHighestPrice 0.999452 0.999698 0.033949 \n", 683 | "d1AgoLowestPrice 1.000000 0.999626 0.038352 \n", 684 | "d1AgoClosingPrice 0.999626 1.000000 0.048075 \n", 685 | "d1AgoClosingDiff 0.038352 0.048075 1.000000 \n", 686 | "d1AgoTransaction 0.408250 0.415308 -0.309428 \n", 687 | "d1AgoMaxDiff 0.409719 0.426655 -0.105086 \n", 688 | "d1DayPriseRise -0.016922 -0.010142 0.690124 \n", 689 | "\n", 690 | " d1AgoTransaction d1AgoMaxDiff d1DayPriseRise \n", 691 | "dDayPriseRise -0.029423 -0.053826 0.021118 \n", 692 | "dDayClosingDiff -0.004265 -0.011269 0.022485 \n", 693 | "totalRise -0.197942 -0.166556 0.714534 \n", 694 | "d1TradeVolume 0.723970 0.502675 -0.075356 \n", 695 | "d1Amount 0.890879 0.641260 -0.076290 \n", 696 | "d1OpeningPrice 0.423086 0.430105 -0.028490 \n", 697 | "d1AgoHighestPrice 0.426283 0.439682 -0.019473 \n", 698 | "d1AgoLowestPrice 0.408250 0.409719 -0.016922 \n", 699 | "d1AgoClosingPrice 0.415308 0.426655 -0.010142 \n", 700 | "d1AgoClosingDiff -0.309428 -0.105086 0.690124 \n", 701 | "d1AgoTransaction 1.000000 0.670530 -0.177660 \n", 702 | "d1AgoMaxDiff 0.670530 1.000000 -0.077514 \n", 703 | "d1DayPriseRise -0.177660 -0.077514 1.000000 " 704 | ] 705 | }, 706 | "execution_count": 22, 707 | "metadata": {}, 708 | "output_type": "execute_result" 709 | } 710 | ], 711 | "source": [ 712 | "df.corr()" 713 | ] 714 | }, 715 | { 716 | "cell_type": "code", 717 | "execution_count": 20, 718 | "id": "37d35692", 719 | "metadata": {}, 720 | "outputs": [ 721 | { 722 | "data": { 723 | "text/html": [ 724 | "
\n", 725 | "\n", 738 | "\n", 739 | " \n", 740 | " \n", 741 | " \n", 742 | " \n", 743 | " \n", 744 | " \n", 745 | " \n", 746 | " \n", 747 | " \n", 748 | " \n", 749 | " \n", 750 | " \n", 751 | " \n", 752 | " \n", 753 | " \n", 754 | " \n", 755 | " \n", 756 | " \n", 757 | " \n", 758 | " \n", 759 | " \n", 760 | " \n", 761 | " \n", 762 | " \n", 763 | " \n", 764 | " \n", 765 | " \n", 766 | " \n", 767 | " \n", 768 | " \n", 769 | " \n", 770 | " \n", 771 | " \n", 772 | " \n", 773 | " \n", 774 | " \n", 775 | " \n", 776 | " \n", 777 | " \n", 778 | " \n", 779 | " \n", 780 | " \n", 781 | " \n", 782 | " \n", 783 | " \n", 784 | " \n", 785 | " \n", 786 | " \n", 787 | " \n", 788 | " \n", 789 | " \n", 790 | " \n", 791 | " \n", 792 | " \n", 793 | " \n", 794 | " \n", 795 | " \n", 796 | " \n", 797 | " \n", 798 | " \n", 799 | " \n", 800 | " \n", 801 | " \n", 802 | " \n", 803 | " \n", 804 | " \n", 805 | " \n", 806 | " \n", 807 | " \n", 808 | " \n", 809 | " \n", 810 | " \n", 811 | " \n", 812 | " \n", 813 | " \n", 814 | " \n", 815 | " \n", 816 | " \n", 817 | " \n", 818 | " \n", 819 | " \n", 820 | " \n", 821 | " \n", 822 | " \n", 823 | " \n", 824 | " \n", 825 | " \n", 826 | " \n", 827 | " \n", 828 | " \n", 829 | " \n", 830 | " \n", 831 | " \n", 832 | " \n", 833 | " \n", 834 | " \n", 835 | " \n", 836 | " \n", 837 | " \n", 838 | " \n", 839 | " \n", 840 | " \n", 841 | " \n", 842 | " \n", 843 | " \n", 844 | " \n", 845 | " \n", 846 | " \n", 847 | " \n", 848 | " \n", 849 | " \n", 850 | " \n", 851 | " \n", 852 | " \n", 853 | " \n", 854 | " \n", 855 | " \n", 856 | " \n", 857 | " \n", 858 | " \n", 859 | " \n", 860 | " \n", 861 | " \n", 862 | " \n", 863 | " \n", 864 | " \n", 865 | " \n", 866 | " \n", 867 | " \n", 868 | " \n", 869 | " \n", 870 | " \n", 871 | " \n", 872 | " \n", 873 | " \n", 874 | " \n", 875 | " \n", 876 | " \n", 877 | " \n", 878 | " \n", 879 | " \n", 880 | " \n", 881 | " \n", 882 | " \n", 883 | " \n", 884 | " \n", 885 | " \n", 886 | " \n", 887 | " \n", 888 | " \n", 889 | " \n", 890 | " \n", 891 | " \n", 892 | " \n", 893 | " \n", 894 | " \n", 895 | " \n", 896 | " \n", 897 | " \n", 898 | " \n", 899 | "
d1TradeVolumed1Amountd1OpeningPriced1AgoHighestPriced1AgoLowestPriced1AgoClosingPriced1AgoClosingDiffd1AgoTransactiond1AgoMaxDiffdDayPriseRise
0-0.675245-0.4563821.4812731.4583941.4845181.468681-0.409883-0.214823-0.0894810
1-0.2768320.0803381.4599911.4541641.4888141.485714-0.079304-0.085389-0.3226960
20.1677770.6834991.4472211.4626241.4845181.4857140.8711110.5516870.0271260
3-0.2775750.0683721.4195531.4182091.4243701.436745-0.1619490.1177980.4352501
4-0.576839-0.3294601.4301951.4203241.4437031.4410030.251275-0.283855-0.0311780
.................................
700-0.577537-0.766784-1.198219-1.187506-1.185610-1.1756410.416565-0.584891-0.5559101
7010.469029-0.001575-1.198219-1.195966-1.185610-1.201190-0.905752-0.096135-0.7891241
702-0.378371-0.613678-1.151397-1.160010-1.149092-1.1586080.209953-0.289391-0.7891240
7031.4662490.745586-1.140756-1.155780-1.168425-1.173512-1.3602980.670539-0.1477851
7040.009871-0.308807-1.087549-1.098674-1.091093-1.107510-1.401621-0.455620-0.6725170
\n", 900 | "

705 rows × 10 columns

\n", 901 | "
" 902 | ], 903 | "text/plain": [ 904 | " d1TradeVolume d1Amount d1OpeningPrice d1AgoHighestPrice \\\n", 905 | "0 -0.675245 -0.456382 1.481273 1.458394 \n", 906 | "1 -0.276832 0.080338 1.459991 1.454164 \n", 907 | "2 0.167777 0.683499 1.447221 1.462624 \n", 908 | "3 -0.277575 0.068372 1.419553 1.418209 \n", 909 | "4 -0.576839 -0.329460 1.430195 1.420324 \n", 910 | ".. ... ... ... ... \n", 911 | "700 -0.577537 -0.766784 -1.198219 -1.187506 \n", 912 | "701 0.469029 -0.001575 -1.198219 -1.195966 \n", 913 | "702 -0.378371 -0.613678 -1.151397 -1.160010 \n", 914 | "703 1.466249 0.745586 -1.140756 -1.155780 \n", 915 | "704 0.009871 -0.308807 -1.087549 -1.098674 \n", 916 | "\n", 917 | " d1AgoLowestPrice d1AgoClosingPrice d1AgoClosingDiff d1AgoTransaction \\\n", 918 | "0 1.484518 1.468681 -0.409883 -0.214823 \n", 919 | "1 1.488814 1.485714 -0.079304 -0.085389 \n", 920 | "2 1.484518 1.485714 0.871111 0.551687 \n", 921 | "3 1.424370 1.436745 -0.161949 0.117798 \n", 922 | "4 1.443703 1.441003 0.251275 -0.283855 \n", 923 | ".. ... ... ... ... \n", 924 | "700 -1.185610 -1.175641 0.416565 -0.584891 \n", 925 | "701 -1.185610 -1.201190 -0.905752 -0.096135 \n", 926 | "702 -1.149092 -1.158608 0.209953 -0.289391 \n", 927 | "703 -1.168425 -1.173512 -1.360298 0.670539 \n", 928 | "704 -1.091093 -1.107510 -1.401621 -0.455620 \n", 929 | "\n", 930 | " d1AgoMaxDiff dDayPriseRise \n", 931 | "0 -0.089481 0 \n", 932 | "1 -0.322696 0 \n", 933 | "2 0.027126 0 \n", 934 | "3 0.435250 1 \n", 935 | "4 -0.031178 0 \n", 936 | ".. ... ... \n", 937 | "700 -0.555910 1 \n", 938 | "701 -0.789124 1 \n", 939 | "702 -0.789124 0 \n", 940 | "703 -0.147785 1 \n", 941 | "704 -0.672517 0 \n", 942 | "\n", 943 | "[705 rows x 10 columns]" 944 | ] 945 | }, 946 | "execution_count": 20, 947 | "metadata": {}, 948 | "output_type": "execute_result" 949 | } 950 | ], 951 | "source": [ 952 | "from sklearn.preprocessing import StandardScaler\n", 953 | "\n", 954 | "scale = StandardScaler()\n", 955 | "\n", 956 | "scale_target = [\n", 957 | " 'd1TradeVolume',\n", 958 | " 'd1Amount',\n", 959 | " 'd1OpeningPrice',\n", 960 | " 'd1AgoHighestPrice',\n", 961 | " 'd1AgoLowestPrice',\n", 962 | " 'd1AgoClosingPrice',\n", 963 | " 'd1AgoClosingDiff',\n", 964 | " 'd1AgoTransaction',\n", 965 | " 'd1AgoMaxDiff',\n", 966 | "]\n", 967 | "\n", 968 | "scaled = pd.DataFrame(scale.fit_transform(df[scale_target]), columns=scale_target)\n", 969 | "scaled['dDayPriseRise'] = df['dDayPriseRise']\n", 970 | "scaled" 971 | ] 972 | }, 973 | { 974 | "cell_type": "code", 975 | "execution_count": 38, 976 | "id": "a5747469", 977 | "metadata": {}, 978 | "outputs": [ 979 | { 980 | "name": "stdout", 981 | "output_type": "stream", 982 | "text": [ 983 | "Number of labeled points out of a total 212 points : 109\n" 984 | ] 985 | } 986 | ], 987 | "source": [ 988 | "from sklearn.naive_bayes import GaussianNB\n", 989 | "from sklearn.model_selection import train_test_split\n", 990 | "\n", 991 | "X_train, X_test, y_train, y_test = train_test_split(scaled[scale_target], scaled['dDayPriseRise'], test_size=0.3, random_state=0)\n", 992 | "gnb = GaussianNB()\n", 993 | "y_pred = gnb.fit(X_train, y_train).predict(X_test)\n", 994 | "\n", 995 | "test_count = X_test.shape[0]\n", 996 | "print(\"Number of labeled points out of a total %d points : %d\" % (test_count, (y_test == y_pred).sum()))\n", 997 | "\n", 998 | "from sklearn.metrics import confusion_matrix\n", 999 | "tn, fp, fn, tp = confusion_matrix(y_test, y_pred).ravel()" 1000 | ] 1001 | }, 1002 | { 1003 | "cell_type": "code", 1004 | "execution_count": 36, 1005 | "id": "2d823316", 1006 | "metadata": {}, 1007 | "outputs": [ 1008 | { 1009 | "name": "stdout", 1010 | "output_type": "stream", 1011 | "text": [ 1012 | " : pn,pp\n", 1013 | "an: 42, 135\n", 1014 | "ap: 37, 139\n" 1015 | ] 1016 | } 1017 | ], 1018 | "source": [ 1019 | "print(f' : pn,pp')\n", 1020 | "print(f'an: {tn}, {fp}')\n", 1021 | "print(f'ap: {fn}, {tp}')" 1022 | ] 1023 | }, 1024 | { 1025 | "cell_type": "code", 1026 | "execution_count": null, 1027 | "id": "123ab836", 1028 | "metadata": {}, 1029 | "outputs": [], 1030 | "source": [] 1031 | } 1032 | ], 1033 | "metadata": { 1034 | "kernelspec": { 1035 | "display_name": "Python 3 (ipykernel)", 1036 | "language": "python", 1037 | "name": "python3" 1038 | }, 1039 | "language_info": { 1040 | "codemirror_mode": { 1041 | "name": "ipython", 1042 | "version": 3 1043 | }, 1044 | "file_extension": ".py", 1045 | "mimetype": "text/x-python", 1046 | "name": "python", 1047 | "nbconvert_exporter": "python", 1048 | "pygments_lexer": "ipython3", 1049 | "version": "3.8.12" 1050 | } 1051 | }, 1052 | "nbformat": 4, 1053 | "nbformat_minor": 5 1054 | } 1055 | -------------------------------------------------------------------------------- /lab/experiment1/three_days_strategy.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "id": "10c819c0", 7 | "metadata": {}, 8 | "outputs": [], 9 | "source": [ 10 | "from pymongo import MongoClient\n", 11 | "from pymongo import DESCENDING, ASCENDING\n", 12 | "\n", 13 | "from urllib.parse import quote_plus" 14 | ] 15 | }, 16 | { 17 | "cell_type": "code", 18 | "execution_count": 2, 19 | "id": "a06c39d0", 20 | "metadata": {}, 21 | "outputs": [], 22 | "source": [ 23 | "uri = \"mongodb://%s:%s@%s/?authSource=stock\" % (\n", 24 | " quote_plus('airflower'), quote_plus('money'), 'localhost')\n", 25 | "client = MongoClient(uri)" 26 | ] 27 | }, 28 | { 29 | "cell_type": "code", 30 | "execution_count": 3, 31 | "id": "47d1ac75", 32 | "metadata": {}, 33 | "outputs": [], 34 | "source": [ 35 | "results = list(client.stock.dailyTransactions.find({'stockCode': '0050'}).sort('intDate', DESCENDING))" 36 | ] 37 | }, 38 | { 39 | "cell_type": "code", 40 | "execution_count": 4, 41 | "id": "fdca3755", 42 | "metadata": {}, 43 | "outputs": [], 44 | "source": [ 45 | "def total_rise(days):\n", 46 | " count = 0\n", 47 | " for day in days:\n", 48 | " count += 1 if day['closingDiff'] > 0 else 0\n", 49 | " \n", 50 | " return count\n", 51 | "\n", 52 | "data = [ \n", 53 | " {\n", 54 | " # target\n", 55 | " 'dDayPriseRise': 1 if results[i]['closingDiff'] > 0 else 0,\n", 56 | " 'dDayClosingDiff': results[i]['closingDiff'],\n", 57 | " # summary\n", 58 | " # 'totalRise': total_rise(results[i+1:i+3]),\n", 59 | " # day1 columns\n", 60 | " 'd1TradeVolume': results[i+1]['tradeVolume'],\n", 61 | " 'd1Amount': results[i+1]['amount'],\n", 62 | " 'd1OpeningPrice': results[i+1]['openingPrice'],\n", 63 | " 'd1AgoHighestPrice': results[i+1]['highestPrice'],\n", 64 | " 'd1AgoLowestPrice': results[i+1]['lowestPrice'],\n", 65 | " 'd1AgoClosingPrice': results[i+1]['closingPrice'],\n", 66 | " 'd1AgoClosingDiff': results[i+1]['closingDiff'],\n", 67 | " 'd1AgoTransaction': results[i+1]['transaction'],\n", 68 | " 'd1AgoMaxDiff': round(results[i+1]['highestPrice'] - results[i+1]['lowestPrice'], 2),\n", 69 | " 'd1AgoDayPriseRise': 1 if results[i+1]['closingDiff'] > 0 else 0,\n", 70 | " # day2 columns\n", 71 | " 'd2TradeVolume': results[i+2]['tradeVolume'],\n", 72 | " 'd2Amount': results[i+2]['amount'],\n", 73 | " 'd2OpeningPrice': results[i+2]['openingPrice'],\n", 74 | " 'd2AgoHighestPrice': results[i+2]['highestPrice'],\n", 75 | " 'd2AgoLowestPrice': results[i+2]['lowestPrice'],\n", 76 | " 'd2AgoClosingPrice': results[i+2]['closingPrice'],\n", 77 | " 'd2AgoClosingDiff': results[i+2]['closingDiff'],\n", 78 | " 'd2AgoTransaction': results[i+2]['transaction'],\n", 79 | " 'd2AgoMaxDiff': round(results[i+2]['highestPrice'] - results[i+2]['lowestPrice'], 2),\n", 80 | " 'd2AgoDayPriseRise': 1 if results[i+2]['closingDiff'] > 0 else 0,\n", 81 | " # day3 columns\n", 82 | " 'd3TradeVolume': results[i+3]['tradeVolume'],\n", 83 | " 'd3Amount': results[i+3]['amount'],\n", 84 | " 'd3OpeningPrice': results[i+3]['openingPrice'],\n", 85 | " 'd3AgoHighestPrice': results[i+3]['highestPrice'],\n", 86 | " 'd3AgoLowestPrice': results[i+3]['lowestPrice'],\n", 87 | " 'd3AgoClosingPrice': results[i+3]['closingPrice'],\n", 88 | " 'd3AgoClosingDiff': results[i+3]['closingDiff'],\n", 89 | " 'd3AgoTransaction': results[i+3]['transaction'],\n", 90 | " 'd3AgoMaxDiff': round(results[i+3]['highestPrice'] - results[i+3]['lowestPrice'], 2),\n", 91 | " 'd3AgoDayPriseRise': 1 if results[i+3]['closingDiff'] > 0 else 0,\n", 92 | " # rise\n", 93 | " 'd1AgoTradeVolumeRise': 1 if results[i+1]['tradeVolume'] > results[i+2]['tradeVolume'] else 0,\n", 94 | " 'd1AgoAmountRise': 1 if results[i+1]['amount'] > results[i+2]['amount'] else 0,\n", 95 | " 'd1AgoTransactionRise': 1 if results[i+1]['transaction'] > results[i+2]['transaction'] else 0,\n", 96 | " 'd2AgoTradeVolumeRise': 1 if results[i+2]['tradeVolume'] > results[i+3]['tradeVolume'] else 0,\n", 97 | " 'd2AgoAmountRise': 1 if results[i+2]['amount'] > results[i+3]['amount'] else 0,\n", 98 | " 'd2AgoTransactionRise': 1 if results[i+2]['transaction'] > results[i+3]['transaction'] else 0,\n", 99 | "\n", 100 | " }\n", 101 | " for i in range(0, len(results)-3)\n", 102 | "]" 103 | ] 104 | }, 105 | { 106 | "cell_type": "code", 107 | "execution_count": 5, 108 | "id": "28d2f155", 109 | "metadata": {}, 110 | "outputs": [], 111 | "source": [ 112 | "import pandas as pd\n", 113 | "df = pd.DataFrame(data)" 114 | ] 115 | }, 116 | { 117 | "cell_type": "code", 118 | "execution_count": 10, 119 | "id": "d9111ddb", 120 | "metadata": {}, 121 | "outputs": [ 122 | { 123 | "data": { 124 | "text/html": [ 125 | "
\n", 126 | "\n", 139 | "\n", 140 | " \n", 141 | " \n", 142 | " \n", 143 | " \n", 144 | " \n", 145 | " \n", 146 | " \n", 147 | " \n", 148 | " \n", 149 | " \n", 150 | " \n", 151 | " \n", 152 | " \n", 153 | " \n", 154 | " \n", 155 | " \n", 156 | " \n", 157 | " \n", 158 | " \n", 159 | " \n", 160 | " \n", 161 | " \n", 162 | " \n", 163 | " \n", 164 | " \n", 165 | " \n", 166 | " \n", 167 | " \n", 168 | " \n", 169 | " \n", 170 | " \n", 171 | " \n", 172 | " \n", 173 | " \n", 174 | " \n", 175 | " \n", 176 | " \n", 177 | " \n", 178 | " \n", 179 | " \n", 180 | " \n", 181 | " \n", 182 | " \n", 183 | " \n", 184 | " \n", 185 | " \n", 186 | " \n", 187 | " \n", 188 | " \n", 189 | " \n", 190 | " \n", 191 | " \n", 192 | " \n", 193 | " \n", 194 | " \n", 195 | " \n", 196 | " \n", 197 | " \n", 198 | " \n", 199 | " \n", 200 | " \n", 201 | " \n", 202 | " \n", 203 | " \n", 204 | " \n", 205 | " \n", 206 | " \n", 207 | " \n", 208 | " \n", 209 | " \n", 210 | " \n", 211 | " \n", 212 | " \n", 213 | " \n", 214 | " \n", 215 | " \n", 216 | " \n", 217 | " \n", 218 | " \n", 219 | " \n", 220 | " \n", 221 | " \n", 222 | " \n", 223 | " \n", 224 | " \n", 225 | " \n", 226 | " \n", 227 | " \n", 228 | " \n", 229 | " \n", 230 | " \n", 231 | " \n", 232 | " \n", 233 | " \n", 234 | " \n", 235 | " \n", 236 | " \n", 237 | " \n", 238 | " \n", 239 | " \n", 240 | " \n", 241 | " \n", 242 | " \n", 243 | " \n", 244 | " \n", 245 | " \n", 246 | " \n", 247 | " \n", 248 | " \n", 249 | " \n", 250 | " \n", 251 | " \n", 252 | " \n", 253 | " \n", 254 | " \n", 255 | " \n", 256 | " \n", 257 | " \n", 258 | " \n", 259 | " \n", 260 | " \n", 261 | " \n", 262 | " \n", 263 | " \n", 264 | " \n", 265 | " \n", 266 | " \n", 267 | " \n", 268 | " \n", 269 | " \n", 270 | " \n", 271 | " \n", 272 | " \n", 273 | " \n", 274 | " \n", 275 | " \n", 276 | " \n", 277 | " \n", 278 | " \n", 279 | " \n", 280 | " \n", 281 | " \n", 282 | " \n", 283 | " \n", 284 | " \n", 285 | " \n", 286 | " \n", 287 | " \n", 288 | " \n", 289 | " \n", 290 | " \n", 291 | " \n", 292 | " \n", 293 | " \n", 294 | " \n", 295 | " \n", 296 | " \n", 297 | " \n", 298 | " \n", 299 | " \n", 300 | " \n", 301 | " \n", 302 | " \n", 303 | " \n", 304 | " \n", 305 | " \n", 306 | " \n", 307 | " \n", 308 | " \n", 309 | " \n", 310 | " \n", 311 | " \n", 312 | " \n", 313 | " \n", 314 | " \n", 315 | " \n", 316 | " \n", 317 | " \n", 318 | " \n", 319 | " \n", 320 | " \n", 321 | " \n", 322 | " \n", 323 | " \n", 324 | " \n", 325 | " \n", 326 | " \n", 327 | " \n", 328 | " \n", 329 | " \n", 330 | " \n", 331 | " \n", 332 | " \n", 333 | " \n", 334 | " \n", 335 | " \n", 336 | " \n", 337 | " \n", 338 | " \n", 339 | " \n", 340 | " \n", 341 | " \n", 342 | " \n", 343 | " \n", 344 | " \n", 345 | " \n", 346 | " \n", 347 | " \n", 348 | " \n", 349 | " \n", 350 | " \n", 351 | " \n", 352 | " \n", 353 | " \n", 354 | " \n", 355 | " \n", 356 | " \n", 357 | " \n", 358 | " \n", 359 | " \n", 360 | " \n", 361 | " \n", 362 | " \n", 363 | " \n", 364 | " \n", 365 | " \n", 366 | " \n", 367 | " \n", 368 | " \n", 369 | " \n", 370 | " \n", 371 | " \n", 372 | " \n", 373 | " \n", 374 | " \n", 375 | " \n", 376 | " \n", 377 | " \n", 378 | " \n", 379 | " \n", 380 | " \n", 381 | " \n", 382 | " \n", 383 | " \n", 384 | " \n", 385 | " \n", 386 | " \n", 387 | " \n", 388 | " \n", 389 | " \n", 390 | " \n", 391 | " \n", 392 | " \n", 393 | " \n", 394 | " \n", 395 | " \n", 396 | " \n", 397 | " \n", 398 | " \n", 399 | " \n", 400 | " \n", 401 | " \n", 402 | " \n", 403 | " \n", 404 | " \n", 405 | " \n", 406 | " \n", 407 | " \n", 408 | " \n", 409 | " \n", 410 | " \n", 411 | " \n", 412 | " \n", 413 | " \n", 414 | " \n", 415 | " \n", 416 | " \n", 417 | " \n", 418 | " \n", 419 | " \n", 420 | " \n", 421 | " \n", 422 | " \n", 423 | " \n", 424 | " \n", 425 | " \n", 426 | " \n", 427 | " \n", 428 | " \n", 429 | " \n", 430 | " \n", 431 | " \n", 432 | " \n", 433 | " \n", 434 | " \n", 435 | " \n", 436 | " \n", 437 | " \n", 438 | " \n", 439 | " \n", 440 | " \n", 441 | " \n", 442 | " \n", 443 | " \n", 444 | " \n", 445 | " \n", 446 | " \n", 447 | " \n", 448 | " \n", 449 | " \n", 450 | " \n", 451 | " \n", 452 | " \n", 453 | " \n", 454 | " \n", 455 | " \n", 456 | " \n", 457 | " \n", 458 | " \n", 459 | " \n", 460 | " \n", 461 | " \n", 462 | " \n", 463 | " \n", 464 | " \n", 465 | " \n", 466 | " \n", 467 | " \n", 468 | " \n", 469 | " \n", 470 | " \n", 471 | " \n", 472 | " \n", 473 | " \n", 474 | " \n", 475 | " \n", 476 | " \n", 477 | " \n", 478 | " \n", 479 | " \n", 480 | " \n", 481 | " \n", 482 | " \n", 483 | " \n", 484 | " \n", 485 | " \n", 486 | " \n", 487 | " \n", 488 | " \n", 489 | " \n", 490 | " \n", 491 | " \n", 492 | " \n", 493 | " \n", 494 | " \n", 495 | " \n", 496 | " \n", 497 | " \n", 498 | " \n", 499 | " \n", 500 | " \n", 501 | " \n", 502 | " \n", 503 | " \n", 504 | " \n", 505 | " \n", 506 | " \n", 507 | " \n", 508 | " \n", 509 | " \n", 510 | " \n", 511 | " \n", 512 | " \n", 513 | " \n", 514 | " \n", 515 | " \n", 516 | " \n", 517 | " \n", 518 | " \n", 519 | " \n", 520 | " \n", 521 | " \n", 522 | " \n", 523 | " \n", 524 | " \n", 525 | " \n", 526 | " \n", 527 | " \n", 528 | " \n", 529 | " \n", 530 | " \n", 531 | " \n", 532 | " \n", 533 | " \n", 534 | " \n", 535 | " \n", 536 | " \n", 537 | " \n", 538 | " \n", 539 | " \n", 540 | " \n", 541 | " \n", 542 | " \n", 543 | " \n", 544 | " \n", 545 | " \n", 546 | " \n", 547 | " \n", 548 | " \n", 549 | " \n", 550 | " \n", 551 | " \n", 552 | " \n", 553 | " \n", 554 | " \n", 555 | " \n", 556 | " \n", 557 | " \n", 558 | " \n", 559 | " \n", 560 | " \n", 561 | " \n", 562 | " \n", 563 | " \n", 564 | " \n", 565 | " \n", 566 | " \n", 567 | " \n", 568 | " \n", 569 | " \n", 570 | " \n", 571 | " \n", 572 | " \n", 573 | " \n", 574 | " \n", 575 | " \n", 576 | " \n", 577 | " \n", 578 | " \n", 579 | " \n", 580 | " \n", 581 | " \n", 582 | " \n", 583 | " \n", 584 | " \n", 585 | " \n", 586 | " \n", 587 | " \n", 588 | " \n", 589 | " \n", 590 | " \n", 591 | " \n", 592 | " \n", 593 | " \n", 594 | " \n", 595 | " \n", 596 | " \n", 597 | " \n", 598 | " \n", 599 | " \n", 600 | " \n", 601 | " \n", 602 | " \n", 603 | " \n", 604 | " \n", 605 | " \n", 606 | " \n", 607 | " \n", 608 | " \n", 609 | " \n", 610 | " \n", 611 | " \n", 612 | " \n", 613 | " \n", 614 | " \n", 615 | " \n", 616 | " \n", 617 | " \n", 618 | " \n", 619 | " \n", 620 | " \n", 621 | " \n", 622 | " \n", 623 | " \n", 624 | " \n", 625 | " \n", 626 | " \n", 627 | " \n", 628 | " \n", 629 | " \n", 630 | " \n", 631 | " \n", 632 | " \n", 633 | " \n", 634 | " \n", 635 | " \n", 636 | " \n", 637 | " \n", 638 | " \n", 639 | " \n", 640 | " \n", 641 | " \n", 642 | " \n", 643 | " \n", 644 | " \n", 645 | " \n", 646 | " \n", 647 | " \n", 648 | " \n", 649 | " \n", 650 | " \n", 651 | " \n", 652 | " \n", 653 | " \n", 654 | " \n", 655 | " \n", 656 | " \n", 657 | " \n", 658 | " \n", 659 | " \n", 660 | " \n", 661 | " \n", 662 | " \n", 663 | " \n", 664 | " \n", 665 | " \n", 666 | " \n", 667 | " \n", 668 | " \n", 669 | " \n", 670 | " \n", 671 | " \n", 672 | " \n", 673 | " \n", 674 | " \n", 675 | " \n", 676 | " \n", 677 | " \n", 678 | " \n", 679 | " \n", 680 | " \n", 681 | " \n", 682 | " \n", 683 | " \n", 684 | " \n", 685 | " \n", 686 | " \n", 687 | " \n", 688 | " \n", 689 | " \n", 690 | " \n", 691 | " \n", 692 | " \n", 693 | " \n", 694 | " \n", 695 | " \n", 696 | " \n", 697 | " \n", 698 | " \n", 699 | " \n", 700 | " \n", 701 | " \n", 702 | " \n", 703 | " \n", 704 | " \n", 705 | " \n", 706 | " \n", 707 | " \n", 708 | " \n", 709 | " \n", 710 | " \n", 711 | " \n", 712 | " \n", 713 | " \n", 714 | " \n", 715 | " \n", 716 | " \n", 717 | " \n", 718 | " \n", 719 | " \n", 720 | " \n", 721 | " \n", 722 | " \n", 723 | " \n", 724 | " \n", 725 | " \n", 726 | " \n", 727 | " \n", 728 | " \n", 729 | " \n", 730 | " \n", 731 | " \n", 732 | " \n", 733 | " \n", 734 | " \n", 735 | " \n", 736 | " \n", 737 | " \n", 738 | " \n", 739 | " \n", 740 | " \n", 741 | " \n", 742 | " \n", 743 | " \n", 744 | " \n", 745 | " \n", 746 | " \n", 747 | " \n", 748 | " \n", 749 | " \n", 750 | " \n", 751 | " \n", 752 | " \n", 753 | " \n", 754 | " \n", 755 | " \n", 756 | " \n", 757 | " \n", 758 | " \n", 759 | " \n", 760 | " \n", 761 | " \n", 762 | " \n", 763 | " \n", 764 | " \n", 765 | " \n", 766 | " \n", 767 | " \n", 768 | " \n", 769 | " \n", 770 | " \n", 771 | " \n", 772 | " \n", 773 | " \n", 774 | " \n", 775 | " \n", 776 | " \n", 777 | " \n", 778 | " \n", 779 | " \n", 780 | " \n", 781 | " \n", 782 | " \n", 783 | " \n", 784 | " \n", 785 | " \n", 786 | " \n", 787 | " \n", 788 | " \n", 789 | " \n", 790 | " \n", 791 | " \n", 792 | " \n", 793 | " \n", 794 | " \n", 795 | " \n", 796 | " \n", 797 | " \n", 798 | " \n", 799 | " \n", 800 | " \n", 801 | " \n", 802 | " \n", 803 | " \n", 804 | " \n", 805 | " \n", 806 | " \n", 807 | " \n", 808 | " \n", 809 | " \n", 810 | " \n", 811 | " \n", 812 | " \n", 813 | " \n", 814 | " \n", 815 | " \n", 816 | " \n", 817 | " \n", 818 | " \n", 819 | " \n", 820 | " \n", 821 | " \n", 822 | " \n", 823 | " \n", 824 | " \n", 825 | " \n", 826 | " \n", 827 | " \n", 828 | " \n", 829 | " \n", 830 | " \n", 831 | " \n", 832 | " \n", 833 | " \n", 834 | " \n", 835 | " \n", 836 | " \n", 837 | " \n", 838 | " \n", 839 | " \n", 840 | " \n", 841 | " \n", 842 | " \n", 843 | " \n", 844 | " \n", 845 | " \n", 846 | " \n", 847 | " \n", 848 | " \n", 849 | " \n", 850 | " \n", 851 | " \n", 852 | " \n", 853 | " \n", 854 | " \n", 855 | " \n", 856 | " \n", 857 | " \n", 858 | " \n", 859 | " \n", 860 | " \n", 861 | " \n", 862 | " \n", 863 | " \n", 864 | " \n", 865 | " \n", 866 | " \n", 867 | " \n", 868 | " \n", 869 | " \n", 870 | " \n", 871 | " \n", 872 | " \n", 873 | " \n", 874 | " \n", 875 | " \n", 876 | " \n", 877 | " \n", 878 | " \n", 879 | " \n", 880 | " \n", 881 | " \n", 882 | " \n", 883 | " \n", 884 | " \n", 885 | " \n", 886 | " \n", 887 | " \n", 888 | " \n", 889 | " \n", 890 | " \n", 891 | " \n", 892 | " \n", 893 | " \n", 894 | " \n", 895 | " \n", 896 | " \n", 897 | " \n", 898 | " \n", 899 | " \n", 900 | " \n", 901 | " \n", 902 | " \n", 903 | " \n", 904 | " \n", 905 | " \n", 906 | " \n", 907 | " \n", 908 | " \n", 909 | " \n", 910 | " \n", 911 | " \n", 912 | " \n", 913 | " \n", 914 | " \n", 915 | " \n", 916 | " \n", 917 | " \n", 918 | " \n", 919 | " \n", 920 | " \n", 921 | " \n", 922 | " \n", 923 | " \n", 924 | " \n", 925 | " \n", 926 | " \n", 927 | " \n", 928 | " \n", 929 | " \n", 930 | " \n", 931 | " \n", 932 | " \n", 933 | " \n", 934 | " \n", 935 | " \n", 936 | " \n", 937 | " \n", 938 | " \n", 939 | " \n", 940 | " \n", 941 | " \n", 942 | " \n", 943 | " \n", 944 | " \n", 945 | " \n", 946 | " \n", 947 | " \n", 948 | " \n", 949 | " \n", 950 | " \n", 951 | " \n", 952 | " \n", 953 | " \n", 954 | " \n", 955 | " \n", 956 | " \n", 957 | " \n", 958 | " \n", 959 | " \n", 960 | " \n", 961 | " \n", 962 | " \n", 963 | " \n", 964 | " \n", 965 | " \n", 966 | " \n", 967 | " \n", 968 | " \n", 969 | " \n", 970 | " \n", 971 | " \n", 972 | " \n", 973 | " \n", 974 | " \n", 975 | " \n", 976 | " \n", 977 | " \n", 978 | " \n", 979 | " \n", 980 | " \n", 981 | " \n", 982 | " \n", 983 | " \n", 984 | " \n", 985 | " \n", 986 | " \n", 987 | " \n", 988 | " \n", 989 | " \n", 990 | " \n", 991 | " \n", 992 | " \n", 993 | " \n", 994 | " \n", 995 | " \n", 996 | " \n", 997 | " \n", 998 | " \n", 999 | " \n", 1000 | " \n", 1001 | " \n", 1002 | " \n", 1003 | " \n", 1004 | " \n", 1005 | " \n", 1006 | " \n", 1007 | " \n", 1008 | " \n", 1009 | " \n", 1010 | " \n", 1011 | " \n", 1012 | " \n", 1013 | " \n", 1014 | " \n", 1015 | " \n", 1016 | " \n", 1017 | " \n", 1018 | " \n", 1019 | " \n", 1020 | " \n", 1021 | " \n", 1022 | " \n", 1023 | " \n", 1024 | " \n", 1025 | " \n", 1026 | " \n", 1027 | " \n", 1028 | " \n", 1029 | " \n", 1030 | " \n", 1031 | " \n", 1032 | " \n", 1033 | " \n", 1034 | " \n", 1035 | " \n", 1036 | " \n", 1037 | " \n", 1038 | " \n", 1039 | " \n", 1040 | " \n", 1041 | " \n", 1042 | " \n", 1043 | " \n", 1044 | " \n", 1045 | " \n", 1046 | " \n", 1047 | " \n", 1048 | " \n", 1049 | " \n", 1050 | " \n", 1051 | " \n", 1052 | " \n", 1053 | " \n", 1054 | " \n", 1055 | " \n", 1056 | " \n", 1057 | " \n", 1058 | " \n", 1059 | " \n", 1060 | " \n", 1061 | " \n", 1062 | " \n", 1063 | " \n", 1064 | " \n", 1065 | " \n", 1066 | " \n", 1067 | " \n", 1068 | " \n", 1069 | " \n", 1070 | " \n", 1071 | " \n", 1072 | " \n", 1073 | " \n", 1074 | " \n", 1075 | " \n", 1076 | " \n", 1077 | " \n", 1078 | " \n", 1079 | " \n", 1080 | "
dDayPriseRisedDayClosingDiffd1TradeVolumed1Amountd1OpeningPriced1AgoHighestPriced1AgoLowestPriced1AgoClosingPriced1AgoClosingDiffd1AgoTransaction...d3AgoClosingDiffd3AgoTransactiond3AgoMaxDiffd3AgoDayPriseRised1AgoTradeVolumeRised1AgoAmountRised1AgoTransactionRised2AgoTradeVolumeRised2AgoAmountRised2AgoTransactionRise
dDayPriseRise1.0000000.692129-0.048711-0.047336-0.047911-0.047099-0.046045-0.0464490.022676-0.023960...-0.042127-0.045853-0.031084-0.0601080.0688810.0568770.043534-0.0085010.007142-0.011731
dDayClosingDiff0.6921291.000000-0.040452-0.026802-0.008208-0.007084-0.006943-0.0077630.015269-0.000756...0.007302-0.0019500.014466-0.0071330.0347180.0271940.018125-0.022222-0.016792-0.014523
d1TradeVolume-0.048711-0.0404521.0000000.915744-0.065934-0.059411-0.078021-0.069193-0.1785450.724128...-0.1361090.3590180.269790-0.0846140.2969030.2980940.2719980.0412150.0428100.064892
d1Amount-0.047336-0.0268020.9157441.0000000.2607990.2670920.2484720.257456-0.1678000.890393...-0.1063050.4797990.379975-0.0737450.3028120.3041050.2685800.0437680.0432570.051107
d1OpeningPrice-0.047911-0.008208-0.0659340.2607991.0000000.9997240.9995320.9993580.0221610.413294...0.0510030.4051680.401764-0.0016850.0177490.019142-0.0108750.0221380.023496-0.008444
d1AgoHighestPrice-0.047099-0.007084-0.0594110.2670920.9997241.0000000.9994730.9997120.0350270.416553...0.0489210.4103200.407927-0.0029610.0193810.021139-0.0099270.0219010.023109-0.008009
d1AgoLowestPrice-0.046045-0.006943-0.0780210.2484720.9995320.9994731.0000000.9996410.0392950.398856...0.0528260.3998280.396589-0.0002190.0159620.017204-0.0143680.0221140.023378-0.010058
d1AgoClosingPrice-0.046449-0.007763-0.0691930.2574560.9993580.9997120.9996411.0000000.0487940.406020...0.0508720.4051580.401867-0.0017680.0173310.018742-0.0132270.0225060.023605-0.008629
d1AgoClosingDiff0.0226760.015269-0.178545-0.1678000.0221610.0350270.0392950.0487941.000000-0.299492...0.042829-0.024235-0.0141380.006713-0.033479-0.016926-0.1009090.0339790.0264310.017356
d1AgoTransaction-0.023960-0.0007560.7241280.8903930.4132940.4165530.3988560.406020-0.2994921.000000...-0.1386200.5531260.431454-0.1069200.2383540.2386610.2506260.0231650.0212300.016530
d1AgoMaxDiff-0.048395-0.0067780.4973700.6285760.4062250.4157370.3859830.402825-0.1054810.662672...-0.0897490.4583690.481108-0.0779680.1035240.1186880.1203750.0028160.0017340.054168
d1AgoDayPriseRise0.0258500.029230-0.065240-0.061991-0.025350-0.016525-0.013928-0.0073510.692225-0.166937...0.024388-0.072995-0.0432980.0063220.0025540.023720-0.0559440.0688810.0568770.043534
d2TradeVolume-0.073006-0.0654810.6063500.524134-0.068497-0.062375-0.076997-0.068852-0.0416710.425635...-0.1544350.4237900.326057-0.089003-0.304293-0.302703-0.2505800.2966960.2978830.271782
d2Amount-0.074072-0.0480260.5322840.5930960.2586440.2646530.2499780.258270-0.0273690.555092...-0.1143380.5458390.448161-0.070167-0.297051-0.295771-0.2558040.3028710.3041660.268642
d2OpeningPrice-0.048322-0.009232-0.0557310.2699840.9986700.9985020.9979600.998063-0.0019450.425951...0.0491540.4080880.402403-0.0027790.0202040.020504-0.0077180.0212650.022718-0.007355
d2AgoHighestPrice-0.048904-0.009287-0.0522680.2736130.9990620.9989350.9983020.998420-0.0008260.428568...0.0460090.4138080.410018-0.0046050.0170240.017532-0.0101460.0229000.024718-0.006404
d2AgoLowestPrice-0.048072-0.008937-0.0646860.2615200.9990870.9986850.9985750.998411-0.0006290.417210...0.0539400.3989510.3951600.0011180.0222400.022675-0.0047800.0195010.020803-0.010827
d2AgoClosingPrice-0.048331-0.009209-0.0595810.2671260.9994020.9990990.9987850.998682-0.0014820.422522...0.0487580.4076570.403072-0.0016540.0190080.019580-0.0079080.0208590.022331-0.009695
d2AgoClosingDiff0.0276430.047588-0.152251-0.1119460.0503020.0472030.0550770.0499340.015998-0.127842...0.014669-0.000457-0.0059390.027539-0.0026290.0062400.025145-0.032911-0.016352-0.100324
d2AgoTransaction-0.072474-0.0233690.4241010.5462890.4076260.4133700.3984790.407206-0.0002920.580575...-0.1295190.5801400.490051-0.097677-0.218559-0.217779-0.2501900.2388030.2391180.251075
d2AgoMaxDiff-0.042953-0.0135450.3262560.4484570.4018770.4095110.3946230.402555-0.0058470.490350...-0.2032910.5822510.580766-0.161927-0.139007-0.136794-0.1541560.1042740.1194440.121124
d2AgoDayPriseRise0.0092450.011447-0.086643-0.067858-0.003661-0.0054440.000208-0.0025200.028420-0.096225...0.020030-0.023915-0.0478580.022930-0.030610-0.037067-0.0283110.0025540.023720-0.055944
d3TradeVolume-0.0360860.0032840.5136580.431778-0.071059-0.065690-0.077706-0.072132-0.0691010.363142...-0.1772890.7245120.497825-0.066060-0.057756-0.060395-0.083526-0.306334-0.304776-0.252622
d3Amount-0.052464-0.0053650.4452190.5006090.2565220.2618820.2501390.255870-0.0503110.492699...-0.1672410.8907120.629058-0.062735-0.052066-0.053862-0.092361-0.298288-0.297028-0.257038
d3OpeningPrice-0.049109-0.012837-0.0500980.2747390.9974570.9973370.9964920.996713-0.0028410.432810...0.0220580.4158230.408731-0.0207160.0179510.018266-0.0117080.0237970.024157-0.004121
d3AgoHighestPrice-0.049489-0.011762-0.0458990.2786720.9977750.9977370.9967600.997052-0.0030380.435525...0.0349800.4190700.418240-0.0118700.0179380.018304-0.0108730.0205310.021097-0.006634
d3AgoLowestPrice-0.049091-0.012449-0.0562170.2690970.9978040.9975450.9969590.997067-0.0025760.426373...0.0392270.4013970.388492-0.0092300.0195110.019571-0.0102210.0258140.026309-0.001202
d3AgoClosingPrice-0.049797-0.012175-0.0516940.2734270.9980480.9978900.9971570.997323-0.0029610.429827...0.0487650.4085460.405333-0.0026530.0186630.018781-0.0096790.0225130.023144-0.004397
d3AgoClosingDiff-0.0421270.007302-0.136109-0.1063050.0510030.0489210.0528260.0508720.042829-0.138620...1.000000-0.298125-0.1045690.692979-0.024608-0.0215230.066537-0.0055970.0032090.022108
d3AgoTransaction-0.045853-0.0019500.3590180.4797990.4051680.4103200.3998280.405158-0.0242350.553126...-0.2981251.0000000.662909-0.166374-0.023891-0.026534-0.084802-0.219041-0.218269-0.250675
d3AgoMaxDiff-0.0310840.0144660.2697900.3799750.4017640.4079270.3965890.401867-0.0141380.431454...-0.1045690.6629091.000000-0.078556-0.036712-0.028022-0.022610-0.139327-0.137119-0.154478
d3AgoDayPriseRise-0.060108-0.007133-0.084614-0.073745-0.001685-0.002961-0.000219-0.0017680.006713-0.106920...0.692979-0.166374-0.0785561.000000-0.038850-0.0398250.024229-0.033321-0.039825-0.031045
d1AgoTradeVolumeRise0.0688810.0347180.2969030.3028120.0177490.0193810.0159620.017331-0.0334790.238354...-0.024608-0.023891-0.036712-0.0388501.0000000.9780630.688219-0.280548-0.274630-0.227909
d1AgoAmountRise0.0568770.0271940.2980940.3041050.0191420.0211390.0172040.018742-0.0169260.238661...-0.021523-0.026534-0.028022-0.0398250.9780631.0000000.693801-0.274630-0.268987-0.222139
d1AgoTransactionRise0.0435340.0181250.2719980.268580-0.010875-0.009927-0.014368-0.013227-0.1009090.250626...0.066537-0.084802-0.0226100.0242290.6882190.6938011.000000-0.238947-0.238692-0.269161
d2AgoTradeVolumeRise-0.008501-0.0222220.0412150.0437680.0221380.0219010.0221140.0225060.0339790.023165...-0.005597-0.219041-0.139327-0.033321-0.280548-0.274630-0.2389471.0000000.9780630.688219
d2AgoAmountRise0.007142-0.0167920.0428100.0432570.0234960.0231090.0233780.0236050.0264310.021230...0.003209-0.218269-0.137119-0.039825-0.274630-0.268987-0.2386920.9780631.0000000.693801
d2AgoTransactionRise-0.011731-0.0145230.0648920.051107-0.008444-0.008009-0.010058-0.0086290.0173560.016530...0.022108-0.250675-0.154478-0.031045-0.227909-0.222139-0.2691610.6882190.6938011.000000
\n", 1081 | "

38 rows × 38 columns

\n", 1082 | "
" 1083 | ], 1084 | "text/plain": [ 1085 | " dDayPriseRise dDayClosingDiff d1TradeVolume d1Amount \\\n", 1086 | "dDayPriseRise 1.000000 0.692129 -0.048711 -0.047336 \n", 1087 | "dDayClosingDiff 0.692129 1.000000 -0.040452 -0.026802 \n", 1088 | "d1TradeVolume -0.048711 -0.040452 1.000000 0.915744 \n", 1089 | "d1Amount -0.047336 -0.026802 0.915744 1.000000 \n", 1090 | "d1OpeningPrice -0.047911 -0.008208 -0.065934 0.260799 \n", 1091 | "d1AgoHighestPrice -0.047099 -0.007084 -0.059411 0.267092 \n", 1092 | "d1AgoLowestPrice -0.046045 -0.006943 -0.078021 0.248472 \n", 1093 | "d1AgoClosingPrice -0.046449 -0.007763 -0.069193 0.257456 \n", 1094 | "d1AgoClosingDiff 0.022676 0.015269 -0.178545 -0.167800 \n", 1095 | "d1AgoTransaction -0.023960 -0.000756 0.724128 0.890393 \n", 1096 | "d1AgoMaxDiff -0.048395 -0.006778 0.497370 0.628576 \n", 1097 | "d1AgoDayPriseRise 0.025850 0.029230 -0.065240 -0.061991 \n", 1098 | "d2TradeVolume -0.073006 -0.065481 0.606350 0.524134 \n", 1099 | "d2Amount -0.074072 -0.048026 0.532284 0.593096 \n", 1100 | "d2OpeningPrice -0.048322 -0.009232 -0.055731 0.269984 \n", 1101 | "d2AgoHighestPrice -0.048904 -0.009287 -0.052268 0.273613 \n", 1102 | "d2AgoLowestPrice -0.048072 -0.008937 -0.064686 0.261520 \n", 1103 | "d2AgoClosingPrice -0.048331 -0.009209 -0.059581 0.267126 \n", 1104 | "d2AgoClosingDiff 0.027643 0.047588 -0.152251 -0.111946 \n", 1105 | "d2AgoTransaction -0.072474 -0.023369 0.424101 0.546289 \n", 1106 | "d2AgoMaxDiff -0.042953 -0.013545 0.326256 0.448457 \n", 1107 | "d2AgoDayPriseRise 0.009245 0.011447 -0.086643 -0.067858 \n", 1108 | "d3TradeVolume -0.036086 0.003284 0.513658 0.431778 \n", 1109 | "d3Amount -0.052464 -0.005365 0.445219 0.500609 \n", 1110 | "d3OpeningPrice -0.049109 -0.012837 -0.050098 0.274739 \n", 1111 | "d3AgoHighestPrice -0.049489 -0.011762 -0.045899 0.278672 \n", 1112 | "d3AgoLowestPrice -0.049091 -0.012449 -0.056217 0.269097 \n", 1113 | "d3AgoClosingPrice -0.049797 -0.012175 -0.051694 0.273427 \n", 1114 | "d3AgoClosingDiff -0.042127 0.007302 -0.136109 -0.106305 \n", 1115 | "d3AgoTransaction -0.045853 -0.001950 0.359018 0.479799 \n", 1116 | "d3AgoMaxDiff -0.031084 0.014466 0.269790 0.379975 \n", 1117 | "d3AgoDayPriseRise -0.060108 -0.007133 -0.084614 -0.073745 \n", 1118 | "d1AgoTradeVolumeRise 0.068881 0.034718 0.296903 0.302812 \n", 1119 | "d1AgoAmountRise 0.056877 0.027194 0.298094 0.304105 \n", 1120 | "d1AgoTransactionRise 0.043534 0.018125 0.271998 0.268580 \n", 1121 | "d2AgoTradeVolumeRise -0.008501 -0.022222 0.041215 0.043768 \n", 1122 | "d2AgoAmountRise 0.007142 -0.016792 0.042810 0.043257 \n", 1123 | "d2AgoTransactionRise -0.011731 -0.014523 0.064892 0.051107 \n", 1124 | "\n", 1125 | " d1OpeningPrice d1AgoHighestPrice d1AgoLowestPrice \\\n", 1126 | "dDayPriseRise -0.047911 -0.047099 -0.046045 \n", 1127 | "dDayClosingDiff -0.008208 -0.007084 -0.006943 \n", 1128 | "d1TradeVolume -0.065934 -0.059411 -0.078021 \n", 1129 | "d1Amount 0.260799 0.267092 0.248472 \n", 1130 | "d1OpeningPrice 1.000000 0.999724 0.999532 \n", 1131 | "d1AgoHighestPrice 0.999724 1.000000 0.999473 \n", 1132 | "d1AgoLowestPrice 0.999532 0.999473 1.000000 \n", 1133 | "d1AgoClosingPrice 0.999358 0.999712 0.999641 \n", 1134 | "d1AgoClosingDiff 0.022161 0.035027 0.039295 \n", 1135 | "d1AgoTransaction 0.413294 0.416553 0.398856 \n", 1136 | "d1AgoMaxDiff 0.406225 0.415737 0.385983 \n", 1137 | "d1AgoDayPriseRise -0.025350 -0.016525 -0.013928 \n", 1138 | "d2TradeVolume -0.068497 -0.062375 -0.076997 \n", 1139 | "d2Amount 0.258644 0.264653 0.249978 \n", 1140 | "d2OpeningPrice 0.998670 0.998502 0.997960 \n", 1141 | "d2AgoHighestPrice 0.999062 0.998935 0.998302 \n", 1142 | "d2AgoLowestPrice 0.999087 0.998685 0.998575 \n", 1143 | "d2AgoClosingPrice 0.999402 0.999099 0.998785 \n", 1144 | "d2AgoClosingDiff 0.050302 0.047203 0.055077 \n", 1145 | "d2AgoTransaction 0.407626 0.413370 0.398479 \n", 1146 | "d2AgoMaxDiff 0.401877 0.409511 0.394623 \n", 1147 | "d2AgoDayPriseRise -0.003661 -0.005444 0.000208 \n", 1148 | "d3TradeVolume -0.071059 -0.065690 -0.077706 \n", 1149 | "d3Amount 0.256522 0.261882 0.250139 \n", 1150 | "d3OpeningPrice 0.997457 0.997337 0.996492 \n", 1151 | "d3AgoHighestPrice 0.997775 0.997737 0.996760 \n", 1152 | "d3AgoLowestPrice 0.997804 0.997545 0.996959 \n", 1153 | "d3AgoClosingPrice 0.998048 0.997890 0.997157 \n", 1154 | "d3AgoClosingDiff 0.051003 0.048921 0.052826 \n", 1155 | "d3AgoTransaction 0.405168 0.410320 0.399828 \n", 1156 | "d3AgoMaxDiff 0.401764 0.407927 0.396589 \n", 1157 | "d3AgoDayPriseRise -0.001685 -0.002961 -0.000219 \n", 1158 | "d1AgoTradeVolumeRise 0.017749 0.019381 0.015962 \n", 1159 | "d1AgoAmountRise 0.019142 0.021139 0.017204 \n", 1160 | "d1AgoTransactionRise -0.010875 -0.009927 -0.014368 \n", 1161 | "d2AgoTradeVolumeRise 0.022138 0.021901 0.022114 \n", 1162 | "d2AgoAmountRise 0.023496 0.023109 0.023378 \n", 1163 | "d2AgoTransactionRise -0.008444 -0.008009 -0.010058 \n", 1164 | "\n", 1165 | " d1AgoClosingPrice d1AgoClosingDiff d1AgoTransaction \\\n", 1166 | "dDayPriseRise -0.046449 0.022676 -0.023960 \n", 1167 | "dDayClosingDiff -0.007763 0.015269 -0.000756 \n", 1168 | "d1TradeVolume -0.069193 -0.178545 0.724128 \n", 1169 | "d1Amount 0.257456 -0.167800 0.890393 \n", 1170 | "d1OpeningPrice 0.999358 0.022161 0.413294 \n", 1171 | "d1AgoHighestPrice 0.999712 0.035027 0.416553 \n", 1172 | "d1AgoLowestPrice 0.999641 0.039295 0.398856 \n", 1173 | "d1AgoClosingPrice 1.000000 0.048794 0.406020 \n", 1174 | "d1AgoClosingDiff 0.048794 1.000000 -0.299492 \n", 1175 | "d1AgoTransaction 0.406020 -0.299492 1.000000 \n", 1176 | "d1AgoMaxDiff 0.402825 -0.105481 0.662672 \n", 1177 | "d1AgoDayPriseRise -0.007351 0.692225 -0.166937 \n", 1178 | "d2TradeVolume -0.068852 -0.041671 0.425635 \n", 1179 | "d2Amount 0.258270 -0.027369 0.555092 \n", 1180 | "d2OpeningPrice 0.998063 -0.001945 0.425951 \n", 1181 | "d2AgoHighestPrice 0.998420 -0.000826 0.428568 \n", 1182 | "d2AgoLowestPrice 0.998411 -0.000629 0.417210 \n", 1183 | "d2AgoClosingPrice 0.998682 -0.001482 0.422522 \n", 1184 | "d2AgoClosingDiff 0.049934 0.015998 -0.127842 \n", 1185 | "d2AgoTransaction 0.407206 -0.000292 0.580575 \n", 1186 | "d2AgoMaxDiff 0.402555 -0.005847 0.490350 \n", 1187 | "d2AgoDayPriseRise -0.002520 0.028420 -0.096225 \n", 1188 | "d3TradeVolume -0.072132 -0.069101 0.363142 \n", 1189 | "d3Amount 0.255870 -0.050311 0.492699 \n", 1190 | "d3OpeningPrice 0.996713 -0.002841 0.432810 \n", 1191 | "d3AgoHighestPrice 0.997052 -0.003038 0.435525 \n", 1192 | "d3AgoLowestPrice 0.997067 -0.002576 0.426373 \n", 1193 | "d3AgoClosingPrice 0.997323 -0.002961 0.429827 \n", 1194 | "d3AgoClosingDiff 0.050872 0.042829 -0.138620 \n", 1195 | "d3AgoTransaction 0.405158 -0.024235 0.553126 \n", 1196 | "d3AgoMaxDiff 0.401867 -0.014138 0.431454 \n", 1197 | "d3AgoDayPriseRise -0.001768 0.006713 -0.106920 \n", 1198 | "d1AgoTradeVolumeRise 0.017331 -0.033479 0.238354 \n", 1199 | "d1AgoAmountRise 0.018742 -0.016926 0.238661 \n", 1200 | "d1AgoTransactionRise -0.013227 -0.100909 0.250626 \n", 1201 | "d2AgoTradeVolumeRise 0.022506 0.033979 0.023165 \n", 1202 | "d2AgoAmountRise 0.023605 0.026431 0.021230 \n", 1203 | "d2AgoTransactionRise -0.008629 0.017356 0.016530 \n", 1204 | "\n", 1205 | " ... d3AgoClosingDiff d3AgoTransaction d3AgoMaxDiff \\\n", 1206 | "dDayPriseRise ... -0.042127 -0.045853 -0.031084 \n", 1207 | "dDayClosingDiff ... 0.007302 -0.001950 0.014466 \n", 1208 | "d1TradeVolume ... -0.136109 0.359018 0.269790 \n", 1209 | "d1Amount ... -0.106305 0.479799 0.379975 \n", 1210 | "d1OpeningPrice ... 0.051003 0.405168 0.401764 \n", 1211 | "d1AgoHighestPrice ... 0.048921 0.410320 0.407927 \n", 1212 | "d1AgoLowestPrice ... 0.052826 0.399828 0.396589 \n", 1213 | "d1AgoClosingPrice ... 0.050872 0.405158 0.401867 \n", 1214 | "d1AgoClosingDiff ... 0.042829 -0.024235 -0.014138 \n", 1215 | "d1AgoTransaction ... -0.138620 0.553126 0.431454 \n", 1216 | "d1AgoMaxDiff ... -0.089749 0.458369 0.481108 \n", 1217 | "d1AgoDayPriseRise ... 0.024388 -0.072995 -0.043298 \n", 1218 | "d2TradeVolume ... -0.154435 0.423790 0.326057 \n", 1219 | "d2Amount ... -0.114338 0.545839 0.448161 \n", 1220 | "d2OpeningPrice ... 0.049154 0.408088 0.402403 \n", 1221 | "d2AgoHighestPrice ... 0.046009 0.413808 0.410018 \n", 1222 | "d2AgoLowestPrice ... 0.053940 0.398951 0.395160 \n", 1223 | "d2AgoClosingPrice ... 0.048758 0.407657 0.403072 \n", 1224 | "d2AgoClosingDiff ... 0.014669 -0.000457 -0.005939 \n", 1225 | "d2AgoTransaction ... -0.129519 0.580140 0.490051 \n", 1226 | "d2AgoMaxDiff ... -0.203291 0.582251 0.580766 \n", 1227 | "d2AgoDayPriseRise ... 0.020030 -0.023915 -0.047858 \n", 1228 | "d3TradeVolume ... -0.177289 0.724512 0.497825 \n", 1229 | "d3Amount ... -0.167241 0.890712 0.629058 \n", 1230 | "d3OpeningPrice ... 0.022058 0.415823 0.408731 \n", 1231 | "d3AgoHighestPrice ... 0.034980 0.419070 0.418240 \n", 1232 | "d3AgoLowestPrice ... 0.039227 0.401397 0.388492 \n", 1233 | "d3AgoClosingPrice ... 0.048765 0.408546 0.405333 \n", 1234 | "d3AgoClosingDiff ... 1.000000 -0.298125 -0.104569 \n", 1235 | "d3AgoTransaction ... -0.298125 1.000000 0.662909 \n", 1236 | "d3AgoMaxDiff ... -0.104569 0.662909 1.000000 \n", 1237 | "d3AgoDayPriseRise ... 0.692979 -0.166374 -0.078556 \n", 1238 | "d1AgoTradeVolumeRise ... -0.024608 -0.023891 -0.036712 \n", 1239 | "d1AgoAmountRise ... -0.021523 -0.026534 -0.028022 \n", 1240 | "d1AgoTransactionRise ... 0.066537 -0.084802 -0.022610 \n", 1241 | "d2AgoTradeVolumeRise ... -0.005597 -0.219041 -0.139327 \n", 1242 | "d2AgoAmountRise ... 0.003209 -0.218269 -0.137119 \n", 1243 | "d2AgoTransactionRise ... 0.022108 -0.250675 -0.154478 \n", 1244 | "\n", 1245 | " d3AgoDayPriseRise d1AgoTradeVolumeRise \\\n", 1246 | "dDayPriseRise -0.060108 0.068881 \n", 1247 | "dDayClosingDiff -0.007133 0.034718 \n", 1248 | "d1TradeVolume -0.084614 0.296903 \n", 1249 | "d1Amount -0.073745 0.302812 \n", 1250 | "d1OpeningPrice -0.001685 0.017749 \n", 1251 | "d1AgoHighestPrice -0.002961 0.019381 \n", 1252 | "d1AgoLowestPrice -0.000219 0.015962 \n", 1253 | "d1AgoClosingPrice -0.001768 0.017331 \n", 1254 | "d1AgoClosingDiff 0.006713 -0.033479 \n", 1255 | "d1AgoTransaction -0.106920 0.238354 \n", 1256 | "d1AgoMaxDiff -0.077968 0.103524 \n", 1257 | "d1AgoDayPriseRise 0.006322 0.002554 \n", 1258 | "d2TradeVolume -0.089003 -0.304293 \n", 1259 | "d2Amount -0.070167 -0.297051 \n", 1260 | "d2OpeningPrice -0.002779 0.020204 \n", 1261 | "d2AgoHighestPrice -0.004605 0.017024 \n", 1262 | "d2AgoLowestPrice 0.001118 0.022240 \n", 1263 | "d2AgoClosingPrice -0.001654 0.019008 \n", 1264 | "d2AgoClosingDiff 0.027539 -0.002629 \n", 1265 | "d2AgoTransaction -0.097677 -0.218559 \n", 1266 | "d2AgoMaxDiff -0.161927 -0.139007 \n", 1267 | "d2AgoDayPriseRise 0.022930 -0.030610 \n", 1268 | "d3TradeVolume -0.066060 -0.057756 \n", 1269 | "d3Amount -0.062735 -0.052066 \n", 1270 | "d3OpeningPrice -0.020716 0.017951 \n", 1271 | "d3AgoHighestPrice -0.011870 0.017938 \n", 1272 | "d3AgoLowestPrice -0.009230 0.019511 \n", 1273 | "d3AgoClosingPrice -0.002653 0.018663 \n", 1274 | "d3AgoClosingDiff 0.692979 -0.024608 \n", 1275 | "d3AgoTransaction -0.166374 -0.023891 \n", 1276 | "d3AgoMaxDiff -0.078556 -0.036712 \n", 1277 | "d3AgoDayPriseRise 1.000000 -0.038850 \n", 1278 | "d1AgoTradeVolumeRise -0.038850 1.000000 \n", 1279 | "d1AgoAmountRise -0.039825 0.978063 \n", 1280 | "d1AgoTransactionRise 0.024229 0.688219 \n", 1281 | "d2AgoTradeVolumeRise -0.033321 -0.280548 \n", 1282 | "d2AgoAmountRise -0.039825 -0.274630 \n", 1283 | "d2AgoTransactionRise -0.031045 -0.227909 \n", 1284 | "\n", 1285 | " d1AgoAmountRise d1AgoTransactionRise \\\n", 1286 | "dDayPriseRise 0.056877 0.043534 \n", 1287 | "dDayClosingDiff 0.027194 0.018125 \n", 1288 | "d1TradeVolume 0.298094 0.271998 \n", 1289 | "d1Amount 0.304105 0.268580 \n", 1290 | "d1OpeningPrice 0.019142 -0.010875 \n", 1291 | "d1AgoHighestPrice 0.021139 -0.009927 \n", 1292 | "d1AgoLowestPrice 0.017204 -0.014368 \n", 1293 | "d1AgoClosingPrice 0.018742 -0.013227 \n", 1294 | "d1AgoClosingDiff -0.016926 -0.100909 \n", 1295 | "d1AgoTransaction 0.238661 0.250626 \n", 1296 | "d1AgoMaxDiff 0.118688 0.120375 \n", 1297 | "d1AgoDayPriseRise 0.023720 -0.055944 \n", 1298 | "d2TradeVolume -0.302703 -0.250580 \n", 1299 | "d2Amount -0.295771 -0.255804 \n", 1300 | "d2OpeningPrice 0.020504 -0.007718 \n", 1301 | "d2AgoHighestPrice 0.017532 -0.010146 \n", 1302 | "d2AgoLowestPrice 0.022675 -0.004780 \n", 1303 | "d2AgoClosingPrice 0.019580 -0.007908 \n", 1304 | "d2AgoClosingDiff 0.006240 0.025145 \n", 1305 | "d2AgoTransaction -0.217779 -0.250190 \n", 1306 | "d2AgoMaxDiff -0.136794 -0.154156 \n", 1307 | "d2AgoDayPriseRise -0.037067 -0.028311 \n", 1308 | "d3TradeVolume -0.060395 -0.083526 \n", 1309 | "d3Amount -0.053862 -0.092361 \n", 1310 | "d3OpeningPrice 0.018266 -0.011708 \n", 1311 | "d3AgoHighestPrice 0.018304 -0.010873 \n", 1312 | "d3AgoLowestPrice 0.019571 -0.010221 \n", 1313 | "d3AgoClosingPrice 0.018781 -0.009679 \n", 1314 | "d3AgoClosingDiff -0.021523 0.066537 \n", 1315 | "d3AgoTransaction -0.026534 -0.084802 \n", 1316 | "d3AgoMaxDiff -0.028022 -0.022610 \n", 1317 | "d3AgoDayPriseRise -0.039825 0.024229 \n", 1318 | "d1AgoTradeVolumeRise 0.978063 0.688219 \n", 1319 | "d1AgoAmountRise 1.000000 0.693801 \n", 1320 | "d1AgoTransactionRise 0.693801 1.000000 \n", 1321 | "d2AgoTradeVolumeRise -0.274630 -0.238947 \n", 1322 | "d2AgoAmountRise -0.268987 -0.238692 \n", 1323 | "d2AgoTransactionRise -0.222139 -0.269161 \n", 1324 | "\n", 1325 | " d2AgoTradeVolumeRise d2AgoAmountRise \\\n", 1326 | "dDayPriseRise -0.008501 0.007142 \n", 1327 | "dDayClosingDiff -0.022222 -0.016792 \n", 1328 | "d1TradeVolume 0.041215 0.042810 \n", 1329 | "d1Amount 0.043768 0.043257 \n", 1330 | "d1OpeningPrice 0.022138 0.023496 \n", 1331 | "d1AgoHighestPrice 0.021901 0.023109 \n", 1332 | "d1AgoLowestPrice 0.022114 0.023378 \n", 1333 | "d1AgoClosingPrice 0.022506 0.023605 \n", 1334 | "d1AgoClosingDiff 0.033979 0.026431 \n", 1335 | "d1AgoTransaction 0.023165 0.021230 \n", 1336 | "d1AgoMaxDiff 0.002816 0.001734 \n", 1337 | "d1AgoDayPriseRise 0.068881 0.056877 \n", 1338 | "d2TradeVolume 0.296696 0.297883 \n", 1339 | "d2Amount 0.302871 0.304166 \n", 1340 | "d2OpeningPrice 0.021265 0.022718 \n", 1341 | "d2AgoHighestPrice 0.022900 0.024718 \n", 1342 | "d2AgoLowestPrice 0.019501 0.020803 \n", 1343 | "d2AgoClosingPrice 0.020859 0.022331 \n", 1344 | "d2AgoClosingDiff -0.032911 -0.016352 \n", 1345 | "d2AgoTransaction 0.238803 0.239118 \n", 1346 | "d2AgoMaxDiff 0.104274 0.119444 \n", 1347 | "d2AgoDayPriseRise 0.002554 0.023720 \n", 1348 | "d3TradeVolume -0.306334 -0.304776 \n", 1349 | "d3Amount -0.298288 -0.297028 \n", 1350 | "d3OpeningPrice 0.023797 0.024157 \n", 1351 | "d3AgoHighestPrice 0.020531 0.021097 \n", 1352 | "d3AgoLowestPrice 0.025814 0.026309 \n", 1353 | "d3AgoClosingPrice 0.022513 0.023144 \n", 1354 | "d3AgoClosingDiff -0.005597 0.003209 \n", 1355 | "d3AgoTransaction -0.219041 -0.218269 \n", 1356 | "d3AgoMaxDiff -0.139327 -0.137119 \n", 1357 | "d3AgoDayPriseRise -0.033321 -0.039825 \n", 1358 | "d1AgoTradeVolumeRise -0.280548 -0.274630 \n", 1359 | "d1AgoAmountRise -0.274630 -0.268987 \n", 1360 | "d1AgoTransactionRise -0.238947 -0.238692 \n", 1361 | "d2AgoTradeVolumeRise 1.000000 0.978063 \n", 1362 | "d2AgoAmountRise 0.978063 1.000000 \n", 1363 | "d2AgoTransactionRise 0.688219 0.693801 \n", 1364 | "\n", 1365 | " d2AgoTransactionRise \n", 1366 | "dDayPriseRise -0.011731 \n", 1367 | "dDayClosingDiff -0.014523 \n", 1368 | "d1TradeVolume 0.064892 \n", 1369 | "d1Amount 0.051107 \n", 1370 | "d1OpeningPrice -0.008444 \n", 1371 | "d1AgoHighestPrice -0.008009 \n", 1372 | "d1AgoLowestPrice -0.010058 \n", 1373 | "d1AgoClosingPrice -0.008629 \n", 1374 | "d1AgoClosingDiff 0.017356 \n", 1375 | "d1AgoTransaction 0.016530 \n", 1376 | "d1AgoMaxDiff 0.054168 \n", 1377 | "d1AgoDayPriseRise 0.043534 \n", 1378 | "d2TradeVolume 0.271782 \n", 1379 | "d2Amount 0.268642 \n", 1380 | "d2OpeningPrice -0.007355 \n", 1381 | "d2AgoHighestPrice -0.006404 \n", 1382 | "d2AgoLowestPrice -0.010827 \n", 1383 | "d2AgoClosingPrice -0.009695 \n", 1384 | "d2AgoClosingDiff -0.100324 \n", 1385 | "d2AgoTransaction 0.251075 \n", 1386 | "d2AgoMaxDiff 0.121124 \n", 1387 | "d2AgoDayPriseRise -0.055944 \n", 1388 | "d3TradeVolume -0.252622 \n", 1389 | "d3Amount -0.257038 \n", 1390 | "d3OpeningPrice -0.004121 \n", 1391 | "d3AgoHighestPrice -0.006634 \n", 1392 | "d3AgoLowestPrice -0.001202 \n", 1393 | "d3AgoClosingPrice -0.004397 \n", 1394 | "d3AgoClosingDiff 0.022108 \n", 1395 | "d3AgoTransaction -0.250675 \n", 1396 | "d3AgoMaxDiff -0.154478 \n", 1397 | "d3AgoDayPriseRise -0.031045 \n", 1398 | "d1AgoTradeVolumeRise -0.227909 \n", 1399 | "d1AgoAmountRise -0.222139 \n", 1400 | "d1AgoTransactionRise -0.269161 \n", 1401 | "d2AgoTradeVolumeRise 0.688219 \n", 1402 | "d2AgoAmountRise 0.693801 \n", 1403 | "d2AgoTransactionRise 1.000000 \n", 1404 | "\n", 1405 | "[38 rows x 38 columns]" 1406 | ] 1407 | }, 1408 | "execution_count": 10, 1409 | "metadata": {}, 1410 | "output_type": "execute_result" 1411 | } 1412 | ], 1413 | "source": [ 1414 | "df.corr()" 1415 | ] 1416 | }, 1417 | { 1418 | "cell_type": "code", 1419 | "execution_count": 34, 1420 | "id": "e3dbc69a", 1421 | "metadata": {}, 1422 | "outputs": [], 1423 | "source": [ 1424 | "trainings = []\n", 1425 | "tests = []\n", 1426 | "\n", 1427 | "for i in range(0, df.count()[0]-151):\n", 1428 | " trainings.append(df[i+1:i+151])\n", 1429 | " tests.append(df[i:i+1])\n", 1430 | " " 1431 | ] 1432 | }, 1433 | { 1434 | "cell_type": "code", 1435 | "execution_count": null, 1436 | "id": "234105dc", 1437 | "metadata": {}, 1438 | "outputs": [], 1439 | "source": [ 1440 | "from sklearn.naive_bayes import GaussianNB\n", 1441 | "from sklearn.linear_model import LogisticRegression\n", 1442 | "from sklearn.model_selection import train_test_split\n", 1443 | "from sklearn.preprocessing import StandardScaler\n", 1444 | "from sklearn.ensemble import RandomForestClassifier\n", 1445 | "\n", 1446 | "# X_train, X_test, y_train, y_test = train_test_split(scaled[train_target], scaled['dDayPriseRise'], test_size=0.6, random_state=0, stratify=scaled['dDayPriseRise'])\n", 1447 | "gnb = GaussianNB()\n", 1448 | "lgr = LogisticRegression()\n", 1449 | "clf = RandomForestClassifier(max_depth=2, n_estimators=100, random_state=0)\n", 1450 | "trainings_len = len(trainings)\n", 1451 | "true_positive = 0\n", 1452 | "false_positive = 0\n", 1453 | "true_negative = 0\n", 1454 | "false_negative = 0\n", 1455 | "true_count = 0\n", 1456 | "\n", 1457 | "scale_target = [\n", 1458 | " # day1\n", 1459 | " 'd1TradeVolume',\n", 1460 | " 'd1Amount',\n", 1461 | " 'd1OpeningPrice',\n", 1462 | " 'd1AgoHighestPrice',\n", 1463 | " 'd1AgoLowestPrice',\n", 1464 | " 'd1AgoClosingPrice',\n", 1465 | " 'd1AgoClosingDiff',\n", 1466 | " 'd1AgoTransaction',\n", 1467 | " 'd1AgoMaxDiff',\n", 1468 | " # day2\n", 1469 | " 'd2TradeVolume',\n", 1470 | " 'd2Amount',\n", 1471 | " 'd2OpeningPrice',\n", 1472 | " 'd2AgoHighestPrice',\n", 1473 | " 'd2AgoLowestPrice',\n", 1474 | " 'd2AgoClosingPrice',\n", 1475 | " 'd2AgoClosingDiff',\n", 1476 | " 'd2AgoTransaction',\n", 1477 | " 'd2AgoMaxDiff',\n", 1478 | " # day3\n", 1479 | " 'd3TradeVolume',\n", 1480 | " 'd3Amount',\n", 1481 | " 'd3OpeningPrice',\n", 1482 | " 'd3AgoHighestPrice',\n", 1483 | " 'd3AgoLowestPrice',\n", 1484 | " 'd3AgoClosingPrice',\n", 1485 | " 'd3AgoClosingDiff',\n", 1486 | " 'd3AgoTransaction',\n", 1487 | " 'd3AgoMaxDiff',\n", 1488 | "]\n", 1489 | "\n", 1490 | "train_target = [\n", 1491 | "# # day1\n", 1492 | "# 'd1TradeVolume',\n", 1493 | "# 'd1Amount',\n", 1494 | "# 'd1OpeningPrice',\n", 1495 | "# 'd1AgoHighestPrice',\n", 1496 | "# 'd1AgoLowestPrice',\n", 1497 | "# 'd1AgoClosingPrice',\n", 1498 | "# 'd1AgoClosingDiff',\n", 1499 | "# 'd1AgoTransaction',\n", 1500 | "# 'd1AgoMaxDiff',\n", 1501 | "# 'd1AgoDayPriseRise',\n", 1502 | "# # day2\n", 1503 | "# 'd2TradeVolume',\n", 1504 | "# 'd2Amount',\n", 1505 | "# 'd2OpeningPrice',\n", 1506 | "# 'd2AgoHighestPrice',\n", 1507 | "# 'd2AgoLowestPrice',\n", 1508 | "# 'd2AgoClosingPrice',\n", 1509 | "# 'd2AgoClosingDiff',\n", 1510 | "# 'd2AgoTransaction',\n", 1511 | "# 'd2AgoMaxDiff',\n", 1512 | "# 'd2AgoDayPriseRise',\n", 1513 | "# # day3\n", 1514 | "# 'd3TradeVolume',\n", 1515 | "# 'd3Amount',\n", 1516 | "# 'd3OpeningPrice',\n", 1517 | "# 'd3AgoHighestPrice',\n", 1518 | "# 'd3AgoLowestPrice',\n", 1519 | "# 'd3AgoClosingPrice',\n", 1520 | "# 'd3AgoClosingDiff',\n", 1521 | "# 'd3AgoTransaction',\n", 1522 | "# 'd3AgoMaxDiff',\n", 1523 | "# 'd3AgoDayPriseRise',\n", 1524 | " # rise\n", 1525 | " 'd1AgoTradeVolumeRise',\n", 1526 | "# 'd1AgoAmountRise',\n", 1527 | "# 'd1AgoTransactionRise'\n", 1528 | "]\n", 1529 | "\n", 1530 | "for i in range(0, trainings_len):\n", 1531 | "# for i in range(0, 1):\n", 1532 | " training = trainings[i]\n", 1533 | " test = tests[i]\n", 1534 | " scale = StandardScaler()\n", 1535 | " \n", 1536 | " train_scaled = pd.DataFrame(scale.fit_transform(training[scale_target]), columns=scale_target, index=training.index)\n", 1537 | " train_scaled['dDayPriseRise'] = training['dDayPriseRise']\n", 1538 | " train_scaled['d1AgoDayPriseRise'] = training['d1AgoDayPriseRise']\n", 1539 | " train_scaled['d1AgoTradeVolumeRise'] = training['d1AgoTradeVolumeRise']\n", 1540 | " train_scaled['d1AgoAmountRise'] = training['d1AgoAmountRise']\n", 1541 | " train_scaled['d1AgoTransactionRise'] = training['d1AgoTransactionRise']\n", 1542 | " train_scaled['d2AgoDayPriseRise'] = training['d2AgoDayPriseRise']\n", 1543 | " train_scaled['d3AgoDayPriseRise'] = training['d3AgoDayPriseRise']\n", 1544 | "\n", 1545 | "\n", 1546 | " test_scaled = pd.DataFrame(scale.transform(test[scale_target]), columns=scale_target, index=test.index)\n", 1547 | " test_scaled['dDayPriseRise'] = test['dDayPriseRise']\n", 1548 | " test_scaled['d1AgoDayPriseRise'] = test['d1AgoDayPriseRise']\n", 1549 | " test_scaled['d1AgoTradeVolumeRise'] = test['d1AgoTradeVolumeRise']\n", 1550 | " test_scaled['d1AgoAmountRise'] = test['d1AgoAmountRise']\n", 1551 | " test_scaled['d1AgoTransactionRise'] = test['d1AgoTransactionRise']\n", 1552 | " test_scaled['d2AgoDayPriseRise'] = test['d2AgoDayPriseRise']\n", 1553 | " test_scaled['d3AgoDayPriseRise'] = test['d3AgoDayPriseRise']\n", 1554 | "\n", 1555 | "\n", 1556 | " \n", 1557 | " \n", 1558 | " y_test = test_scaled[[\"dDayPriseRise\"]].values[0][0]\n", 1559 | "\n", 1560 | " y_pred = clf.fit(train_scaled[train_target], train_scaled['dDayPriseRise']).predict(test_scaled[train_target])\n", 1561 | " \n", 1562 | " if y_pred == 1:\n", 1563 | " true_positive += 1 if y_test == y_pred else 0\n", 1564 | " false_positive += 1 if y_test != y_pred else 0\n", 1565 | " else:\n", 1566 | " true_negative += 1 if y_test == y_pred else 0\n", 1567 | " false_negative += 1 if y_test != y_pred else 0\n", 1568 | " \n", 1569 | "\n", 1570 | "positive_count = true_positive + false_positive\n", 1571 | "negative_count = true_negative + false_negative\n", 1572 | "\n", 1573 | "\n", 1574 | "print(f'count: {len(trainings)}, accuracy: {(true_positive + true_negative) / trainings_len}')\n", 1575 | "print(f'true_positive: {true_positive}, false_positive: {false_positive}')\n", 1576 | "print(f'true_negative: {true_negative}, false_negative: {false_negative}')\n", 1577 | "\n", 1578 | "print(f'true_positive_rate: {(true_positive/positive_count)}, false_positive_rate: {(false_positive/positive_count)}')\n", 1579 | "print(f'true_positive_rate: {(true_negative/negative_count)}, false_positive_rate: {(false_negative/positive_count)}')\n", 1580 | "\n", 1581 | "\n", 1582 | "\n", 1583 | "# test_count = X_test.shape[0]\n", 1584 | "# print(\"Number of labeled points out of a total %d points : %d\" % (test_count, (y_test == y_pred).sum()))\n", 1585 | "\n", 1586 | "# from sklearn.metrics import confusion_matrix \n", 1587 | "# tn, fp, fn, tp = confusion_matrix(y_test, y_pred).ravel()\n" 1588 | ] 1589 | }, 1590 | { 1591 | "cell_type": "markdown", 1592 | "id": "1ebd8df4", 1593 | "metadata": {}, 1594 | "source": [ 1595 | "150\n", 1596 | "2 only day1 d1AgoTradeVolumeRise\n", 1597 | "\n", 1598 | "\n", 1599 | "2\n", 1600 | "count: 574, accuracy: 0.5348432055749129\n", 1601 | "true_positive: 200, false_positive: 159\n", 1602 | "true_negative: 107, false_negative: 108\n", 1603 | "true_positive_rate: 0.5571030640668524, false_positive_rate: 0.4428969359331476\n", 1604 | "true_positive_rate: 0.49767441860465117, false_positive_rate: 0.3008356545961003\n", 1605 | "\n", 1606 | "\n", 1607 | "3\n", 1608 | "count: 574, accuracy: 0.5087108013937283\n", 1609 | "true_positive: 175, false_positive: 149\n", 1610 | "true_negative: 117, false_negative: 133\n", 1611 | "true_positive_rate: 0.5401234567901234, false_positive_rate: 0.45987654320987653\n", 1612 | "true_positive_rate: 0.468, false_positive_rate: 0.4104938271604938\n", 1613 | "\n", 1614 | "5\n", 1615 | "count: 574, accuracy: 0.5069686411149826\n", 1616 | "true_positive: 170, false_positive: 145\n", 1617 | "true_negative: 121, false_negative: 138\n", 1618 | "true_positive_rate: 0.5396825396825397, false_positive_rate: 0.4603174603174603\n", 1619 | "true_positive_rate: 0.4671814671814672, false_positive_rate: 0.4380952380952381\n", 1620 | "\n", 1621 | "\n", 1622 | "zzzzzzzzzzzz\n", 1623 | "100day clf, dep=2, day2, trade, amount accuracy: 0.5512820512820513\n", 1624 | "count: 624, \n", 1625 | "true_positive: 232, false_positive: 181\n", 1626 | "true_negative: 112, false_negative: 99\n", 1627 | "true_positive_rate: 0.5617433414043583, false_positive_rate: 0.43825665859564167\n", 1628 | "true_positive_rate: 0.5308056872037915, false_positive_rate: 0.2397094430992736\n", 1629 | "\n", 1630 | "100day, day2 clf\n", 1631 | "count: 624\n", 1632 | "true_positive: 186, false_positive: 148\n", 1633 | "true_negative: 145, false_negative: 145\n", 1634 | "true_positive_rate: 0.5568862275449101, false_positive_rate: 0.4431137724550898\n", 1635 | "true_positive_rate: 0.5, false_positive_rate: 0.4341317365269461\n", 1636 | "\n", 1637 | "100day, day2, trade, amount accuracy: 0.5368589743589743\n", 1638 | "clf\n", 1639 | "count: 624\n", 1640 | "true_positive: 220, false_positive: 178\n", 1641 | "true_negative: 115, false_negative: 111\n", 1642 | "true_positive_rate: 0.5527638190954773, false_positive_rate: 0.4472361809045226\n", 1643 | "true_positive_rate: 0.5088495575221239, false_positive_rate: 0.27889447236180903\n", 1644 | "\n" 1645 | ] 1646 | }, 1647 | { 1648 | "cell_type": "code", 1649 | "execution_count": 40, 1650 | "id": "abd0b573", 1651 | "metadata": {}, 1652 | "outputs": [ 1653 | { 1654 | "data": { 1655 | "text/plain": [ 1656 | "1" 1657 | ] 1658 | }, 1659 | "execution_count": 40, 1660 | "metadata": {}, 1661 | "output_type": "execute_result" 1662 | } 1663 | ], 1664 | "source": [ 1665 | "y_test" 1666 | ] 1667 | }, 1668 | { 1669 | "cell_type": "code", 1670 | "execution_count": 14, 1671 | "id": "8595eb11", 1672 | "metadata": {}, 1673 | "outputs": [ 1674 | { 1675 | "name": "stdout", 1676 | "output_type": "stream", 1677 | "text": [ 1678 | " : pn,pp\n", 1679 | "an: 43, 6\n", 1680 | "ap: 12, 1\n" 1681 | ] 1682 | } 1683 | ], 1684 | "source": [ 1685 | "print(f' : pn,pp')\n", 1686 | "print(f'an: {tn}, {fp}')\n", 1687 | "print(f'ap: {fn}, {tp}')" 1688 | ] 1689 | }, 1690 | { 1691 | "cell_type": "code", 1692 | "execution_count": 21, 1693 | "id": "20e52c03", 1694 | "metadata": {}, 1695 | "outputs": [], 1696 | "source": [ 1697 | "from sklearn.model_selection import cross_val_score\n", 1698 | "scores = cross_val_score(lgr, scaled[train_target], scaled['dDayPriseRise'], cv=4, scoring='roc_auc')" 1699 | ] 1700 | }, 1701 | { 1702 | "cell_type": "code", 1703 | "execution_count": 22, 1704 | "id": "cd224d5c", 1705 | "metadata": {}, 1706 | "outputs": [ 1707 | { 1708 | "data": { 1709 | "text/plain": [ 1710 | "array([0.59047619, 0.41904762, 0.60833333, 0.73 ])" 1711 | ] 1712 | }, 1713 | "execution_count": 22, 1714 | "metadata": {}, 1715 | "output_type": "execute_result" 1716 | } 1717 | ], 1718 | "source": [ 1719 | "scores" 1720 | ] 1721 | }, 1722 | { 1723 | "cell_type": "code", 1724 | "execution_count": null, 1725 | "id": "b9f96d45", 1726 | "metadata": {}, 1727 | "outputs": [], 1728 | "source": [] 1729 | } 1730 | ], 1731 | "metadata": { 1732 | "kernelspec": { 1733 | "display_name": "Python 3 (ipykernel)", 1734 | "language": "python", 1735 | "name": "python3" 1736 | }, 1737 | "language_info": { 1738 | "codemirror_mode": { 1739 | "name": "ipython", 1740 | "version": 3 1741 | }, 1742 | "file_extension": ".py", 1743 | "mimetype": "text/x-python", 1744 | "name": "python", 1745 | "nbconvert_exporter": "python", 1746 | "pygments_lexer": "ipython3", 1747 | "version": "3.8.12" 1748 | } 1749 | }, 1750 | "nbformat": 4, 1751 | "nbformat_minor": 5 1752 | } 1753 | --------------------------------------------------------------------------------