├── .DS_Store ├── .firebaserc ├── .gitignore ├── .python-version ├── LICENSE ├── README.md ├── firebase.json ├── functions ├── .DS_Store ├── .gitignore ├── bootstrap_tables.py ├── env-example ├── ingest_economic_calendar.py ├── macrosurfer │ ├── __init__.py │ ├── agent │ │ ├── __init__.py │ │ └── query_agent.py │ ├── data_ingestion │ │ ├── __init__.py │ │ ├── data_ingestor.py │ │ ├── fmp │ │ │ ├── __init__.py │ │ │ ├── directory │ │ │ │ ├── __init__.py │ │ │ │ ├── cik_list_ingestor.py │ │ │ │ ├── company_symbol_list_ingestor.py │ │ │ │ └── financial_statement_symbols_ingestor.py │ │ │ ├── economics │ │ │ │ ├── __init__.py │ │ │ │ └── economic_calendar_ingestor.py │ │ │ ├── fmp_data_ingestor.py │ │ │ └── stocks │ │ │ │ ├── stock_data_1_day_ingestor.py │ │ │ │ └── stock_data_1_min_ingestor.py │ │ └── ingest_economic_calendar.py │ ├── database.py │ └── models │ │ ├── __init__.py │ │ ├── fmp │ │ ├── __init__.py │ │ ├── directory.py │ │ ├── economics.py │ │ └── stocks.py │ │ └── metadata.py ├── main.py ├── requirements-base.txt ├── requirements.txt ├── test_agent.py └── test_ingestor.py └── images └── illustration.jpg /.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MacroSurfer/MacroSurferFunctions/b9d3f14f0feae4873ab85bb1b786b98892c4688f/.DS_Store -------------------------------------------------------------------------------- /.firebaserc: -------------------------------------------------------------------------------- 1 | { 2 | "projects": { 3 | "default": "macrosurfer" 4 | } 5 | } 6 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | share/python-wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | MANIFEST 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .nox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | *.py,cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | cover/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | .pybuilder/ 76 | target/ 77 | 78 | # Jupyter Notebook 79 | .ipynb_checkpoints 80 | 81 | # IPython 82 | profile_default/ 83 | ipython_config.py 84 | 85 | # pyenv 86 | # For a library or package, you might want to ignore these files since the code is 87 | # intended to run in multiple environments; otherwise, check them in: 88 | # .python-version 89 | 90 | # pipenv 91 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 92 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 93 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 94 | # install all needed dependencies. 95 | #Pipfile.lock 96 | 97 | # poetry 98 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 99 | # This is especially recommended for binary packages to ensure reproducibility, and is more 100 | # commonly ignored for libraries. 101 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 102 | #poetry.lock 103 | 104 | # pdm 105 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 106 | #pdm.lock 107 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 108 | # in version control. 109 | # https://pdm.fming.dev/latest/usage/project/#working-with-version-control 110 | .pdm.toml 111 | .pdm-python 112 | .pdm-build/ 113 | 114 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 115 | __pypackages__/ 116 | 117 | # Celery stuff 118 | celerybeat-schedule 119 | celerybeat.pid 120 | 121 | # SageMath parsed files 122 | *.sage.py 123 | 124 | # Environments 125 | .env 126 | .venv 127 | env/ 128 | venv/ 129 | ENV/ 130 | env.bak/ 131 | venv.bak/ 132 | 133 | # Spyder project settings 134 | .spyderproject 135 | .spyproject 136 | 137 | # Rope project settings 138 | .ropeproject 139 | 140 | # mkdocs documentation 141 | /site 142 | 143 | # mypy 144 | .mypy_cache/ 145 | .dmypy.json 146 | dmypy.json 147 | 148 | # Pyre type checker 149 | .pyre/ 150 | 151 | # pytype static type analyzer 152 | .pytype/ 153 | 154 | # Cython debug symbols 155 | cython_debug/ 156 | 157 | # PyCharm 158 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 159 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 160 | # and can be added to the global gitignore or merged into this file. For a more nuclear 161 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 162 | #.idea/ 163 | 164 | # ignore all .DS_Store file 165 | .DS_Store 166 | # ignore all .DS_Store file in any subdirectory 167 | **/.DS_Store -------------------------------------------------------------------------------- /.python-version: -------------------------------------------------------------------------------- 1 | 3.12 2 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 MacroSurfer 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # 🌊 MacroSurfer Scout 2 | 3 | ***Built for traders, quants, and macro thinkers. Surf the data tide.*** 🌊 4 | 5 | ## 👀 What is MacroSurfer Scout? 6 | 7 | ![Alt text](./images/illustration.jpg) 8 | 9 | 10 | **MacroSurfer Scout** is an open-source intelligence layer designed for quantitative developers, financial engineers, and traders who want to build powerful data pipelines and leverage LLMs for fast insights into financial and economic data. 11 | 12 | Scout makes it easy to: 13 | - 🏗️ Build your own **big data curation pipeline** to store economic, fundamental, and news data into an SQL database. 14 | - 🤖 Query complex datasets using a **LLM-powered natural language agent**, skipping manual SQL writing. 15 | - 🚀 Deploy a **cloud-native API** to Google Cloud in just a few clicks — ready to be connected to any UI, terminal, or automated workflow. 16 | 17 | --- 18 | 19 | ## 🧠 Why MacroSurfer Scout? 20 | 21 | Modern quantitative investing requires a seamless flow from raw data to actionable insights. Scout is built to help you: 22 | 23 | - **Ingest and manage** vast financial data sources at scale. 24 | - **Ask questions like a human**, and get structured answers with SQL-driven accuracy. 25 | - **Deploy in the cloud**, and integrate with dashboards, bots, notebooks, or mobile apps. 26 | 27 | Whether you're building a systematic macro model, a real-time alert system, or your own Bloomberg Terminal — Scout gives you the foundation. 28 | 29 | --- 30 | 31 | ## 🚀 Features 32 | 33 | ### 📊 1. Scalable Data Curation Engine 34 | - Ingest data from APIs (e.g., FMP, EDGAR, News APIs, Alpha Vantage, etc.). 35 | - Normalize and store in **PostgreSQL** or any compatible SQL database. 36 | - Modular connectors for custom data sources. 37 | - Timestamped and structured for **quantitative analytics**. 38 | 39 | ### 🤖 2. LLM-Powered Agent 40 | - Chat-style interface or API call to **query data in natural language**. 41 | - Powered by OpenAI / local models (configurable). 42 | - Handles multi-table joins, summaries, trend spotting, and anomaly detection. 43 | - Designed for **research acceleration** and exploration. 44 | 45 | ### ☁️ 3. Cloud-Native Deployment (GCP) 46 | - One-command deployment to **Google Cloud Run / App Engine**. 47 | - Auto-scales and integrates with **GCP SQL**, **IAM**, and **logging**. 48 | - Exposes REST API endpoints to hook into **Streamlit**, **React**, **CLI tools**, or **trading bots**. 49 | 50 | --- 51 | 52 | ## 🗺️ Roadmap 53 | 54 | Stay tuned for exciting new features and improvements! Here's what's coming up for MacroSurfer Scout: 55 | 56 | | Feature | Description | Timeline | 57 | |---------|-------------|----------| 58 | | **📈 Enhanced agent with quantative analytics** | Enhanced the agent to add quantative analytics capability, not just querying database. | Q2 2025 | 59 | | **📊 Recurrent Data backfill** | Automatically batch request and backfill new data in without manual effort. | Q2 2025 | 60 | | **🔍 Visualization** | Add visualization from agent's analysis via charting in the conversation. | Q4 2025 | 61 | | **🔒 Security Enhancements** | Add UI login and API login token verification so it can run not just from your localhost. | Q4 2025 | 62 | | **🌐 Global market data support** | Support stock data storage in major global markets (US, CA, UK, EU, JP, HK, etc). | Q2 2026 | 63 | 64 | --- 65 | 66 | ## 📦 Getting Started 67 | 68 | ### 🔧 Prerequisites 69 | 70 | - Python 3.13+ 71 | - PostgreSQL instance (local or cloud) we recommend Supabase to get you started free 72 | - GCP account & project (for deployment) 73 | - [OpenAI API key](https://platform.openai.com/) (or custom LLM) 74 | 75 | ### 🐍 Install Locally 76 | 77 | ```bash 78 | git clone git@github.com:MacroSurfer/MacroSurferFunctions.git 79 | cd MacroSurferFunctions 80 | python -m venv .venv 81 | source .venv/bin/activate 82 | pip install -r requirements.txt 83 | ``` 84 | 85 | ### ⚙️ Setup Config 86 | Create a .env file in the /functions (example in functions/env-example): 87 | ```bash 88 | DB_USERNAME="" 89 | DB_PASSWORD="" 90 | DB_HOST_NAME="" 91 | DB_PORT="" 92 | DB_NAME="" 93 | FINANCIAL_MODELINGPREP_API_KEY="" 94 | OPENAI_API_KEY="" 95 | ``` 96 | 97 | ## 🖥 Setup your SQL database with tables 98 | 99 | ```bash 100 | python functions/bootstrap_tables.py 101 | ``` 102 | 103 | ### 🧪 Run the Agent Locally 104 | ```bash 105 | python functions/test_agent.py 106 | ``` 107 | 108 | Ask questions like: 109 | ```vbnet 110 | > What is the GDP growth trend for the past 5 years? 111 | > Show me all companies with increasing ROE in the last 3 quarters. 112 | > What's the inflation rate spike during oil shocks since 2000? 113 | ``` 114 | 115 | ## ☁️ Deploy to Google Cloud 116 | --- 117 | ```bash 118 | firebase login 119 | firebase deploy --only functions 120 | ``` 121 | Now your API endpoint is live and scalable. 🎉 122 | 123 | ## 🧩 Integrations 124 | * 🖥️ Connect to dashboards (Streamlit, React, Grafana). 125 | 126 | * 🔗 Use in Jupyter Notebooks or trading pipelines. 127 | 128 | * ⚡ Webhooks or Slack bot integration ready. 129 | 130 | ## 🤝 Contributing 131 | We welcome contributors of all experience levels! Here's how to get involved: 132 | 133 | 1. Fork the repo and clone it locally. 134 | 135 | 2. Create a new branch: git checkout -b feature-your-feature-name 136 | 137 | 3. Make your changes and commit: git commit -m "Add feature" 138 | 139 | 4. Push and submit a PR: git push origin feature-your-feature-name 140 | 141 | Please follow the [Conventional Commits](https://www.conventionalcommits.org/) format and include test coverage where possible. 142 | 143 | ## 📄 License 144 | ``` 145 | MIT License 146 | 147 | Copyright (c) 2025 MacroSurfer 148 | 149 | Permission is hereby granted, free of charge, to any person obtaining a copy 150 | of this software and associated documentation files (the "Software"), to deal 151 | in the Software without restriction... 152 | 153 | 154 | -------------------------------------------------------------------------------- /firebase.json: -------------------------------------------------------------------------------- 1 | { 2 | "functions": [ 3 | { 4 | "source": "functions", 5 | "codebase": "default", 6 | "runtime": "python312", 7 | "ignore": [ 8 | "venv", 9 | ".git", 10 | "firebase-debug.log", 11 | "firebase-debug.*.log", 12 | "*.local" 13 | ] 14 | } 15 | ] 16 | } 17 | -------------------------------------------------------------------------------- /functions/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MacroSurfer/MacroSurferFunctions/b9d3f14f0feae4873ab85bb1b786b98892c4688f/functions/.DS_Store -------------------------------------------------------------------------------- /functions/.gitignore: -------------------------------------------------------------------------------- 1 | *.local 2 | **/.DS_Store 3 | *.DS_Store -------------------------------------------------------------------------------- /functions/bootstrap_tables.py: -------------------------------------------------------------------------------- 1 | # Create all tables if not exists in the database using sqlalchemy 2 | from dotenv import load_dotenv 3 | from macrosurfer.database import Database 4 | 5 | if __name__ == "__main__": 6 | load_dotenv() 7 | print("Bootstrapping tables...") 8 | db = Database() 9 | engine = db.get_engine() 10 | session = db.get_session() 11 | 12 | # Create all tables if not exists in the database using sqlalchemy 13 | metadata = db.get_metadata() 14 | metadata.create_all(engine) 15 | print("Tables created successfully") 16 | -------------------------------------------------------------------------------- /functions/env-example: -------------------------------------------------------------------------------- 1 | DB_USERNAME="" 2 | DB_PASSWORD="" 3 | DB_HOST_NAME="" 4 | DB_PORT="" 5 | DB_NAME="" 6 | FINANCIAL_MODELINGPREP_API_KEY="" 7 | OPENAI_API_KEY="" -------------------------------------------------------------------------------- /functions/ingest_economic_calendar.py: -------------------------------------------------------------------------------- 1 | from dotenv import load_dotenv 2 | from macrosurfer.data_ingestion.ingest_economic_calendar import ingest_incoming_month_economic_calendar 3 | from datetime import datetime, timedelta 4 | from macrosurfer.database import Database 5 | 6 | load_dotenv() 7 | 8 | if __name__ == "__main__": 9 | db = Database() 10 | 11 | end_date = datetime(2015, 1, 1, 0, 0, 0) 12 | start_date = datetime(2005, 1, 1, 0, 0, 0) 13 | while start_date < end_date: 14 | cur_end_date = min(start_date + timedelta(days=3), end_date) 15 | print(f"Ingesting economic calendar from {start_date} to {cur_end_date}") 16 | ingest_incoming_month_economic_calendar(db, start_date, cur_end_date) 17 | start_date = cur_end_date 18 | -------------------------------------------------------------------------------- /functions/macrosurfer/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MacroSurfer/MacroSurferFunctions/b9d3f14f0feae4873ab85bb1b786b98892c4688f/functions/macrosurfer/__init__.py -------------------------------------------------------------------------------- /functions/macrosurfer/agent/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MacroSurfer/MacroSurferFunctions/b9d3f14f0feae4873ab85bb1b786b98892c4688f/functions/macrosurfer/agent/__init__.py -------------------------------------------------------------------------------- /functions/macrosurfer/agent/query_agent.py: -------------------------------------------------------------------------------- 1 | from langchain.agents import create_sql_agent 2 | from langchain.agents.agent_toolkits import SQLDatabaseToolkit 3 | from langchain.agents.agent_types import AgentType 4 | from langchain.chat_models import ChatOpenAI 5 | from langchain.sql_database import SQLDatabase 6 | from macrosurfer.database import Database 7 | 8 | class QueryAgent: 9 | def __init__(self, database: Database, llm: ChatOpenAI): 10 | self.__db = database 11 | self.__llm = llm 12 | db = SQLDatabase(engine=self.__db.get_engine()) 13 | self.__toolkit = SQLDatabaseToolkit(db=db, llm=self.__llm) 14 | 15 | def query(self, question: str) -> str: 16 | 17 | agent = create_sql_agent( 18 | llm=self.__llm, 19 | toolkit=self.__toolkit, 20 | verbose=True, 21 | agent_type=AgentType.OPENAI_FUNCTIONS, 22 | ) 23 | 24 | return agent.run(question) 25 | -------------------------------------------------------------------------------- /functions/macrosurfer/data_ingestion/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MacroSurfer/MacroSurferFunctions/b9d3f14f0feae4873ab85bb1b786b98892c4688f/functions/macrosurfer/data_ingestion/__init__.py -------------------------------------------------------------------------------- /functions/macrosurfer/data_ingestion/data_ingestor.py: -------------------------------------------------------------------------------- 1 | from abc import ABC, abstractmethod 2 | from datetime import datetime 3 | from macrosurfer.database import Database 4 | 5 | class DataIngestor(ABC): 6 | FMP_ENDPOINT = "https://financialmodelingprep.com/api/v3" 7 | 8 | def __init__(self, db: Database, table: str, batch_size: int = 100): 9 | self.__db = db 10 | self.__table = table 11 | self.__batch_size = batch_size 12 | 13 | @abstractmethod 14 | def ingest(self, start_date: datetime, end_date: datetime): 15 | pass 16 | -------------------------------------------------------------------------------- /functions/macrosurfer/data_ingestion/fmp/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MacroSurfer/MacroSurferFunctions/b9d3f14f0feae4873ab85bb1b786b98892c4688f/functions/macrosurfer/data_ingestion/fmp/__init__.py -------------------------------------------------------------------------------- /functions/macrosurfer/data_ingestion/fmp/directory/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MacroSurfer/MacroSurferFunctions/b9d3f14f0feae4873ab85bb1b786b98892c4688f/functions/macrosurfer/data_ingestion/fmp/directory/__init__.py -------------------------------------------------------------------------------- /functions/macrosurfer/data_ingestion/fmp/directory/cik_list_ingestor.py: -------------------------------------------------------------------------------- 1 | from macrosurfer.data_ingestion.fmp.fmp_data_ingestor import FMPDataIngestor 2 | from macrosurfer.database import Database 3 | from macrosurfer.models.fmp import CIK_LIST 4 | from datetime import datetime 5 | from typing import Any, override 6 | from sqlalchemy.dialects.postgresql import insert as pg_insert 7 | class CIKListIngestor(FMPDataIngestor): 8 | 9 | def __init__(self, db: Database): 10 | super().__init__(db, CIK_LIST) 11 | 12 | @override 13 | def ingest(self, start_date: datetime, end_date: datetime): 14 | url = self._get_url_with_api_key(start_date, end_date) 15 | data = self._get_data(url) 16 | self._execute_batch(data) 17 | 18 | @override 19 | def _get_stmt(self, event: Any) -> Any: 20 | return pg_insert(self._table).values( 21 | cik=event['cik'], 22 | company_name=event['companyName'], 23 | 24 | ).on_conflict_do_update( 25 | index_elements=['cik'], 26 | set_=dict( 27 | company_name=event['companyName'], 28 | ) 29 | ) 30 | 31 | @override 32 | def _get_url(self, start_date: datetime, end_date: datetime) -> str: 33 | return f"{self.FMP_ENDPOINT}/cik-list" 34 | 35 | -------------------------------------------------------------------------------- /functions/macrosurfer/data_ingestion/fmp/directory/company_symbol_list_ingestor.py: -------------------------------------------------------------------------------- 1 | from typing import Any, override 2 | from macrosurfer.data_ingestion.fmp.fmp_data_ingestor import FMPDataIngestor 3 | from datetime import datetime 4 | from sqlalchemy.dialects.postgresql import insert as pg_insert 5 | from macrosurfer.database import Database 6 | from macrosurfer.models.fmp import COMPANY_SYMBOLS 7 | 8 | class CompanySymbolListIngestor(FMPDataIngestor): 9 | 10 | def __init__(self, db: Database, batch_size: int = 100): 11 | super().__init__(db, COMPANY_SYMBOLS, batch_size) 12 | 13 | @override 14 | def ingest(self, start_date: datetime, end_date: datetime): 15 | url = self._get_url_with_api_key(start_date, end_date) 16 | data = self._get_data(url) 17 | self._execute_batch(data) 18 | 19 | @override 20 | def _get_stmt(self, event: Any) -> Any: 21 | return pg_insert(self._table).values( 22 | symbol=event['symbol'], 23 | company_name=event['companyName'] 24 | ).on_conflict_do_update( 25 | index_elements=['symbol'], 26 | set_=dict( 27 | company_name=event['companyName'] 28 | ) 29 | ) 30 | 31 | @override 32 | def _get_url(self, start_date: datetime, end_date: datetime) -> str: 33 | return f"{self.FMP_ENDPOINT}/stock-list" 34 | -------------------------------------------------------------------------------- /functions/macrosurfer/data_ingestion/fmp/directory/financial_statement_symbols_ingestor.py: -------------------------------------------------------------------------------- 1 | from macrosurfer.data_ingestion.fmp.fmp_data_ingestor import FMPDataIngestor 2 | from macrosurfer.database import Database 3 | from macrosurfer.models.fmp import FINANCIAL_STATEMENT_SYMBOLS 4 | from datetime import datetime 5 | from sqlalchemy.dialects.postgresql import insert as pg_insert 6 | 7 | from typing import Any, override 8 | 9 | class FinancialStatementSymbolsIngestor(FMPDataIngestor): 10 | def __init__(self, db: Database, batch_size: int = 100): 11 | super().__init__(db, FINANCIAL_STATEMENT_SYMBOLS, batch_size) 12 | 13 | @override 14 | def ingest(self, start_date: datetime, end_date: datetime): 15 | url = self._get_url_with_api_key(start_date, end_date) 16 | data = self._get_data(url) 17 | self._execute_batch(data) 18 | 19 | @override 20 | def _get_stmt(self, event: Any) -> Any: 21 | return pg_insert(self._table).values( 22 | symbol=event['symbol'], 23 | company_name=event['companyName'], 24 | trading_currency=event['tradingCurrency'], 25 | reporting_currency=event['reportingCurrency'] 26 | ).on_conflict_do_update( 27 | index_elements=['symbol'], 28 | set_=dict( 29 | company_name=event['companyName'], 30 | trading_currency=event['tradingCurrency'], 31 | reporting_currency=event['reportingCurrency'] 32 | ) 33 | ) 34 | 35 | @override 36 | def _get_url(self, start_date: datetime, end_date: datetime) -> str: 37 | return f"{self.FMP_ENDPOINT}/financial-statement-symbol-list" 38 | 39 | -------------------------------------------------------------------------------- /functions/macrosurfer/data_ingestion/fmp/economics/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MacroSurfer/MacroSurferFunctions/b9d3f14f0feae4873ab85bb1b786b98892c4688f/functions/macrosurfer/data_ingestion/fmp/economics/__init__.py -------------------------------------------------------------------------------- /functions/macrosurfer/data_ingestion/fmp/economics/economic_calendar_ingestor.py: -------------------------------------------------------------------------------- 1 | from typing import Any, override 2 | from macrosurfer.data_ingestion.fmp.fmp_data_ingestor import FMPDataIngestor 3 | from datetime import datetime 4 | from sqlalchemy.dialects.postgresql import insert as pg_insert 5 | from macrosurfer.database import Database 6 | from macrosurfer.models.fmp import ECONOMIC_CALENDAR_TABLE 7 | 8 | class EconomicCalendarIngestor(FMPDataIngestor): 9 | 10 | def __init__(self, db: Database, batch_size: int = 100): 11 | super().__init__(db, ECONOMIC_CALENDAR_TABLE, batch_size) 12 | 13 | @override 14 | def ingest(self, start_date: datetime, end_date: datetime): 15 | url = self._get_url_with_api_key(start_date, end_date) 16 | data = self._get_data(url) 17 | self._execute_batch(data) 18 | 19 | @override 20 | def _get_stmt(self, event: Any) -> Any: 21 | event_date = datetime.strptime(event['date'], '%Y-%m-%d %H:%M:%S') 22 | return pg_insert(self._table).values( 23 | event=event['event'], 24 | event_date=event_date, 25 | country=event['country'], 26 | currency=event['currency'], 27 | previous=event['previous'], 28 | estimate=event['estimate'], 29 | actual=event['actual'], 30 | change=event['change'], 31 | impact=event['impact'], 32 | change_percentage=event['changePercentage'], 33 | unit=event['unit'] 34 | ).on_conflict_do_update( 35 | index_elements=['event', 'event_date'], 36 | set_=dict( 37 | country=event['country'], 38 | currency=event['currency'], 39 | previous=event['previous'], 40 | estimate=event['estimate'], 41 | actual=event['actual'], 42 | change=event['change'], 43 | impact=event['impact'], 44 | change_percentage=event['changePercentage'], 45 | unit=event['unit'] 46 | ) 47 | ) 48 | 49 | @override 50 | def _get_url(self, start_date: datetime, end_date: datetime) -> str: 51 | from_date = self.strf_date(start_date) 52 | to_date = self.strf_date(end_date) 53 | return f"{self.FMP_ENDPOINT}/economic-calendar?from={from_date}&to={to_date}" 54 | -------------------------------------------------------------------------------- /functions/macrosurfer/data_ingestion/fmp/fmp_data_ingestor.py: -------------------------------------------------------------------------------- 1 | from abc import ABC, abstractmethod 2 | from datetime import datetime 3 | from macrosurfer.database import Database 4 | import os 5 | import requests 6 | from typing import Any, List 7 | from sqlalchemy.exc import SQLAlchemyError 8 | 9 | class FMPDataIngestor(ABC): 10 | FMP_ENDPOINT = "https://financialmodelingprep.com/stable" 11 | 12 | def __init__(self, db: Database, table: str, batch_size: int = 100): 13 | self._db = db 14 | self._table = table 15 | self._batch_size = batch_size 16 | self._api_key = os.getenv("FINANCIAL_MODELINGPREP_API_KEY") 17 | 18 | @abstractmethod 19 | def ingest(self, start_date: datetime, end_date: datetime): 20 | pass 21 | 22 | @abstractmethod 23 | def _get_url(self, start_date: datetime, end_date: datetime) -> str: 24 | pass 25 | 26 | @staticmethod 27 | def strf_date(date: datetime) -> str: 28 | return date.strftime('%Y-%m-%d') 29 | 30 | @abstractmethod 31 | def _get_stmt(self, event: Any) -> Any: 32 | pass 33 | 34 | def _get_data(self, url: str) -> Any: 35 | response = requests.get(url) 36 | response.raise_for_status() 37 | return response.json() 38 | 39 | def _get_url_with_api_key(self, start_date: datetime, end_date: datetime) -> str: 40 | url = self._get_url(start_date, end_date) 41 | if '?' in url: 42 | return f"{url}&apikey={self._api_key}" 43 | else: 44 | return f"{url}?apikey={self._api_key}" 45 | 46 | def _execute_batch(self, data: List[Any]): 47 | session = self._db.get_session() 48 | try: 49 | for i in range(0, len(data), self._batch_size): 50 | batch = data[i:i + self._batch_size] 51 | stmts = [] 52 | for event in batch: 53 | stmt = self._get_stmt(event) 54 | if stmt is not None: 55 | stmts.append(stmt) 56 | 57 | for stmt in stmts: 58 | session.execute(stmt) 59 | 60 | session.commit() 61 | print(f"Processed batch {i//self._batch_size + 1} of {(len(data) + self._batch_size - 1)//self._batch_size}") 62 | 63 | except requests.HTTPError as http_err: 64 | print(f"HTTP error occurred: {http_err}") 65 | except SQLAlchemyError as db_err: 66 | print(f"Database error occurred: {db_err}") 67 | except Exception as err: 68 | print(f"An error occurred: {err}") 69 | finally: 70 | session.close() -------------------------------------------------------------------------------- /functions/macrosurfer/data_ingestion/fmp/stocks/stock_data_1_day_ingestor.py: -------------------------------------------------------------------------------- 1 | from macrosurfer.data_ingestion.fmp.fmp_data_ingestor import FMPDataIngestor 2 | from macrosurfer.database import Database 3 | from macrosurfer.models.fmp import STOCK_PRICE_1_DAY_TABLE 4 | from datetime import datetime, timezone, timedelta 5 | from typing import List, override, Any 6 | from sqlalchemy.dialects.postgresql import insert as pg_insert 7 | 8 | class StockData1DayIngestor(FMPDataIngestor): 9 | def __init__(self, db: Database, symbol: str, batch_size: int = 100): 10 | super().__init__(db, STOCK_PRICE_1_DAY_TABLE, batch_size) 11 | self._symbol = symbol 12 | 13 | @override 14 | def ingest(self, start_date: datetime, end_date: datetime): 15 | url = self._get_url_with_api_key(start_date, end_date) 16 | data = self._get_data(url) 17 | self._execute_batch(data) 18 | 19 | @override 20 | def _get_stmt(self, event: Any) -> Any: 21 | event_date = datetime.strptime(event['date'], '%Y-%m-%d') 22 | return pg_insert(self._table).values( 23 | symbol=self._symbol, 24 | date=event_date, 25 | open=event['open'], 26 | high=event['high'], 27 | low=event['low'], 28 | close=event['close'], 29 | volume=event['volume'], 30 | change=event['change'], 31 | change_percentage=event['changePercent'], 32 | vwap=event['vwap'] 33 | ).on_conflict_do_update( 34 | index_elements=['symbol', 'date'], 35 | set_=dict( 36 | open=event['open'], 37 | high=event['high'], 38 | low=event['low'], 39 | close=event['close'], 40 | volume=event['volume'], 41 | change=event['change'], 42 | change_percentage=event['changePercent'], 43 | vwap=event['vwap'] 44 | ) 45 | ) 46 | 47 | @override 48 | def _get_url(self, start_date: datetime, end_date: datetime) -> str: 49 | start_date_str = start_date.strftime('%Y-%m-%d') 50 | end_date_str = end_date.strftime('%Y-%m-%d') 51 | return f"https://financialmodelingprep.com/stable/historical-price-eod/full?symbol={self._symbol}&from={start_date_str}&to={end_date_str}" 52 | -------------------------------------------------------------------------------- /functions/macrosurfer/data_ingestion/fmp/stocks/stock_data_1_min_ingestor.py: -------------------------------------------------------------------------------- 1 | from macrosurfer.data_ingestion.fmp.fmp_data_ingestor import FMPDataIngestor 2 | from macrosurfer.database import Database 3 | from macrosurfer.models.fmp import STOCK_PRICE_1_MIN_TABLE 4 | from datetime import datetime, timezone, timedelta 5 | from typing import List, override, Any 6 | from sqlalchemy.dialects.postgresql import insert as pg_insert 7 | 8 | class StockData1MinIngestor(FMPDataIngestor): 9 | def __init__(self, db: Database, symbol: str, backfill: bool = False, batch_size: int = 100): 10 | super().__init__(db, STOCK_PRICE_1_MIN_TABLE, batch_size) 11 | self._symbol = symbol 12 | self._backfill = backfill 13 | self._start_time = (datetime.now(timezone.utc) - timedelta(minutes=5)).strftime('%Y-%m-%d %H:%M:%S') 14 | 15 | @override 16 | def ingest(self, start_date: datetime, end_date: datetime): 17 | url = self._get_url_with_api_key(start_date, end_date) 18 | data = self._get_data(url) 19 | self._execute_batch(data) 20 | 21 | @override 22 | def _get_stmt(self, event: Any) -> Any: 23 | event_date = datetime.strptime(event['date'], '%Y-%m-%d %H:%M:%S') 24 | return pg_insert(self._table).values( 25 | symbol=self._symbol, 26 | date=event_date, 27 | open=event['open'], 28 | high=event['high'], 29 | low=event['low'], 30 | close=event['close'], 31 | volume=event['volume'] 32 | ).on_conflict_do_update( 33 | index_elements=['symbol', 'date'], 34 | set_=dict( 35 | open=event['open'], 36 | high=event['high'], 37 | low=event['low'], 38 | close=event['close'], 39 | volume=event['volume'] 40 | ) 41 | ) 42 | 43 | @override 44 | def _get_url(self, start_date: datetime, end_date: datetime) -> str: 45 | start_date_str = start_date.strftime('%Y-%m-%d') 46 | end_date_str = end_date.strftime('%Y-%m-%d') 47 | return f"https://financialmodelingprep.com/stable/historical-chart/1min?symbol={self._symbol}&from={start_date_str}&to={end_date_str}" 48 | 49 | -------------------------------------------------------------------------------- /functions/macrosurfer/data_ingestion/ingest_economic_calendar.py: -------------------------------------------------------------------------------- 1 | import os 2 | from sqlalchemy.orm import sessionmaker, Session 3 | from sqlalchemy import create_engine 4 | from pydantic import BaseModel 5 | from macrosurfer.database import Database 6 | from typing import Tuple, Optional 7 | from dotenv import load_dotenv 8 | from sqlalchemy.exc import SQLAlchemyError 9 | from macrosurfer.models.fmp import ECONOMIC_CALENDAR_TABLE_PROCESSED, ECONOMIC_CALENDAR_TABLE_RAW 10 | import requests 11 | from datetime import datetime, timedelta 12 | from sqlalchemy.dialects.postgresql import insert as pg_insert 13 | import re 14 | load_dotenv() 15 | 16 | def split_title_and_month(text: str) -> Tuple[str, Optional[str]]: 17 | match = re.match(r"^(.*?)\s*\(([^)]+)\)\s*$", text) 18 | if match: 19 | title, month = match.group(1), match.group(2) 20 | return title.strip(), month.strip() 21 | return text.strip(), "" # No parentheses found 22 | 23 | def ingest_incoming_month_economic_calendar(db: Database, start_date: datetime, end_date: datetime): 24 | # Format dates for the API 25 | from_date = start_date.strftime('%Y-%m-%d') 26 | to_date = end_date.strftime('%Y-%m-%d') 27 | try: 28 | # Fetch data from the API 29 | api_key = os.getenv("FINANCIAL_MODELINGPREP_API_KEY") 30 | url = f'https://financialmodelingprep.com/api/v3/economic_calendar?from={from_date}&to={to_date}&apikey={api_key}' 31 | response = requests.get(url) 32 | response.raise_for_status() 33 | data = response.json() 34 | 35 | # Process data in chunks of 1000 records 36 | BATCH_SIZE = 100 37 | with db.get_session() as session: 38 | for i in range(0, len(data), BATCH_SIZE): 39 | batch = data[i:i + BATCH_SIZE] 40 | stmts = [] 41 | 42 | for event in batch: 43 | event_date = datetime.strptime(event['date'], '%Y-%m-%d %H:%M:%S') 44 | stmt = pg_insert(ECONOMIC_CALENDAR_TABLE_RAW).values( 45 | event=event['event'], 46 | event_date=event_date, 47 | country=event['country'], 48 | currency=event['currency'], 49 | previous=event['previous'], 50 | estimate=event['estimate'], 51 | actual=event['actual'], 52 | change=event['change'], 53 | impact=event['impact'], 54 | change_percentage=event['changePercentage'], 55 | unit=event['unit'] 56 | ).on_conflict_do_update( 57 | index_elements=['event', 'event_date', 'country'], 58 | set_=dict( 59 | currency=event['currency'], 60 | previous=event['previous'], 61 | estimate=event['estimate'], 62 | actual=event['actual'], 63 | change=event['change'], 64 | impact=event['impact'], 65 | change_percentage=event['changePercentage'], 66 | unit=event['unit'] 67 | ) 68 | ) 69 | stmts.append(stmt) 70 | 71 | # Execute batch 72 | for stmt in stmts: 73 | session.execute(stmt) 74 | session.commit() 75 | print(f"Processed batch {i//BATCH_SIZE + 1} of {(len(data) + BATCH_SIZE - 1)//BATCH_SIZE}") 76 | for i in range(0, len(data), BATCH_SIZE): 77 | batch = data[i:i + BATCH_SIZE] 78 | stmts = [] 79 | 80 | for event in batch: 81 | event_date = datetime.strptime(event['date'], '%Y-%m-%d %H:%M:%S') 82 | # Event string exxtract any name in format " ()" 83 | event_title, additional_identifier = split_title_and_month(event['event']) 84 | stmt = pg_insert(ECONOMIC_CALENDAR_TABLE_PROCESSED).values( 85 | event=event_title, 86 | event_date=event_date, 87 | additional_identifier=additional_identifier, 88 | country=event['country'], 89 | currency=event['currency'], 90 | previous=event['previous'], 91 | estimate=event['estimate'], 92 | actual=event['actual'], 93 | change=event['change'], 94 | impact=event['impact'], 95 | change_percentage=event['changePercentage'], 96 | unit=event['unit'] 97 | ).on_conflict_do_update( 98 | index_elements=['event', 'event_date', 'country', 'additional_identifier'], 99 | set_=dict( 100 | currency=event['currency'], 101 | previous=event['previous'], 102 | estimate=event['estimate'], 103 | actual=event['actual'], 104 | change=event['change'], 105 | impact=event['impact'], 106 | change_percentage=event['changePercentage'], 107 | unit=event['unit'] 108 | ) 109 | ) 110 | stmts.append(stmt) 111 | 112 | # Execute batch 113 | for stmt in stmts: 114 | session.execute(stmt) 115 | session.commit() 116 | print(f"Processed batch {i//BATCH_SIZE + 1} of {(len(data) + BATCH_SIZE - 1)//BATCH_SIZE}") 117 | 118 | except requests.HTTPError as http_err: 119 | print(f"HTTP error occurred: {http_err}") 120 | except SQLAlchemyError as db_err: 121 | print(f"Database error occurred: {db_err}") 122 | session.rollback() 123 | except Exception as err: 124 | print(f"An error occurred: {err}") 125 | finally: 126 | session.close() 127 | 128 | -------------------------------------------------------------------------------- /functions/macrosurfer/database.py: -------------------------------------------------------------------------------- 1 | import os 2 | from sqlalchemy import create_engine 3 | from sqlalchemy.orm import sessionmaker 4 | from sqlalchemy.engine import Engine 5 | from sqlalchemy.orm import Session 6 | from sqlalchemy import MetaData 7 | from macrosurfer.models.metadata import METADATA 8 | from sqlalchemy.orm import scoped_session 9 | from macrosurfer.models.fmp import * 10 | 11 | class Database: 12 | def __init__(self): 13 | self.DATABASE_URL = f'postgresql+psycopg2://{os.getenv("DB_USERNAME")}:{os.getenv("DB_PASSWORD")}@{os.getenv("DB_HOST_NAME")}:{os.getenv("DB_PORT")}/{os.getenv("DB_NAME")}' 14 | print(self.DATABASE_URL) 15 | self.engine = create_engine(self.DATABASE_URL) 16 | self.__session_factory = sessionmaker(autocommit=False, autoflush=False, bind=self.engine) 17 | 18 | def get_engine(self) -> Engine: 19 | return self.engine 20 | 21 | def get_session(self) -> Session: 22 | SC = scoped_session(self.__session_factory) 23 | return SC() 24 | 25 | def get_metadata(self) -> MetaData: 26 | return METADATA 27 | -------------------------------------------------------------------------------- /functions/macrosurfer/models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MacroSurfer/MacroSurferFunctions/b9d3f14f0feae4873ab85bb1b786b98892c4688f/functions/macrosurfer/models/__init__.py -------------------------------------------------------------------------------- /functions/macrosurfer/models/fmp/__init__.py: -------------------------------------------------------------------------------- 1 | from .directory import * 2 | from .economics import * 3 | from .stocks import * -------------------------------------------------------------------------------- /functions/macrosurfer/models/fmp/directory.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import Table, Column, String, TIMESTAMP 2 | 3 | from macrosurfer.models.metadata import METADATA 4 | 5 | COMPANY_SYMBOLS = Table( 6 | 'company_symbols', METADATA, 7 | Column('symbol', String, primary_key=True), 8 | Column('company_name', String) 9 | ) 10 | 11 | FINANCIAL_STATEMENT_SYMBOLS = Table( 12 | 'financial_statement_symbols', METADATA, 13 | Column('symbol', String, primary_key=True), 14 | Column('company_name', String, index=True), 15 | Column('trading_currency', String), 16 | Column('reporting_currency', String) 17 | ) 18 | 19 | CIK_LIST = Table( 20 | 'cik_list', METADATA, 21 | Column('cik', String, primary_key=True), 22 | Column('company_name', String, index=True) 23 | ) 24 | 25 | SYMBOL_CHANGE_LIST = Table( 26 | 'symbol_change_list', METADATA, 27 | Column('new_symbol', String, primary_key=True), 28 | Column('old_symbol', String, index=True), 29 | Column('company_name', String, index=True), 30 | Column('date', TIMESTAMP, index=True) 31 | ) 32 | 33 | ETF_LIST = Table( 34 | 'etf_list', METADATA, 35 | Column('symbol', String, primary_key=True), 36 | Column('name', String, index=True) 37 | ) 38 | 39 | ACTIVE_TRADING_LIST = Table( 40 | 'active_trading_list', METADATA, 41 | Column('symbol', String, primary_key=True), 42 | Column('name', String, index=True) 43 | ) 44 | 45 | AVAILABLE_EXCHANGES = Table( 46 | 'available_exchanges', METADATA, 47 | Column('exchange', String, primary_key=True) 48 | ) 49 | 50 | AVAILABLE_SECTORS = Table( 51 | 'available_sectors', METADATA, 52 | Column('sector', String, primary_key=True) 53 | ) 54 | 55 | AVAILABLE_INDUSTRIES = Table( 56 | 'available_industries', METADATA, 57 | Column('industry', String, primary_key=True) 58 | ) 59 | 60 | AVAILABLE_COUNTRIES = Table( 61 | 'available_countries', METADATA, 62 | Column('country', String, primary_key=True) 63 | ) 64 | -------------------------------------------------------------------------------- /functions/macrosurfer/models/fmp/economics.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import Table, Column, String, Boolean, Double, TIMESTAMP 2 | from macrosurfer.models.metadata import METADATA 3 | 4 | ECONOMIC_CALENDAR_TABLE = Table( 5 | 'economic_calendar', METADATA, 6 | Column('event', String, primary_key=True), 7 | Column('event_date', TIMESTAMP, primary_key=True), 8 | Column('country', String), 9 | Column('currency', String), 10 | Column('previous', Double), 11 | Column('estimate', Double), 12 | Column('actual', Double), 13 | Column('change', Double), 14 | Column('impact', String), 15 | Column('change_percentage', Double), 16 | Column('unit', String), 17 | ) 18 | 19 | ECONOMIC_CALENDAR_TABLE_RAW = Table( 20 | 'economic_calendar_raw', METADATA, 21 | Column('event', String, primary_key=True), 22 | Column('event_date', TIMESTAMP, primary_key=True), 23 | Column('country', String, primary_key=True), 24 | Column('currency', String), 25 | Column('previous', Double), 26 | Column('estimate', Double), 27 | Column('actual', Double), 28 | Column('change', Double), 29 | Column('impact', String), 30 | Column('change_percentage', Double), 31 | Column('unit', String), 32 | ) 33 | 34 | ECONOMIC_CALENDAR_TABLE_PROCESSED = Table( 35 | 'economic_calendar_processed', METADATA, 36 | Column('event', String, primary_key=True), 37 | Column('event_date', TIMESTAMP, primary_key=True), 38 | Column('additional_identifier', String, primary_key=True), 39 | Column('country', String, primary_key=True), 40 | Column('currency', String), 41 | Column('previous', Double), 42 | Column('estimate', Double), 43 | Column('actual', Double), 44 | Column('change', Double), 45 | Column('impact', String), 46 | Column('change_percentage', Double), 47 | Column('unit', String), 48 | ) 49 | 50 | EVENT_DETAILS = Table( 51 | 'event_details', METADATA, 52 | Column('event', String, primary_key=True), 53 | Column('country', String, primary_key=True), 54 | Column('period', String), 55 | Column('official_data', Boolean), 56 | Column('detail', String), 57 | ) 58 | -------------------------------------------------------------------------------- /functions/macrosurfer/models/fmp/stocks.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import Table, Column, String, Boolean, Double, TIMESTAMP 2 | from macrosurfer.models.metadata import METADATA 3 | 4 | STOCK_PRICE_1_MIN_TABLE = Table( 5 | 'stock_price_1_min', METADATA, 6 | Column('symbol', String, primary_key=True), 7 | Column('date', TIMESTAMP, primary_key=True), 8 | Column('open', Double), 9 | Column('high', Double), 10 | Column('low', Double), 11 | Column('close', Double), 12 | Column('volume', Double)) 13 | 14 | STOCK_PRICE_1_DAY_TABLE = Table( 15 | 'stock_price_1_day', METADATA, 16 | Column('symbol', String, primary_key=True), 17 | Column('date', TIMESTAMP, primary_key=True), 18 | Column('open', Double), 19 | Column('high', Double), 20 | Column('low', Double), 21 | Column('close', Double), 22 | Column('volume', Double), 23 | Column('change', Double), 24 | Column('change_percentage', Double), 25 | Column('vwap', Double) 26 | ) 27 | -------------------------------------------------------------------------------- /functions/macrosurfer/models/metadata.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import Table, Column, String, Boolean, Double, TIMESTAMP, MetaData 2 | 3 | METADATA = MetaData() -------------------------------------------------------------------------------- /functions/main.py: -------------------------------------------------------------------------------- 1 | # Welcome to Cloud Functions for Firebase for Python! 2 | # To get started, simply uncomment the below code or create your own. 3 | # Deploy with `firebase deploy` 4 | import json 5 | import os 6 | from datetime import datetime, timedelta, timezone 7 | from decimal import Decimal 8 | import time 9 | from threading import Thread 10 | 11 | # The Cloud Functions for Firebase SDK to create Cloud Functions and set up triggers. 12 | from firebase_functions import https_fn, options, scheduler_fn 13 | from dotenv import load_dotenv 14 | from firebase_functions.options import MemoryOption 15 | # The Firebase Admin SDK to access Cloud Firestore. 16 | from firebase_admin import initialize_app 17 | from sqlalchemy.sql import text 18 | from sqlalchemy import create_engine, select, and_ 19 | from macrosurfer.models.fmp.economics import ECONOMIC_CALENDAR_TABLE, EVENT_DETAILS 20 | from macrosurfer.database import Database 21 | from langchain.chat_models import ChatOpenAI 22 | from macrosurfer.agent.query_agent import QueryAgent 23 | from macrosurfer.data_ingestion.ingest_economic_calendar import ingest_incoming_month_economic_calendar 24 | from macrosurfer.data_ingestion.fmp.stocks.stock_data_1_day_ingestor import StockData1DayIngestor 25 | from macrosurfer.data_ingestion.fmp.stocks.stock_data_1_min_ingestor import StockData1MinIngestor 26 | initialize_app() 27 | load_dotenv() 28 | 29 | # Replace the direct database connection with Database class 30 | db = Database() 31 | engine = db.get_engine() 32 | os.environ["OPENAI_API_KEY"] = os.getenv("OPENAI_API_KEY") 33 | llm = ChatOpenAI(temperature=0, model="gpt-4o") 34 | query_agent = QueryAgent(db, llm) 35 | 36 | class CustomJSONEncoder(json.JSONEncoder): 37 | def default(self, obj): 38 | if isinstance(obj, datetime): 39 | return obj.isoformat() 40 | if isinstance(obj, Decimal): 41 | return float(obj) 42 | return super().default(obj) 43 | 44 | @https_fn.on_request(cors=options.CorsOptions(cors_origins="*", cors_methods=["get", "post"])) 45 | def on_request_example(req: https_fn.Request) -> https_fn.Response: 46 | return https_fn.Response("Hello world!") 47 | 48 | def deserialize_row(row): 49 | return { 50 | "event": row.event, 51 | "event_date": row.event_date.strftime('%Y-%m-%dT%H:%M:%SZ'), 52 | "country": row.country, 53 | "currency": row.currency, 54 | "previous": row.previous, 55 | "estimate": row.estimate, 56 | "actual": row.actual, 57 | "change": row.change, 58 | "impact": row.impact, 59 | "change_percentage": row.change_percentage, 60 | "unit": row.unit 61 | } 62 | 63 | @https_fn.on_request(cors=options.CorsOptions(cors_origins="*", cors_methods=["get", "post"])) 64 | def getEventsInDateRange(req: https_fn.Request) -> https_fn.Response: 65 | """Take the text parameter passed to this HTTP endpoint and insert it into 66 | a new document in the messages collection.""" 67 | # Grab the text parameter. 68 | start_date = req.args.get("startDate") 69 | end_date = req.args.get("endDate") 70 | country = req.args.get("country", "US") 71 | 72 | print("start date: ", start_date) 73 | print("end date: ", end_date) 74 | print("country: ", country) 75 | 76 | if not start_date or not end_date: 77 | return https_fn.Response("startDate and endDate are required", status=400) 78 | 79 | with db.get_engine().connect() as connection: 80 | query = select( 81 | ECONOMIC_CALENDAR_TABLE 82 | 83 | ).where( 84 | and_( 85 | ECONOMIC_CALENDAR_TABLE.c.event_date >= text(f"'{start_date}'"), 86 | ECONOMIC_CALENDAR_TABLE.c.event_date <= text(f"'{end_date}'") 87 | ) 88 | ) 89 | 90 | if country: 91 | query = query.where(ECONOMIC_CALENDAR_TABLE.c.country == country) 92 | 93 | result = connection.execute(query) 94 | events = [deserialize_row(row) for row in result][:15] 95 | return https_fn.Response(json.dumps(events), status=200) 96 | 97 | @https_fn.on_request(cors=options.CorsOptions(cors_origins="*", cors_methods=["get", "post"])) 98 | def getHistoryForEvent(req: https_fn.Request) -> https_fn.Response: 99 | """Take the text parameter passed to this HTTP endpoint and insert it into 100 | a new document in the messages collection.""" 101 | # Grab the text parameter. 102 | event = req.args.get("event") 103 | country = req.args.get("country", "US") 104 | end_date = req.args.get("endDate") 105 | start_date = req.args.get("startDate") 106 | 107 | print("start date: ", start_date) 108 | print("end date: ", end_date) 109 | print("event: ", event) 110 | 111 | if not event or not country: 112 | return https_fn.Response("Event and country name are required", status=400) 113 | 114 | with db.get_engine().connect() as connection: 115 | query = select( 116 | ECONOMIC_CALENDAR_TABLE 117 | 118 | ).where( 119 | and_( 120 | ECONOMIC_CALENDAR_TABLE.c.country == country, 121 | ECONOMIC_CALENDAR_TABLE.c.event.like(f"%{event}%") 122 | ) 123 | ) 124 | 125 | if start_date and end_date: 126 | query = query.where( 127 | and_( 128 | ECONOMIC_CALENDAR_TABLE.c.event_date >= text(f"'{start_date}'"), 129 | ECONOMIC_CALENDAR_TABLE.c.event_date <= text(f"'{end_date}'") 130 | ) 131 | ) 132 | 133 | result = connection.execute(query) 134 | events = [deserialize_row(row) for row in result][:15] 135 | return https_fn.Response(json.dumps(events), status=200) 136 | 137 | 138 | @https_fn.on_request(cors=options.CorsOptions(cors_origins="*", cors_methods=["get", "post"])) 139 | def getEventDetails(req: https_fn.Request) -> https_fn.Response: 140 | """Take the text parameter passed to this HTTP endpoint and insert it into 141 | a new document in the messages collection.""" 142 | # Grab the text parameter. 143 | event = req.args.get("event") 144 | country = req.args.get("country") 145 | 146 | if not event or not country: 147 | return https_fn.Response("Event and country name are required", status=400) 148 | 149 | with db.get_engine().connect() as connection: 150 | query = select( 151 | EVENT_DETAILS 152 | 153 | ).where( 154 | and_( 155 | EVENT_DETAILS.c.country == country, 156 | EVENT_DETAILS.c.event == event 157 | ) 158 | ) 159 | result = connection.execute(query) 160 | events = [dict(row._mapping) for row in result] 161 | if len(events) == 0: 162 | return https_fn.Response("No details found", status=404) 163 | return https_fn.Response(json.dumps(events[0]), status=200) 164 | 165 | @https_fn.on_request(cors=options.CorsOptions(cors_origins="*", cors_methods=["get", "post"])) 166 | def chat(req: https_fn.Request) -> https_fn.Response: 167 | """Take the text parameter passed to this HTTP endpoint and insert it into 168 | a new document in the messages collection.""" 169 | # Grab the text parameter. 170 | question = req.args.get("question") 171 | 172 | if not question: 173 | # Check request body 174 | body = req.json 175 | if not body: 176 | return https_fn.Response("Please ask a question", status=400) 177 | question = body.get("question") 178 | 179 | result = query_agent.query(question + "Please respond in html format.") 180 | if not result: 181 | return https_fn.Response("No answer found", status=404) 182 | return https_fn.Response(result, status=200) 183 | 184 | # Function to run the recurrent job 185 | @scheduler_fn.on_schedule(schedule="*/10 * * * *", memory=MemoryOption(512), timeout_sec=3600) 186 | def update_economic_calendar(req): 187 | # Set timezone if needed 188 | current_time = datetime.now() 189 | current_time.replace(tzinfo=timezone.utc) 190 | 191 | start_time = current_time - timedelta(days=1) 192 | end_time = current_time + timedelta(days=1) 193 | 194 | ingest_incoming_month_economic_calendar(db, start_time, end_time) 195 | return "Ingestion triggered", 200 196 | 197 | 198 | # Runs every hour on the hour 199 | @scheduler_fn.on_schedule(schedule="0 * * * *", memory=MemoryOption(2048), timeout_sec=3600) 200 | def update_economic_calendar_every_day_on_new_event(req): 201 | # Set timezone if needed 202 | current_time = datetime.now() 203 | current_time.replace(tzinfo=timezone.utc) 204 | 205 | one_day_before = (current_time - timedelta(days=1)).replace(hour=0, minute=0, second=0, microsecond=0) 206 | end_time = current_time + timedelta(days=8) 207 | today_end_time = current_time.replace(hour=23, minute=59, second=59, microsecond=999999) 208 | 209 | 210 | ingestor = StockData1DayIngestor(db, 'SPY') 211 | ingestor.ingest(one_day_before, today_end_time) 212 | 213 | ingest_incoming_month_economic_calendar(db, one_day_before, end_time) 214 | return "Ingestion triggered", 200 215 | 216 | # runs every 2 minutes 217 | @scheduler_fn.on_schedule(schedule="*/2 * * * 1-5", memory=MemoryOption(512), timeout_sec=3600, timezone="America/New_York") 218 | def every_2_min_jobs(req): 219 | # early exit if outside of market hours and non trading day 220 | current_time = datetime.now() 221 | current_time.replace(tzinfo=timezone.utc) 222 | one_day_before = (current_time - timedelta(days=1)).replace(hour=0, minute=0, second=0, microsecond=0) 223 | today_start_time = current_time.replace(hour=0, minute=0, second=0, microsecond=0) 224 | today_end_time = current_time.replace(hour=23, minute=59, second=59, microsecond=999999) 225 | ingestor = StockData1MinIngestor(db, 'SPY', backfill=False) 226 | ingestor.ingest(one_day_before, today_end_time) 227 | return "Ingestion triggered", 200 228 | -------------------------------------------------------------------------------- /functions/requirements-base.txt: -------------------------------------------------------------------------------- 1 | firebase_functions 2 | psycopg2-binary 3 | pydantic 4 | pydantic_core 5 | python-dotenv 6 | PyYAML 7 | requests 8 | SQLAlchemy 9 | typing_extensions 10 | urllib3 11 | openai 12 | langchain 13 | langchain-community 14 | -------------------------------------------------------------------------------- /functions/requirements.txt: -------------------------------------------------------------------------------- 1 | aiohappyeyeballs==2.6.1 2 | aiohttp==3.11.16 3 | aiosignal==1.3.2 4 | annotated-types==0.7.0 5 | anyio==4.9.0 6 | attrs==25.3.0 7 | blinker==1.9.0 8 | CacheControl==0.14.2 9 | cachetools==5.5.2 10 | certifi==2025.1.31 11 | cffi==1.17.1 12 | charset-normalizer==3.4.1 13 | click==8.1.8 14 | cloudevents==1.9.0 15 | cryptography==44.0.2 16 | dataclasses-json==0.6.7 17 | deprecation==2.1.0 18 | distro==1.9.0 19 | firebase-admin==6.7.0 20 | firebase-functions==0.4.2 21 | Flask==3.1.0 22 | flask-cors==5.0.1 23 | frozenlist==1.5.0 24 | functions-framework==3.8.2 25 | google-api-core==2.24.2 26 | google-api-python-client==2.166.0 27 | google-auth==2.38.0 28 | google-auth-httplib2==0.2.0 29 | google-cloud-core==2.4.3 30 | google-cloud-firestore==2.20.1 31 | google-cloud-storage==3.1.0 32 | google-crc32c==1.7.1 33 | google-events==0.14.0 34 | google-resumable-media==2.7.2 35 | googleapis-common-protos==1.69.2 36 | grpcio==1.72.0rc1 37 | grpcio-status==1.71.0 38 | gunicorn==23.0.0 39 | h11==0.14.0 40 | httpcore==1.0.8 41 | httplib2==0.22.0 42 | httpx==0.28.1 43 | httpx-sse==0.4.0 44 | idna==3.10 45 | itsdangerous==2.2.0 46 | Jinja2==3.1.6 47 | jiter==0.9.0 48 | jsonpatch==1.33 49 | jsonpointer==3.0.0 50 | langchain==0.3.23 51 | langchain-community==0.3.21 52 | langchain-core==0.3.51 53 | langchain-text-splitters==0.3.8 54 | langsmith==0.3.30 55 | MarkupSafe==3.0.2 56 | marshmallow==3.26.1 57 | msgpack==1.1.0 58 | multidict==6.4.3 59 | mypy-extensions==1.0.0 60 | numpy==2.2.4 61 | openai==1.73.0 62 | orjson==3.10.16 63 | packaging==24.2 64 | propcache==0.3.1 65 | proto-plus==1.26.1 66 | protobuf==5.29.4 67 | psycopg2-binary==2.9.10 68 | pyasn1==0.6.1 69 | pyasn1_modules==0.4.2 70 | pycparser==2.22 71 | pydantic==2.11.3 72 | pydantic-settings==2.8.1 73 | pydantic_core==2.33.1 74 | PyJWT==2.10.1 75 | pyparsing==3.2.3 76 | python-dotenv==1.1.0 77 | PyYAML==6.0.2 78 | requests==2.32.3 79 | requests-toolbelt==1.0.0 80 | rsa==4.9 81 | sniffio==1.3.1 82 | SQLAlchemy==2.0.40 83 | tenacity==9.1.2 84 | tqdm==4.67.1 85 | typing-inspect==0.9.0 86 | typing-inspection==0.4.0 87 | typing_extensions==4.13.2 88 | uritemplate==4.1.1 89 | urllib3==2.4.0 90 | watchdog==6.0.0 91 | Werkzeug==3.1.3 92 | yarl==1.19.0 93 | zstandard==0.23.0 94 | -------------------------------------------------------------------------------- /functions/test_agent.py: -------------------------------------------------------------------------------- 1 | import os 2 | from langchain.agents import create_sql_agent 3 | from langchain.agents.agent_toolkits import SQLDatabaseToolkit 4 | from langchain.agents.agent_types import AgentType 5 | from langchain.chat_models import ChatOpenAI 6 | from langchain.sql_database import SQLDatabase 7 | from macrosurfer.database import Database 8 | from dotenv import load_dotenv 9 | from macrosurfer.agent.query_agent import QueryAgent 10 | load_dotenv() 11 | 12 | db = Database() 13 | 14 | # Load environment variable 15 | os.environ["OPENAI_API_KEY"] = os.getenv("OPENAI_API_KEY") 16 | # Create LLM (you can switch to any model) 17 | llm = ChatOpenAI(temperature=0, model="gpt-4o") 18 | 19 | # Create SQL Agent 20 | query_agent = QueryAgent(db, llm) 21 | 22 | # Run your agent 23 | question = "What are some economic events coming up next week in the US?" 24 | response = query_agent.query(question) 25 | 26 | print(response) 27 | -------------------------------------------------------------------------------- /functions/test_ingestor.py: -------------------------------------------------------------------------------- 1 | # Example usage 2 | from datetime import datetime, timedelta 3 | from macrosurfer.data_ingestion.fmp.economics.economic_calendar_ingestor import EconomicCalendarIngestor 4 | from macrosurfer.database import Database 5 | from macrosurfer.data_ingestion.fmp.directory.company_symbol_list_ingestor import CompanySymbolListIngestor 6 | from macrosurfer.data_ingestion.fmp.directory.financial_statement_symbols_ingestor import FinancialStatementSymbolsIngestor 7 | from macrosurfer.data_ingestion.fmp.directory.cik_list_ingestor import CIKListIngestor 8 | from macrosurfer.models.fmp import * 9 | from dotenv import load_dotenv 10 | from macrosurfer.data_ingestion.fmp.stocks.stock_data_1_day_ingestor import StockData1DayIngestor 11 | from macrosurfer.data_ingestion.fmp.stocks.stock_data_1_min_ingestor import StockData1MinIngestor 12 | 13 | load_dotenv() 14 | 15 | db = Database() 16 | start_date = datetime(2023, 5, 11, 0, 0, 0) 17 | end_date = datetime(2024, 5, 10, 0, 0, 0) 18 | 19 | # ingestor = FinancialStatementSymbolsIngestor(db) 20 | # ingestor.ingest(start_date, end_date) 21 | 22 | # ingestor = CompanySymbolListIngestor(db) 23 | # ingestor.ingest(datetime.now(), datetime.now() + timedelta(days=30)) 24 | 25 | # ingestor = CIKListIngestor(db) 26 | # ingestor.ingest(datetime.now(), datetime.now() + timedelta(days=30)) 27 | 28 | ingestor = StockData1DayIngestor(db, 'SPY') 29 | ingestor.ingest(start_date, end_date) 30 | 31 | while start_date < end_date: 32 | ingestor = StockData1MinIngestor(db, 'SPY', backfill=True) 33 | ingestor.ingest(start_date, start_date + timedelta(days=1)) 34 | start_date += timedelta(days=1) 35 | 36 | -------------------------------------------------------------------------------- /images/illustration.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MacroSurfer/MacroSurferFunctions/b9d3f14f0feae4873ab85bb1b786b98892c4688f/images/illustration.jpg --------------------------------------------------------------------------------