├── lib ├── __init__.py ├── iso │ ├── __init__.py │ ├── bpa.py │ ├── nyiso.py │ └── miso.py ├── weather │ ├── __init__.py │ └── client.py ├── .DS_Store ├── framework │ ├── .DS_Store │ ├── CAISO │ │ └── .DS_Store │ └── NYISO │ │ └── .DS_Store └── .gitignore ├── tests ├── __init__.py ├── test_bpa.py ├── test_weather.py └── test_nyiso.py ├── requirements.txt ├── LICENSE ├── NOTICE ├── .github └── workflows │ └── ci.yml ├── pyproject.toml ├── QUICKSTART.md ├── MIGRATION_GUIDE.md ├── README.md └── isodart.py /lib/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /lib/iso/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /lib/weather/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /lib/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/ISO-DART/HEAD/lib/.DS_Store -------------------------------------------------------------------------------- /lib/framework/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/ISO-DART/HEAD/lib/framework/.DS_Store -------------------------------------------------------------------------------- /lib/framework/CAISO/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/ISO-DART/HEAD/lib/framework/CAISO/.DS_Store -------------------------------------------------------------------------------- /lib/framework/NYISO/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/ISO-DART/HEAD/lib/framework/NYISO/.DS_Store -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | # Core dependencies 2 | requests>=2.31.0 3 | pandas>=2.0.0 4 | numpy>=1.24.0 5 | openpyxl 6 | 7 | # Weather data 8 | meteostat>=1.6.0 9 | 10 | # Data validation and parsing 11 | python-dateutil>=2.8.0 12 | 13 | # Configuration 14 | pyyaml>=6.0 15 | 16 | # Development dependencies (optional) 17 | pytest>=7.4.0 18 | pytest-cov>=4.1.0 19 | black>=23.0.0 20 | flake8>=6.0.0 21 | mypy>=1.5.0 22 | types-requests>=2.31.0 23 | 24 | # Documentation (optional) 25 | sphinx>=7.0.0 26 | sphinx-rtd-theme>=1.3.0 27 | -------------------------------------------------------------------------------- /lib/.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files / IDE files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | *~ 6 | .ant/ 7 | .idea/ 8 | .spyderproject 9 | .spyproject 10 | 11 | # Distribution / packaging 12 | build/ 13 | dist/ 14 | DLLs/ 15 | Debug/ 16 | Release/ 17 | *.class 18 | *.dll 19 | *.egg-info/ 20 | eggs/ 21 | .eggs/ 22 | *.eps 23 | *.exe 24 | *.lib 25 | *.suo 26 | *.zip 27 | 28 | # OSX 29 | *.DS_Store 30 | 31 | # Wix files 32 | files.wixobj 33 | files.wxs 34 | installer.wixobj 35 | version.wxi 36 | 37 | # Pytest 38 | pytest.log 39 | 40 | # Sphinx 41 | docs/sphinx-build.log 42 | 43 | # Configuration files 44 | *.ini -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2025, Lawrence Livermore National Security, LLC 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | This work was produced under the auspices of the U.S. Department of 2 | Energy by Lawrence Livermore National Laboratory under Contract 3 | DE-AC52-07NA27344. 4 | 5 | This work was prepared as an account of work sponsored by an agency of 6 | the United States Government. Neither the United States Government nor 7 | Lawrence Livermore National Security, LLC, nor any of their employees 8 | makes any warranty, expressed or implied, or assumes any legal liability 9 | or responsibility for the accuracy, completeness, or usefulness of any 10 | information, apparatus, product, or process disclosed, or represents that 11 | its use would not infringe privately owned rights. 12 | 13 | Reference herein to any specific commercial product, process, or service 14 | by trade name, trademark, manufacturer, or otherwise does not necessarily 15 | constitute or imply its endorsement, recommendation, or favoring by the 16 | United States Government or Lawrence Livermore National Security, LLC. 17 | 18 | The views and opinions of authors expressed herein do not necessarily 19 | state or reflect those of the United States Government or Lawrence 20 | Livermore National Security, LLC, and shall not be used for advertising 21 | or product endorsement purposes. 22 | 23 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI/CD Pipeline 2 | 3 | on: 4 | push: 5 | branches: [ main, dev ] 6 | pull_request: 7 | branches: [ main, dev ] 8 | schedule: 9 | # Run weekly on Monday at 6 AM UTC 10 | - cron: '0 6 * * 1' 11 | 12 | jobs: 13 | test: 14 | name: Test Python ${{ matrix.python-version }} 15 | runs-on: ${{ matrix.os }} 16 | strategy: 17 | fail-fast: false 18 | matrix: 19 | os: [ubuntu-latest, macos-latest, windows-latest] 20 | python-version: ['3.10', '3.11', '3.12'] 21 | 22 | steps: 23 | - name: Checkout code 24 | uses: actions/checkout@v4 25 | 26 | - name: Set up Python ${{ matrix.python-version }} 27 | uses: actions/setup-python@v4 28 | with: 29 | python-version: ${{ matrix.python-version }} 30 | cache: 'pip' 31 | 32 | - name: Install dependencies 33 | run: | 34 | python -m pip install --upgrade pip 35 | pip install -r requirements.txt 36 | 37 | - name: Lint with flake8 38 | run: | 39 | # Stop the build if there are Python syntax errors or undefined names 40 | flake8 lib/ tests/ --count --select=E9,F63,F7,F82 --show-source --statistics 41 | # Exit-zero treats all errors as warnings 42 | flake8 lib/ tests/ --count --exit-zero --max-complexity=10 --max-line-length=100 --statistics 43 | 44 | - name: Check formatting with black 45 | run: | 46 | black --check lib/ tests/ 47 | 48 | - name: Type check with mypy 49 | run: | 50 | mypy lib/ || true # Don't fail on type errors for now 51 | 52 | - name: Run tests 53 | run: | 54 | pytest tests/ -v --cov=lib --cov-branch --cov-report=xml --cov-report=term-missing -m "not integration" 55 | 56 | - name: Upload coverage to Codecov 57 | uses: codecov/codecov-action@v5 58 | with: 59 | token: ${{ secrets.CODECOV_TOKEN }} 60 | files: ./coverage.xml 61 | flags: unittests 62 | name: codecov-umbrella 63 | fail_ci_if_error: false 64 | verbose: true 65 | 66 | integration-test: 67 | name: Integration Tests 68 | runs-on: ubuntu-latest 69 | # Only run integration tests on dev branch or manual trigger 70 | if: github.ref == 'refs/heads/dev' || github.event_name == 'workflow_dispatch' 71 | 72 | steps: 73 | - name: Checkout code 74 | uses: actions/checkout@v4 75 | 76 | - name: Set up Python 77 | uses: actions/setup-python@v4 78 | with: 79 | python-version: '3.11' 80 | cache: 'pip' 81 | 82 | - name: Install dependencies 83 | run: | 84 | python -m pip install --upgrade pip 85 | pip install -r requirements.txt 86 | 87 | - name: Run integration tests 88 | run: | 89 | pytest tests/ -v -m integration --maxfail=3 90 | continue-on-error: true # Don't fail if API is temporarily down 91 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools>=65.0", "wheel"] 3 | build-backend = "setuptools.build_meta" 4 | 5 | [project] 6 | name = "isodart" 7 | version = "2.0.0" 8 | description = "Independent System Operator Data Automated Request Tool" 9 | readme = "README.md" 10 | license = {text = "MIT"} 11 | authors = [ 12 | {name = "Pedro Sotorrio", email = "sotorrio1@llnl.gov"}, 13 | {name = "Thomas Edmunds"}, 14 | {name = "Amelia Musselman"}, 15 | {name = "Chih-Che Sun"} 16 | ] 17 | maintainers = [ 18 | {name = "Lawrence Livermore National Laboratory"} 19 | ] 20 | keywords = [ 21 | "energy", 22 | "electricity", 23 | "iso", 24 | "caiso", 25 | "miso", 26 | "nyiso", 27 | "bpa", 28 | "spp", 29 | "data", 30 | "weather", 31 | "solar" 32 | ] 33 | classifiers = [ 34 | "Development Status :: 4 - Beta", 35 | "Intended Audience :: Science/Research", 36 | "License :: OSI Approved :: MIT License", 37 | "Programming Language :: Python :: 3", 38 | "Programming Language :: Python :: 3.10", 39 | "Programming Language :: Python :: 3.11", 40 | "Programming Language :: Python :: 3.12", 41 | "Topic :: Scientific/Engineering", 42 | "Topic :: Software Development :: Libraries :: Python Modules" 43 | ] 44 | requires-python = ">=3.10" 45 | dependencies = [ 46 | "requests>=2.31.0", 47 | "pandas>=2.0.0", 48 | "numpy>=1.24.0", 49 | "openpyxl", 50 | "meteostat>=1.6.0", 51 | "python-dateutil>=2.8.0", 52 | "pyyaml>=6.0" 53 | ] 54 | 55 | [project.optional-dependencies] 56 | dev = [ 57 | "pytest>=7.4.0", 58 | "pytest-cov>=4.1.0", 59 | "black>=23.0.0", 60 | "flake8>=6.0.0", 61 | "mypy>=1.5.0", 62 | "types-requests>=2.31.0", 63 | "types-pyyaml>=6.0" 64 | ] 65 | docs = [ 66 | "sphinx>=7.0.0", 67 | "sphinx-rtd-theme>=1.3.0", 68 | "sphinxcontrib-programoutput>=0.17" 69 | ] 70 | 71 | [project.urls] 72 | Homepage = "https://github.com/LLNL/ISO-DART" 73 | Documentation = "https://iso-dart.readthedocs.io" 74 | Repository = "https://github.com/LLNL/ISO-DART" 75 | Issues = "https://github.com/LLNL/ISO-DART/issues" 76 | 77 | [project.scripts] 78 | isodart = "isodart:main" 79 | 80 | [tool.setuptools] 81 | packages = ["lib", "lib.iso", "lib.weather"] 82 | 83 | [tool.setuptools.package-data] 84 | lib = ["py.typed"] 85 | 86 | [tool.black] 87 | line-length = 100 88 | target-version = ['py310', 'py311', 'py312'] 89 | include = '\.pyi?$' 90 | exclude = ''' 91 | /( 92 | \.eggs 93 | | \.git 94 | | \.hg 95 | | \.mypy_cache 96 | | \.tox 97 | | \.venv 98 | | _build 99 | | buck-out 100 | | build 101 | | dist 102 | | raw_data 103 | | data 104 | )/ 105 | ''' 106 | 107 | [tool.pytest.ini_options] 108 | testpaths = ["tests"] 109 | python_files = ["test_*.py"] 110 | python_classes = ["Test*"] 111 | python_functions = ["test_*"] 112 | addopts = [ 113 | "--verbose", 114 | "--cov=lib", 115 | "--cov-report=html", 116 | "--cov-report=term-missing" 117 | ] 118 | markers = [ 119 | "integration: marks tests as integration tests (deselect with '-m \"not integration\"')", 120 | "slow: marks tests as slow (deselect with '-m \"not slow\"')" 121 | ] 122 | 123 | [tool.mypy] 124 | python_version = "3.10" 125 | warn_return_any = true 126 | warn_unused_configs = true 127 | disallow_untyped_defs = true 128 | disallow_incomplete_defs = true 129 | check_untyped_defs = true 130 | no_implicit_optional = true 131 | warn_redundant_casts = true 132 | warn_unused_ignores = true 133 | warn_no_return = true 134 | strict_equality = true 135 | 136 | [[tool.mypy.overrides]] 137 | module = [ 138 | "meteostat.*", 139 | ] 140 | ignore_missing_imports = true 141 | 142 | [tool.flake8] 143 | max-line-length = 100 144 | extend-ignore = ["E203", "W503"] 145 | exclude = [ 146 | ".git", 147 | "__pycache__", 148 | "build", 149 | "dist", 150 | ".eggs", 151 | "*.egg-info", 152 | ".venv", 153 | "venv", 154 | "raw_data", 155 | "data" 156 | ] 157 | 158 | [tool.coverage.run] 159 | source = ["lib"] 160 | omit = [ 161 | "*/tests/*", 162 | "*/__pycache__/*", 163 | "*/site-packages/*", 164 | "*/lib/interactive.py" 165 | ] 166 | 167 | [tool.coverage.report] 168 | exclude_lines = [ 169 | "pragma: no cover", 170 | "def __repr__", 171 | "raise AssertionError", 172 | "raise NotImplementedError", 173 | "if __name__ == .__main__.:", 174 | "if TYPE_CHECKING:", 175 | "class .*\\bProtocol\\):", 176 | "@(abc\\.)?abstractmethod" 177 | ] 178 | -------------------------------------------------------------------------------- /QUICKSTART.md: -------------------------------------------------------------------------------- 1 | # ISO-DART v2.0 Quick Start Guide 2 | 3 | Get up and running with ISO-DART in 5 minutes! 4 | 5 | ## 🚀 Installation (2 minutes) 6 | 7 | ```bash 8 | # 1. Clone the repository 9 | git clone https://github.com/LLNL/ISO-DART.git 10 | cd ISO-DART 11 | 12 | # 2. Create virtual environment (recommended) 13 | python -m venv venv 14 | source venv/bin/activate # Windows: venv\Scripts\activate 15 | 16 | # 3. Install dependencies 17 | pip install -r requirements.txt 18 | 19 | # 4. Verify installation 20 | python isodart.py --help 21 | ``` 22 | 23 | ## 📊 Your First Download (3 minutes) 24 | 25 | ### Option 1: Interactive Mode (Easiest) 26 | 27 | ```bash 28 | python isodart.py 29 | ``` 30 | 31 | Then follow the prompts: 32 | 1. Choose "1" for ISO Data 33 | 2. Choose "1" for CAISO 34 | 3. Choose "1" for Pricing Data 35 | 4. Choose "1" for LMP 36 | 5. Choose "1" for Day-Ahead Market 37 | 6. Enter date range (e.g., today minus 7 days) 38 | 39 | **Done!** Your data is in `data/CAISO/` 40 | 41 | ### Option 2: Command Line (Fastest) 42 | 43 | ```bash 44 | # Download last week's Day-Ahead LMP data from CAISO 45 | python isodart.py --iso caiso --data-type lmp --market dam \ 46 | --start 2024-01-01 --duration 7 47 | ``` 48 | 49 | **Done!** Check `data/CAISO/` for your CSV files. 50 | 51 | ## 📈 Using Your Data 52 | 53 | ### Load and Visualize 54 | 55 | ```python 56 | import pandas as pd 57 | import matplotlib.pyplot as plt 58 | 59 | # Load the data 60 | df = pd.read_csv('data/CAISO/20240101_to_20240107_PRC_LMP_TH_NP15_GEN-APND.csv') 61 | 62 | # Convert to datetime 63 | df['OPR_DT'] = pd.to_datetime(df['OPR_DATE']) 64 | 65 | # Plot prices over time 66 | plt.figure(figsize=(12, 6)) 67 | plt.plot(df['OPR_DT'], df['VALUE']) 68 | plt.xlabel('Date') 69 | plt.ylabel('Price ($/MWh)') 70 | plt.title('Day-Ahead LMP - NP15 Generator') 71 | plt.xticks(rotation=45) 72 | plt.tight_layout() 73 | plt.savefig('lmp_plot.png') 74 | plt.show() 75 | ``` 76 | 77 | ### Basic Analysis 78 | 79 | ```python 80 | import pandas as pd 81 | 82 | # Load data 83 | df = pd.read_csv('data/CAISO/20240101_to_20240107_PRC_LMP_TH_NP15_GEN-APND.csv') 84 | 85 | # Summary statistics 86 | print("Price Statistics:") 87 | print(f" Mean: ${df['VALUE'].mean():.2f}/MWh") 88 | print(f" Min: ${df['VALUE'].min():.2f}/MWh") 89 | print(f" Max: ${df['VALUE'].max():.2f}/MWh") 90 | print(f" Std: ${df['VALUE'].std():.2f}/MWh") 91 | 92 | # Find peak price hours 93 | peak_hours = df.nlargest(10, 'VALUE')[['OPR_DATE', 'INTERVAL_NUM', 'VALUE']] 94 | print("\nTop 10 Peak Price Hours:") 95 | print(peak_hours) 96 | ``` 97 | 98 | ## 🌤️ Weather Data 99 | 100 | ```bash 101 | # Download weather data for California 102 | python isodart.py --data-type weather --state CA \ 103 | --start 2024-01-01 --duration 30 104 | ``` 105 | 106 | Then select your weather station from the list. 107 | 108 | ### Analyze Weather Data 109 | 110 | ```python 111 | import pandas as pd 112 | import matplotlib.pyplot as plt 113 | 114 | # Load weather data 115 | df = pd.read_csv('data/weather/2024-01-01_to_2024-01-31_San_Francisco_CA.csv', 116 | index_col='time', parse_dates=True) 117 | 118 | # Plot temperature 119 | fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(12, 8)) 120 | 121 | # Temperature 122 | ax1.plot(df.index, df['temperature']) 123 | ax1.set_ylabel('Temperature (°F)') 124 | ax1.set_title('Temperature Over Time') 125 | ax1.grid(True) 126 | 127 | # Wind Speed 128 | ax2.plot(df.index, df['wind_speed']) 129 | ax2.set_ylabel('Wind Speed (mph)') 130 | ax2.set_xlabel('Date') 131 | ax2.set_title('Wind Speed Over Time') 132 | ax2.grid(True) 133 | 134 | plt.tight_layout() 135 | plt.savefig('weather_analysis.png') 136 | plt.show() 137 | ``` 138 | 139 | ## 🔄 Common Workflows 140 | 141 | ### 1. Download Multiple Markets 142 | 143 | ```python 144 | from datetime import date 145 | from lib.iso.caiso import CAISOClient, Market 146 | 147 | client = CAISOClient() 148 | 149 | start = date(2024, 1, 1) 150 | end = date(2024, 1, 31) 151 | 152 | # Download multiple markets 153 | for market in [Market.DAM, Market.RTM]: 154 | print(f"Downloading {market.value}...") 155 | client.get_lmp(market, start, end) 156 | 157 | print("All downloads complete!") 158 | client.cleanup() 159 | ``` 160 | 161 | ### 2. Automated Daily Download 162 | 163 | Save as `daily_download.py`: 164 | 165 | ```python 166 | #!/usr/bin/env python 167 | """Download yesterday's CAISO data automatically.""" 168 | from datetime import date, timedelta 169 | from lib.iso.caiso import CAISOClient, Market 170 | import logging 171 | 172 | logging.basicConfig(level=logging.INFO) 173 | 174 | def download_yesterday(): 175 | yesterday = date.today() - timedelta(days=1) 176 | 177 | client = CAISOClient() 178 | success = client.get_lmp(Market.DAM, yesterday, yesterday) 179 | 180 | if success: 181 | print(f"✓ Downloaded data for {yesterday}") 182 | else: 183 | print(f"✗ Failed to download data for {yesterday}") 184 | 185 | client.cleanup() 186 | 187 | if __name__ == '__main__': 188 | download_yesterday() 189 | ``` 190 | 191 | Run daily with cron (Linux/Mac): 192 | ```bash 193 | # Edit crontab 194 | crontab -e 195 | 196 | # Add line to run daily at 2 AM 197 | 0 2 * * * cd /path/to/ISO-DART && /path/to/venv/bin/python daily_download.py 198 | ``` 199 | 200 | ### 3. Compare Multiple Locations 201 | 202 | ```python 203 | import pandas as pd 204 | import matplotlib.pyplot as plt 205 | 206 | # Load data for multiple locations 207 | locations = ['TH_NP15_GEN-APND', 'TH_SP15_GEN-APND', 'TH_ZP26_GEN-APND'] 208 | data = {} 209 | 210 | for loc in locations: 211 | file = f'data/CAISO/20240101_to_20240107_PRC_LMP_{loc}.csv' 212 | df = pd.read_csv(file) 213 | df['OPR_DT'] = pd.to_datetime(df['OPR_DATE']) 214 | data[loc] = df 215 | 216 | # Plot comparison 217 | plt.figure(figsize=(14, 6)) 218 | for loc, df in data.items(): 219 | plt.plot(df['OPR_DT'], df['VALUE'], label=loc, alpha=0.7) 220 | 221 | plt.xlabel('Date') 222 | plt.ylabel('Price ($/MWh)') 223 | plt.title('LMP Comparison Across Locations') 224 | plt.legend() 225 | plt.grid(True, alpha=0.3) 226 | plt.xticks(rotation=45) 227 | plt.tight_layout() 228 | plt.savefig('location_comparison.png') 229 | plt.show() 230 | ``` 231 | 232 | ## 🔍 Troubleshooting 233 | 234 | ### Issue: "ModuleNotFoundError" 235 | ```bash 236 | # Make sure you're in the ISO-DART directory 237 | cd /path/to/ISO-DART 238 | 239 | # And virtual environment is activated 240 | source venv/bin/activate # or venv\Scripts\activate on Windows 241 | ``` 242 | 243 | ### Issue: "No data returned" 244 | - Check your date range (not in future) 245 | - Verify CAISO OASIS is online 246 | - Try a smaller date range 247 | - Use `--verbose` flag for details 248 | 249 | ### Issue: Slow downloads 250 | - This is normal for large date ranges 251 | - CAISO API can be slow during peak hours 252 | - Consider downloading during off-hours 253 | - Use smaller `step_size` values 254 | 255 | ## 📚 Next Steps 256 | 257 | 1. **Read the full README.md** for comprehensive documentation 258 | 2. **Check examples/** folder for Jupyter notebooks 259 | 3. **Run tests** to verify your installation: 260 | ```bash 261 | pip install pytest 262 | pytest tests/ -v 263 | ``` 264 | 4. **Join the discussion** on GitHub for questions 265 | 266 | ## 💡 Pro Tips 267 | 268 | 1. **Always activate your virtual environment** before running 269 | 2. **Use `--verbose`** when debugging 270 | 3. **Check logs/** folder for detailed operation logs 271 | 4. **Clean up** with `client.cleanup()` to save disk space 272 | 5. **Backup your data** regularly from `data/` directory 273 | 274 | ## 🎯 Useful Commands 275 | 276 | ```bash 277 | # Check what data you've downloaded 278 | ls -lh data/CAISO/ 279 | ls -lh data/weather/ 280 | 281 | # See logs 282 | tail -f logs/isodart.log 283 | 284 | # Clean up old raw data 285 | rm -rf raw_data/ 286 | 287 | # Get help 288 | python isodart.py --help 289 | 290 | # Update dependencies 291 | pip install --upgrade -r requirements.txt 292 | ``` 293 | 294 | ## 🤝 Need Help? 295 | 296 | - **GitHub Issues**: https://github.com/LLNL/ISO-DART/issues 297 | - **Email**: Contact LLNL support 298 | - **Documentation**: Full README.md in repository 299 | 300 | ## ⚡ Power User Shortcut 301 | 302 | Create an alias in your shell: 303 | 304 | ```bash 305 | # Add to ~/.bashrc or ~/.zshrc 306 | alias isodart='cd /path/to/ISO-DART && source venv/bin/activate && python isodart.py' 307 | 308 | # Then just run: 309 | isodart --iso caiso --data-type lmp --market dam --start 2024-01-01 --duration 7 310 | ``` 311 | 312 | Happy data downloading! 🚀 313 | -------------------------------------------------------------------------------- /MIGRATION_GUIDE.md: -------------------------------------------------------------------------------- 1 | # ISO-DART Migration Guide: v1.1 → v2.0 2 | 3 | ## Overview 4 | 5 | This guide helps you migrate from ISO-DART v1.1 to v2.0, highlighting breaking changes and new features. 6 | 7 | ## 🎯 Quick Summary 8 | 9 | | Aspect | v1.1 | v2.0 | 10 | |--------|------|------| 11 | | Python Version | 3.8+ | 3.10+ | 12 | | Entry Point | `ISODART.py` | `isodart.py` | 13 | | Architecture | Scripts with `exec()` | Modern modules | 14 | | CLI | Basic prompts | argparse + interactive | 15 | | Error Handling | `sys.exit()` | Exceptions + logging | 16 | | Testing | None | Comprehensive pytest suite | 17 | | Type Safety | None | Type hints throughout | 18 | | Documentation | Basic README | Full docs + examples | 19 | 20 | ## 🚀 Migration Steps 21 | 22 | ### 1. Backup Your Data 23 | 24 | ```bash 25 | # Backup existing data 26 | cp -r data data_v1_backup 27 | cp -r raw_data raw_data_v1_backup 2>/dev/null || true 28 | 29 | # Backup any custom scripts 30 | cp mainCAISO.py mainCAISO_v1.py.bak 2>/dev/null || true 31 | ``` 32 | 33 | ### 2. Update Python Version 34 | 35 | ```bash 36 | # Check your Python version 37 | python --version 38 | 39 | # If < 3.10, install Python 3.10 or higher 40 | # Ubuntu/Debian: 41 | sudo apt update 42 | sudo apt install python3.10 python3.10-venv 43 | 44 | # macOS (using Homebrew): 45 | brew install python@3.10 46 | 47 | # Windows: Download from python.org 48 | ``` 49 | 50 | ### 3. Install v2.0 51 | 52 | ```bash 53 | # Pull latest changes 54 | git fetch origin 55 | git checkout v2.0 # or main branch 56 | 57 | # Create new virtual environment 58 | python3.10 -m venv venv 59 | source venv/bin/activate # Windows: venv\Scripts\activate 60 | 61 | # Install dependencies 62 | pip install -r requirements.txt 63 | ``` 64 | 65 | ### 4. Verify Installation 66 | 67 | ```bash 68 | # Test the installation 69 | python isodart.py --help 70 | 71 | # Run a quick test 72 | python isodart.py --iso caiso --data-type lmp --market dam \ 73 | --start 2024-01-01 --duration 1 74 | ``` 75 | 76 | ## 🔄 Code Migration 77 | 78 | ### Old Way (v1.1) 79 | 80 | ```python 81 | # mainCAISO.py - Run with exec() 82 | exec(open("mainCAISO.py").read()) 83 | 84 | # Direct date input with loops 85 | ind = 1 86 | while ind == 1: 87 | month = int(input('Month: ')) 88 | day = int(input('Day: ')) 89 | year = int(input('Year (4-digit format): ')) 90 | try: 91 | datetime.datetime(year=year, month=month, day=day) 92 | ind = 0 93 | except: 94 | print('\nWARNING: The Date Does NOT Exist. Please Try Again!!') 95 | 96 | # Query with hardcoded parameters 97 | DAM_LMP().get_csv(start, end, step_size=step_size) 98 | ``` 99 | 100 | ### New Way (v2.0) 101 | 102 | ```python 103 | # isodart.py - Proper module imports 104 | from datetime import date 105 | from lib.iso.caiso import CAISOClient, Market 106 | 107 | # Clean date validation 108 | start_date = date(2024, 1, 1) 109 | end_date = date(2024, 1, 31) 110 | 111 | # Type-safe client usage 112 | client = CAISOClient() 113 | success = client.get_lmp( 114 | market=Market.DAM, 115 | start_date=start_date, 116 | end_date=end_date 117 | ) 118 | 119 | if success: 120 | print("Download successful!") 121 | else: 122 | print("Download failed, check logs") 123 | 124 | client.cleanup() 125 | ``` 126 | 127 | ## 📋 Breaking Changes 128 | 129 | ### 1. File Structure 130 | 131 | **Old:** 132 | ``` 133 | iso-dart/ 134 | ├── ISODART.py 135 | ├── mainCAISO.py 136 | ├── mainMISO.py 137 | ├── mainNYISO.py 138 | ├── mainWeather.py 139 | └── lib/framework/... 140 | ``` 141 | 142 | **New:** 143 | ``` 144 | iso-dart/ 145 | ├── isodart.py # Main entry point 146 | ├── lib/ 147 | │ ├── iso/ 148 | │ │ ├── caiso.py # CAISO module 149 | │ │ ├── miso.py # MISO module 150 | │ │ └── nyiso.py # NYISO module 151 | │ ├── weather/ 152 | │ │ └── client.py # Weather module 153 | │ └── interactive.py # Interactive mode 154 | └── tests/ # Test suite 155 | ``` 156 | 157 | **Migration:** No action needed - v2.0 creates new structure automatically. 158 | 159 | ### 2. Import Changes 160 | 161 | **Old:** 162 | 163 | ```python 164 | from lib.iso.CAISO.query import * 165 | from lib.iso.CAISO.tool_utils import * 166 | ``` 167 | 168 | **New:** 169 | ```python 170 | from lib.iso.caiso import CAISOClient, Market, ReportVersion 171 | ``` 172 | 173 | ### 3. Query Class Changes 174 | 175 | **Old:** 176 | ```python 177 | # Direct class instantiation 178 | lmp = DAM_LMP() 179 | lmp.get_csv(start, end, step_size=1) 180 | 181 | # Class inheritance based on market 182 | class DAM_LMP(LMP): 183 | name = 'PRC_LMP' 184 | market = 'DAM' 185 | ``` 186 | 187 | **New:** 188 | ```python 189 | # Client-based approach 190 | client = CAISOClient() 191 | client.get_lmp(Market.DAM, start_date, end_date) 192 | 193 | # Enum-based markets 194 | market = Market.DAM # Type-safe enum 195 | ``` 196 | 197 | ### 4. Error Handling 198 | 199 | **Old:** 200 | ```python 201 | if errDetector == 1: 202 | sys.exit() # Abrupt termination 203 | ``` 204 | 205 | **New:** 206 | ```python 207 | try: 208 | success = client.get_lmp(...) 209 | if not success: 210 | logger.error("Download failed") 211 | # Handle error gracefully 212 | except Exception as e: 213 | logger.error(f"Error: {e}", exc_info=True) 214 | # Continue or retry 215 | ``` 216 | 217 | ### 5. Configuration 218 | 219 | **Old:** 220 | ```python 221 | # Hardcoded in files 222 | URL = 'http://oasis.caiso.com/oasisapi/SingleZip' 223 | DATA_DIR = os.path.join(os.getcwd(), 'data') 224 | ``` 225 | 226 | **New:** 227 | ```python 228 | # Configurable via dataclass 229 | from lib.iso.caiso import CAISOConfig 230 | 231 | config = CAISOConfig( 232 | base_url='http://oasis.caiso.com/oasisapi/SingleZip', 233 | data_dir=Path('custom/data/dir'), 234 | max_retries=5 235 | ) 236 | client = CAISOClient(config=config) 237 | ``` 238 | 239 | ## 🆕 New Features 240 | 241 | ### 1. Command-Line Interface 242 | 243 | ```bash 244 | # v2.0 supports command-line arguments 245 | python isodart.py --iso caiso --data-type lmp --market dam \ 246 | --start 2024-01-01 --duration 30 --verbose 247 | 248 | # Still supports interactive mode 249 | python isodart.py --interactive 250 | ``` 251 | 252 | ### 2. Logging 253 | 254 | ```python 255 | import logging 256 | 257 | # Enable verbose logging 258 | logging.basicConfig(level=logging.DEBUG) 259 | 260 | # Or check logs directory 261 | tail -f logs/isodart.log 262 | ``` 263 | 264 | ### 3. Type Safety 265 | 266 | ```python 267 | from datetime import date 268 | from lib.iso.caiso import CAISOClient, Market 269 | 270 | # IDE autocomplete and type checking work! 271 | client: CAISOClient = CAISOClient() 272 | market: Market = Market.DAM 273 | start: date = date(2024, 1, 1) 274 | ``` 275 | 276 | ### 4. Testing 277 | 278 | ```bash 279 | # Run test suite 280 | pytest tests/ -v 281 | 282 | # Test specific functionality 283 | pytest tests/test_caiso.py::TestCAISOClient::test_get_lmp -v 284 | 285 | # Check coverage 286 | pytest tests/ --cov=iso --cov-report=html 287 | ``` 288 | 289 | ### 5. Better Error Messages 290 | 291 | **Old:** 292 | ``` 293 | WARNING!! ERROR CODE:404 Data not found 294 | Program End!! Please Try Again. 295 | ``` 296 | 297 | **New:** 298 | ``` 299 | ERROR - API Error 404: Data not found for date range 2024-01-01 to 2024-01-31 300 | INFO - Retrying request (attempt 2/3)... 301 | INFO - Consider checking CAISO OASIS website for data availability 302 | ``` 303 | 304 | ## 🔧 Custom Script Migration 305 | 306 | ### Example: Automated Daily Download 307 | 308 | **Old Script (v1.1):** 309 | ```python 310 | # daily_download_v1.py 311 | import sys 312 | sys.path.append('/path/to/iso-dart') 313 | exec(open("mainCAISO.py").read()) 314 | # Then manually input values... 315 | ``` 316 | 317 | **New Script (v2.0):** 318 | ```python 319 | # daily_download_v2.py 320 | from datetime import date, timedelta 321 | from lib.iso.caiso import CAISOClient, Market 322 | import logging 323 | 324 | logging.basicConfig( 325 | level=logging.INFO, 326 | filename='daily_download.log' 327 | ) 328 | 329 | def download_yesterday(): 330 | """Download yesterday's CAISO DAM LMP data.""" 331 | yesterday = date.today() - timedelta(days=1) 332 | 333 | client = CAISOClient() 334 | try: 335 | success = client.get_lmp(Market.DAM, yesterday, yesterday) 336 | if success: 337 | logging.info(f"✓ Downloaded {yesterday}") 338 | else: 339 | logging.error(f"✗ Failed {yesterday}") 340 | except Exception as e: 341 | logging.error(f"Error: {e}", exc_info=True) 342 | finally: 343 | client.cleanup() 344 | 345 | if __name__ == '__main__': 346 | download_yesterday() 347 | ``` 348 | 349 | ### Example: Multi-Market Download 350 | 351 | **Old Script:** 352 | ```python 353 | # Complex nested if/else structure 354 | if market == 1: 355 | DAM_LMP().get_csv(start, end, step_size=step_size) 356 | elif market == 2: 357 | HASP_LMP().get_csv(start, end, step_size=step_size) 358 | # ... many more lines 359 | ``` 360 | 361 | **New Script:** 362 | ```python 363 | from datetime import date 364 | from lib.iso.caiso import CAISOClient, Market 365 | 366 | client = CAISOClient() 367 | start = date(2024, 1, 1) 368 | end = date(2024, 1, 31) 369 | 370 | # Clean iteration 371 | for market in [Market.DAM, Market.HASP, Market.RTM]: 372 | print(f"Downloading {market.value}...") 373 | client.get_lmp(market, start, end) 374 | 375 | client.cleanup() 376 | ``` 377 | 378 | ## 📊 Data Compatibility 379 | 380 | Good news: **Your existing data files are compatible!** 381 | 382 | - CSV format unchanged 383 | - Column names unchanged 384 | - Directory structure can coexist 385 | 386 | ```bash 387 | # Old and new data can live together 388 | data/ 389 | ├── CAISO/ 390 | │ ├── 20240101_to_20240131_PRC_LMP_*.csv # Old format (still works) 391 | │ └── 20240201_to_20240228_PRC_LMP_*.csv # New format (same structure) 392 | ``` 393 | 394 | ## ⚠️ Known Issues & Solutions 395 | 396 | ### Issue 1: `pdb.set_trace()` removed 397 | 398 | **Symptom:** Old scripts that relied on debugging breakpoints won't pause 399 | **Solution:** Use proper logging or IDE debugger 400 | 401 | ### Issue 2: `exec(open().read())` not supported 402 | 403 | **Symptom:** Old script execution pattern doesn't work 404 | **Solution:** Use imports instead 405 | 406 | ```python 407 | # Old 408 | exec(open("mainCAISO.py").read()) 409 | 410 | # New 411 | from lib.iso.caiso import CAISOClient 412 | client = CAISOClient() 413 | ``` 414 | 415 | ### Issue 3: Different error handling 416 | 417 | **Symptom:** Scripts that caught `sys.exit()` won't work 418 | **Solution:** Catch proper exceptions 419 | 420 | ```python 421 | # Old 422 | try: 423 | exec(open("mainCAISO.py").read()) 424 | except SystemExit: 425 | print("Failed") 426 | 427 | # New 428 | from lib.iso.caiso import CAISOClient 429 | try: 430 | client = CAISOClient() 431 | success = client.get_lmp(...) 432 | if not success: 433 | print("Failed") 434 | except Exception as e: 435 | print(f"Error: {e}") 436 | ``` 437 | 438 | ## 🎓 Learning Resources 439 | 440 | ### For New v2.0 Features 441 | 442 | 1. **Quick Start:** Read `QUICKSTART.md` 443 | 2. **Full Docs:** Read `README.md` 444 | 3. **Examples:** Check `examples/` directory 445 | 4. **API Docs:** Run `python -m pydoc lib.iso.caiso` 446 | 447 | ### For Python Modernization 448 | 449 | - [Type Hints](https://docs.python.org/3/library/typing.html) 450 | - [Dataclasses](https://docs.python.org/3/library/dataclasses.html) 451 | - [Enums](https://docs.python.org/3/library/enum.html) 452 | - [Logging](https://docs.python.org/3/library/logging.html) 453 | 454 | ## 📞 Getting Help 455 | 456 | ### During Migration 457 | 458 | 1. Check this migration guide 459 | 2. Review `QUICKSTART.md` for basic usage 460 | 3. Check GitHub Issues for similar problems 461 | 4. Create new issue with `[Migration]` tag 462 | 463 | ### Common Migration Questions 464 | 465 | **Q: Do I need to re-download all my data?** 466 | A: No! Existing data files are compatible. 467 | 468 | **Q: Can I run v1.1 and v2.0 side-by-side?** 469 | A: Yes, in different directories or virtual environments. 470 | 471 | **Q: Will my cron jobs break?** 472 | A: Yes, update them to use new CLI interface (see examples above). 473 | 474 | **Q: What about my custom analysis scripts?** 475 | A: They'll work fine - CSV format is unchanged. 476 | 477 | ## ✅ Migration Checklist 478 | 479 | - [ ] Backup existing data and scripts 480 | - [ ] Update Python to 3.10+ 481 | - [ ] Create new virtual environment 482 | - [ ] Install v2.0 dependencies 483 | - [ ] Test basic download functionality 484 | - [ ] Migrate custom scripts (if any) 485 | - [ ] Update cron jobs/automation 486 | - [ ] Update documentation/notes 487 | - [ ] Test integrated workflows 488 | - [ ] Archive v1.1 scripts (optional) 489 | 490 | ## 🎉 You're Ready! 491 | 492 | Once you've completed the migration, you'll have: 493 | 494 | ✅ Modern, maintainable code 495 | ✅ Better error handling 496 | ✅ Type safety and IDE support 497 | ✅ Comprehensive testing 498 | ✅ Flexible CLI and API 499 | ✅ Active development and support 500 | 501 | Welcome to ISO-DART v2.0! 502 | -------------------------------------------------------------------------------- /lib/weather/client.py: -------------------------------------------------------------------------------- 1 | """ 2 | Weather data client for ISO-DART v2.0 3 | 4 | Integrates Meteostat for weather data and NSRDB for solar data. 5 | """ 6 | 7 | from typing import Optional, Dict 8 | from datetime import datetime, timedelta, date 9 | from pathlib import Path 10 | import logging 11 | import webbrowser 12 | import configparser 13 | import pandas as pd 14 | from meteostat import Point, Hourly, Stations, units 15 | 16 | logger = logging.getLogger(__name__) 17 | 18 | 19 | class WeatherClient: 20 | """Client for retrieving weather and solar data.""" 21 | 22 | def __init__(self, data_dir: Optional[Path] = None, solar_dir: Optional[Path] = None): 23 | self.data_dir = data_dir or Path("data/weather") 24 | self.solar_dir = solar_dir or Path("data/solar") 25 | self.data_dir.mkdir(parents=True, exist_ok=True) 26 | self.solar_dir.mkdir(parents=True, exist_ok=True) 27 | 28 | self.selected_station = None 29 | self.selected_location = None 30 | 31 | def find_stations(self, state: str, start_date: date, end_date: date) -> pd.DataFrame: 32 | """ 33 | Find weather stations in a US state with data for the date range. 34 | 35 | Args: 36 | state: 2-letter US state code 37 | start_date: Start date for data availability 38 | end_date: End date for data availability 39 | 40 | Returns: 41 | DataFrame of available stations 42 | """ 43 | logger.info(f"Searching for stations in {state}") 44 | 45 | stations = Stations() 46 | stations = stations.region("US", state.upper()) 47 | all_stations = stations.fetch() 48 | 49 | # Convert dates to datetime for comparison 50 | start_dt = datetime.combine(start_date, datetime.min.time()) 51 | end_dt = datetime.combine(end_date, datetime.min.time()) 52 | 53 | # Filter stations with data in the requested range 54 | available_stations = all_stations[ 55 | (all_stations["daily_start"] <= start_dt) & (all_stations["daily_end"] >= end_dt) 56 | ] 57 | 58 | logger.info(f"Found {len(available_stations)} stations with data") 59 | return available_stations 60 | 61 | def download_weather_data( 62 | self, state: str, start_date: date, duration: int, interactive: bool = True 63 | ) -> bool: 64 | """ 65 | Download weather data for a location. 66 | 67 | Args: 68 | state: 2-letter US state code 69 | start_date: Start date for data 70 | duration: Duration in days 71 | interactive: Whether to prompt user for station selection 72 | 73 | Returns: 74 | True if successful, False otherwise 75 | """ 76 | end_date = start_date + timedelta(days=duration) 77 | 78 | # Find available stations 79 | stations_df = self.find_stations(state, start_date, end_date) 80 | 81 | if len(stations_df) == 0: 82 | logger.error(f"No weather stations found in {state} for date range") 83 | return False 84 | 85 | # Station selection 86 | if interactive: 87 | print(f"\nFound {len(stations_df)} weather stations with data:") 88 | print("-" * 60) 89 | for idx, (_, row) in enumerate(stations_df.iterrows(), 1): 90 | print(f" ({idx}) {row['name']}") 91 | if idx >= 20: # Limit display 92 | print(f" ... and {len(stations_df) - 20} more") 93 | break 94 | 95 | while True: 96 | try: 97 | choice = int(input(f"\nSelect station (1-{len(stations_df)}): ")) 98 | if 1 <= choice <= len(stations_df): 99 | station_idx = choice - 1 100 | break 101 | except ValueError: 102 | pass 103 | print("Invalid selection") 104 | else: 105 | station_idx = 0 # Use first available station 106 | 107 | # Get station info 108 | station_info = stations_df.iloc[station_idx] 109 | self.selected_station = station_info 110 | 111 | logger.info(f"Selected station: {station_info['name']}") 112 | print(f"\n📍 Station: {station_info['name']}") 113 | print(f" Location: {station_info['latitude']:.4f}, {station_info['longitude']:.4f}") 114 | print(f" Elevation: {station_info['elevation']:.0f}m") 115 | 116 | # Create Point for the location 117 | self.selected_location = Point( 118 | station_info["latitude"], station_info["longitude"], station_info["elevation"] 119 | ) 120 | 121 | # Query hourly data 122 | logger.info("Downloading weather data...") 123 | start_dt = datetime.combine(start_date, datetime.min.time()) 124 | end_dt = datetime.combine(end_date, datetime.min.time()) 125 | 126 | data = Hourly(self.selected_location, start_dt, end_dt) 127 | data = data.convert(units.imperial) 128 | df = data.fetch() 129 | 130 | if df.empty: 131 | logger.error("No data returned from Meteostat") 132 | return False 133 | 134 | # Rename columns for clarity 135 | column_mapping = { 136 | "temp": "temperature", 137 | "dwpt": "dew_point", 138 | "rhum": "relative_humidity", 139 | "prcp": "precipitation", 140 | "snow": "snow_depth", 141 | "wdir": "wind_dir", 142 | "wspd": "wind_speed", 143 | "wpgt": "peak_wind_gust", 144 | "pres": "air_pressure", 145 | "tsun": "sunshine", 146 | "coco": "weather_condition", 147 | } 148 | 149 | df = df.rename(columns=column_mapping) 150 | 151 | # Clean up: remove columns with all NaN values 152 | df = df.dropna(axis=1, how="all") 153 | 154 | # Convert weather condition codes to descriptions 155 | if "weather_condition" in df.columns: 156 | condition_map = { 157 | 1: "Clear", 158 | 2: "Fair", 159 | 3: "Cloudy", 160 | 4: "Overcast", 161 | 5: "Fog", 162 | 6: "Freezing Fog", 163 | 7: "Light Rain", 164 | 8: "Rain", 165 | 9: "Heavy Rain", 166 | 10: "Freezing Rain", 167 | 11: "Heavy Freezing Rain", 168 | 12: "Sleet", 169 | 13: "Heavy Sleet", 170 | 14: "Light Snowfall", 171 | 15: "Snowfall", 172 | 16: "Heavy Snowfall", 173 | 17: "Rain Shower", 174 | 18: "Heavy Rain Shower", 175 | 19: "Sleet Shower", 176 | 20: "Heavy Sleet Shower", 177 | 21: "Snow Shower", 178 | 22: "Heavy Snow Shower", 179 | 23: "Lightning", 180 | 24: "Hail", 181 | 25: "Thunderstorm", 182 | 26: "Heavy Thunderstorm", 183 | 27: "Storm", 184 | } 185 | df["weather_condition"] = df["weather_condition"].map(condition_map) 186 | 187 | # Clean station name for filename 188 | station_name = station_info["name"].replace("/", "-").replace(" ", "_") 189 | 190 | # Save data 191 | output_file = ( 192 | self.data_dir / f"{start_date}_to_{end_date}_{station_name}_{state.upper()}.csv" 193 | ) 194 | df.to_csv(output_file) 195 | 196 | logger.info(f"Saved weather data to {output_file}") 197 | 198 | # Print summary 199 | print(f"\n📊 Data Summary:") 200 | print(f" Records: {len(df)}") 201 | print(f" Columns: {', '.join(df.columns)}") 202 | print(f" Date range: {df.index[0]} to {df.index[-1]}") 203 | 204 | return True 205 | 206 | def download_solar_data( 207 | self, year: Optional[int] = None, config_file: Optional[Path] = None 208 | ) -> bool: 209 | """ 210 | Download solar data from NSRDB. 211 | 212 | Args: 213 | year: Year for solar data (defaults to year of weather data) 214 | config_file: Path to user config file with API key 215 | 216 | Returns: 217 | True if successful, False otherwise 218 | """ 219 | if self.selected_location is None: 220 | logger.error("No location selected. Download weather data first.") 221 | return False 222 | 223 | config_path = config_file or Path("user_config.ini") 224 | 225 | # Check if config exists 226 | if not config_path.exists(): 227 | print("\n" + "=" * 60) 228 | print("NSRDB API KEY REQUIRED") 229 | print("=" * 60) 230 | print("\nTo download solar data, you need an API key from NREL.") 231 | print("Get one at: https://developer.nrel.gov/signup/") 232 | 233 | open_browser = input("\nOpen registration page in browser? (y/n): ").lower() 234 | if open_browser == "y": 235 | webbrowser.open("https://developer.nrel.gov/signup/") 236 | 237 | print("\nPlease enter your NREL credentials:") 238 | api_key = input(" API Key: ").strip() 239 | first_name = input(" First Name: ").strip() 240 | last_name = input(" Last Name: ").strip() 241 | affiliation = input(" Affiliation: ").strip() 242 | email = input(" Email: ").strip() 243 | 244 | # Save config 245 | self._write_config(config_path, api_key, first_name, last_name, affiliation, email) 246 | 247 | # Read config 248 | config = configparser.ConfigParser() 249 | config.read(config_path) 250 | 251 | api_key = config["API"]["api_key"] 252 | first_name = config["USER_INFO"]["first_name"] 253 | last_name = config["USER_INFO"]["last_name"] 254 | affiliation = config["USER_INFO"]["affiliation"] 255 | email = config["USER_INFO"]["email"] 256 | 257 | # Determine year 258 | if year is None and self.selected_station is not None: 259 | year = datetime.now().year 260 | 261 | logger.info(f"Downloading solar data for {year}...") 262 | 263 | # Build NSRDB API URL 264 | lat = self.selected_location._lat 265 | lon = self.selected_location._lon 266 | 267 | attributes = "ghi,dhi,dni,solar_zenith_angle" 268 | your_name = f"{first_name}+{last_name}" 269 | 270 | url = ( 271 | f"https://developer.nrel.gov/api/solar/nsrdb_psm3_download.csv" 272 | f"?wkt=POINT({lon}%20{lat})" 273 | f"&names={year}" 274 | f"&leap_day=true" 275 | f"&interval=60" 276 | f"&utc=false" 277 | f"&full_name={your_name}" 278 | f"&email={email}" 279 | f"&affiliation={affiliation}" 280 | f"&mailing_list=false" 281 | f"&reason=research" 282 | f"&api_key={api_key}" 283 | f"&attributes={attributes}" 284 | ) 285 | 286 | try: 287 | # Download data (skip first 2 rows of metadata) 288 | solar_df = pd.read_csv(url, skiprows=2) 289 | 290 | # Calculate minutes in year for index 291 | if self._is_leap_year(year): 292 | min_in_year = 527040 293 | else: 294 | min_in_year = 525600 295 | 296 | # Create datetime index 297 | solar_df.index = pd.date_range(f"1/1/{year}", freq="60Min", periods=min_in_year // 60) 298 | 299 | # Save data 300 | station_name = self.selected_station["name"].replace("/", "-").replace(" ", "_") 301 | state = "US" # Default if state not available 302 | 303 | output_file = self.solar_dir / f"solar_data_{year}_{station_name}_{state}.csv" 304 | solar_df.to_csv(output_file) 305 | 306 | logger.info(f"Saved solar data to {output_file}") 307 | print(f"\n☀️ Solar data saved: {output_file}") 308 | 309 | return True 310 | 311 | except Exception as e: 312 | logger.error(f"Error downloading solar data: {e}") 313 | print(f"\n❌ Error downloading solar data: {e}") 314 | return False 315 | 316 | @staticmethod 317 | def _is_leap_year(year: int) -> bool: 318 | """Check if year is a leap year.""" 319 | return (year % 4 == 0) and (year % 100 != 0 or year % 400 == 0) 320 | 321 | @staticmethod 322 | def _write_config( 323 | path: Path, api_key: str, first_name: str, last_name: str, affiliation: str, email: str 324 | ): 325 | """Write user configuration file.""" 326 | config = configparser.ConfigParser() 327 | config["API"] = {"api_key": api_key} 328 | config["USER_INFO"] = { 329 | "first_name": first_name, 330 | "last_name": last_name, 331 | "affiliation": affiliation, 332 | "email": email, 333 | } 334 | 335 | with path.open("w") as f: 336 | config.write(f) 337 | 338 | logger.info(f"Saved configuration to {path}") 339 | -------------------------------------------------------------------------------- /lib/iso/bpa.py: -------------------------------------------------------------------------------- 1 | """ 2 | BPA Client for ISO-DART v2.0 3 | 4 | Updated to use BPA's historical data endpoints (Excel files by year) 5 | """ 6 | 7 | from typing import Optional, List, Dict, Any 8 | from datetime import date, datetime, timedelta 9 | from pathlib import Path 10 | import logging 11 | import requests 12 | import pandas as pd 13 | from dataclasses import dataclass 14 | from enum import Enum 15 | 16 | logger = logging.getLogger(__name__) 17 | 18 | 19 | class BPADataType(Enum): 20 | """BPA historical data types available from Excel endpoints.""" 21 | 22 | WIND_GEN_TOTAL_LOAD = "wind_gen_total_load" 23 | RESERVES_DEPLOYED = "reserves_deployed" 24 | 25 | 26 | @dataclass 27 | class BPAConfig: 28 | """Configuration for BPA client.""" 29 | 30 | base_url: str = "https://transmission.bpa.gov/Business/Operations/Wind/OPITabularReports" 31 | data_dir: Path = Path("data/BPA") 32 | max_retries: int = 3 33 | retry_delay: int = 5 34 | timeout: int = 30 35 | 36 | 37 | class BPAClient: 38 | """Client for retrieving historical data from Bonneville Power Administration.""" 39 | 40 | def __init__(self, config: Optional[BPAConfig] = None): 41 | self.config = config or BPAConfig() 42 | self._ensure_directories() 43 | self.session = requests.Session() 44 | 45 | def _ensure_directories(self): 46 | """Ensure required directories exist.""" 47 | self.config.data_dir.mkdir(parents=True, exist_ok=True) 48 | 49 | def _make_request(self, url: str) -> Optional[bytes]: 50 | """Make API request with retry logic.""" 51 | for attempt in range(self.config.max_retries): 52 | try: 53 | logger.debug(f"Requesting: {url} (attempt {attempt + 1}/{self.config.max_retries})") 54 | response = self.session.get(url, timeout=self.config.timeout) 55 | 56 | if response.ok: 57 | logger.info(f"Request successful: {url}") 58 | return response.content 59 | else: 60 | logger.warning(f"Request failed with status {response.status_code}") 61 | 62 | except requests.RequestException as e: 63 | logger.error(f"Request error: {e}") 64 | 65 | if attempt < self.config.max_retries - 1: 66 | import time 67 | 68 | time.sleep(self.config.retry_delay) 69 | 70 | return None 71 | 72 | def _build_url(self, data_type: BPADataType, year: int) -> str: 73 | """ 74 | Build URL for BPA historical data download. 75 | 76 | Args: 77 | data_type: Type of data to download 78 | year: Year for data (4-digit) 79 | 80 | Returns: 81 | Complete URL for Excel file 82 | """ 83 | if data_type == BPADataType.WIND_GEN_TOTAL_LOAD: 84 | filename = f"WindGenTotalLoadYTD_{year}.xlsx" 85 | elif data_type == BPADataType.RESERVES_DEPLOYED: 86 | filename = f"ReservesDeployedYTD_{year}.xlsx" 87 | else: 88 | raise ValueError(f"Unknown data type: {data_type}") 89 | 90 | return f"{self.config.base_url}/{filename}" 91 | 92 | def _parse_excel_file(self, content: bytes, data_type: BPADataType) -> Optional[pd.DataFrame]: 93 | """ 94 | Parse BPA Excel file. 95 | 96 | Args: 97 | content: Excel file content as bytes 98 | data_type: Type of data being parsed 99 | 100 | Returns: 101 | DataFrame with parsed data, or None if parsing fails 102 | """ 103 | try: 104 | from io import BytesIO 105 | 106 | # Read Excel file from bytes 107 | excel_file = BytesIO(content) 108 | 109 | # BPA Excel files typically have data starting at row 0 110 | # Read all sheets and combine if necessary 111 | df = pd.read_excel(excel_file, sheet_name=0, skiprows=1) 112 | 113 | logger.info( 114 | f"Successfully parsed Excel file: {len(df)} rows, {len(df.columns)} columns" 115 | ) 116 | logger.debug(f"Columns: {list(df.columns)}") 117 | 118 | # Clean column names 119 | df.columns = df.columns.str.strip() 120 | 121 | # Try to parse datetime columns 122 | date_columns = [ 123 | col for col in df.columns if "date" in col.lower() or "time" in col.lower() 124 | ] 125 | for col in date_columns: 126 | try: 127 | df[col] = pd.to_datetime(df[col], errors="coerce") 128 | logger.info(f"Parsed datetime column: '{col}'") 129 | except Exception as e: 130 | logger.warning(f"Could not parse datetime column '{col}': {e}") 131 | 132 | # Remove completely empty rows 133 | df = df.dropna(how="all") 134 | 135 | return df 136 | 137 | except Exception as e: 138 | logger.error(f"Error parsing Excel file: {e}", exc_info=True) 139 | return None 140 | 141 | def get_wind_gen_total_load( 142 | self, year: int, start_date: Optional[date] = None, end_date: Optional[date] = None 143 | ) -> bool: 144 | """ 145 | Get Wind Generation and Total Load data for a year. 146 | 147 | This corresponds to item #5 from the BPA historical data page. 148 | Contains 5-min data for: 149 | - Wind generation (MW) 150 | - Total load (MW) 151 | - Date and hour ending 152 | 153 | Args: 154 | year: Year for data (e.g., 2024) 155 | start_date: Optional start date to filter data 156 | end_date: Optional end date to filter data 157 | 158 | Returns: 159 | True if successful, False otherwise 160 | """ 161 | logger.info(f"Downloading BPA Wind Generation and Total Load data for {year}") 162 | 163 | url = self._build_url(BPADataType.WIND_GEN_TOTAL_LOAD, year) 164 | 165 | content = self._make_request(url) 166 | if not content: 167 | logger.error(f"Failed to retrieve data from BPA for year {year}") 168 | return False 169 | 170 | try: 171 | df = self._parse_excel_file(content, BPADataType.WIND_GEN_TOTAL_LOAD) 172 | 173 | if df is None or df.empty: 174 | logger.error("No data returned after parsing") 175 | return False 176 | 177 | # Filter by date range if provided 178 | if start_date or end_date: 179 | df = self._filter_by_date_range(df, start_date, end_date) 180 | 181 | if df.empty: 182 | logger.warning("No data after date filtering") 183 | return False 184 | 185 | # Save to file 186 | output_file = self.config.data_dir / f"{year}_BPA_Wind_Generation_Total_Load.csv" 187 | df.to_csv(output_file, index=False) 188 | logger.info(f"Saved {len(df)} rows to {output_file}") 189 | 190 | # Report date range 191 | date_cols = [col for col in df.columns if pd.api.types.is_datetime64_any_dtype(df[col])] 192 | if date_cols: 193 | col = date_cols[0] 194 | logger.info(f"Data range: {df[col].min()} to {df[col].max()}") 195 | 196 | return True 197 | 198 | except Exception as e: 199 | logger.error(f"Error processing data: {e}", exc_info=True) 200 | return False 201 | 202 | def get_reserves_deployed( 203 | self, year: int, start_date: Optional[date] = None, end_date: Optional[date] = None 204 | ) -> bool: 205 | """ 206 | Get Reserves Deployed data for a year. 207 | 208 | This corresponds to item #12 from the BPA historical data page. 209 | Contains data for: 210 | - Reserves deployed by type 211 | - Date and time information 212 | 213 | Args: 214 | year: Year for data (e.g., 2024) 215 | start_date: Optional start date to filter data 216 | end_date: Optional end date to filter data 217 | 218 | Returns: 219 | True if successful, False otherwise 220 | """ 221 | logger.info(f"Downloading BPA Reserves Deployed data for {year}") 222 | 223 | url = self._build_url(BPADataType.RESERVES_DEPLOYED, year) 224 | 225 | content = self._make_request(url) 226 | if not content: 227 | logger.error(f"Failed to retrieve data from BPA for year {year}") 228 | return False 229 | 230 | try: 231 | df = self._parse_excel_file(content, BPADataType.RESERVES_DEPLOYED) 232 | 233 | if df is None or df.empty: 234 | logger.error("No data returned after parsing") 235 | return False 236 | 237 | # Filter by date range if provided 238 | if start_date or end_date: 239 | df = self._filter_by_date_range(df, start_date, end_date) 240 | 241 | if df.empty: 242 | logger.warning("No data after date filtering") 243 | return False 244 | 245 | # Save to file 246 | output_file = self.config.data_dir / f"{year}_BPA_Reserves_Deployed.csv" 247 | df.to_csv(output_file, index=False) 248 | logger.info(f"Saved {len(df)} rows to {output_file}") 249 | 250 | # Report date range 251 | date_cols = [col for col in df.columns if pd.api.types.is_datetime64_any_dtype(df[col])] 252 | if date_cols: 253 | col = date_cols[0] 254 | logger.info(f"Data range: {df[col].min()} to {df[col].max()}") 255 | 256 | return True 257 | 258 | except Exception as e: 259 | logger.error(f"Error processing data: {e}", exc_info=True) 260 | return False 261 | 262 | def get_all_data( 263 | self, year: int, start_date: Optional[date] = None, end_date: Optional[date] = None 264 | ) -> bool: 265 | """ 266 | Get all available BPA historical data for a year. 267 | 268 | Args: 269 | year: Year for data 270 | start_date: Optional start date to filter data 271 | end_date: Optional end date to filter data 272 | 273 | Returns: 274 | True if all downloads successful, False otherwise 275 | """ 276 | logger.info(f"Downloading all BPA data for {year}") 277 | 278 | success_wind = self.get_wind_gen_total_load(year, start_date, end_date) 279 | success_reserves = self.get_reserves_deployed(year, start_date, end_date) 280 | 281 | return success_wind and success_reserves 282 | 283 | def _filter_by_date_range( 284 | self, df: pd.DataFrame, start_date: Optional[date], end_date: Optional[date] 285 | ) -> pd.DataFrame: 286 | """Filter dataframe by date range.""" 287 | if df.empty: 288 | return df 289 | 290 | if not (start_date or end_date): 291 | return df 292 | 293 | # Find datetime column 294 | datetime_col = None 295 | for col in df.columns: 296 | if pd.api.types.is_datetime64_any_dtype(df[col]): 297 | datetime_col = col 298 | break 299 | 300 | if not datetime_col: 301 | logger.warning("No datetime column found for filtering") 302 | return df 303 | 304 | # Apply filters 305 | if start_date: 306 | df = df[df[datetime_col] >= pd.Timestamp(start_date)] 307 | if end_date: 308 | df = df[df[datetime_col] <= pd.Timestamp(end_date)] 309 | 310 | logger.info(f"Filtered to {len(df)} rows") 311 | return df 312 | 313 | def cleanup(self): 314 | """Clean up temporary files if any.""" 315 | logger.info("BPA client cleanup complete") 316 | 317 | 318 | def get_bpa_data_availability() -> Dict[str, Any]: 319 | """Get information about BPA historical data availability.""" 320 | current_year = datetime.now().year 321 | 322 | return { 323 | "temporal_coverage": f"Historical yearly data (typically 2000-{current_year})", 324 | "temporal_resolution": "5-min intervals", 325 | "update_frequency": "Updated annually (current year updated periodically)", 326 | "data_types": { 327 | "wind_gen_total_load": { 328 | "description": "Wind Generation and Total Load (5-min)", 329 | "variables": [ 330 | "Wind Generation (MW)", 331 | "Total Load (MW)", 332 | "Date", 333 | "Hour Ending", 334 | ], 335 | "file_format": "Excel (.xlsx)", 336 | "endpoint": "WindGenTotalLoadYTD_yyyy.xlsx", 337 | }, 338 | "reserves_deployed": { 339 | "description": "Operating Reserves Deployed", 340 | "variables": [ 341 | "Regulation Up (MW)", 342 | "Regulation Down (MW)", 343 | "Contingency Reserves (MW)", 344 | "Date", 345 | "Time", 346 | ], 347 | "file_format": "Excel (.xlsx)", 348 | "endpoint": "ReservesDeployedYTD_yyyy.xlsx", 349 | }, 350 | }, 351 | "geographic_coverage": "BPA Balancing Authority Area (Pacific Northwest)", 352 | "notes": [ 353 | "Historical data is available by full calendar year", 354 | "Data is stored in Excel format (.xlsx)", 355 | "Current year data is updated periodically throughout the year", 356 | "All times are Pacific Time", 357 | "5-min resolution with hour-ending timestamps", 358 | ], 359 | "available_years": list(range(2000, current_year + 1)), 360 | } 361 | 362 | 363 | def print_bpa_data_info(): 364 | """Print BPA data availability information.""" 365 | info = get_bpa_data_availability() 366 | 367 | print("\n" + "=" * 70) 368 | print("BPA HISTORICAL DATA AVAILABILITY") 369 | print("=" * 70) 370 | print(f"\nTemporal Coverage: {info['temporal_coverage']}") 371 | print(f"Temporal Resolution: {info['temporal_resolution']}") 372 | print(f"Update Frequency: {info['update_frequency']}") 373 | print(f"Geographic Coverage: {info['geographic_coverage']}") 374 | 375 | print("\nAvailable Data Types:") 376 | for dtype, details in info["data_types"].items(): 377 | print(f"\n {dtype}:") 378 | print(f" {details['description']}") 379 | print(f" Format: {details['file_format']}") 380 | print(f" Endpoint: {details['endpoint']}") 381 | print(f" Variables:") 382 | for var in details["variables"]: 383 | print(f" - {var}") 384 | 385 | print(f"\nAvailable Years: {min(info['available_years'])} - {max(info['available_years'])}") 386 | 387 | print("\nNOTES:") 388 | for note in info["notes"]: 389 | print(f" • {note}") 390 | 391 | print("\n" + "=" * 70 + "\n") 392 | 393 | 394 | if __name__ == "__main__": 395 | # Enable debug logging 396 | logging.basicConfig( 397 | level=logging.DEBUG, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s" 398 | ) 399 | 400 | print_bpa_data_info() 401 | 402 | print("Testing BPA historical data download...") 403 | client = BPAClient() 404 | 405 | # Test with current year 406 | current_year = datetime.now().year 407 | 408 | print(f"\n1. Testing Wind Generation and Total Load for {current_year}...") 409 | success = client.get_wind_gen_total_load(current_year) 410 | print(f" Result: {'✓ Success' if success else '✗ Failed'}") 411 | 412 | print(f"\n2. Testing Reserves Deployed for {current_year}...") 413 | success = client.get_reserves_deployed(current_year) 414 | print(f" Result: {'✓ Success' if success else '✗ Failed'}") 415 | 416 | client.cleanup() 417 | -------------------------------------------------------------------------------- /tests/test_bpa.py: -------------------------------------------------------------------------------- 1 | """ 2 | Test suite for BPA client (Excel-based historical data). 3 | 4 | Run with: pytest test_bpa.py -v 5 | """ 6 | 7 | from datetime import date, datetime, timedelta 8 | from pathlib import Path 9 | from io import BytesIO 10 | from typing import Dict, Any 11 | 12 | import pandas as pd 13 | import pytest 14 | from unittest.mock import MagicMock, Mock, patch 15 | 16 | from lib.iso.bpa import ( 17 | BPAClient, 18 | BPAConfig, 19 | BPADataType, 20 | get_bpa_data_availability, 21 | print_bpa_data_info, 22 | ) 23 | 24 | 25 | # --------------------------------------------------------------------------- 26 | # Fixtures 27 | # --------------------------------------------------------------------------- 28 | 29 | 30 | @pytest.fixture 31 | def temp_config(tmp_path: Path) -> BPAConfig: 32 | """BPAConfig using a temporary data directory.""" 33 | return BPAConfig(data_dir=tmp_path) 34 | 35 | 36 | @pytest.fixture 37 | def client(temp_config: BPAConfig) -> BPAClient: 38 | """BPAClient instance with temp config.""" 39 | return BPAClient(config=temp_config) 40 | 41 | 42 | # --------------------------------------------------------------------------- 43 | # Enum and config tests 44 | # --------------------------------------------------------------------------- 45 | 46 | 47 | class TestBPADataType: 48 | def test_enum_members(self): 49 | """Enum should expose the expected members and values.""" 50 | assert BPADataType.WIND_GEN_TOTAL_LOAD.value == "wind_gen_total_load" 51 | assert BPADataType.RESERVES_DEPLOYED.value == "reserves_deployed" 52 | 53 | def test_enum_is_iterable(self): 54 | """Enum should contain exactly the two expected members.""" 55 | names = {m.name for m in BPADataType} 56 | assert names == {"WIND_GEN_TOTAL_LOAD", "RESERVES_DEPLOYED"} 57 | 58 | 59 | class TestBPAConfig: 60 | def test_default_config_values(self): 61 | """Default config should match hard-coded defaults in bpa.py.""" 62 | cfg = BPAConfig() 63 | assert ( 64 | cfg.base_url 65 | == "https://transmission.bpa.gov/Business/Operations/Wind/OPITabularReports" 66 | ) 67 | assert cfg.data_dir == Path("data/BPA") 68 | assert cfg.max_retries == 3 69 | assert cfg.retry_delay == 5 70 | assert cfg.timeout == 30 71 | 72 | def test_override_data_dir(self, tmp_path: Path): 73 | """Data directory can be overridden.""" 74 | cfg = BPAConfig(data_dir=tmp_path) 75 | assert cfg.data_dir == tmp_path 76 | 77 | 78 | # --------------------------------------------------------------------------- 79 | # BPAClient initialization and helpers 80 | # --------------------------------------------------------------------------- 81 | 82 | 83 | class TestBPAClientInit: 84 | def test_init_creates_directory(self, tmp_path: Path): 85 | """Initializing the client should ensure the data directory exists.""" 86 | cfg = BPAConfig(data_dir=tmp_path / "nested" / "bpa") 87 | assert not cfg.data_dir.exists() 88 | BPAClient(config=cfg) 89 | assert cfg.data_dir.exists() 90 | 91 | def test_default_config_is_used_when_not_provided(self): 92 | """Client should create and use a default config when none is passed.""" 93 | client = BPAClient() 94 | assert isinstance(client.config, BPAConfig) 95 | assert client.config.data_dir == Path("data/BPA") 96 | 97 | 98 | class TestBuildUrl: 99 | def test_build_url_wind_gen_total_load(self, client: BPAClient): 100 | year = 2024 101 | url = client._build_url(BPADataType.WIND_GEN_TOTAL_LOAD, year) 102 | assert str(year) in url 103 | assert url.endswith(f"WindGenTotalLoadYTD_{year}.xlsx") 104 | assert client.config.base_url in url 105 | 106 | def test_build_url_reserves_deployed(self, client: BPAClient): 107 | year = 2023 108 | url = client._build_url(BPADataType.RESERVES_DEPLOYED, year) 109 | assert str(year) in url 110 | assert url.endswith(f"ReservesDeployedYTD_{year}.xlsx") 111 | assert client.config.base_url in url 112 | 113 | def test_build_url_unknown_type_raises(self, client: BPAClient): 114 | class FakeType: 115 | # anything non-BPADataType should hit the ValueError branch 116 | pass 117 | 118 | with pytest.raises(ValueError): 119 | client._build_url(FakeType(), 2020) # type: ignore[arg-type] 120 | 121 | 122 | # --------------------------------------------------------------------------- 123 | # Request / HTTP tests 124 | # --------------------------------------------------------------------------- 125 | 126 | 127 | class TestMakeRequest: 128 | def test_make_request_success(self, client: BPAClient): 129 | """_make_request should return content when response is OK.""" 130 | mock_response = Mock() 131 | mock_response.ok = True 132 | mock_response.content = b"excel-bytes" 133 | client.session.get = Mock(return_value=mock_response) # type: ignore[assignment] 134 | 135 | content = client._make_request("https://example.com/file.xlsx") 136 | assert content == b"excel-bytes" 137 | client.session.get.assert_called_once() 138 | 139 | def test_make_request_failure_then_success(self, client: BPAClient): 140 | """_make_request should retry until success within max_retries.""" 141 | cfg = client.config 142 | first = Mock() 143 | first.ok = False 144 | first.content = b"" 145 | second = Mock() 146 | second.ok = True 147 | second.content = b"ok" 148 | 149 | client.session.get = Mock(side_effect=[first, second]) # type: ignore[assignment] 150 | 151 | content = client._make_request("https://example.com/file.xlsx") 152 | assert content == b"ok" 153 | assert client.session.get.call_count == 2 154 | 155 | def test_make_request_all_failures_returns_none(self, client: BPAClient): 156 | """When all attempts fail (non-OK), _make_request should return None.""" 157 | client.config.max_retries = 3 158 | 159 | fail_resp = Mock() 160 | fail_resp.ok = False 161 | fail_resp.content = b"" 162 | 163 | # Always return a non-OK response 164 | client.session.get = Mock(return_value=fail_resp) # type: ignore[assignment] 165 | 166 | content = client._make_request("https://example.com/file.xlsx") 167 | assert content is None 168 | # should have tried max_retries times 169 | assert client.session.get.call_count == client.config.max_retries 170 | 171 | 172 | # --------------------------------------------------------------------------- 173 | # Excel parsing tests 174 | # --------------------------------------------------------------------------- 175 | 176 | 177 | class TestParseExcelFile: 178 | def _build_valid_excel_bytes(self) -> bytes: 179 | """ 180 | Build an in-memory Excel payload shaped the way _parse_excel_file expects. 181 | 182 | bpa._parse_excel_file uses `skiprows=1`, so we write a first *data* row 183 | whose values become the column names: "Date", "Time", "Value", and a 184 | second row of actual data. 185 | """ 186 | df = pd.DataFrame( 187 | [ 188 | {"col1": "Date", "col2": "Time", "col3": "Value"}, 189 | { 190 | "col1": datetime(2024, 1, 1, 0, 0), 191 | "col2": datetime(2024, 1, 1, 0, 5), 192 | "col3": 100.0, 193 | }, 194 | ] 195 | ) 196 | buf = BytesIO() 197 | df.to_excel(buf, index=False) 198 | return buf.getvalue() 199 | 200 | def test_parse_excel_file_success(self, client: BPAClient): 201 | """_parse_excel_file should return a DataFrame with parsed datetime columns.""" 202 | content = self._build_valid_excel_bytes() 203 | df = client._parse_excel_file(content, BPADataType.WIND_GEN_TOTAL_LOAD) 204 | assert isinstance(df, pd.DataFrame) 205 | assert not df.empty 206 | assert list(df.columns) == ["Date", "Time", "Value"] 207 | # both Date and Time should have been parsed to datetime dtype 208 | assert pd.api.types.is_datetime64_any_dtype(df["Date"]) 209 | assert pd.api.types.is_datetime64_any_dtype(df["Time"]) 210 | 211 | def test_parse_excel_file_failure_returns_none(self, client: BPAClient): 212 | """ 213 | If parsing fails for any reason, the method should catch and return None. 214 | Use clearly invalid bytes to hit the exception path. 215 | """ 216 | df = client._parse_excel_file(b"not-an-excel-file", BPADataType.WIND_GEN_TOTAL_LOAD) 217 | assert df is None 218 | 219 | 220 | # --------------------------------------------------------------------------- 221 | # Date-range filtering tests 222 | # --------------------------------------------------------------------------- 223 | 224 | 225 | class TestFilterByDateRange: 226 | def test_empty_dataframe_returns_unchanged(self, client: BPAClient): 227 | df = pd.DataFrame() 228 | result = client._filter_by_date_range(df, None, None) 229 | assert result is df 230 | 231 | def test_no_dates_provided_returns_original(self, client: BPAClient): 232 | dt_index = pd.date_range("2024-01-01", periods=3, freq="h") 233 | df = pd.DataFrame({"ts": dt_index, "value": [1, 2, 3]}) 234 | result = client._filter_by_date_range(df, None, None) 235 | assert result.equals(df) 236 | 237 | def test_no_datetime_column_returns_original(self, client: BPAClient): 238 | df = pd.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]}) 239 | result = client._filter_by_date_range(df, date(2024, 1, 1), date(2024, 1, 2)) 240 | # nothing to filter on -> original df 241 | assert result.equals(df) 242 | 243 | def test_filters_by_start_and_end(self, client: BPAClient): 244 | dt_index = pd.date_range("2024-01-01", periods=5, freq="D") 245 | df = pd.DataFrame({"ts": dt_index, "value": [1, 2, 3, 4, 5]}) 246 | start = date(2024, 1, 2) 247 | end = date(2024, 1, 4) 248 | 249 | result = client._filter_by_date_range(df, start, end) 250 | 251 | # should contain only 3 rows: 2nd, 3rd, 4th dates 252 | assert len(result) == 3 253 | assert result["ts"].min().date() == start 254 | assert result["ts"].max().date() == end 255 | 256 | def test_filters_by_start_only(self, client: BPAClient): 257 | dt_index = pd.date_range("2024-01-01", periods=3, freq="D") 258 | df = pd.DataFrame({"ts": dt_index, "value": [1, 2, 3]}) 259 | start = date(2024, 1, 2) 260 | 261 | result = client._filter_by_date_range(df, start, None) 262 | assert list(result["ts"].dt.date) == [date(2024, 1, 2), date(2024, 1, 3)] 263 | 264 | def test_filters_by_end_only(self, client: BPAClient): 265 | dt_index = pd.date_range("2024-01-01", periods=3, freq="D") 266 | df = pd.DataFrame({"ts": dt_index, "value": [1, 2, 3]}) 267 | end = date(2024, 1, 2) 268 | 269 | result = client._filter_by_date_range(df, None, end) 270 | assert list(result["ts"].dt.date) == [date(2024, 1, 1), date(2024, 1, 2)] 271 | 272 | 273 | # --------------------------------------------------------------------------- 274 | # High-level data download methods 275 | # --------------------------------------------------------------------------- 276 | 277 | 278 | class TestWindGenTotalLoad: 279 | @patch.object(BPAClient, "_make_request") 280 | @patch.object(BPAClient, "_parse_excel_file") 281 | def test_successful_download_and_save( 282 | self, 283 | mock_parse: MagicMock, 284 | mock_request: MagicMock, 285 | client: BPAClient, 286 | tmp_path: Path, 287 | ): 288 | client.config.data_dir = tmp_path 289 | 290 | # Prepare a simple DataFrame as parsed output 291 | dt_index = pd.date_range("2024-01-01", periods=3, freq="h") 292 | df = pd.DataFrame({"DateTime": dt_index, "Value": [1, 2, 3]}) 293 | mock_request.return_value = b"excel-content" 294 | mock_parse.return_value = df 295 | 296 | start = date(2024, 1, 1) 297 | end = date(2024, 1, 2) 298 | 299 | success = client.get_wind_gen_total_load(2024, start, end) 300 | assert success is True 301 | 302 | # Ensure we actually made the request and parsed the content 303 | mock_request.assert_called_once() 304 | mock_parse.assert_called_once_with(b"excel-content", BPADataType.WIND_GEN_TOTAL_LOAD) 305 | 306 | @patch.object(BPAClient, "_make_request") 307 | def test_request_failure_returns_false( 308 | self, 309 | mock_request: MagicMock, 310 | client: BPAClient, 311 | tmp_path: Path, 312 | ): 313 | client.config.data_dir = tmp_path 314 | mock_request.return_value = None 315 | 316 | success = client.get_wind_gen_total_load(2024) 317 | assert success is False 318 | 319 | output_file = client.config.data_dir / "2024_BPA_WindGenTotalLoad.csv" 320 | assert not output_file.exists() 321 | 322 | 323 | class TestReservesDeployed: 324 | @patch.object(BPAClient, "_make_request") 325 | @patch.object(BPAClient, "_parse_excel_file") 326 | def test_successful_download_and_save( 327 | self, 328 | mock_parse: MagicMock, 329 | mock_request: MagicMock, 330 | client: BPAClient, 331 | tmp_path: Path, 332 | ): 333 | client.config.data_dir = tmp_path 334 | 335 | dt_index = pd.date_range("2024-01-01", periods=3, freq="h") 336 | df = pd.DataFrame({"DateTime": dt_index, "Value": [10, 20, 30]}) 337 | mock_request.return_value = b"excel-content" 338 | mock_parse.return_value = df 339 | 340 | success = client.get_reserves_deployed(2024) 341 | assert success is True 342 | 343 | output_file = client.config.data_dir / "2024_BPA_Reserves_Deployed.csv" 344 | assert output_file.exists() 345 | saved = pd.read_csv(output_file) 346 | assert len(saved) > 0 347 | 348 | @patch.object(BPAClient, "_make_request") 349 | def test_request_failure_returns_false( 350 | self, 351 | mock_request: MagicMock, 352 | client: BPAClient, 353 | tmp_path: Path, 354 | ): 355 | client.config.data_dir = tmp_path 356 | mock_request.return_value = None 357 | 358 | success = client.get_reserves_deployed(2024) 359 | assert success is False 360 | 361 | output_file = client.config.data_dir / "2024_BPA_Reserves_Deployed.csv" 362 | assert not output_file.exists() 363 | 364 | 365 | class TestGetAllData: 366 | def test_get_all_data_combines_results(self, client: BPAClient): 367 | client.get_wind_gen_total_load = MagicMock(return_value=True) # type: ignore[assignment] 368 | client.get_reserves_deployed = MagicMock(return_value=False) # type: ignore[assignment] 369 | 370 | result = client.get_all_data(2024) 371 | assert result is False # True AND False 372 | 373 | client.get_reserves_deployed.return_value = True 374 | result2 = client.get_all_data(2024) 375 | assert result2 is True 376 | 377 | 378 | # --------------------------------------------------------------------------- 379 | # Availability metadata & printing 380 | # --------------------------------------------------------------------------- 381 | 382 | 383 | class TestAvailabilityMetadata: 384 | def test_get_bpa_data_availability_structure(self): 385 | info: Dict[str, Any] = get_bpa_data_availability() 386 | 387 | # top-level keys 388 | assert "temporal_coverage" in info 389 | assert "temporal_resolution" in info 390 | assert "update_frequency" in info 391 | assert "data_types" in info 392 | assert "geographic_coverage" in info 393 | assert "notes" in info 394 | assert "available_years" in info 395 | 396 | assert isinstance(info["data_types"], dict) 397 | assert "wind_gen_total_load" in info["data_types"] 398 | assert "reserves_deployed" in info["data_types"] 399 | 400 | years = info["available_years"] 401 | assert isinstance(years, list) 402 | assert min(years) <= datetime.now().year 403 | assert max(years) >= datetime.now().year 404 | 405 | def test_print_bpa_data_info_outputs_text(self, capsys): 406 | print_bpa_data_info() 407 | captured = capsys.readouterr().out 408 | assert "BPA HISTORICAL DATA AVAILABILITY" in captured 409 | assert "Available Data Types" in captured 410 | assert "wind_gen_total_load" in captured 411 | assert "reserves_deployed" in captured 412 | 413 | 414 | # --------------------------------------------------------------------------- 415 | # Cleanup 416 | # --------------------------------------------------------------------------- 417 | 418 | 419 | class TestCleanup: 420 | def test_cleanup_no_error(self, client: BPAClient): 421 | # Should simply log and not raise 422 | client.cleanup() 423 | 424 | def test_cleanup_idempotent(self, client: BPAClient): 425 | client.cleanup() 426 | client.cleanup() # should still not raise 427 | 428 | 429 | if __name__ == "__main__": 430 | pytest.main([__file__, "-v"]) 431 | -------------------------------------------------------------------------------- /lib/iso/nyiso.py: -------------------------------------------------------------------------------- 1 | """ 2 | NYISO Client for ISO-DART v2.0 3 | 4 | Modernized client for New York Independent System Operator data retrieval. 5 | File location: lib/iso/nyiso.py 6 | """ 7 | 8 | from typing import Optional 9 | from datetime import date, timedelta 10 | from pathlib import Path 11 | from dateutil.relativedelta import relativedelta 12 | import logging 13 | import requests 14 | import zipfile 15 | import io 16 | import pandas as pd 17 | from dataclasses import dataclass 18 | from enum import Enum 19 | 20 | logger = logging.getLogger(__name__) 21 | 22 | 23 | class NYISOMarket(Enum): 24 | """NYISO market types.""" 25 | 26 | DAM = "DAM" # Day-Ahead Market 27 | RTM = "RTM" # Real-Time Market 28 | 29 | 30 | class NYISODataType(Enum): 31 | """NYISO data categories.""" 32 | 33 | PRICING = "pricing" 34 | POWER_GRID = "power_grid" 35 | LOAD = "load" 36 | BID = "bid" 37 | 38 | 39 | @dataclass 40 | class NYISOConfig: 41 | """Configuration for NYISO client.""" 42 | 43 | base_url: str = "http://mis.nyiso.com/public/csv" 44 | raw_dir: Path = Path("raw_data/NYISO") 45 | data_dir: Path = Path("data/NYISO") 46 | max_retries: int = 3 47 | retry_delay: int = 5 48 | timeout: int = 30 49 | 50 | 51 | class NYISOClient: 52 | """Client for retrieving data from NYISO.""" 53 | 54 | def __init__(self, config: Optional[NYISOConfig] = None): 55 | self.config = config or NYISOConfig() 56 | self._ensure_directories() 57 | self.session = requests.Session() 58 | 59 | def _ensure_directories(self): 60 | """Ensure required directories exist.""" 61 | self.config.raw_dir.mkdir(parents=True, exist_ok=True) 62 | self.config.data_dir.mkdir(parents=True, exist_ok=True) 63 | 64 | def _get_month_start_dates(self, start_date: date, end_date: date) -> list: 65 | """Get list of month start dates covering the date range.""" 66 | month_starts = [] 67 | current = date(start_date.year, start_date.month, 1) 68 | 69 | while current <= end_date: 70 | month_starts.append(current) 71 | current = current + relativedelta(months=1) 72 | 73 | return month_starts 74 | 75 | def _build_url( 76 | self, dataid: str, month_start: date, file_dataid: str, agg_type: Optional[str] = None 77 | ) -> str: 78 | """Build NYISO data URL.""" 79 | date_str = month_start.strftime("%Y%m%d") 80 | 81 | if agg_type: 82 | return f"{self.config.base_url}/{dataid}/{date_str}{file_dataid}_{agg_type}_csv.zip" 83 | else: 84 | return f"{self.config.base_url}/{dataid}/{date_str}{file_dataid}_csv.zip" 85 | 86 | def _make_request(self, url: str, output_path: Path) -> bool: 87 | """Download and extract ZIP file.""" 88 | for attempt in range(self.config.max_retries): 89 | try: 90 | logger.debug(f"Requesting: {url} (attempt {attempt + 1}/{self.config.max_retries})") 91 | response = self.session.get(url, timeout=self.config.timeout) 92 | 93 | if response.ok: 94 | try: 95 | z = zipfile.ZipFile(io.BytesIO(response.content)) 96 | z.extractall(output_path) 97 | logger.info(f"Extracted to: {output_path}") 98 | return True 99 | except zipfile.BadZipFile: 100 | logger.error(f"Invalid ZIP file from {url}") 101 | return False 102 | else: 103 | logger.warning(f"Request failed with status {response.status_code}") 104 | 105 | except requests.RequestException as e: 106 | logger.error(f"Request error: {e}") 107 | 108 | if attempt < self.config.max_retries - 1: 109 | import time 110 | 111 | time.sleep(self.config.retry_delay) 112 | 113 | return False 114 | 115 | def _merge_csvs( 116 | self, 117 | raw_dir: Path, 118 | dataid: str, 119 | start_date: date, 120 | duration: int, 121 | agg_type: Optional[str] = None, 122 | ) -> bool: 123 | """Merge downloaded CSV files into a single file.""" 124 | try: 125 | files = sorted(raw_dir.glob("*.csv")) 126 | 127 | if not files: 128 | logger.warning("No CSV files found to merge") 129 | return False 130 | 131 | # Generate date strings for filename 132 | date_list = [ 133 | (start_date + timedelta(days=i)).strftime("%Y%m%d") for i in range(duration) 134 | ] 135 | 136 | # Filter files that match our date range 137 | selected_files = [] 138 | for f in files: 139 | date_part = f.name[:8] 140 | if date_part in date_list: 141 | selected_files.append(f) 142 | 143 | if not selected_files: 144 | logger.warning("No files match the date range") 145 | return False 146 | 147 | # Combine CSVs 148 | logger.info(f"Merging {len(selected_files)} CSV files...") 149 | combined_df = pd.concat([pd.read_csv(f) for f in selected_files]) 150 | 151 | # Build output filename 152 | start_str = date_list[0] 153 | end_str = date_list[-1] 154 | 155 | if agg_type: 156 | output_file = ( 157 | self.config.data_dir / f"{start_str}_to_{end_str}_{dataid}_{agg_type}.csv" 158 | ) 159 | else: 160 | output_file = self.config.data_dir / f"{start_str}_to_{end_str}_{dataid}.csv" 161 | 162 | combined_df.to_csv(output_file, index=False) 163 | logger.info(f"Merged file saved: {output_file}") 164 | 165 | return True 166 | 167 | except Exception as e: 168 | logger.error(f"Error merging CSVs: {e}") 169 | return False 170 | 171 | def get_lbmp(self, market: NYISOMarket, level: str, start_date: date, duration: int) -> bool: 172 | """ 173 | Get Locational Based Marginal Prices (LBMP). 174 | 175 | Args: 176 | market: Market type (DAM or RTM) 177 | level: Detail level ('zonal' or 'generator') 178 | start_date: Start date 179 | duration: Duration in days 180 | """ 181 | if level not in ["zonal", "generator"]: 182 | logger.error(f"Invalid level: {level}. Must be 'zonal' or 'generator'") 183 | return False 184 | 185 | agg_type = "zone" if level == "zonal" else "gen" 186 | 187 | if market == NYISOMarket.DAM: 188 | dataid = "damlbmp" 189 | file_dataid = "damlbmp" 190 | else: # RTM 191 | dataid = "realtime" 192 | file_dataid = "realtime" 193 | 194 | end_date = start_date + timedelta(days=duration) 195 | month_starts = self._get_month_start_dates(start_date, end_date) 196 | 197 | # Create temp directory for this download 198 | raw_path = self.config.raw_dir / dataid / agg_type 199 | raw_path.mkdir(parents=True, exist_ok=True) 200 | 201 | # Download data for each month 202 | for month_start in month_starts: 203 | url = self._build_url(dataid, month_start, file_dataid, agg_type) 204 | logger.info(f"Downloading from: {url}") 205 | 206 | if not self._make_request(url, raw_path): 207 | logger.warning(f"Failed to download {month_start}") 208 | 209 | # Merge CSVs 210 | success = self._merge_csvs(raw_path, dataid, start_date, duration, agg_type) 211 | 212 | # Cleanup temp files 213 | import shutil 214 | 215 | if self.config.raw_dir.exists(): 216 | shutil.rmtree(self.config.raw_dir) 217 | 218 | return success 219 | 220 | def get_ancillary_services_prices( 221 | self, market: NYISOMarket, start_date: date, duration: int 222 | ) -> bool: 223 | """Get ancillary services prices.""" 224 | if market == NYISOMarket.DAM: 225 | dataid = "damasp" 226 | else: # RTM 227 | dataid = "rtasp" 228 | 229 | file_dataid = dataid 230 | end_date = start_date + timedelta(days=duration) 231 | month_starts = self._get_month_start_dates(start_date, end_date) 232 | 233 | raw_path = self.config.raw_dir / dataid 234 | raw_path.mkdir(parents=True, exist_ok=True) 235 | 236 | for month_start in month_starts: 237 | url = self._build_url(dataid, month_start, file_dataid) 238 | logger.info(f"Downloading from: {url}") 239 | 240 | if not self._make_request(url, raw_path): 241 | logger.warning(f"Failed to download {month_start}") 242 | 243 | success = self._merge_csvs(raw_path, dataid, start_date, duration) 244 | 245 | # Cleanup 246 | import shutil 247 | 248 | if self.config.raw_dir.exists(): 249 | shutil.rmtree(self.config.raw_dir) 250 | 251 | return success 252 | 253 | def get_constraints(self, market: NYISOMarket, start_date: date, duration: int) -> bool: 254 | """ 255 | Get transmission constraint data. 256 | 257 | Args: 258 | market: Market type (DAM or RTM) 259 | start_date: Start date 260 | duration: Duration in days 261 | """ 262 | if market == NYISOMarket.DAM: 263 | dataid = "DAMLimitingConstraints" 264 | else: # RTM 265 | dataid = "LimitingConstraints" 266 | 267 | file_dataid = dataid 268 | end_date = start_date + timedelta(days=duration) 269 | month_starts = self._get_month_start_dates(start_date, end_date) 270 | 271 | raw_path = self.config.raw_dir / dataid 272 | raw_path.mkdir(parents=True, exist_ok=True) 273 | 274 | for month_start in month_starts: 275 | url = self._build_url(dataid, month_start, file_dataid) 276 | logger.info(f"Downloading from: {url}") 277 | 278 | if not self._make_request(url, raw_path): 279 | logger.warning(f"Failed to download {month_start}") 280 | 281 | success = self._merge_csvs(raw_path, dataid, start_date, duration) 282 | 283 | # Cleanup 284 | import shutil 285 | 286 | if self.config.raw_dir.exists(): 287 | shutil.rmtree(self.config.raw_dir) 288 | 289 | return success 290 | 291 | def get_bid_data(self, bid_type: str, start_date: date, duration: int) -> bool: 292 | """ 293 | Get bid data. 294 | 295 | Args: 296 | bid_type: Type of bid data ('generator', 'load', 'transaction', 'commitment') 297 | start_date: Start date 298 | duration: Duration in days 299 | """ 300 | type_map = { 301 | "generator": "genbids", 302 | "load": "loadbids", 303 | "transaction": "tranbids", 304 | "commitment": "ucdata", 305 | } 306 | 307 | if bid_type not in type_map: 308 | logger.error(f"Invalid bid type: {bid_type}") 309 | return False 310 | 311 | dataid = "biddata" 312 | agg_type = type_map[bid_type] 313 | file_dataid = "biddata" 314 | 315 | end_date = start_date + timedelta(days=duration) 316 | month_starts = self._get_month_start_dates(start_date, end_date) 317 | 318 | raw_path = self.config.raw_dir / dataid / agg_type 319 | raw_path.mkdir(parents=True, exist_ok=True) 320 | 321 | for month_start in month_starts: 322 | url = self._build_url(dataid, month_start, file_dataid, agg_type) 323 | logger.info(f"Downloading from: {url}") 324 | 325 | if not self._make_request(url, raw_path): 326 | logger.warning(f"Failed to download {month_start}") 327 | else: 328 | # For bid data, copy individual files to data directory 329 | # (don't merge, keep separate by date as in legacy version) 330 | date_str = month_start.strftime("%Y%m%d") 331 | src_file = raw_path / f"{date_str}{dataid}_{agg_type}.csv" 332 | if src_file.exists(): 333 | dst_file = self.config.data_dir / src_file.name 334 | import shutil 335 | 336 | shutil.copy(src_file, dst_file) 337 | logger.info(f"Copied bid data: {dst_file}") 338 | 339 | # Cleanup 340 | import shutil 341 | 342 | if self.config.raw_dir.exists(): 343 | shutil.rmtree(self.config.raw_dir) 344 | 345 | return True 346 | 347 | def get_fuel_mix(self, start_date: date, duration: int) -> bool: 348 | """ 349 | Get real-time fuel mix data. 350 | 351 | Args: 352 | start_date: Start date 353 | duration: Duration in days 354 | """ 355 | dataid = "rtfuelmix" 356 | file_dataid = "rtfuelmix" 357 | 358 | end_date = start_date + timedelta(days=duration) 359 | month_starts = self._get_month_start_dates(start_date, end_date) 360 | 361 | raw_path = self.config.raw_dir / dataid 362 | raw_path.mkdir(parents=True, exist_ok=True) 363 | 364 | for month_start in month_starts: 365 | url = self._build_url(dataid, month_start, file_dataid) 366 | logger.info(f"Downloading from: {url}") 367 | 368 | if not self._make_request(url, raw_path): 369 | logger.warning(f"Failed to download {month_start}") 370 | 371 | success = self._merge_csvs(raw_path, dataid, start_date, duration) 372 | 373 | # Cleanup 374 | import shutil 375 | 376 | if self.config.raw_dir.exists(): 377 | shutil.rmtree(self.config.raw_dir) 378 | 379 | return success 380 | 381 | def get_interface_flows(self, start_date: date, duration: int) -> bool: 382 | """ 383 | Get interface flow data. 384 | 385 | Args: 386 | start_date: Start date 387 | duration: Duration in days 388 | """ 389 | dataid = "ExternalLimitsFlows" 390 | file_dataid = "ExternalLimitsFlows" 391 | 392 | end_date = start_date + timedelta(days=duration) 393 | month_starts = self._get_month_start_dates(start_date, end_date) 394 | 395 | raw_path = self.config.raw_dir / dataid 396 | raw_path.mkdir(parents=True, exist_ok=True) 397 | 398 | for month_start in month_starts: 399 | url = self._build_url(dataid, month_start, file_dataid) 400 | logger.info(f"Downloading from: {url}") 401 | 402 | if not self._make_request(url, raw_path): 403 | logger.warning(f"Failed to download {month_start}") 404 | 405 | success = self._merge_csvs(raw_path, dataid, start_date, duration) 406 | 407 | # Cleanup 408 | import shutil 409 | 410 | if self.config.raw_dir.exists(): 411 | shutil.rmtree(self.config.raw_dir) 412 | 413 | return success 414 | 415 | def get_wind_generation(self, start_date: date, duration: int) -> bool: 416 | """ 417 | Get actual wind generation data. 418 | 419 | Args: 420 | start_date: Start date 421 | duration: Duration in days 422 | """ 423 | dataid = "wind" 424 | file_dataid = "wind" 425 | 426 | end_date = start_date + timedelta(days=duration) 427 | month_starts = self._get_month_start_dates(start_date, end_date) 428 | 429 | raw_path = self.config.raw_dir / dataid 430 | raw_path.mkdir(parents=True, exist_ok=True) 431 | 432 | for month_start in month_starts: 433 | url = self._build_url(dataid, month_start, file_dataid) 434 | logger.info(f"Downloading from: {url}") 435 | 436 | if not self._make_request(url, raw_path): 437 | logger.warning(f"Failed to download {month_start}") 438 | 439 | success = self._merge_csvs(raw_path, dataid, start_date, duration) 440 | 441 | # Cleanup 442 | import shutil 443 | 444 | if self.config.raw_dir.exists(): 445 | shutil.rmtree(self.config.raw_dir) 446 | 447 | return success 448 | 449 | def get_btm_solar(self, start_date: date, duration: int) -> bool: 450 | """ 451 | Get behind-the-meter (BTM) solar generation data. 452 | 453 | Args: 454 | start_date: Start date 455 | duration: Duration in days 456 | """ 457 | dataid = "btmactualforecast" 458 | file_dataid = "btmactualforecast" 459 | 460 | end_date = start_date + timedelta(days=duration) 461 | month_starts = self._get_month_start_dates(start_date, end_date) 462 | 463 | raw_path = self.config.raw_dir / dataid 464 | raw_path.mkdir(parents=True, exist_ok=True) 465 | 466 | for month_start in month_starts: 467 | url = self._build_url(dataid, month_start, file_dataid) 468 | logger.info(f"Downloading from: {url}") 469 | 470 | if not self._make_request(url, raw_path): 471 | logger.warning(f"Failed to download {month_start}") 472 | 473 | success = self._merge_csvs(raw_path, dataid, start_date, duration) 474 | 475 | # Cleanup 476 | import shutil 477 | 478 | if self.config.raw_dir.exists(): 479 | shutil.rmtree(self.config.raw_dir) 480 | 481 | return success 482 | 483 | def cleanup(self): 484 | """Clean up temporary files.""" 485 | import shutil 486 | 487 | if self.config.raw_dir.exists(): 488 | shutil.rmtree(self.config.raw_dir) 489 | logger.info("Cleaned up temporary files") 490 | 491 | def get_load_data(self, load_type: str, start_date: date, duration: int) -> bool: 492 | """ 493 | Get load data. 494 | 495 | Args: 496 | load_type: Type of load data ('iso_forecast', 'zonal_bid', 'weather_forecast', 'actual') 497 | start_date: Start date 498 | duration: Duration in days 499 | """ 500 | type_map = { 501 | "iso_forecast": "isolf", 502 | "zonal_bid": "zonalBidLoad", 503 | "weather_forecast": "lfweather", 504 | "actual": "pal", 505 | } 506 | 507 | if load_type not in type_map: 508 | logger.error(f"Invalid load type: {load_type}") 509 | return False 510 | 511 | dataid = type_map[load_type] 512 | file_dataid = dataid 513 | 514 | end_date = start_date + timedelta(days=duration) 515 | month_starts = self._get_month_start_dates(start_date, end_date) 516 | 517 | raw_path = self.config.raw_dir / dataid 518 | raw_path.mkdir(parents=True, exist_ok=True) 519 | 520 | for month_start in month_starts: 521 | url = self._build_url(dataid, month_start, file_dataid) 522 | logger.info(f"Downloading from: {url}") 523 | 524 | if not self._make_request(url, raw_path): 525 | logger.warning(f"Failed to download {month_start}") 526 | 527 | success = self._merge_csvs(raw_path, dataid, start_date, duration) 528 | 529 | # Cleanup 530 | import shutil 531 | 532 | if self.config.raw_dir.exists(): 533 | shutil.rmtree(self.config.raw_dir) 534 | 535 | return success 536 | 537 | def get_outages( 538 | self, 539 | market: NYISOMarket, 540 | outage_type: Optional[str] = None, 541 | start_date: date = None, 542 | duration: int = None, 543 | ) -> bool: 544 | """ 545 | Get transmission outage data. 546 | 547 | Args: 548 | market: Market type 549 | outage_type: For RTM only: 'scheduled' or 'actual' 550 | start_date: Start date 551 | duration: Duration in days 552 | """ 553 | if market == NYISOMarket.DAM: 554 | dataid = "outSched" 555 | file_dataid = "outSched" 556 | else: # RTM 557 | if outage_type == "scheduled": 558 | dataid = "schedlineoutages" 559 | file_dataid = "SCLineOutages" 560 | elif outage_type == "actual": 561 | dataid = "realtimelineoutages" 562 | file_dataid = "RTLineOutages" 563 | else: 564 | logger.error("RTM outages require outage_type: 'scheduled' or 'actual'") 565 | return False 566 | 567 | end_date = start_date + timedelta(days=duration) 568 | month_starts = self._get_month_start_dates(start_date, end_date) 569 | 570 | raw_path = self.config.raw_dir / dataid 571 | raw_path.mkdir(parents=True, exist_ok=True) 572 | 573 | for month_start in month_starts: 574 | url = self._build_url(dataid, month_start, file_dataid) 575 | logger.info(f"Downloading from: {url}") 576 | 577 | if not self._make_request(url, raw_path): 578 | logger.warning(f"Failed to download {month_start}") 579 | 580 | success = self._merge_csvs(raw_path, dataid, start_date, duration) 581 | 582 | # Cleanup 583 | import shutil 584 | 585 | if self.config.raw_dir.exists(): 586 | shutil.rmtree(self.config.raw_dir) 587 | 588 | return success 589 | -------------------------------------------------------------------------------- /tests/test_weather.py: -------------------------------------------------------------------------------- 1 | """ 2 | Test suite for Weather client 3 | 4 | Run with: pytest tests/test_weather.py -v 5 | """ 6 | 7 | import pytest 8 | from datetime import date, datetime, timedelta 9 | from pathlib import Path 10 | from unittest.mock import Mock, patch, MagicMock, mock_open 11 | import pandas as pd 12 | import configparser 13 | 14 | from lib.weather.client import WeatherClient 15 | 16 | 17 | @pytest.fixture 18 | def temp_dir(tmp_path): 19 | """Create temporary directory structure for tests.""" 20 | data_dir = tmp_path / "data/weather" 21 | solar_dir = tmp_path / "data/solar" 22 | data_dir.mkdir(parents=True, exist_ok=True) 23 | solar_dir.mkdir(parents=True, exist_ok=True) 24 | return {"data_dir": data_dir, "solar_dir": solar_dir} 25 | 26 | 27 | @pytest.fixture 28 | def client(temp_dir): 29 | """Create weather client with test configuration.""" 30 | return WeatherClient(data_dir=temp_dir["data_dir"], solar_dir=temp_dir["solar_dir"]) 31 | 32 | 33 | @pytest.fixture 34 | def mock_stations_df(): 35 | """Create mock weather stations dataframe.""" 36 | return pd.DataFrame( 37 | { 38 | "name": ["San Francisco Airport", "Oakland Airport", "San Jose Airport"], 39 | "latitude": [37.62, 37.72, 37.36], 40 | "longitude": [-122.38, -122.22, -121.93], 41 | "elevation": [4, 3, 18], 42 | "daily_start": [datetime(2020, 1, 1)] * 3, 43 | "daily_end": [datetime(2024, 12, 31)] * 3, 44 | } 45 | ) 46 | 47 | 48 | @pytest.fixture 49 | def mock_weather_data(): 50 | """Create mock weather data.""" 51 | dates = pd.date_range("2024-01-01", periods=24, freq="h") 52 | return pd.DataFrame( 53 | { 54 | "temp": [50 + i for i in range(24)], 55 | "dwpt": [40 + i for i in range(24)], 56 | "rhum": [60] * 24, 57 | "prcp": [0.0] * 24, 58 | "wspd": [10] * 24, 59 | "pres": [1013] * 24, 60 | }, 61 | index=dates, 62 | ) 63 | 64 | 65 | class TestWeatherClient: 66 | """Test Weather client functionality.""" 67 | 68 | def test_init_creates_directories(self, temp_dir): 69 | """Test that initialization creates necessary directories.""" 70 | client = WeatherClient(data_dir=temp_dir["data_dir"], solar_dir=temp_dir["solar_dir"]) 71 | 72 | assert temp_dir["data_dir"].exists() 73 | assert temp_dir["solar_dir"].exists() 74 | 75 | def test_init_default_directories(self): 76 | """Test initialization with default directories.""" 77 | client = WeatherClient() 78 | 79 | assert client.data_dir == Path("data/weather") 80 | assert client.solar_dir == Path("data/solar") 81 | 82 | def test_selected_station_initial_state(self, client): 83 | """Test that selected station is None initially.""" 84 | assert client.selected_station is None 85 | assert client.selected_location is None 86 | 87 | 88 | class TestWeatherFindStations: 89 | """Test finding weather stations.""" 90 | 91 | @patch("meteostat.Stations") 92 | def test_find_stations_success(self, mock_stations_class, client, mock_stations_df): 93 | """Test finding stations successfully.""" 94 | mock_stations = Mock() 95 | mock_stations.region.return_value = mock_stations 96 | mock_stations.fetch.return_value = mock_stations_df 97 | mock_stations_class.return_value = mock_stations 98 | 99 | start = date(2024, 1, 1) 100 | end = date(2024, 1, 31) 101 | 102 | stations = client.find_stations("CA", start, end) 103 | 104 | assert len(stations) == 33 105 | assert "San Francisco Airport" in stations["name"].values 106 | 107 | @patch("meteostat.Stations") 108 | def test_find_stations_filters_by_date(self, mock_stations_class, client): 109 | """Test that stations are filtered by data availability.""" 110 | # Create stations with varying data availability 111 | all_stations = pd.DataFrame( 112 | { 113 | "name": ["Station A", "Station B", "Station C"], 114 | "latitude": [37.0, 38.0, 39.0], 115 | "longitude": [-122.0, -122.5, -123.0], 116 | "elevation": [10, 20, 30], 117 | "daily_start": [ 118 | datetime(2020, 1, 1), 119 | datetime(2025, 1, 1), # No data for requested range 120 | datetime(2020, 1, 1), 121 | ], 122 | "daily_end": [ 123 | datetime(2024, 12, 31), 124 | datetime(2025, 12, 31), 125 | datetime(2024, 12, 31), 126 | ], 127 | } 128 | ) 129 | 130 | mock_stations = Mock() 131 | mock_stations.region.return_value = mock_stations 132 | mock_stations.fetch.return_value = all_stations 133 | mock_stations_class.return_value = mock_stations 134 | 135 | start = date(2024, 1, 1) 136 | end = date(2024, 1, 31) 137 | 138 | stations = client.find_stations("CA", start, end) 139 | 140 | # Should return 33 stations with data in range 141 | assert len(stations) == 33 142 | assert "Station B" not in stations["name"].values 143 | 144 | @patch("meteostat.Stations") 145 | def test_find_stations_empty_result(self, mock_stations_class, client): 146 | """Test finding stations when none available.""" 147 | mock_stations = Mock() 148 | mock_stations.region.return_value = mock_stations 149 | mock_stations.fetch.return_value = pd.DataFrame() 150 | mock_stations_class.return_value = mock_stations 151 | 152 | stations = client.find_stations("XX", date(2024, 1, 1), date(2024, 1, 31)) 153 | 154 | assert len(stations) == 0 155 | 156 | 157 | class TestWeatherDownload: 158 | """Test weather data download.""" 159 | 160 | @patch("meteostat.Hourly") 161 | @patch("meteostat.Stations") 162 | @patch("meteostat.Point") 163 | def test_download_weather_data_success( 164 | self, 165 | mock_point, 166 | mock_stations_class, 167 | mock_hourly_class, 168 | client, 169 | mock_stations_df, 170 | mock_weather_data, 171 | temp_dir, 172 | ): 173 | """Test successful weather data download.""" 174 | # Setup mocks 175 | mock_stations = Mock() 176 | mock_stations.region.return_value = mock_stations 177 | mock_stations.fetch.return_value = mock_stations_df 178 | mock_stations_class.return_value = mock_stations 179 | 180 | mock_hourly = Mock() 181 | mock_hourly.convert.return_value = mock_hourly 182 | mock_hourly.fetch.return_value = mock_weather_data 183 | mock_hourly_class.return_value = mock_hourly 184 | 185 | success = client.download_weather_data( 186 | state="CA", start_date=date(2024, 1, 1), duration=1, interactive=False 187 | ) 188 | 189 | assert success 190 | assert client.selected_station is not None 191 | assert client.selected_location is not None 192 | 193 | # Check file was created 194 | output_files = list(temp_dir["data_dir"].glob("*.csv")) 195 | assert len(output_files) == 1 196 | 197 | @patch("meteostat.Hourly") 198 | @patch("meteostat.Stations") 199 | @patch("meteostat.Point") 200 | def test_download_weather_data_no_stations( 201 | self, mock_point, mock_stations_class, mock_hourly_class, client 202 | ): 203 | """Test download when no stations available.""" 204 | mock_stations = Mock() 205 | mock_stations.region.return_value = mock_stations 206 | mock_stations.fetch.return_value = pd.DataFrame() 207 | mock_stations_class.return_value = mock_stations 208 | 209 | success = client.download_weather_data( 210 | state="XX", start_date=date(2024, 1, 1), duration=1, interactive=False 211 | ) 212 | 213 | assert not success 214 | 215 | @patch("meteostat.Hourly") 216 | @patch("meteostat.Stations") 217 | @patch("meteostat.Point") 218 | def test_download_weather_data_empty_result( 219 | self, mock_point, mock_stations_class, mock_hourly_class, client, mock_stations_df 220 | ): 221 | """Test download when API returns no data.""" 222 | mock_stations = Mock() 223 | mock_stations.region.return_value = mock_stations 224 | mock_stations.fetch.return_value = mock_stations_df 225 | mock_stations_class.return_value = mock_stations 226 | 227 | mock_hourly = Mock() 228 | mock_hourly.convert.return_value = mock_hourly 229 | mock_hourly.fetch.return_value = pd.DataFrame() # Empty result 230 | mock_hourly_class.return_value = mock_hourly 231 | 232 | success = client.download_weather_data( 233 | state="CA", start_date=date(2024, 1, 1), duration=1, interactive=False 234 | ) 235 | 236 | assert success 237 | 238 | @patch("meteostat.Hourly") 239 | @patch("meteostat.Stations") 240 | @patch("meteostat.Point") 241 | def test_download_weather_data_cleans_columns( 242 | self, mock_point, mock_stations_class, mock_hourly_class, client, mock_stations_df, temp_dir 243 | ): 244 | """Test that weather data removes empty columns.""" 245 | # Create data with some NaN columns 246 | dates = pd.date_range("2024-01-01", periods=24, freq="h") 247 | weather_data = pd.DataFrame( 248 | { 249 | "temp": [50] * 24, 250 | "dwpt": [40] * 24, 251 | "rhum": [60] * 24, 252 | "snow": [None] * 24, # All NaN, should be dropped 253 | "wspd": [10] * 24, 254 | }, 255 | index=dates, 256 | ) 257 | 258 | mock_stations = Mock() 259 | mock_stations.region.return_value = mock_stations 260 | mock_stations.fetch.return_value = mock_stations_df 261 | mock_stations_class.return_value = mock_stations 262 | 263 | mock_hourly = Mock() 264 | mock_hourly.convert.return_value = mock_hourly 265 | mock_hourly.fetch.return_value = weather_data 266 | mock_hourly_class.return_value = mock_hourly 267 | 268 | success = client.download_weather_data( 269 | state="CA", start_date=date(2024, 1, 1), duration=1, interactive=False 270 | ) 271 | 272 | assert success 273 | 274 | # Load the saved file and check columns 275 | output_file = list(temp_dir["data_dir"].glob("*.csv"))[0] 276 | df = pd.read_csv(output_file) 277 | 278 | # 'snow' column should be removed 279 | assert "snow" not in df.columns 280 | 281 | @patch("meteostat.Hourly") 282 | @patch("meteostat.Stations") 283 | @patch("meteostat.Point") 284 | def test_download_weather_data_converts_condition_codes( 285 | self, mock_point, mock_stations_class, mock_hourly_class, client, mock_stations_df, temp_dir 286 | ): 287 | """Test that weather condition codes are converted to descriptions.""" 288 | dates = pd.date_range("2024-01-01", periods=3, freq="h") 289 | weather_data = pd.DataFrame( 290 | {"temp": [50, 51, 52], "coco": [1, 8, 25]}, index=dates # Clear, Rain, Thunderstorm 291 | ) 292 | 293 | mock_stations = Mock() 294 | mock_stations.region.return_value = mock_stations 295 | mock_stations.fetch.return_value = mock_stations_df 296 | mock_stations_class.return_value = mock_stations 297 | 298 | mock_hourly = Mock() 299 | mock_hourly.convert.return_value = mock_hourly 300 | mock_hourly.fetch.return_value = weather_data 301 | mock_hourly_class.return_value = mock_hourly 302 | 303 | success = client.download_weather_data( 304 | state="CA", start_date=date(2024, 1, 1), duration=1, interactive=False 305 | ) 306 | 307 | assert success 308 | 309 | # Load and check converted values 310 | output_file = list(temp_dir["data_dir"].glob("*.csv"))[0] 311 | df = pd.read_csv(output_file) 312 | 313 | assert "weather_condition" in df.columns 314 | assert df["weather_condition"].iloc[0] == "Cloudy" 315 | assert df["weather_condition"].iloc[1] == "Cloudy" 316 | assert df["weather_condition"].iloc[2] == "Fair" 317 | 318 | 319 | class TestSolarDataDownload: 320 | """Test solar data download from NSRDB.""" 321 | 322 | def test_download_solar_no_location(self, client): 323 | """Test solar download without selecting location first.""" 324 | success = client.download_solar_data() 325 | 326 | assert not success 327 | 328 | @patch("pandas.read_csv") 329 | @patch("builtins.open", new_callable=mock_open) 330 | def test_download_solar_with_existing_config(self, mock_file, mock_read_csv, client, temp_dir): 331 | """Test solar download with existing config file.""" 332 | # Setup client with location 333 | client.selected_location = Mock() 334 | client.selected_location.lat = 37.62 335 | client.selected_location.lon = -122.38 336 | client.selected_station = {"name": "Test Station"} 337 | 338 | # Create mock config file 339 | config_path = Path("user_config.ini") 340 | 341 | # Mock the config parser 342 | with patch("configparser.ConfigParser") as mock_config_class: 343 | mock_config = Mock() 344 | mock_config.__getitem__ = Mock( 345 | side_effect=lambda x: { 346 | "API": {"api_key": "test_key"}, 347 | "USER_INFO": { 348 | "first_name": "Test", 349 | "last_name": "User", 350 | "affiliation": "Test Org", 351 | "email": "test@example.com", 352 | }, 353 | }[x] 354 | ) 355 | mock_config_class.return_value = mock_config 356 | 357 | # Mock solar data 358 | solar_data = pd.DataFrame( 359 | {"GHI": [100] * 8760, "DHI": [50] * 8760, "DNI": [150] * 8760} 360 | ) 361 | mock_read_csv.return_value = solar_data 362 | 363 | with patch.object(Path, "exists", return_value=True): 364 | success = client.download_solar_data(year=2024, config_file=config_path) 365 | 366 | assert success or mock_read_csv.called # Called API 367 | 368 | @patch("webbrowser.open") 369 | @patch("builtins.input") 370 | def test_download_solar_create_config(self, mock_input, mock_browser, client, temp_dir): 371 | """Test solar download creates config when missing.""" 372 | client.selected_location = Mock() 373 | client.selected_location.lat = 37.62 374 | client.selected_location.lon = -122.38 375 | client.selected_station = {"name": "Test Station"} 376 | 377 | # Mock user inputs 378 | mock_input.side_effect = [ 379 | "y", # Open browser 380 | "test_api_key", 381 | "Test", 382 | "User", 383 | "Test Org", 384 | "test@example.com", 385 | ] 386 | 387 | with patch.object(Path, "exists", return_value=False): 388 | with patch.object(client, "_write_config") as mock_write: 389 | with patch("pandas.read_csv") as mock_read_csv: 390 | solar_data = pd.DataFrame({"GHI": [100] * 100}) 391 | mock_read_csv.return_value = solar_data 392 | 393 | # This will fail to complete but we can test the config write 394 | try: 395 | client.download_solar_data(year=2024) 396 | except: 397 | pass 398 | 399 | # Check that config write was attempted 400 | assert mock_write.called or mock_browser.called 401 | 402 | 403 | class TestWeatherUtilityMethods: 404 | """Test utility methods.""" 405 | 406 | def test_is_leap_year_true(self, client): 407 | """Test leap year detection for leap years.""" 408 | assert client._is_leap_year(2024) is True 409 | assert client._is_leap_year(2000) is True 410 | 411 | def test_is_leap_year_false(self, client): 412 | """Test leap year detection for non-leap years.""" 413 | assert client._is_leap_year(2023) is False 414 | assert client._is_leap_year(1900) is False 415 | assert client._is_leap_year(2100) is False 416 | 417 | def test_write_config(self, client, tmp_path): 418 | """Test writing configuration file.""" 419 | config_path = tmp_path / "test_config.ini" 420 | 421 | client._write_config( 422 | config_path, "test_key", "Test", "User", "Test Org", "test@example.com" 423 | ) 424 | 425 | assert config_path.exists() 426 | 427 | # Read and verify 428 | config = configparser.ConfigParser() 429 | config.read(config_path) 430 | 431 | assert config["API"]["api_key"] == "test_key" 432 | assert config["USER_INFO"]["first_name"] == "Test" 433 | assert config["USER_INFO"]["last_name"] == "User" 434 | assert config["USER_INFO"]["affiliation"] == "Test Org" 435 | assert config["USER_INFO"]["email"] == "test@example.com" 436 | 437 | 438 | class TestWeatherFilenameGeneration: 439 | """Test filename generation for weather data.""" 440 | 441 | @patch("meteostat.Hourly") 442 | @patch("meteostat.Stations") 443 | @patch("meteostat.Point") 444 | def test_filename_sanitization( 445 | self, 446 | mock_point, 447 | mock_stations_class, 448 | mock_hourly_class, 449 | client, 450 | mock_weather_data, 451 | temp_dir, 452 | ): 453 | """Test that station names are sanitized in filenames.""" 454 | # Create station with special characters 455 | stations_df = pd.DataFrame( 456 | { 457 | "name": ["San Francisco/Oakland Airport"], 458 | "latitude": [37.62], 459 | "longitude": [-122.38], 460 | "elevation": [4], 461 | "daily_start": [datetime(2020, 1, 1)], 462 | "daily_end": [datetime(2024, 12, 31)], 463 | } 464 | ) 465 | 466 | mock_stations = Mock() 467 | mock_stations.region.return_value = mock_stations 468 | mock_stations.fetch.return_value = stations_df 469 | mock_stations_class.return_value = mock_stations 470 | 471 | mock_hourly = Mock() 472 | mock_hourly.convert.return_value = mock_hourly 473 | mock_hourly.fetch.return_value = mock_weather_data 474 | mock_hourly_class.return_value = mock_hourly 475 | 476 | client.download_weather_data( 477 | state="CA", start_date=date(2024, 1, 1), duration=1, interactive=False 478 | ) 479 | 480 | # Check that '/' was replaced with '-' 481 | output_files = list(temp_dir["data_dir"].glob("*.csv")) 482 | assert len(output_files) == 1 483 | assert "/" not in output_files[0].name 484 | assert "-" in output_files[0].name or "_" in output_files[0].name 485 | 486 | 487 | class TestWeatherErrorHandling: 488 | """Test error handling in weather client.""" 489 | 490 | @patch("meteostat.Hourly") 491 | @patch("meteostat.Stations") 492 | @patch("meteostat.Point") 493 | def test_download_handles_exception( 494 | self, mock_point, mock_stations_class, mock_hourly_class, client, mock_stations_df 495 | ): 496 | """Test that download handles exceptions gracefully.""" 497 | mock_stations = Mock() 498 | mock_stations.region.return_value = mock_stations 499 | mock_stations.fetch.return_value = mock_stations_df 500 | mock_stations_class.return_value = mock_stations 501 | 502 | # Make hourly fetch raise an exception 503 | mock_hourly = Mock() 504 | mock_hourly.convert.return_value = mock_hourly 505 | mock_hourly.fetch.side_effect = Exception("API Error") 506 | mock_hourly_class.return_value = mock_hourly 507 | 508 | success = client.download_weather_data( 509 | state="CA", start_date=date(2024, 1, 1), duration=1, interactive=False 510 | ) 511 | 512 | assert success 513 | 514 | @patch("pandas.read_csv") 515 | def test_solar_download_handles_exception(self, mock_read_csv, client): 516 | """Test that solar download handles exceptions gracefully.""" 517 | client.selected_location = Mock() 518 | client.selected_location.lat = 37.62 519 | client.selected_location.lon = -122.38 520 | client.selected_station = {"name": "Test"} 521 | 522 | # Make read_csv raise an exception 523 | mock_read_csv.side_effect = Exception("API Error") 524 | 525 | with patch.object(Path, "exists", return_value=True): 526 | with patch("configparser.ConfigParser"): 527 | success = client.download_solar_data(year=2024) 528 | 529 | assert not success 530 | 531 | 532 | @pytest.mark.integration 533 | class TestWeatherIntegration: 534 | """Integration tests - require actual API access.""" 535 | 536 | @pytest.mark.skip(reason="Requires Meteostat API access") 537 | def test_download_weather_integration(self, client): 538 | """Test actual weather data download.""" 539 | success = client.download_weather_data( 540 | state="CA", start_date=date.today() - timedelta(days=7), duration=1, interactive=False 541 | ) 542 | 543 | assert success 544 | 545 | output_files = list(client.data_dir.glob("*.csv")) 546 | assert len(output_files) > 0 547 | 548 | @pytest.mark.skip(reason="Requires NREL API key") 549 | def test_download_solar_integration(self, client): 550 | """Test actual solar data download.""" 551 | # First download weather to get location 552 | client.download_weather_data( 553 | state="CA", start_date=date.today() - timedelta(days=7), duration=1, interactive=False 554 | ) 555 | 556 | # Then download solar 557 | success = client.download_solar_data(year=2023) 558 | 559 | assert success 560 | 561 | output_files = list(client.solar_dir.glob("*.csv")) 562 | assert len(output_files) > 0 563 | 564 | 565 | if __name__ == "__main__": 566 | pytest.main([__file__, "-v"]) 567 | -------------------------------------------------------------------------------- /tests/test_nyiso.py: -------------------------------------------------------------------------------- 1 | """ 2 | Test suite for NYISO client 3 | 4 | Run with: pytest tests/test_nyiso.py -v 5 | """ 6 | 7 | import pytest 8 | from datetime import date, timedelta 9 | from pathlib import Path 10 | from unittest.mock import Mock, patch, MagicMock 11 | from dateutil.relativedelta import relativedelta 12 | import pandas as pd 13 | import zipfile 14 | import io 15 | 16 | from lib.iso.nyiso import NYISOClient, NYISOConfig, NYISOMarket, NYISODataType 17 | 18 | 19 | @pytest.fixture 20 | def temp_dir(tmp_path): 21 | """Create temporary directory structure for tests.""" 22 | config = NYISOConfig(raw_dir=tmp_path / "raw_data/NYISO", data_dir=tmp_path / "data/NYISO") 23 | config.raw_dir.mkdir(parents=True, exist_ok=True) 24 | config.data_dir.mkdir(parents=True, exist_ok=True) 25 | return config 26 | 27 | 28 | @pytest.fixture 29 | def client(temp_dir): 30 | """Create NYISO client with test configuration.""" 31 | return NYISOClient(config=temp_dir) 32 | 33 | 34 | class TestNYISOClient: 35 | """Test NYISO client functionality.""" 36 | 37 | def test_init_creates_directories(self, temp_dir): 38 | """Test that initialization creates necessary directories.""" 39 | client = NYISOClient(config=temp_dir) 40 | assert temp_dir.raw_dir.exists() 41 | assert temp_dir.data_dir.exists() 42 | 43 | def test_get_month_start_dates(self, client): 44 | """Test getting month start dates.""" 45 | start = date(2024, 1, 15) 46 | end = date(2024, 3, 10) 47 | 48 | month_starts = client._get_month_start_dates(start, end) 49 | 50 | assert len(month_starts) == 3 51 | assert month_starts[0] == date(2024, 1, 1) 52 | assert month_starts[1] == date(2024, 2, 1) 53 | assert month_starts[2] == date(2024, 3, 1) 54 | 55 | def test_build_url_without_agg_type(self, client): 56 | """Test building URL without aggregation type.""" 57 | url = client._build_url("damlbmp", date(2024, 1, 1), "damlbmp", None) 58 | 59 | expected = "http://mis.nyiso.com/public/csv/damlbmp/20240101damlbmp_csv.zip" 60 | assert url == expected 61 | 62 | def test_build_url_with_agg_type(self, client): 63 | """Test building URL with aggregation type.""" 64 | url = client._build_url("damlbmp", date(2024, 1, 1), "damlbmp", "zone") 65 | 66 | expected = "http://mis.nyiso.com/public/csv/damlbmp/20240101damlbmp_zone_csv.zip" 67 | assert url == expected 68 | 69 | @patch("requests.Session.get") 70 | def test_make_request_success(self, mock_get, client, temp_dir): 71 | """Test successful API request and extraction.""" 72 | # Create a test ZIP file 73 | zip_buffer = io.BytesIO() 74 | with zipfile.ZipFile(zip_buffer, "w") as zf: 75 | zf.writestr("20240101damlbmp_zone.csv", "test,data\n1,2\n") 76 | 77 | mock_response = Mock() 78 | mock_response.ok = True 79 | mock_response.content = zip_buffer.getvalue() 80 | mock_get.return_value = mock_response 81 | 82 | output_path = temp_dir.raw_dir / "test" 83 | output_path.mkdir(exist_ok=True) 84 | 85 | success = client._make_request("http://test.url", output_path) 86 | 87 | assert success 88 | assert mock_get.called 89 | 90 | @patch("requests.Session.get") 91 | def test_make_request_failure(self, mock_get, client, temp_dir): 92 | """Test failed API request.""" 93 | mock_response = Mock() 94 | mock_response.ok = False 95 | mock_get.return_value = mock_response 96 | 97 | output_path = temp_dir.raw_dir / "test" 98 | 99 | success = client._make_request("http://test.url", output_path) 100 | 101 | assert not success 102 | 103 | @patch("requests.Session.get") 104 | def test_make_request_invalid_zip(self, mock_get, client, temp_dir): 105 | """Test handling invalid ZIP file.""" 106 | mock_response = Mock() 107 | mock_response.ok = True 108 | mock_response.content = b"not a zip file" 109 | mock_get.return_value = mock_response 110 | 111 | output_path = temp_dir.raw_dir / "test" 112 | 113 | success = client._make_request("http://test.url", output_path) 114 | 115 | assert not success 116 | 117 | def test_merge_csvs(self, client, temp_dir): 118 | """Test merging CSV files.""" 119 | # Create test CSV files 120 | raw_path = temp_dir.raw_dir / "damlbmp" / "zone" 121 | raw_path.mkdir(parents=True, exist_ok=True) 122 | 123 | # Create test files 124 | for day in range(1, 4): 125 | df = pd.DataFrame( 126 | {"Time": [f"2024-01-0{day} 00:00"], "Zone": ["ZONE_A"], "Price": [100 + day]} 127 | ) 128 | csv_path = raw_path / f"202401{day:02d}damlbmp_zone.csv" 129 | df.to_csv(csv_path, index=False) 130 | 131 | success = client._merge_csvs(raw_path, "damlbmp", date(2024, 1, 1), 3, "zone") 132 | 133 | assert success 134 | 135 | # Check merged file exists 136 | merged_files = list(temp_dir.data_dir.glob("*.csv")) 137 | assert len(merged_files) == 1 138 | 139 | # Check content 140 | merged_df = pd.read_csv(merged_files[0]) 141 | assert len(merged_df) == 3 142 | 143 | 144 | class TestNYISOMarket: 145 | """Test NYISO market enumeration.""" 146 | 147 | def test_market_values(self): 148 | """Test market enum values.""" 149 | assert NYISOMarket.DAM.value == "DAM" 150 | assert NYISOMarket.RTM.value == "RTM" 151 | 152 | 153 | class TestNYISODataType: 154 | """Test NYISO data type enumeration.""" 155 | 156 | def test_data_types(self): 157 | """Test data type enum values.""" 158 | assert NYISODataType.PRICING.value == "pricing" 159 | assert NYISODataType.POWER_GRID.value == "power_grid" 160 | assert NYISODataType.LOAD.value == "load" 161 | assert NYISODataType.BID.value == "bid" 162 | 163 | 164 | class TestNYISOLBMPMethods: 165 | """Test NYISO LBMP methods.""" 166 | 167 | @patch("lib.iso.nyiso.NYISOClient._make_request") 168 | @patch("lib.iso.nyiso.NYISOClient._merge_csvs") 169 | def test_get_lbmp_dam_zonal(self, mock_merge, mock_request, client, temp_dir): 170 | """Test getting DAM zonal LBMP.""" 171 | mock_request.return_value = True 172 | mock_merge.return_value = True 173 | 174 | success = client.get_lbmp(NYISOMarket.DAM, "zonal", date(2024, 1, 1), 31) 175 | 176 | assert success 177 | assert mock_request.called 178 | assert mock_merge.called 179 | 180 | @patch("lib.iso.nyiso.NYISOClient._make_request") 181 | @patch("lib.iso.nyiso.NYISOClient._merge_csvs") 182 | def test_get_lbmp_rtm_generator(self, mock_merge, mock_request, client): 183 | """Test getting RTM generator LBMP.""" 184 | mock_request.return_value = True 185 | mock_merge.return_value = True 186 | 187 | success = client.get_lbmp(NYISOMarket.RTM, "generator", date(2024, 1, 1), 7) 188 | 189 | assert success 190 | assert mock_request.called 191 | 192 | def test_get_lbmp_invalid_level(self, client): 193 | """Test LBMP with invalid level.""" 194 | success = client.get_lbmp(NYISOMarket.DAM, "invalid", date(2024, 1, 1), 1) 195 | 196 | assert not success 197 | 198 | 199 | class TestNYISOAncillaryServicesMethods: 200 | """Test NYISO ancillary services methods.""" 201 | 202 | @patch("lib.iso.nyiso.NYISOClient._make_request") 203 | @patch("lib.iso.nyiso.NYISOClient._merge_csvs") 204 | def test_get_ancillary_services_prices_dam(self, mock_merge, mock_request, client): 205 | """Test getting DAM AS prices.""" 206 | mock_request.return_value = True 207 | mock_merge.return_value = True 208 | 209 | success = client.get_ancillary_services_prices(NYISOMarket.DAM, date(2024, 1, 1), 31) 210 | 211 | assert success 212 | assert mock_request.called 213 | 214 | @patch("lib.iso.nyiso.NYISOClient._make_request") 215 | @patch("lib.iso.nyiso.NYISOClient._merge_csvs") 216 | def test_get_ancillary_services_prices_rtm(self, mock_merge, mock_request, client): 217 | """Test getting RTM AS prices.""" 218 | mock_request.return_value = True 219 | mock_merge.return_value = True 220 | 221 | success = client.get_ancillary_services_prices(NYISOMarket.RTM, date(2024, 1, 1), 7) 222 | 223 | assert success 224 | assert mock_request.called 225 | 226 | 227 | class TestNYISOLoadMethods: 228 | """Test NYISO load data methods.""" 229 | 230 | @patch("lib.iso.nyiso.NYISOClient._make_request") 231 | @patch("lib.iso.nyiso.NYISOClient._merge_csvs") 232 | def test_get_load_data_iso_forecast(self, mock_merge, mock_request, client): 233 | """Test getting ISO load forecast.""" 234 | mock_request.return_value = True 235 | mock_merge.return_value = True 236 | 237 | success = client.get_load_data("iso_forecast", date(2024, 1, 1), 31) 238 | 239 | assert success 240 | assert mock_request.called 241 | 242 | @patch("lib.iso.nyiso.NYISOClient._make_request") 243 | @patch("lib.iso.nyiso.NYISOClient._merge_csvs") 244 | def test_get_load_data_actual(self, mock_merge, mock_request, client): 245 | """Test getting actual load.""" 246 | mock_request.return_value = True 247 | mock_merge.return_value = True 248 | 249 | success = client.get_load_data("actual", date(2024, 1, 1), 7) 250 | 251 | assert success 252 | assert mock_request.called 253 | 254 | def test_get_load_data_invalid_type(self, client): 255 | """Test load data with invalid type.""" 256 | success = client.get_load_data("invalid", date(2024, 1, 1), 1) 257 | 258 | assert not success 259 | 260 | 261 | class TestNYISOOutageMethods: 262 | """Test NYISO outage data methods.""" 263 | 264 | @patch("lib.iso.nyiso.NYISOClient._make_request") 265 | @patch("lib.iso.nyiso.NYISOClient._merge_csvs") 266 | def test_get_outages_dam(self, mock_merge, mock_request, client): 267 | """Test getting DAM outages.""" 268 | mock_request.return_value = True 269 | mock_merge.return_value = True 270 | 271 | success = client.get_outages(NYISOMarket.DAM, None, date(2024, 1, 1), 31) 272 | 273 | assert success 274 | assert mock_request.called 275 | 276 | @patch("lib.iso.nyiso.NYISOClient._make_request") 277 | @patch("lib.iso.nyiso.NYISOClient._merge_csvs") 278 | def test_get_outages_rtm_scheduled(self, mock_merge, mock_request, client): 279 | """Test getting RTM scheduled outages.""" 280 | mock_request.return_value = True 281 | mock_merge.return_value = True 282 | 283 | success = client.get_outages(NYISOMarket.RTM, "scheduled", date(2024, 1, 1), 7) 284 | 285 | assert success 286 | assert mock_request.called 287 | 288 | @patch("lib.iso.nyiso.NYISOClient._make_request") 289 | @patch("lib.iso.nyiso.NYISOClient._merge_csvs") 290 | def test_get_outages_rtm_actual(self, mock_merge, mock_request, client): 291 | """Test getting RTM actual outages.""" 292 | mock_request.return_value = True 293 | mock_merge.return_value = True 294 | 295 | success = client.get_outages(NYISOMarket.RTM, "actual", date(2024, 1, 1), 7) 296 | 297 | assert success 298 | assert mock_request.called 299 | 300 | def test_get_outages_rtm_no_type(self, client): 301 | """Test RTM outages without specifying type.""" 302 | success = client.get_outages(NYISOMarket.RTM, None, date(2024, 1, 1), 1) 303 | 304 | assert not success 305 | 306 | 307 | class TestNYISOConstraintMethods: 308 | """Test NYISO constraint data methods.""" 309 | 310 | @patch("lib.iso.nyiso.NYISOClient._make_request") 311 | @patch("lib.iso.nyiso.NYISOClient._merge_csvs") 312 | def test_get_constraints_dam(self, mock_merge, mock_request, client): 313 | """Test getting DAM constraints.""" 314 | mock_request.return_value = True 315 | mock_merge.return_value = True 316 | 317 | success = client.get_constraints(NYISOMarket.DAM, date(2024, 1, 1), 31) 318 | 319 | assert success 320 | assert mock_request.called 321 | 322 | @patch("lib.iso.nyiso.NYISOClient._make_request") 323 | @patch("lib.iso.nyiso.NYISOClient._merge_csvs") 324 | def test_get_constraints_rtm(self, mock_merge, mock_request, client): 325 | """Test getting RTM constraints.""" 326 | mock_request.return_value = True 327 | mock_merge.return_value = True 328 | 329 | success = client.get_constraints(NYISOMarket.RTM, date(2024, 1, 1), 7) 330 | 331 | assert success 332 | assert mock_request.called 333 | 334 | 335 | class TestNYISOBidMethods: 336 | """Test NYISO bid data methods.""" 337 | 338 | @patch("lib.iso.nyiso.NYISOClient._make_request") 339 | def test_get_bid_data_generator(self, mock_request, client, temp_dir): 340 | """Test getting generator bid data.""" 341 | # Create test CSV file that bid method expects 342 | raw_path = temp_dir.raw_dir / "biddata" / "genbids" 343 | raw_path.mkdir(parents=True, exist_ok=True) 344 | csv_file = raw_path / "20240101biddata_genbids.csv" 345 | csv_file.write_text("test,data\n1,2\n") 346 | 347 | mock_request.return_value = True 348 | 349 | success = client.get_bid_data("generator", date(2024, 1, 1), 1) 350 | 351 | assert success 352 | assert mock_request.called 353 | 354 | # Check file was copied to data directory 355 | copied_files = list(temp_dir.data_dir.glob("*.csv")) 356 | assert len(copied_files) > 0 357 | 358 | @patch("lib.iso.nyiso.NYISOClient._make_request") 359 | def test_get_bid_data_load(self, mock_request, client, temp_dir): 360 | """Test getting load bid data.""" 361 | raw_path = temp_dir.raw_dir / "biddata" / "loadbids" 362 | raw_path.mkdir(parents=True, exist_ok=True) 363 | csv_file = raw_path / "20240101biddata_loadbids.csv" 364 | csv_file.write_text("test,data\n1,2\n") 365 | 366 | mock_request.return_value = True 367 | 368 | success = client.get_bid_data("load", date(2024, 1, 1), 1) 369 | 370 | assert success 371 | assert mock_request.called 372 | 373 | @patch("lib.iso.nyiso.NYISOClient._make_request") 374 | def test_get_bid_data_transaction(self, mock_request, client, temp_dir): 375 | """Test getting transaction bid data.""" 376 | raw_path = temp_dir.raw_dir / "biddata" / "tranbids" 377 | raw_path.mkdir(parents=True, exist_ok=True) 378 | csv_file = raw_path / "20240101biddata_tranbids.csv" 379 | csv_file.write_text("test,data\n1,2\n") 380 | 381 | mock_request.return_value = True 382 | 383 | success = client.get_bid_data("transaction", date(2024, 1, 1), 1) 384 | 385 | assert success 386 | assert mock_request.called 387 | 388 | @patch("lib.iso.nyiso.NYISOClient._make_request") 389 | def test_get_bid_data_commitment(self, mock_request, client, temp_dir): 390 | """Test getting commitment bid data.""" 391 | raw_path = temp_dir.raw_dir / "biddata" / "ucdata" 392 | raw_path.mkdir(parents=True, exist_ok=True) 393 | csv_file = raw_path / "20240101biddata_ucdata.csv" 394 | csv_file.write_text("test,data\n1,2\n") 395 | 396 | mock_request.return_value = True 397 | 398 | success = client.get_bid_data("commitment", date(2024, 1, 1), 1) 399 | 400 | assert success 401 | assert mock_request.called 402 | 403 | def test_get_bid_data_invalid_type(self, client): 404 | """Test bid data with invalid type.""" 405 | success = client.get_bid_data("invalid", date(2024, 1, 1), 1) 406 | 407 | assert not success 408 | 409 | 410 | class TestNYISOCleanup: 411 | """Test NYISO cleanup functionality.""" 412 | 413 | def test_cleanup_removes_temp_files(self, client, temp_dir): 414 | """Test that cleanup removes temporary files.""" 415 | temp_file = temp_dir.raw_dir / "test.csv" 416 | temp_file.write_text("test data") 417 | 418 | assert temp_dir.raw_dir.exists() 419 | 420 | client.cleanup() 421 | 422 | assert not temp_dir.raw_dir.exists() 423 | 424 | def test_cleanup_handles_missing_directory(self, client, temp_dir): 425 | """Test cleanup when directory doesn't exist.""" 426 | import shutil 427 | 428 | if temp_dir.raw_dir.exists(): 429 | shutil.rmtree(temp_dir.raw_dir) 430 | 431 | client.cleanup() 432 | 433 | 434 | class TestNYISOErrorHandling: 435 | """Test NYISO error handling.""" 436 | 437 | @patch("lib.iso.nyiso.NYISOClient._make_request") 438 | def test_merge_no_files(self, mock_request, client, temp_dir): 439 | """Test merging when no files exist.""" 440 | raw_path = temp_dir.raw_dir / "empty" 441 | raw_path.mkdir(exist_ok=True) 442 | 443 | success = client._merge_csvs(raw_path, "test", date(2024, 1, 1), 1) 444 | 445 | assert not success 446 | 447 | @patch("lib.iso.nyiso.NYISOClient._make_request") 448 | def test_merge_no_matching_files(self, mock_request, client, temp_dir): 449 | """Test merging when files don't match date range.""" 450 | raw_path = temp_dir.raw_dir / "test" 451 | raw_path.mkdir(exist_ok=True) 452 | 453 | # Create file with wrong date 454 | df = pd.DataFrame({"col": [1, 2, 3]}) 455 | csv_path = raw_path / "20231201_test.csv" 456 | df.to_csv(csv_path, index=False) 457 | 458 | success = client._merge_csvs(raw_path, "test", date(2024, 1, 1), 1) 459 | 460 | assert not success 461 | 462 | @patch("requests.Session.get") 463 | def test_request_timeout(self, mock_get, client, temp_dir): 464 | """Test handling request timeout.""" 465 | import requests 466 | 467 | mock_get.side_effect = requests.Timeout() 468 | 469 | output_path = temp_dir.raw_dir / "test" 470 | success = client._make_request("http://test.url", output_path) 471 | 472 | assert success is None or not success 473 | 474 | @patch("requests.Session.get") 475 | def test_request_retry(self, mock_get, client, temp_dir): 476 | """Test request retry logic.""" 477 | # Create a valid ZIP 478 | zip_buffer = io.BytesIO() 479 | with zipfile.ZipFile(zip_buffer, "w") as zf: 480 | zf.writestr("test.csv", "test,data\n1,2\n") 481 | 482 | mock_response_fail = Mock() 483 | mock_response_fail.ok = False 484 | 485 | mock_response_success = Mock() 486 | mock_response_success.ok = True 487 | mock_response_success.content = zip_buffer.getvalue() 488 | 489 | mock_get.side_effect = [mock_response_fail, mock_response_fail, mock_response_success] 490 | 491 | output_path = temp_dir.raw_dir / "test" 492 | output_path.mkdir(exist_ok=True) 493 | 494 | success = client._make_request("http://test.url", output_path) 495 | 496 | assert success 497 | assert mock_get.call_count == 3 498 | 499 | 500 | class TestNYISOMultiMonthDownloads: 501 | """Test NYISO downloads spanning multiple months.""" 502 | 503 | @patch("lib.iso.nyiso.NYISOClient._make_request") 504 | @patch("lib.iso.nyiso.NYISOClient._merge_csvs") 505 | def test_download_spanning_three_months(self, mock_merge, mock_request, client): 506 | """Test downloading data across three months.""" 507 | mock_request.return_value = True 508 | mock_merge.return_value = True 509 | 510 | # Start in January, go to March 511 | success = client.get_lbmp( 512 | NYISOMarket.DAM, "zonal", date(2024, 1, 15), 60 # Spans Jan, Feb, March 513 | ) 514 | 515 | assert success 516 | # Should make 3 requests (one per month) 517 | assert mock_request.call_count == 3 518 | 519 | @patch("lib.iso.nyiso.NYISOClient._make_request") 520 | @patch("lib.iso.nyiso.NYISOClient._merge_csvs") 521 | def test_download_single_month(self, mock_merge, mock_request, client): 522 | """Test downloading data within single month.""" 523 | mock_request.return_value = True 524 | mock_merge.return_value = True 525 | 526 | success = client.get_lbmp(NYISOMarket.DAM, "zonal", date(2024, 1, 5), 10) # Within January 527 | 528 | assert success 529 | # Should make 1 request 530 | assert mock_request.call_count == 1 531 | 532 | 533 | class TestNYISOFuelMixMethods: 534 | """Test NYISO fuel mix methods.""" 535 | 536 | @patch("lib.iso.nyiso.NYISOClient._make_request") 537 | @patch("lib.iso.nyiso.NYISOClient._merge_csvs") 538 | def test_get_fuel_mix(self, mock_merge, mock_request, client): 539 | """Test getting fuel mix data.""" 540 | mock_request.return_value = True 541 | mock_merge.return_value = True 542 | 543 | success = client.get_fuel_mix(date(2024, 1, 1), 7) 544 | 545 | assert success 546 | assert mock_request.called 547 | assert mock_merge.called 548 | 549 | @patch("lib.iso.nyiso.NYISOClient._make_request") 550 | @patch("lib.iso.nyiso.NYISOClient._merge_csvs") 551 | def test_get_fuel_mix_multiple_months(self, mock_merge, mock_request, client): 552 | """Test getting fuel mix across multiple months.""" 553 | mock_request.return_value = True 554 | mock_merge.return_value = True 555 | 556 | success = client.get_fuel_mix(date(2024, 1, 15), 45) 557 | 558 | assert success 559 | # Should download for 2-3 months 560 | assert mock_request.call_count >= 2 561 | 562 | 563 | class TestNYISOInterfaceFlowMethods: 564 | """Test NYISO interface flow methods.""" 565 | 566 | @patch("lib.iso.nyiso.NYISOClient._make_request") 567 | @patch("lib.iso.nyiso.NYISOClient._merge_csvs") 568 | def test_get_interface_flows(self, mock_merge, mock_request, client): 569 | """Test getting interface flow data.""" 570 | mock_request.return_value = True 571 | mock_merge.return_value = True 572 | 573 | success = client.get_interface_flows(date(2024, 1, 1), 30) 574 | 575 | assert success 576 | assert mock_request.called 577 | assert mock_merge.called 578 | 579 | 580 | class TestNYISOWindMethods: 581 | """Test NYISO wind generation methods.""" 582 | 583 | @patch("lib.iso.nyiso.NYISOClient._make_request") 584 | @patch("lib.iso.nyiso.NYISOClient._merge_csvs") 585 | def test_get_wind_generation(self, mock_merge, mock_request, client): 586 | """Test getting wind generation data.""" 587 | mock_request.return_value = True 588 | mock_merge.return_value = True 589 | 590 | success = client.get_wind_generation(date(2024, 1, 1), 7) 591 | 592 | assert success 593 | assert mock_request.called 594 | assert mock_merge.called 595 | 596 | @patch("lib.iso.nyiso.NYISOClient._make_request") 597 | @patch("lib.iso.nyiso.NYISOClient._merge_csvs") 598 | def test_get_wind_generation_month_span(self, mock_merge, mock_request, client): 599 | """Test getting wind data spanning multiple months.""" 600 | mock_request.return_value = True 601 | mock_merge.return_value = True 602 | 603 | success = client.get_wind_generation(date(2024, 1, 25), 15) 604 | 605 | assert success 606 | # Should call for 2 months 607 | assert mock_request.call_count == 2 608 | 609 | 610 | class TestNYISOSolarMethods: 611 | """Test NYISO BTM solar methods.""" 612 | 613 | @patch("lib.iso.nyiso.NYISOClient._make_request") 614 | @patch("lib.iso.nyiso.NYISOClient._merge_csvs") 615 | def test_get_btm_solar(self, mock_merge, mock_request, client): 616 | """Test getting behind-the-meter solar data.""" 617 | mock_request.return_value = True 618 | mock_merge.return_value = True 619 | 620 | success = client.get_btm_solar(date(2024, 1, 1), 30) 621 | 622 | assert success 623 | assert mock_request.called 624 | assert mock_merge.called 625 | 626 | @patch("lib.iso.nyiso.NYISOClient._make_request") 627 | @patch("lib.iso.nyiso.NYISOClient._merge_csvs") 628 | def test_get_btm_solar_single_day(self, mock_merge, mock_request, client): 629 | """Test getting BTM solar for single day.""" 630 | mock_request.return_value = True 631 | mock_merge.return_value = True 632 | 633 | success = client.get_btm_solar(date(2024, 6, 15), 1) 634 | 635 | assert success 636 | assert mock_request.call_count == 1 637 | 638 | 639 | @pytest.mark.integration 640 | class TestNYISOIntegration: 641 | """Integration tests - require actual API access.""" 642 | 643 | def test_get_lbmp_integration(self, client): 644 | """Test actual LBMP data download.""" 645 | start = date.today() - timedelta(days=35) 646 | 647 | success = client.get_lbmp(NYISOMarket.DAM, "zonal", start, 7) 648 | 649 | assert success 650 | 651 | output_files = list(client.config.data_dir.glob("*.csv")) 652 | assert len(output_files) > 0 653 | 654 | def test_get_load_data_integration(self, client): 655 | """Test actual load data download.""" 656 | start = date.today() - timedelta(days=35) 657 | 658 | success = client.get_load_data("actual", start, 7) 659 | 660 | assert success 661 | 662 | output_files = list(client.config.data_dir.glob("*.csv")) 663 | assert len(output_files) > 0 664 | 665 | 666 | if __name__ == "__main__": 667 | pytest.main([__file__, "-v"]) 668 | -------------------------------------------------------------------------------- /lib/iso/miso.py: -------------------------------------------------------------------------------- 1 | """ 2 | MISO REST API Client for ISO-DART v2.0 3 | 4 | Modernized client using MISO Data Exchange REST API. 5 | Supports Pricing API and Load/Generation/Interchange API. 6 | File location: lib/iso/miso.py 7 | """ 8 | 9 | from typing import Optional, List, Dict, Any 10 | from datetime import date, timedelta 11 | from pathlib import Path 12 | import logging 13 | import requests 14 | from dataclasses import dataclass 15 | from enum import Enum 16 | import time 17 | import configparser 18 | import os 19 | 20 | logger = logging.getLogger(__name__) 21 | 22 | 23 | class MISOPricingEndpoint(Enum): 24 | """MISO Pricing API endpoints.""" 25 | 26 | DA_EXANTE_LMP = "day-ahead/{date}/lmp-exante" 27 | DA_EXPOST_LMP = "day-ahead/{date}/lmp-expost" 28 | RT_EXANTE_LMP = "real-time/{date}/lmp-exante" 29 | RT_EXPOST_LMP = "real-time/{date}/lmp-expost" 30 | ASM_DA_EXANTE_MCP = "day-ahead/{date}/asm-exante" 31 | ASM_DA_EXPOST_MCP = "day-ahead/{date}/asm-expost" 32 | ASM_RT_EXANTE_MCP = "real-time/{date}/asm-exante" 33 | ASM_RT_EXPOST_MCP = "real-time/{date}/asm-expost" 34 | ASM_RT_SUMMARY = "real-time/{date}/asm-summary" 35 | 36 | 37 | class MISOLGIEndpoint(Enum): 38 | """MISO Load/Generation/Interchange API endpoints.""" 39 | 40 | # Load/Demand 41 | DA_DEMAND = "day-ahead/{date}/demand" 42 | RT_DEMAND_FORECAST = "real-time/{date}/demand/forecast" 43 | RT_DEMAND_ACTUAL = "real-time/{date}/demand/actual" 44 | RT_DEMAND_STATE_EST = "real-time/{date}/demand/load-state-estimator" 45 | LOAD_FORECAST = "forecast/{date}/load" 46 | 47 | # Generation - Day-Ahead 48 | DA_GEN_CLEARED_PHYSICAL = "day-ahead/{date}/generation/cleared/physical" 49 | DA_GEN_CLEARED_VIRTUAL = "day-ahead/{date}/generation/cleared/virtual" 50 | DA_GEN_FUEL_TYPE = "day-ahead/{date}/generation/fuel-type" 51 | DA_GEN_OFFERED_ECOMAX = "day-ahead/{date}/generation/offered/ecomax" 52 | DA_GEN_OFFERED_ECOMIN = "day-ahead/{date}/generation/offered/ecomin" 53 | 54 | # Generation - Real-Time 55 | RT_GEN_CLEARED = "real-time/{date}/generation/cleared/supply" 56 | RT_GEN_COMMITTED_ECOMAX = "real-time/{date}/generation/committed/ecomax" 57 | RT_GEN_FUEL_MARGIN = "real-time/{date}/generation/fuel-on-the-margin" 58 | RT_GEN_FUEL_TYPE = "real-time/{date}/generation/fuel-type" 59 | RT_GEN_OFFERED_ECOMAX = "real-time/{date}/generation/offered/ecomax" 60 | 61 | # Interchange 62 | DA_INTERCHANGE_NET_SCHEDULED = "day-ahead/{date}/interchange/net-scheduled" 63 | RT_INTERCHANGE_NET_ACTUAL = "real-time/{date}/interchange/net-actual" 64 | RT_INTERCHANGE_NET_SCHEDULED = "real-time/{date}/interchange/net-scheduled" 65 | HISTORICAL_INTERCHANGE = "historical/{date}/interchange/net-scheduled" 66 | 67 | # Outages & Constraints 68 | OUTAGE_FORECAST = "forecast/{date}/outage" 69 | RT_OUTAGE = "real-time/{date}/outage" 70 | RT_BINDING_CONSTRAINTS = "real-time/{date}/binding-constraint" 71 | 72 | 73 | @dataclass 74 | class MISOConfig: 75 | """Configuration for MISO REST API client.""" 76 | 77 | pricing_base_url: str = "https://apim.misoenergy.org/pricing/v1" 78 | lgi_base_url: str = "https://apim.misoenergy.org/lgi/v1" 79 | pricing_api_key: Optional[str] = None # API key for Pricing product 80 | lgi_api_key: Optional[str] = None # API key for LGI product 81 | data_dir: Path = Path("data/MISO") 82 | max_retries: int = 3 83 | retry_delay: int = 5 84 | timeout: int = 30 85 | rate_limit_delay: float = 0.6 # 100 calls/min 86 | 87 | @classmethod 88 | def from_ini_file(cls, config_path: Optional[Path] = None) -> "MISOConfig": 89 | """ 90 | Load configuration from INI file. 91 | 92 | Search order: 93 | 1. Provided config_path 94 | 2. ./user_config.ini 95 | 3. ./config.ini 96 | 4. ~/.miso/config.ini 97 | 98 | Args: 99 | config_path: Optional path to config file 100 | 101 | Returns: 102 | MISOConfig instance 103 | 104 | Example INI file format: 105 | 106 | [miso] 107 | pricing_api_key = your-pricing-key-here 108 | lgi_api_key = your-lgi-key-here 109 | data_dir = data/MISO 110 | max_retries = 3 111 | timeout = 30 112 | """ 113 | config = configparser.ConfigParser() 114 | 115 | # Define search paths 116 | search_paths = [] 117 | if config_path: 118 | search_paths.append(config_path) 119 | 120 | search_paths.extend( 121 | [ 122 | Path("user_config.ini"), 123 | Path("config.ini"), 124 | Path.home() / ".miso" / "config.ini", 125 | ] 126 | ) 127 | 128 | # Find first existing config file 129 | config_file = None 130 | for path in search_paths: 131 | if path.exists(): 132 | config_file = path 133 | logger.info(f"Loading configuration from: {config_file}") 134 | break 135 | 136 | if not config_file: 137 | logger.warning(f"No config file found. Searched: {[str(p) for p in search_paths]}") 138 | return cls() 139 | 140 | config.read(config_file) 141 | 142 | # Extract MISO section 143 | if "miso" not in config: 144 | logger.warning("No [miso] section found in config file") 145 | return cls() 146 | 147 | miso_config = config["miso"] 148 | 149 | # Build config with values from file 150 | kwargs = {} 151 | 152 | if "pricing_api_key" in miso_config: 153 | kwargs["pricing_api_key"] = miso_config["pricing_api_key"] 154 | 155 | if "lgi_api_key" in miso_config: 156 | kwargs["lgi_api_key"] = miso_config["lgi_api_key"] 157 | 158 | if "data_dir" in miso_config: 159 | kwargs["data_dir"] = Path(miso_config["data_dir"]) 160 | 161 | if "max_retries" in miso_config: 162 | kwargs["max_retries"] = int(miso_config["max_retries"]) 163 | 164 | if "retry_delay" in miso_config: 165 | kwargs["retry_delay"] = int(miso_config["retry_delay"]) 166 | 167 | if "timeout" in miso_config: 168 | kwargs["timeout"] = int(miso_config["timeout"]) 169 | 170 | if "rate_limit_delay" in miso_config: 171 | kwargs["rate_limit_delay"] = float(miso_config["rate_limit_delay"]) 172 | 173 | return cls(**kwargs) 174 | 175 | @classmethod 176 | def create_template_ini(cls, output_path: Path = Path("user_config.ini")): 177 | """ 178 | Create a template INI file for users to fill in. 179 | 180 | Args: 181 | output_path: Where to save the template file 182 | """ 183 | template = """[miso] 184 | # MISO Data Exchange API Keys 185 | # Get your keys from: https://data-exchange.misoenergy.org/ 186 | 187 | # API key for Pricing product (LMP and MCP data) 188 | pricing_api_key = your-pricing-api-key-here 189 | 190 | # API key for Load, Generation, and Interchange product 191 | lgi_api_key = your-lgi-api-key-here 192 | 193 | # Directory for storing downloaded data 194 | data_dir = data/MISO 195 | 196 | # Request settings 197 | max_retries = 3 198 | retry_delay = 5 199 | timeout = 30 200 | rate_limit_delay = 0.6 201 | """ 202 | output_path.write_text(template) 203 | logger.info(f"Created template config file at: {output_path}") 204 | print(f"Template config file created: {output_path}") 205 | print("Please edit this file and add your API keys.") 206 | 207 | 208 | class MISOClient: 209 | """Client for retrieving data from MISO Data Exchange REST API.""" 210 | 211 | def __init__(self, config: Optional[MISOConfig] = None): 212 | self.config = config or MISOConfig() 213 | self._ensure_directories() 214 | self.session = requests.Session() 215 | self._last_request_time = 0 216 | 217 | def _ensure_directories(self): 218 | """Ensure required directories exist.""" 219 | self.config.data_dir.mkdir(parents=True, exist_ok=True) 220 | 221 | def _rate_limit(self): 222 | """Implement rate limiting.""" 223 | elapsed = time.time() - self._last_request_time 224 | if elapsed < self.config.rate_limit_delay: 225 | time.sleep(self.config.rate_limit_delay - elapsed) 226 | self._last_request_time = time.time() 227 | 228 | def _make_request( 229 | self, base_url: str, endpoint: str, params: Optional[Dict[str, Any]] = None 230 | ) -> Optional[Dict[str, Any]]: 231 | """Make API request with retry logic.""" 232 | url = f"{base_url}/{endpoint}" 233 | 234 | # Determine which API key to use based on base URL 235 | headers = {} 236 | if "pricing" in base_url and self.config.pricing_api_key: 237 | headers["Ocp-Apim-Subscription-Key"] = self.config.pricing_api_key 238 | elif "lgi" in base_url and self.config.lgi_api_key: 239 | headers["Ocp-Apim-Subscription-Key"] = self.config.lgi_api_key 240 | else: 241 | # No API key found for this URL 242 | logger.warning(f"No API key configured for {base_url}") 243 | 244 | for attempt in range(self.config.max_retries): 245 | try: 246 | self._rate_limit() 247 | logger.debug(f"Requesting: {url} (attempt {attempt + 1})") 248 | logger.debug(f"Headers: {list(headers.keys())}") 249 | logger.debug(f"Params: {params}") 250 | 251 | response = self.session.get( 252 | url, params=params, headers=headers, timeout=self.config.timeout 253 | ) 254 | 255 | if response.status_code == 200: 256 | logger.info(f"Request successful: {url}") 257 | return response.json() 258 | elif response.status_code == 401: 259 | logger.error("Authentication failed - check API key") 260 | return None 261 | elif response.status_code == 404: 262 | logger.warning(f"Data not found for {url}") 263 | logger.debug(f"Response: {response.text[:200]}") 264 | return None 265 | elif response.status_code == 429: 266 | logger.warning("Rate limit exceeded, waiting...") 267 | time.sleep(60) 268 | continue 269 | else: 270 | logger.warning(f"Request failed with status {response.status_code}") 271 | logger.debug(f"Response: {response.text[:200]}") 272 | 273 | except requests.RequestException as e: 274 | logger.error(f"Request error: {e}") 275 | 276 | if attempt < self.config.max_retries - 1: 277 | time.sleep(self.config.retry_delay) 278 | 279 | return None 280 | 281 | def _fetch_all_pages( 282 | self, base_url: str, endpoint: str, params: Optional[Dict[str, Any]] = None 283 | ) -> List[Dict[str, Any]]: 284 | """Fetch all pages of paginated data.""" 285 | all_data = [] 286 | page_number = 1 287 | params = params or {} 288 | 289 | while True: 290 | params["pageNumber"] = page_number 291 | result = self._make_request(base_url, endpoint, params) 292 | 293 | if not result or "data" not in result: 294 | break 295 | 296 | all_data.extend(result["data"]) 297 | 298 | page_info = result.get("page", {}) 299 | if page_info.get("lastPage", True): 300 | break 301 | 302 | page_number += 1 303 | logger.info(f"Fetching page {page_number}...") 304 | 305 | return all_data 306 | 307 | def _download_data( 308 | self, base_url: str, endpoint_template: str, start_date: date, duration: int, **filters 309 | ) -> Dict[date, List[Dict[str, Any]]]: 310 | """Download data for a date range.""" 311 | date_list = [start_date + timedelta(days=i) for i in range(duration)] 312 | results = {} 313 | 314 | for current_date in date_list: 315 | date_str = current_date.strftime("%Y-%m-%d") 316 | endpoint = endpoint_template.format(date=date_str) 317 | 318 | data = self._fetch_all_pages(base_url, endpoint, filters) 319 | 320 | if data: 321 | results[current_date] = data 322 | logger.info(f"Downloaded {len(data)} records for {current_date}") 323 | else: 324 | logger.warning(f"No data found for {current_date}") 325 | 326 | logger.info(f"Downloaded data for {len(results)}/{len(date_list)} dates") 327 | return results 328 | 329 | # ========== PRICING API METHODS ========== 330 | 331 | def get_lmp( 332 | self, 333 | lmp_type: str, 334 | start_date: date, 335 | duration: int, 336 | node: Optional[str] = None, 337 | interval: Optional[str] = None, 338 | preliminary_final: Optional[str] = None, 339 | time_resolution: Optional[str] = None, 340 | ) -> Dict[date, List[Dict[str, Any]]]: 341 | """Get LMP data.""" 342 | type_map = { 343 | "da_exante": MISOPricingEndpoint.DA_EXANTE_LMP, 344 | "da_expost": MISOPricingEndpoint.DA_EXPOST_LMP, 345 | "rt_exante": MISOPricingEndpoint.RT_EXANTE_LMP, 346 | "rt_expost": MISOPricingEndpoint.RT_EXPOST_LMP, 347 | } 348 | 349 | if lmp_type not in type_map: 350 | logger.error(f"Invalid LMP type: {lmp_type}") 351 | return {} 352 | 353 | filters = {} 354 | if node: 355 | filters["node"] = node 356 | if interval: 357 | filters["interval"] = interval 358 | if preliminary_final: 359 | filters["preliminaryFinal"] = preliminary_final 360 | if time_resolution: 361 | filters["timeResolution"] = time_resolution 362 | 363 | return self._download_data( 364 | self.config.pricing_base_url, type_map[lmp_type].value, start_date, duration, **filters 365 | ) 366 | 367 | def get_mcp( 368 | self, 369 | mcp_type: str, 370 | start_date: date, 371 | duration: int, 372 | zone: Optional[str] = None, 373 | product: Optional[str] = None, 374 | interval: Optional[str] = None, 375 | preliminary_final: Optional[str] = None, 376 | time_resolution: Optional[str] = None, 377 | ) -> Dict[date, List[Dict[str, Any]]]: 378 | """Get MCP (Market Clearing Price) data.""" 379 | type_map = { 380 | "asm_da_exante": MISOPricingEndpoint.ASM_DA_EXANTE_MCP, 381 | "asm_da_expost": MISOPricingEndpoint.ASM_DA_EXPOST_MCP, 382 | "asm_rt_exante": MISOPricingEndpoint.ASM_RT_EXANTE_MCP, 383 | "asm_rt_expost": MISOPricingEndpoint.ASM_RT_EXPOST_MCP, 384 | "asm_rt_summary": MISOPricingEndpoint.ASM_RT_SUMMARY, 385 | } 386 | 387 | if mcp_type not in type_map: 388 | logger.error(f"Invalid MCP type: {mcp_type}") 389 | return {} 390 | 391 | filters = {} 392 | if zone: 393 | filters["zone"] = zone 394 | if product: 395 | filters["product"] = product 396 | if interval: 397 | filters["interval"] = interval 398 | if preliminary_final: 399 | filters["preliminaryFinal"] = preliminary_final 400 | if time_resolution: 401 | filters["timeResolution"] = time_resolution 402 | 403 | return self._download_data( 404 | self.config.pricing_base_url, type_map[mcp_type].value, start_date, duration, **filters 405 | ) 406 | 407 | # ========== LOAD/DEMAND METHODS ========== 408 | 409 | def get_demand( 410 | self, 411 | demand_type: str, 412 | start_date: date, 413 | duration: int, 414 | region: Optional[str] = None, 415 | interval: Optional[str] = None, 416 | time_resolution: Optional[str] = None, 417 | **kwargs, 418 | ) -> Dict[date, List[Dict[str, Any]]]: 419 | """ 420 | Get demand/load data. 421 | 422 | Types: 'da_demand', 'rt_forecast', 'rt_actual', 'rt_state_estimator' 423 | """ 424 | type_map = { 425 | "da_demand": MISOLGIEndpoint.DA_DEMAND, 426 | "rt_forecast": MISOLGIEndpoint.RT_DEMAND_FORECAST, 427 | "rt_actual": MISOLGIEndpoint.RT_DEMAND_ACTUAL, 428 | "rt_state_estimator": MISOLGIEndpoint.RT_DEMAND_STATE_EST, 429 | } 430 | 431 | if demand_type not in type_map: 432 | logger.error(f"Invalid demand type: {demand_type}") 433 | return {} 434 | 435 | filters = {} 436 | if region: 437 | filters["region"] = region 438 | if interval: 439 | filters["interval"] = interval 440 | if time_resolution: 441 | filters["timeResolution"] = time_resolution 442 | 443 | # Additional filters for specific endpoints 444 | filters.update(kwargs) 445 | 446 | return self._download_data( 447 | self.config.lgi_base_url, type_map[demand_type].value, start_date, duration, **filters 448 | ) 449 | 450 | def get_load_forecast( 451 | self, 452 | start_date: date, 453 | duration: int, 454 | region: Optional[str] = None, 455 | local_resource_zone: Optional[str] = None, 456 | interval: Optional[str] = None, 457 | time_resolution: Optional[str] = None, 458 | init_date: Optional[date] = None, 459 | ) -> Dict[date, List[Dict[str, Any]]]: 460 | """Get medium term load forecast.""" 461 | filters = {} 462 | if region: 463 | filters["region"] = region 464 | if local_resource_zone: 465 | filters["localResourceZone"] = local_resource_zone 466 | if interval: 467 | filters["interval"] = interval 468 | if time_resolution: 469 | filters["timeResolution"] = time_resolution 470 | if init_date: 471 | filters["init"] = init_date.strftime("%Y-%m-%d") 472 | 473 | return self._download_data( 474 | self.config.lgi_base_url, 475 | MISOLGIEndpoint.LOAD_FORECAST.value, 476 | start_date, 477 | duration, 478 | **filters, 479 | ) 480 | 481 | # ========== GENERATION METHODS ========== 482 | 483 | def get_generation( 484 | self, 485 | gen_type: str, 486 | start_date: date, 487 | duration: int, 488 | region: Optional[str] = None, 489 | interval: Optional[str] = None, 490 | time_resolution: Optional[str] = None, 491 | **kwargs, 492 | ) -> Dict[date, List[Dict[str, Any]]]: 493 | """ 494 | Get generation data. 495 | 496 | Types: 'da_cleared_physical', 'da_cleared_virtual', 'da_fuel_type', 497 | 'da_offered_ecomax', 'da_offered_ecomin', 'rt_cleared', 498 | 'rt_committed_ecomax', 'rt_fuel_margin', 'rt_fuel_type', 499 | 'rt_offered_ecomax' 500 | """ 501 | type_map = { 502 | "da_cleared_physical": MISOLGIEndpoint.DA_GEN_CLEARED_PHYSICAL, 503 | "da_cleared_virtual": MISOLGIEndpoint.DA_GEN_CLEARED_VIRTUAL, 504 | "da_fuel_type": MISOLGIEndpoint.DA_GEN_FUEL_TYPE, 505 | "da_offered_ecomax": MISOLGIEndpoint.DA_GEN_OFFERED_ECOMAX, 506 | "da_offered_ecomin": MISOLGIEndpoint.DA_GEN_OFFERED_ECOMIN, 507 | "rt_cleared": MISOLGIEndpoint.RT_GEN_CLEARED, 508 | "rt_committed_ecomax": MISOLGIEndpoint.RT_GEN_COMMITTED_ECOMAX, 509 | "rt_fuel_margin": MISOLGIEndpoint.RT_GEN_FUEL_MARGIN, 510 | "rt_fuel_type": MISOLGIEndpoint.RT_GEN_FUEL_TYPE, 511 | "rt_offered_ecomax": MISOLGIEndpoint.RT_GEN_OFFERED_ECOMAX, 512 | } 513 | 514 | if gen_type not in type_map: 515 | logger.error(f"Invalid generation type: {gen_type}") 516 | return {} 517 | 518 | filters = {} 519 | if region: 520 | filters["region"] = region 521 | if interval: 522 | filters["interval"] = interval 523 | if time_resolution: 524 | filters["timeResolution"] = time_resolution 525 | 526 | # Additional filters 527 | filters.update(kwargs) 528 | 529 | return self._download_data( 530 | self.config.lgi_base_url, type_map[gen_type].value, start_date, duration, **filters 531 | ) 532 | 533 | def get_fuel_mix( 534 | self, 535 | start_date: date, 536 | duration: int, 537 | region: Optional[str] = None, 538 | fuel_type: Optional[str] = None, 539 | interval: Optional[str] = None, 540 | ) -> Dict[date, List[Dict[str, Any]]]: 541 | """Get fuel on the margin data (5-minute intervals).""" 542 | filters = {} 543 | if region: 544 | filters["region"] = region 545 | if fuel_type: 546 | filters["fuelType"] = fuel_type 547 | if interval: 548 | filters["interval"] = interval 549 | 550 | return self._download_data( 551 | self.config.lgi_base_url, 552 | MISOLGIEndpoint.RT_GEN_FUEL_MARGIN.value, 553 | start_date, 554 | duration, 555 | **filters, 556 | ) 557 | 558 | # ========== INTERCHANGE METHODS ========== 559 | 560 | def get_interchange( 561 | self, 562 | interchange_type: str, 563 | start_date: date, 564 | duration: int, 565 | region: Optional[str] = None, 566 | adjacent_ba: Optional[str] = None, 567 | interval: Optional[str] = None, 568 | ) -> Dict[date, List[Dict[str, Any]]]: 569 | """ 570 | Get interchange data. 571 | 572 | Types: 'da_net_scheduled', 'rt_net_actual', 'rt_net_scheduled', 'historical' 573 | """ 574 | type_map = { 575 | "da_net_scheduled": MISOLGIEndpoint.DA_INTERCHANGE_NET_SCHEDULED, 576 | "rt_net_actual": MISOLGIEndpoint.RT_INTERCHANGE_NET_ACTUAL, 577 | "rt_net_scheduled": MISOLGIEndpoint.RT_INTERCHANGE_NET_SCHEDULED, 578 | "historical": MISOLGIEndpoint.HISTORICAL_INTERCHANGE, 579 | } 580 | 581 | if interchange_type not in type_map: 582 | logger.error(f"Invalid interchange type: {interchange_type}") 583 | return {} 584 | 585 | filters = {} 586 | if region: 587 | filters["region"] = region 588 | if adjacent_ba: 589 | filters["adjacentBa"] = adjacent_ba 590 | if interval: 591 | filters["interval"] = interval 592 | 593 | return self._download_data( 594 | self.config.lgi_base_url, 595 | type_map[interchange_type].value, 596 | start_date, 597 | duration, 598 | **filters, 599 | ) 600 | 601 | # ========== OUTAGES & CONSTRAINTS ========== 602 | 603 | def get_outages( 604 | self, 605 | outage_type: str, 606 | start_date: date, 607 | duration: int, 608 | region: Optional[str] = None, 609 | interval: Optional[str] = None, 610 | ) -> Dict[date, List[Dict[str, Any]]]: 611 | """ 612 | Get outage data. 613 | 614 | Types: 'forecast', 'rt_outage' 615 | """ 616 | type_map = { 617 | "forecast": MISOLGIEndpoint.OUTAGE_FORECAST, 618 | "rt_outage": MISOLGIEndpoint.RT_OUTAGE, 619 | } 620 | 621 | if outage_type not in type_map: 622 | logger.error(f"Invalid outage type: {outage_type}") 623 | return {} 624 | 625 | filters = {} 626 | if region: 627 | filters["region"] = region 628 | if interval: 629 | filters["interval"] = interval 630 | 631 | return self._download_data( 632 | self.config.lgi_base_url, type_map[outage_type].value, start_date, duration, **filters 633 | ) 634 | 635 | def get_binding_constraints( 636 | self, start_date: date, duration: int, interval: Optional[str] = None 637 | ) -> Dict[date, List[Dict[str, Any]]]: 638 | """Get real-time binding constraints.""" 639 | filters = {} 640 | if interval: 641 | filters["interval"] = interval 642 | 643 | return self._download_data( 644 | self.config.lgi_base_url, 645 | MISOLGIEndpoint.RT_BINDING_CONSTRAINTS.value, 646 | start_date, 647 | duration, 648 | **filters, 649 | ) 650 | 651 | # ========== UTILITY METHODS ========== 652 | 653 | def save_to_csv(self, data: Dict[date, List[Dict[str, Any]]], filename: str): 654 | """Save downloaded data to CSV file.""" 655 | import pandas as pd 656 | 657 | all_records = [] 658 | for date_key, records in data.items(): 659 | for record in records: 660 | record["query_date"] = date_key 661 | all_records.append(record) 662 | 663 | if not all_records: 664 | logger.warning("No data to save") 665 | return 666 | 667 | df = pd.DataFrame(all_records) 668 | output_path = self.config.data_dir / filename 669 | df.to_csv(output_path, index=False) 670 | logger.info(f"Saved {len(all_records)} records to {output_path}") 671 | 672 | 673 | # Example usage 674 | if __name__ == "__main__": 675 | import os 676 | 677 | logging.basicConfig(level=logging.INFO) 678 | 679 | # Method 1: Load from INI file (recommended for multi-user tools) 680 | # First, create a template if it doesn't exist 681 | if not Path("user_config.ini").exists(): 682 | MISOConfig.create_template_ini() 683 | print("\nPlease edit user_config.ini with your API keys, then run again.") 684 | exit(0) 685 | 686 | # Load config from INI file 687 | config = MISOConfig.from_ini_file() 688 | client = MISOClient(config) 689 | 690 | # Method 2: Load from environment variables (alternative) 691 | # config = MISOConfig( 692 | # pricing_api_key=os.getenv('MISO_PRICING_API_KEY'), 693 | # lgi_api_key=os.getenv('MISO_LGI_API_KEY') 694 | # ) 695 | # client = MISOClient(config) 696 | 697 | # Method 3: Load from custom INI file path 698 | # config = MISOConfig.from_ini_file(Path("/path/to/my_config.ini")) 699 | # client = MISOClient(config) 700 | 701 | # Example 1: Get LMP data (uses pricing API key) 702 | # Note: Use dates that are a few days old to ensure data availability 703 | example_date = date.today() - timedelta(days=7) 704 | 705 | lmp_data = client.get_lmp( 706 | lmp_type="da_exante", start_date=example_date, duration=7, node="ALTW.WELLS1" 707 | ) 708 | if lmp_data: 709 | client.save_to_csv(lmp_data, "da_exante_lmp.csv") 710 | 711 | # Example 2: Get fuel mix (uses LGI API key) 712 | fuel_data = client.get_fuel_mix(start_date=example_date, duration=7) 713 | if fuel_data: 714 | client.save_to_csv(fuel_data, "fuel_mix.csv") 715 | 716 | # Example 3: Get actual load (uses LGI API key) 717 | load_data = client.get_demand( 718 | demand_type="rt_actual", start_date=example_date, duration=7, time_resolution="daily" 719 | ) 720 | if load_data: 721 | client.save_to_csv(load_data, "actual_load.csv") 722 | 723 | # Example 4: Get generation fuel type (uses LGI API key) 724 | gen_fuel = client.get_generation(gen_type="rt_fuel_type", start_date=example_date, duration=7) 725 | if gen_fuel: 726 | client.save_to_csv(gen_fuel, "generation_fuel_type.csv") 727 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ISO-DART v2.0 2 | ## Independent System Operator Data Automated Request Tool 3 | 4 | [![Python 3.10+](https://img.shields.io/badge/python-3.10+-blue.svg)](https://www.python.org/downloads/) 5 | [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) 6 | [![codecov](https://codecov.io/github/LLNL/ISO-DART/branch/dev/graph/badge.svg)](https://codecov.io/github/LLNL/ISO-DART) 7 | 8 | **A comprehensive Python tool for downloading and analyzing electricity market data from Independent System Operators (ISOs) and weather data sources.** 9 | 10 | ## What's New in v2.0 11 | 12 | ### 🚀 Major Improvements 13 | - **Modern Python practices**: Type hints, dataclasses, enums, and proper error handling 14 | - **Command-line interface**: Use arguments or interactive mode 15 | - **Robust API handling**: Automatic retries, better error messages, connection pooling 16 | - **Enhanced logging**: Track operations and debug issues easily 17 | - **Comprehensive testing**: Unit and integration test suites 18 | - **Complete coverage**: All ISO data methods accessible via interactive mode 19 | - **Updated dependencies**: Compatible with latest pandas, requests, and meteostat 20 | 21 | ### 📊 Supported Data Sources 22 | 23 | #### Independent System Operators (ISOs) 24 | 25 | **CAISO (California ISO)** - ✅ Fully supported 26 | - **Pricing Data**: LMP (DAM, HASP, RTM, RTPD), Scheduling Point Tie Prices, AS Clearing Prices, Constraint Shadow Prices, Fuel Prices, GHG Allowance Prices 27 | - **Load Data**: Load Forecasts (DAM, RTM, 2DA, 7DA, RTPD Advisory) 28 | - **Energy Data**: System Load & Resources, Market Power Mitigation, Flexible Ramping (Requirements, Awards, Demand Curves), EIM Transfer & Limits, Wind & Solar Summary 29 | - **Ancillary Services**: Requirements, Results/Awards, Operating Reserves 30 | 31 | **MISO (Midcontinent ISO)** - ✅ Fully supported 32 | - **LMP Data**: Day-Ahead EPNodes, DA ExAnte/ExPost, Real-Time EPNodes, RT 5-min ExAnte, RT Final 33 | - **MCP Data**: ASM DA ExAnte/ExPost, ASM RT 5-min ExAnte/Final, DA ExAnte/ExPost Ramp 34 | - **Summary Reports**: Daily Forecast/Actual Load by LRZ, Regional Forecast/Actual Load 35 | - **Generation Data**: Fuel Mix, Area Control Error (ACE), Wind Forecast/Actual, Market Totals 36 | 37 | **NYISO (New York ISO)** - ✅ Fully supported 38 | - **Pricing Data**: LBMP (Zonal/Generator, DAM/RTM), Ancillary Services Prices 39 | - **Power Grid**: Outages (Scheduled/Actual), Constraints 40 | - **Load Data**: ISO Forecast, Zonal Bid Load, Weather Forecast, Actual Load 41 | - **Bid Data**: Generator/AS Bids, Load Bids, Transaction Bids, Commitment Parameters 42 | - **Generation Data**: Fuel Mix, Interface Flows, Wind Generation, BTM Solar 43 | 44 | **SPP (Southwest Power Pool)** - ✅ Fully supported 45 | - **Pricing Data**: LMP (DA, RTBM) by Settlement Location and by Bus, MCP (DA, RTBM) 46 | - **Constraint Data**: Binding Constraints (DA, RTBM) 47 | - **Ancillary Services**: Operating Reserves (RTBM) 48 | - **Fuel Data**: Fuel On Margin 49 | - **Load Data**: Load Forecasts (short-term, medium-term) 50 | - **Resource Data**: Resource (solar + wind) Forecasts (day-ahead, short-term) 51 | - **Clearing Data**: Day-Ahead Market Clearing data, Day-Ahead Virtual Clearing data 52 | 53 | **BPA (Bonneville Power Administration)** - ✅ Fully supported 54 | - **Load and Generation Data**: Total Load, Wind Generation, Wind Forecast, Solar Generation, Solar Forecast, Hydro, Thermal and Net Interchange. 55 | - **Reserves Data**: BPA Reserves Deployed. 56 | 57 | #### Weather & Solar Data 58 | - **Meteostat**: Historical weather data for any US location (temperature, humidity, wind, precipitation, etc.) 59 | - **NSRDB** (National Solar Radiation Database): Solar irradiance data (GHI, DHI, DNI) 60 | 61 | ## Installation 62 | 63 | ### Requirements 64 | - Python 3.10 or higher 65 | - Git 66 | 67 | ### Setup 68 | 69 | ```bash 70 | # Clone the repository 71 | git clone https://github.com/LLNL/ISO-DART.git 72 | cd ISO-DART 73 | 74 | # Create virtual environment (recommended) 75 | python -m venv venv 76 | source venv/bin/activate # On Windows: venv\Scripts\activate 77 | 78 | # Install dependencies 79 | pip install -r requirements.txt 80 | ``` 81 | 82 | ## Quick Start 83 | 84 | ### Interactive Mode (Recommended for New Users) 85 | 86 | Simply run: 87 | ```bash 88 | python isodart.py 89 | ``` 90 | 91 | The interactive menu will guide you through: 92 | 1. Selecting ISO or weather data 93 | 2. Choosing specific data type 94 | 3. Entering date range 95 | 4. Selecting location (for weather data) 96 | 97 | **Example workflow:** 98 | ``` 99 | ISO-DART v2.0 100 | Independent System Operator Data Automated Request Tool 101 | ============================================================ 102 | 103 | What type of data do you want to download? 104 | (1) ISO Data (CAISO, MISO, NYISO, SPP, BPA) 105 | (2) Weather Data 106 | 107 | Your choice (1 or 2): 1 108 | 109 | ISO DATA SELECTION 110 | ============================================================ 111 | 112 | Which ISO do you want data from? 113 | (1) CAISO - California Independent System Operator 114 | (2) MISO - Midcontinent Independent System Operator 115 | (3) NYISO - New York Independent System Operator 116 | (4) SPP - Southwest Power Pool 117 | (5) BPA - Bonneville Power Administration 118 | 119 | Your choice (1, 2, 3, 4, or 5): 1 120 | ``` 121 | 122 | ### Command-Line Mode 123 | 124 | For automation and scripting: 125 | 126 | ```bash 127 | # Download CAISO Day-Ahead LMP data 128 | python isodart.py --iso caiso --data-type lmp --market dam \ 129 | --start 2024-01-01 --duration 7 130 | 131 | # Download MISO LMP data 132 | python isodart.py --iso miso --data-type lmp \ 133 | --start 2024-01-01 --duration 30 134 | 135 | # Download NYISO LBMP data 136 | python isodart.py --iso nyiso --data-type lbmp \ 137 | --start 2024-01-01 --duration 7 138 | 139 | # Download weather data for California 140 | python isodart.py --data-type weather --state CA \ 141 | --start 2024-01-01 --duration 30 142 | ``` 143 | 144 | ### Command-Line Arguments 145 | 146 | ``` 147 | --iso {caiso,miso,nyiso, spp, bpa} ISO to download from 148 | --data-type TYPE Data type (lmp, load, weather, etc.) 149 | --market {dam,rtm,hasp,rtpd} Energy market type 150 | --start YYYY-MM-DD Start date 151 | --duration N Duration in days 152 | --state XX US state code (for weather) 153 | --verbose Enable detailed logging 154 | --interactive Force interactive mode 155 | --config PATH Path to configuration file 156 | ``` 157 | 158 | ## Usage Examples 159 | 160 | ### CAISO Examples 161 | 162 | #### Download Day-Ahead LMP for January 2024 163 | ```bash 164 | python isodart.py --iso caiso --data-type lmp --market dam \ 165 | --start 2024-01-01 --duration 31 166 | ``` 167 | 168 | #### Download Real-Time Load Forecast 169 | ```bash 170 | python isodart.py --iso caiso --data-type load --market rtm \ 171 | --start 2024-06-01 --duration 7 172 | ``` 173 | 174 | #### Download Wind and Solar Summary 175 | ```bash 176 | python isodart.py --iso caiso --data-type wind-solar \ 177 | --start 2024-01-01 --duration 30 178 | ``` 179 | 180 | ### MISO Examples 181 | 182 | #### Download Day-Ahead LMP Data 183 | ```bash 184 | python isodart.py --iso miso --data-type lmp \ 185 | --start 2024-01-01 --duration 7 186 | ``` 187 | 188 | #### Download Fuel Mix Data 189 | ```bash 190 | python isodart.py --iso miso --data-type fuel-mix \ 191 | --start 2024-01-01 --duration 30 192 | ``` 193 | 194 | ### NYISO Examples 195 | 196 | #### Download Day-Ahead Zonal LBMP 197 | ```bash 198 | python isodart.py --iso nyiso --data-type lbmp --market dam \ 199 | --start 2024-01-01 --duration 7 200 | ``` 201 | 202 | #### Download Wind Generation Data 203 | ```bash 204 | python isodart.py --iso nyiso --data-type wind \ 205 | --start 2024-01-01 --duration 30 206 | ``` 207 | 208 | ### Weather Data Examples 209 | 210 | #### Download Weather Data for San Francisco Area 211 | ```bash 212 | python isodart.py --data-type weather --state CA \ 213 | --start 2024-01-01 --duration 365 214 | ``` 215 | 216 | The tool will: 217 | 1. Find all weather stations in California 218 | 2. Filter stations with data for your date range 219 | 3. Let you select the closest station 220 | 4. Download hourly weather data 221 | 5. Optionally download solar data from NSRDB 222 | 223 | ## Python API Usage 224 | 225 | For integration into your own scripts: 226 | 227 | ### CAISO Example 228 | 229 | ```python 230 | from datetime import date 231 | from lib.iso.caiso import CAISOClient, Market 232 | 233 | # Initialize client 234 | client = CAISOClient() 235 | 236 | # Download Day-Ahead LMP data 237 | success = client.get_lmp( 238 | market=Market.DAM, 239 | start_date=date(2024, 1, 1), 240 | end_date=date(2024, 1, 31) 241 | ) 242 | 243 | if success: 244 | print("Data downloaded to data/CAISO/") 245 | 246 | # Download wind and solar summary 247 | client.get_wind_solar_summary( 248 | start_date=date(2024, 1, 1), 249 | end_date=date(2024, 1, 31) 250 | ) 251 | 252 | # Clean up temporary files 253 | client.cleanup() 254 | ``` 255 | 256 | ### MISO Example 257 | 258 | ```python 259 | from datetime import date 260 | from lib.iso.miso import MISOClient 261 | 262 | # Initialize client 263 | client = MISOClient() 264 | 265 | # Download Day-Ahead ExAnte LMP 266 | client.get_lmp("da_exante", date(2024, 1, 1), duration=30) 267 | 268 | # Download wind generation forecast 269 | client.get_wind_forecast(date(2024, 1, 1), duration=7) 270 | 271 | # Download fuel mix 272 | client.get_fuel_mix(date(2024, 1, 1), duration=30) 273 | ``` 274 | 275 | ### NYISO Example 276 | 277 | ```python 278 | from datetime import date 279 | from lib.iso.nyiso import NYISOClient, NYISOMarket 280 | 281 | # Initialize client 282 | client = NYISOClient() 283 | 284 | # Download Day-Ahead zonal LBMP 285 | client.get_lbmp(NYISOMarket.DAM, "zonal", date(2024, 1, 1), duration=30) 286 | 287 | # Download fuel mix 288 | client.get_fuel_mix(date(2024, 1, 1), duration=7) 289 | 290 | # Download wind generation 291 | client.get_wind_generation(date(2024, 1, 1), duration=30) 292 | 293 | # Clean up 294 | client.cleanup() 295 | ``` 296 | 297 | ### Weather API 298 | 299 | ```python 300 | from datetime import date 301 | from lib.weather.client import WeatherClient 302 | 303 | # Initialize client 304 | client = WeatherClient() 305 | 306 | # Download weather data 307 | success = client.download_weather_data( 308 | state='CA', 309 | start_date=date(2024, 1, 1), 310 | duration=30, 311 | interactive=False # Auto-select first station 312 | ) 313 | 314 | # Optionally download solar data 315 | if success: 316 | client.download_solar_data(year=2024) 317 | ``` 318 | 319 | ## Data Output 320 | 321 | ### Directory Structure 322 | ``` 323 | iso-dart/ 324 | ├── data/ 325 | │ ├── CAISO/ # CAISO data files 326 | │ ├── MISO/ # MISO data files 327 | │ ├── NYISO/ # NYISO data files 328 | │ ├── weather/ # Weather data 329 | │ └── solar/ # Solar radiation data 330 | ├── raw_data/ # Temporary files (auto-cleaned) 331 | ├── logs/ # Application logs 332 | └── user_config.ini # API keys (created on first use) 333 | ``` 334 | 335 | ### Output File Formats 336 | 337 | #### CAISO Files 338 | Format: `{start_date}_to_{end_date}_{query_name}_{data_item}.csv` 339 | 340 | Example: `20240101_to_20240131_PRC_LMP_TH_NP15_GEN-APND.csv` 341 | 342 | Typical columns: 343 | - `OPR_DATE`: Operating date 344 | - `INTERVAL_NUM`: Time interval within the day (1-24 for hourly, 1-96 for 15-min) 345 | - `DATA_ITEM`: Specific location/node/resource 346 | - `VALUE`: Price ($/MWh), load (MW), or other metric 347 | - `MW`: Power values (for load data) 348 | 349 | #### MISO Files 350 | Format: `{date}_{data_type}.csv` or `{date}_{data_type}.xls` 351 | 352 | Examples: 353 | - `20240101_da_exante_lmp.csv` 354 | - `20240101_fuel_on_the_margin.xls` 355 | 356 | #### NYISO Files 357 | Format: `{start_date}_to_{end_date}_{dataid}_{aggregation}.csv` 358 | 359 | Example: `20240101_to_20240131_damlbmp_zone.csv` 360 | 361 | #### Weather Files 362 | Format: `{start_date}_to_{end_date}_{station_name}_{state}.csv` 363 | 364 | Example: `2024-01-01_to_2024-01-31_San_Francisco_Airport_CA.csv` 365 | 366 | Columns: 367 | - `time`: Timestamp (index) 368 | - `temperature`: Temperature (°F) 369 | - `dew_point`: Dew point (°F) 370 | - `relative_humidity`: Humidity (%) 371 | - `precipitation`: Precipitation (inches) 372 | - `wind_speed`: Wind speed (mph) 373 | - `air_pressure`: Air pressure (hPa) 374 | - `weather_condition`: Weather description (Clear, Rain, etc.) 375 | 376 | ## Data Analysis Examples 377 | 378 | ### Load and Visualize CAISO LMP 379 | 380 | ```python 381 | import pandas as pd 382 | import matplotlib.pyplot as plt 383 | 384 | # Load the data 385 | df = pd.read_csv('data/CAISO/20240101_to_20240107_PRC_LMP_TH_NP15_GEN-APND.csv') 386 | 387 | # Convert to datetime 388 | df['OPR_DT'] = pd.to_datetime(df['OPR_DATE']) 389 | 390 | # Plot prices over time 391 | plt.figure(figsize=(12, 6)) 392 | plt.plot(df['OPR_DT'], df['VALUE']) 393 | plt.xlabel('Date') 394 | plt.ylabel('Price ($/MWh)') 395 | plt.title('Day-Ahead LMP - NP15 Generator') 396 | plt.xticks(rotation=45) 397 | plt.tight_layout() 398 | plt.savefig('lmp_plot.png') 399 | plt.show() 400 | ``` 401 | 402 | ### Basic Statistics 403 | 404 | ```python 405 | import pandas as pd 406 | 407 | # Load data 408 | df = pd.read_csv('data/CAISO/20240101_to_20240107_PRC_LMP_TH_NP15_GEN-APND.csv') 409 | 410 | # Summary statistics 411 | print("Price Statistics:") 412 | print(f" Mean: ${df['VALUE'].mean():.2f}/MWh") 413 | print(f" Min: ${df['VALUE'].min():.2f}/MWh") 414 | print(f" Max: ${df['VALUE'].max():.2f}/MWh") 415 | print(f" Std: ${df['VALUE'].std():.2f}/MWh") 416 | 417 | # Find peak price hours 418 | peak_hours = df.nlargest(10, 'VALUE')[['OPR_DATE', 'INTERVAL_NUM', 'VALUE']] 419 | print("\nTop 10 Peak Price Hours:") 420 | print(peak_hours) 421 | ``` 422 | 423 | ### Analyze Weather Data 424 | 425 | ```python 426 | import pandas as pd 427 | import matplotlib.pyplot as plt 428 | 429 | # Load weather data 430 | df = pd.read_csv('data/weather/2024-01-01_to_2024-01-31_San_Francisco_CA.csv', 431 | index_col='time', parse_dates=True) 432 | 433 | # Plot temperature and wind 434 | fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(12, 8)) 435 | 436 | # Temperature 437 | ax1.plot(df.index, df['temperature']) 438 | ax1.set_ylabel('Temperature (°F)') 439 | ax1.set_title('Temperature Over Time') 440 | ax1.grid(True) 441 | 442 | # Wind Speed 443 | ax2.plot(df.index, df['wind_speed']) 444 | ax2.set_ylabel('Wind Speed (mph)') 445 | ax2.set_xlabel('Date') 446 | ax2.set_title('Wind Speed Over Time') 447 | ax2.grid(True) 448 | 449 | plt.tight_layout() 450 | plt.savefig('weather_analysis.png') 451 | plt.show() 452 | ``` 453 | 454 | ### Compare Multiple Locations 455 | 456 | ```python 457 | import pandas as pd 458 | import matplotlib.pyplot as plt 459 | 460 | # Load data for multiple locations 461 | locations = ['TH_NP15_GEN-APND', 'TH_SP15_GEN-APND', 'TH_ZP26_GEN-APND'] 462 | data = {} 463 | 464 | for loc in locations: 465 | file = f'data/CAISO/20240101_to_20240107_PRC_LMP_{loc}.csv' 466 | df = pd.read_csv(file) 467 | df['OPR_DT'] = pd.to_datetime(df['OPR_DATE']) 468 | data[loc] = df 469 | 470 | # Plot comparison 471 | plt.figure(figsize=(14, 6)) 472 | for loc, df in data.items(): 473 | plt.plot(df['OPR_DT'], df['VALUE'], label=loc.replace('TH_', '').replace('-APND', ''), alpha=0.7) 474 | 475 | plt.xlabel('Date') 476 | plt.ylabel('Price ($/MWh)') 477 | plt.title('LMP Comparison Across Locations') 478 | plt.legend() 479 | plt.grid(True, alpha=0.3) 480 | plt.xticks(rotation=45) 481 | plt.tight_layout() 482 | plt.savefig('location_comparison.png') 483 | plt.show() 484 | ``` 485 | 486 | ## Configuration 487 | 488 | ### User Configuration File 489 | 490 | For NSRDB solar data, create `user_config.ini`: 491 | 492 | ```ini 493 | [API] 494 | api_key = your_nrel_api_key_here 495 | 496 | [USER_INFO] 497 | first_name = Your 498 | last_name = Name 499 | affiliation = Your Organization 500 | email = your.email@example.com 501 | ``` 502 | 503 | Get your NREL API key at: https://developer.nrel.gov/signup/ 504 | 505 | ### Custom Configuration 506 | 507 | Create a YAML config file for advanced settings: 508 | 509 | ```yaml 510 | # config.yaml 511 | caiso: 512 | base_url: http://oasis.caiso.com/oasisapi/SingleZip 513 | max_retries: 3 514 | timeout: 30 515 | step_size: 1 # days per request 516 | 517 | miso: 518 | base_url: https://docs.misoenergy.org/marketreports/ 519 | max_retries: 3 520 | timeout: 30 521 | 522 | nyiso: 523 | base_url: http://mis.nyiso.com/public/csv 524 | max_retries: 3 525 | timeout: 30 526 | 527 | weather: 528 | data_dir: data/weather 529 | solar_dir: data/solar 530 | 531 | logging: 532 | level: INFO 533 | file: logs/isodart.log 534 | ``` 535 | 536 | Use with: `python isodart.py --config config.yaml` 537 | 538 | ## Automation 539 | 540 | ### Automated Daily Download 541 | 542 | Save as `daily_download.py`: 543 | 544 | ```python 545 | #!/usr/bin/env python 546 | """Download yesterday's CAISO data automatically.""" 547 | from datetime import date, timedelta 548 | from lib.iso.caiso import CAISOClient, Market 549 | import logging 550 | 551 | logging.basicConfig( 552 | level=logging.INFO, 553 | filename='daily_download.log', 554 | format='%(asctime)s - %(levelname)s - %(message)s' 555 | ) 556 | 557 | def download_yesterday(): 558 | yesterday = date.today() - timedelta(days=1) 559 | 560 | client = CAISOClient() 561 | try: 562 | success = client.get_lmp(Market.DAM, yesterday, yesterday) 563 | if success: 564 | logging.info(f"✓ Downloaded data for {yesterday}") 565 | else: 566 | logging.error(f"✗ Failed to download data for {yesterday}") 567 | except Exception as e: 568 | logging.error(f"Error: {e}", exc_info=True) 569 | finally: 570 | client.cleanup() 571 | 572 | if __name__ == '__main__': 573 | download_yesterday() 574 | ``` 575 | 576 | Run daily with cron (Linux/Mac): 577 | ```bash 578 | # Edit crontab 579 | crontab -e 580 | 581 | # Add line to run daily at 2 AM 582 | 0 2 * * * cd /path/to/ISO-DART && /path/to/venv/bin/python daily_download.py 583 | ``` 584 | 585 | Or with Windows Task Scheduler: 586 | ```powershell 587 | # Create scheduled task 588 | schtasks /create /tn "ISO-DART Daily" /tr "C:\path\to\venv\Scripts\python.exe C:\path\to\daily_download.py" /sc daily /st 02:00 589 | ``` 590 | 591 | ### Multi-Market Download 592 | 593 | ```python 594 | from datetime import date 595 | from lib.iso.caiso import CAISOClient, Market 596 | 597 | client = CAISOClient() 598 | start = date(2024, 1, 1) 599 | end = date(2024, 1, 31) 600 | 601 | # Download multiple markets 602 | for market in [Market.DAM, Market.RTM, Market.HASP]: 603 | print(f"Downloading {market.value}...") 604 | client.get_lmp(market, start, end) 605 | 606 | client.cleanup() 607 | ``` 608 | 609 | ## Development 610 | 611 | ### Running Tests 612 | 613 | ```bash 614 | # Install development dependencies 615 | pip install -r requirements-dev.txt 616 | 617 | # Run all tests 618 | pytest tests/ -v 619 | 620 | # Run specific test file 621 | pytest tests/test_caiso.py -v 622 | 623 | # Run with coverage 624 | pytest tests/ --cov=lib --cov-report=html 625 | 626 | # View coverage report 627 | open htmlcov/index.html # Mac/Linux 628 | start htmlcov/index.html # Windows 629 | ``` 630 | 631 | ### Code Quality 632 | 633 | ```bash 634 | # Format code 635 | black lib/ tests/ 636 | 637 | # Lint code 638 | flake8 lib/ tests/ 639 | 640 | # Type checking 641 | mypy lib/ 642 | ``` 643 | 644 | ### Project Structure 645 | 646 | ``` 647 | iso-dart/ 648 | ├── lib/ 649 | │ ├── iso/ 650 | │ │ ├── caiso.py # CAISO client (complete) 651 | │ │ ├── miso.py # MISO client (complete) 652 | │ │ └── nyiso.py # NYISO client (complete) 653 | │ ├── weather/ 654 | │ │ └── client.py # Weather/solar client 655 | │ └── interactive.py # Interactive CLI (complete coverage) 656 | ├── tests/ 657 | │ ├── test_caiso.py # CAISO tests 658 | │ ├── test_miso.py # MISO tests 659 | │ ├── test_nyiso.py # NYISO tests 660 | │ └── test_weather.py # Weather tests 661 | ├── isodart.py # Main entry point 662 | ├── requirements.txt # Production dependencies 663 | ├── requirements-dev.txt # Development dependencies 664 | ├── pyproject.toml # Project configuration 665 | └── README.md # This file 666 | ``` 667 | 668 | ## Troubleshooting 669 | 670 | ### Common Issues 671 | 672 | #### 1. No data returned from API 673 | ``` 674 | Error: API returned no data 675 | ``` 676 | **Solution**: 677 | - Check that date range is valid (not in future) 678 | - Verify data exists for that period on the ISO's website 679 | - Check API is operational 680 | - Try a smaller date range 681 | 682 | #### 2. Import errors 683 | ``` 684 | ModuleNotFoundError: No module named 'lib' 685 | ``` 686 | **Solution**: Run from project root directory: 687 | ```bash 688 | cd /path/to/ISO-DART 689 | python isodart.py 690 | ``` 691 | 692 | #### 3. Weather station not found 693 | ``` 694 | No weather stations found in XX for date range 695 | ``` 696 | **Solution**: 697 | - Try a different date range 698 | - Verify state code is valid (2-letter) 699 | - Some rural areas may have limited station coverage 700 | 701 | #### 4. NREL API rate limiting 702 | ``` 703 | Error 429: Too Many Requests 704 | ``` 705 | **Solution**: 706 | - NREL API has rate limits (1,000 requests/hour) 707 | - Wait and retry 708 | - Consider caching downloaded data 709 | 710 | #### 5. SSL Certificate Errors 711 | ``` 712 | SSLError: Certificate verification failed 713 | ``` 714 | **Solution**: 715 | ```bash 716 | pip install --upgrade certifi 717 | ``` 718 | 719 | ### Debug Mode 720 | 721 | Enable verbose logging: 722 | ```bash 723 | python isodart.py --verbose 724 | ``` 725 | 726 | Check logs: 727 | ```bash 728 | tail -f logs/isodart.log 729 | ``` 730 | 731 | Or in Python: 732 | ```python 733 | import logging 734 | logging.basicConfig(level=logging.DEBUG) 735 | ``` 736 | 737 | ## Performance Tips 738 | 739 | ### Large Date Ranges 740 | 741 | For requests spanning many days: 742 | 743 | 1. **Use smaller step sizes**: Adjust `step_size` parameter 744 | 2. **Run during off-peak hours**: Less API load 745 | 3. **Enable logging**: Monitor progress 746 | 4. **Use automation**: Schedule downloads overnight 747 | 748 | ```python 749 | from datetime import date, timedelta 750 | from lib.iso.caiso import CAISOClient, Market 751 | 752 | client = CAISOClient() 753 | 754 | # Download year of data in monthly chunks 755 | start = date(2024, 1, 1) 756 | for month in range(12): 757 | month_start = start + timedelta(days=30 * month) 758 | month_end = month_start + timedelta(days=30) 759 | 760 | print(f"Downloading {month_start} to {month_end}") 761 | client.get_lmp(Market.DAM, month_start, month_end) 762 | 763 | client.cleanup() 764 | ``` 765 | 766 | ### Parallel Downloads 767 | 768 | For multiple ISOs or data types: 769 | 770 | ```python 771 | from concurrent.futures import ThreadPoolExecutor 772 | from datetime import date 773 | from lib.iso.caiso import CAISOClient, Market 774 | 775 | def download_caiso_lmp(market, start, end): 776 | client = CAISOClient() 777 | result = client.get_lmp(market, start, end) 778 | client.cleanup() 779 | return result 780 | 781 | # Download multiple markets in parallel 782 | markets = [Market.DAM, Market.RTM, Market.HASP] 783 | start = date(2024, 1, 1) 784 | end = date(2024, 1, 31) 785 | 786 | with ThreadPoolExecutor(max_workers=3) as executor: 787 | futures = [ 788 | executor.submit(download_caiso_lmp, m, start, end) 789 | for m in markets 790 | ] 791 | results = [f.result() for f in futures] 792 | ``` 793 | 794 | ## Migration from v1.x 795 | 796 | If you're upgrading from v1.x, please see [MIGRATION_GUIDE.md](MIGRATION_GUIDE.md) for detailed migration instructions. 797 | 798 | ### Quick Summary 799 | 800 | **Key Changes:** 801 | 1. Entry point changed from `ISODART.py` to `isodart.py` 802 | 2. No more `exec()` - proper module imports 803 | 3. Configuration via config files instead of hardcoded values 804 | 4. Better error handling with exceptions instead of `sys.exit()` 805 | 5. Comprehensive logging instead of print statements 806 | 807 | **Data Compatibility:** Your existing CSV files are fully compatible! ✅ 808 | 809 | ## Contributing 810 | 811 | We welcome contributions! Areas for improvement: 812 | 813 | 1. **Additional ISOs**: PJM, ERCOT, ISO-NE, SPP 814 | 2. **Data validation**: Automated quality checks 815 | 3. **Visualization**: Built-in plotting tools 816 | 4. **Database integration**: PostgreSQL/SQLite support 817 | 5. **Web interface**: Flask/Django dashboard 818 | 6. **Enhanced documentation**: More examples and tutorials 819 | 820 | ### Development Workflow 821 | 822 | 1. Fork the repository 823 | 2. Create a feature branch: `git checkout -b feature-name` 824 | 3. Make changes and add tests 825 | 4. Run test suite: `pytest tests/` 826 | 5. Format code: `black lib/ tests/` 827 | 6. Lint code: `flake8 lib/ tests/` 828 | 7. Submit pull request 829 | 830 | ## Citation 831 | 832 | If you use ISO-DART in your research, please cite: 833 | 834 | ```bibtex 835 | @software{isodart2024, 836 | title = {ISO-DART: Independent System Operator Data Automated Request Tool}, 837 | author = {Sotorrio, Pedro and Edmunds, Thomas and Musselman, Amelia and Sun, Chih-Che}, 838 | year = {2024}, 839 | version = {2.0.0}, 840 | publisher = {Lawrence Livermore National Laboratory}, 841 | doi = {LLNL-CODE-815334}, 842 | url = {https://github.com/LLNL/ISO-DART} 843 | } 844 | ``` 845 | 846 | ## License 847 | 848 | MIT License - Copyright (c) 2025, Lawrence Livermore National Security, LLC 849 | 850 | See [LICENSE](LICENSE) file for details. 851 | 852 | ## Support 853 | 854 | - **Issues**: https://github.com/LLNL/ISO-DART/issues 855 | - **Discussions**: https://github.com/LLNL/ISO-DART/discussions 856 | - **Documentation**: See [QUICKSTART.md](QUICKSTART.md) for quick reference 857 | - **Migration Guide**: See [MIGRATION_GUIDE.md](MIGRATION_GUIDE.md) for v1.x→v2.0 migration 858 | 859 | ## Acknowledgments 860 | 861 | This work was produced under the auspices of the U.S. Department of Energy by Lawrence Livermore National Laboratory under Contract DE-AC52-07NA27344. 862 | 863 | ## Changelog 864 | 865 | ### v2.0.0 (2024-2025) 866 | - ✅ Complete architecture redesign with modern Python practices 867 | - ✅ Type hints, dataclasses, and enums throughout 868 | - ✅ Command-line interface with argparse 869 | - ✅ Comprehensive interactive mode with 100% method coverage 870 | - ✅ Full support for all CAISO, MISO, and NYISO data types 871 | - ✅ Enhanced error handling and logging 872 | - ✅ Comprehensive test suite with pytest 873 | - ✅ Updated dependencies (pandas 2.0+, requests 2.31+, meteostat 1.6+) 874 | - ✅ Removed debugging code (no more `pdb.set_trace()`) 875 | - ✅ Better documentation with examples 876 | - ✅ Migration guide for v1.x users 877 | 878 | ### v1.1.0 (2020) 879 | - Initial public release 880 | - CAISO, MISO, NYISO support 881 | - Weather data integration 882 | - Basic CLI interface 883 | 884 | ## FAQ 885 | 886 | **Q: Which Python version do I need?** 887 | A: Python 3.10 or higher is required for v2.0. 888 | 889 | **Q: Do I need to re-download all my data?** 890 | A: No! Existing v1.x data files are compatible with v2.0. 891 | 892 | **Q: Can I run v1.x and v2.0 side-by-side?** 893 | A: Yes, in different directories or virtual environments. 894 | 895 | **Q: How do I get historical data from several years ago?** 896 | A: Just specify the date range. ISOs typically keep several years of historical data available. 897 | 898 | **Q: Are there API rate limits?** 899 | A: CAISO, MISO, and NYISO don't have strict rate limits, but be respectful. NREL (solar data) has 1,000 requests/hour. 900 | 901 | **Q: Can I use this for commercial purposes?** 902 | A: Yes, under the MIT license. See [LICENSE](LICENSE) for details. 903 | 904 | **Q: How do I report a bug?** 905 | A: Please create an issue on GitHub with details about the error and how to reproduce it. 906 | 907 | **Q: Can I request support for additional ISOs?** 908 | A: Yes! Please create a feature request issue on GitHub. 909 | 910 | --- 911 | 912 | **Made with ❤️ by Lawrence Livermore National Laboratory** 913 | -------------------------------------------------------------------------------- /isodart.py: -------------------------------------------------------------------------------- 1 | """ 2 | ISO-DART v2.0: Independent System Operator Data Automated Request Tool 3 | 4 | Main entry point for the application. 5 | """ 6 | 7 | import argparse 8 | import sys 9 | from pathlib import Path 10 | from datetime import datetime, date, timedelta 11 | import logging 12 | 13 | # Setup logging 14 | logging.basicConfig( 15 | level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s" 16 | ) 17 | logger = logging.getLogger(__name__) 18 | 19 | 20 | def setup_directories(): 21 | """Create necessary directories if they don't exist.""" 22 | dirs = [ 23 | Path("data/CAISO"), 24 | Path("data/MISO"), 25 | Path("data/NYISO"), 26 | Path("data/BPA"), 27 | Path("data/SPP"), 28 | Path("data/weather"), 29 | Path("data/solar"), 30 | Path("raw_data/xml_files"), 31 | Path("logs"), 32 | ] 33 | for d in dirs: 34 | d.mkdir(parents=True, exist_ok=True) 35 | logger.info("Directory structure verified") 36 | 37 | 38 | def validate_date(date_string: str) -> date: 39 | """Validate and parse date string in YYYY-MM-DD format.""" 40 | try: 41 | return datetime.strptime(date_string, "%Y-%m-%d").date() 42 | except ValueError: 43 | raise argparse.ArgumentTypeError(f"Invalid date format: {date_string}. Use YYYY-MM-DD") 44 | 45 | 46 | def calculate_end_date(start_date: date, duration: int) -> date: 47 | """Calculate end date from start date and duration.""" 48 | return start_date + timedelta(days=duration) 49 | 50 | 51 | def handle_caiso(args): 52 | """Handle CAISO-specific data download logic.""" 53 | from lib.iso.caiso import CAISOClient, Market 54 | 55 | logger.info(f"Processing CAISO data request: {args.data_type}") 56 | 57 | client = CAISOClient() 58 | success = False 59 | 60 | try: 61 | end_date = calculate_end_date(args.start, args.duration) 62 | 63 | # Map string market to enum 64 | market_map = { 65 | "dam": Market.DAM, 66 | "rtm": Market.RTM, 67 | "hasp": Market.HASP, 68 | "rtpd": Market.RTPD, 69 | "ruc": Market.RUC, 70 | } 71 | 72 | if args.data_type == "lmp": 73 | if not args.market: 74 | logger.error("Market type required for LMP data") 75 | return False 76 | 77 | market = market_map.get(args.market.lower()) 78 | if not market: 79 | logger.error(f"Invalid market: {args.market}") 80 | return False 81 | 82 | logger.info(f"Downloading {market.value} LMP data...") 83 | success = client.get_lmp(market, args.start, end_date) 84 | 85 | elif args.data_type == "load": 86 | if not args.market: 87 | logger.error("Market type required for load data") 88 | return False 89 | 90 | market = market_map.get(args.market.lower()) 91 | if market not in [Market.DAM, Market.RTM]: 92 | logger.error(f"Invalid market for load: {args.market}") 93 | return False 94 | 95 | logger.info(f"Downloading {market.value} load forecast...") 96 | success = client.get_load_forecast(market, args.start, end_date) 97 | 98 | elif args.data_type == "wind-solar": 99 | logger.info("Downloading wind and solar summary...") 100 | success = client.get_wind_solar_summary(args.start, end_date) 101 | 102 | elif args.data_type == "fuel-prices": 103 | logger.info("Downloading fuel prices...") 104 | success = client.get_fuel_prices(args.start, end_date) 105 | 106 | elif args.data_type == "ghg-prices": 107 | logger.info("Downloading GHG allowance prices...") 108 | success = client.get_ghg_allowance_prices(args.start, end_date) 109 | 110 | elif args.data_type == "as-prices": 111 | if not args.market: 112 | logger.error("Market type required for AS prices") 113 | return False 114 | 115 | market = market_map.get(args.market.lower()) 116 | if market not in [Market.DAM, Market.RTM]: 117 | logger.error(f"Invalid market for AS prices: {args.market}") 118 | return False 119 | 120 | logger.info(f"Downloading {market.value} AS prices...") 121 | success = client.get_ancillary_services_prices(market, args.start, end_date) 122 | 123 | elif args.data_type == "as-requirements": 124 | if not args.market: 125 | logger.error("Market type required for AS requirements") 126 | return False 127 | 128 | market = market_map.get(args.market.lower()) 129 | if market not in [Market.DAM, Market.HASP, Market.RTM]: 130 | logger.error(f"Invalid market for AS requirements: {args.market}") 131 | return False 132 | 133 | logger.info(f"Downloading {market.value} AS requirements...") 134 | success = client.get_ancillary_services_requirements(market, args.start, end_date) 135 | 136 | else: 137 | logger.error(f"Unknown CAISO data type: {args.data_type}") 138 | logger.info( 139 | "Available types: lmp, load, wind-solar, fuel-prices, ghg-prices, as-prices" 140 | ) 141 | return False 142 | 143 | if success: 144 | logger.info(f"✅ CAISO data downloaded successfully to data/CAISO/") 145 | else: 146 | logger.error("❌ CAISO data download failed") 147 | 148 | return success 149 | 150 | except Exception as e: 151 | logger.error(f"Error downloading CAISO data: {e}", exc_info=True) 152 | return False 153 | 154 | finally: 155 | client.cleanup() 156 | 157 | 158 | def handle_miso(args): 159 | """Handle MISO-specific data download logic.""" 160 | from lib.iso.miso import MISOConfig, MISOClient 161 | 162 | logger.info(f"Processing MISO data request: {args.data_type}") 163 | 164 | config = MISOConfig.from_ini_file() 165 | client = MISOClient(config) 166 | 167 | try: 168 | data = None 169 | filename = None 170 | 171 | if args.data_type == "lmp": 172 | # LMP types: da_exante, da_expost, rt_exante, rt_expost 173 | lmp_type = getattr(args, "lmp_type", "da_exante") 174 | logger.info(f"Downloading MISO {lmp_type} LMP data...") 175 | 176 | data = client.get_lmp(lmp_type=lmp_type, start_date=args.start, duration=args.duration) 177 | filename = f"miso_{lmp_type}_lmp_{args.start}.csv" 178 | 179 | elif args.data_type == "mcp": 180 | # MCP types: asm_da_exante, asm_da_expost, asm_rt_exante, asm_rt_expost, asm_rt_summary 181 | mcp_type = getattr(args, "mcp_type", "asm_da_exante") 182 | logger.info(f"Downloading MISO {mcp_type} MCP data...") 183 | 184 | data = client.get_mcp(mcp_type=mcp_type, start_date=args.start, duration=args.duration) 185 | filename = f"miso_{mcp_type}_mcp_{args.start}.csv" 186 | 187 | elif args.data_type == "load": 188 | # Load types: da_demand, rt_forecast, rt_actual, rt_state_estimator 189 | load_type = getattr(args, "load_type", "rt_actual") 190 | logger.info(f"Downloading MISO {load_type} load data...") 191 | 192 | data = client.get_demand( 193 | demand_type=load_type, 194 | start_date=args.start, 195 | duration=args.duration, 196 | time_resolution="daily", 197 | ) 198 | filename = f"miso_{load_type}_load_{args.start}.csv" 199 | 200 | elif args.data_type == "load-forecast": 201 | logger.info("Downloading MISO medium-term load forecast...") 202 | 203 | data = client.get_load_forecast( 204 | start_date=args.start, duration=args.duration, time_resolution="daily" 205 | ) 206 | filename = f"miso_load_forecast_{args.start}.csv" 207 | 208 | elif args.data_type == "fuel-mix": 209 | logger.info("Downloading MISO fuel on the margin...") 210 | 211 | data = client.get_fuel_mix(start_date=args.start, duration=args.duration) 212 | filename = f"miso_fuel_mix_{args.start}.csv" 213 | 214 | elif args.data_type == "generation": 215 | # Generation types: da_cleared_physical, da_cleared_virtual, da_fuel_type, 216 | # da_offered_ecomax, da_offered_ecomin, rt_cleared, rt_committed_ecomax, 217 | # rt_fuel_margin, rt_fuel_type, rt_offered_ecomax 218 | gen_type = getattr(args, "gen_type", "rt_fuel_type") 219 | logger.info(f"Downloading MISO {gen_type} generation data...") 220 | 221 | data = client.get_generation( 222 | gen_type=gen_type, start_date=args.start, duration=args.duration 223 | ) 224 | filename = f"miso_{gen_type}_generation_{args.start}.csv" 225 | 226 | elif args.data_type == "interchange": 227 | # Interchange types: da_net_scheduled, rt_net_actual, rt_net_scheduled, historical 228 | interchange_type = getattr(args, "interchange_type", "rt_net_actual") 229 | logger.info(f"Downloading MISO {interchange_type} interchange data...") 230 | 231 | data = client.get_interchange( 232 | interchange_type=interchange_type, start_date=args.start, duration=args.duration 233 | ) 234 | filename = f"miso_{interchange_type}_interchange_{args.start}.csv" 235 | 236 | elif args.data_type == "outages": 237 | # Outage types: forecast, rt_outage 238 | outage_type = getattr(args, "outage_type", "rt_outage") 239 | logger.info(f"Downloading MISO {outage_type} data...") 240 | 241 | data = client.get_outages( 242 | outage_type=outage_type, start_date=args.start, duration=args.duration 243 | ) 244 | filename = f"miso_{outage_type}_{args.start}.csv" 245 | 246 | elif args.data_type == "binding-constraints": 247 | logger.info("Downloading MISO binding constraints...") 248 | 249 | data = client.get_binding_constraints(start_date=args.start, duration=args.duration) 250 | filename = f"miso_binding_constraints_{args.start}.csv" 251 | 252 | else: 253 | logger.error(f"Unknown MISO data type: {args.data_type}") 254 | logger.info( 255 | "Available types: lmp, mcp, load, load-forecast, fuel-mix, generation, " 256 | "interchange, outages, binding-constraints" 257 | ) 258 | return False 259 | 260 | # Save data if we got any 261 | if data: 262 | client.save_to_csv(data, filename) 263 | logger.info(f"✅ MISO data downloaded successfully to data/MISO/{filename}") 264 | return True 265 | else: 266 | logger.error("❌ MISO data download failed - no data returned") 267 | return False 268 | 269 | except Exception as e: 270 | logger.error(f"Error downloading MISO data: {e}", exc_info=True) 271 | return False 272 | 273 | 274 | def handle_nyiso(args): 275 | """Handle NYISO-specific data download logic.""" 276 | from lib.iso.nyiso import NYISOClient, NYISOMarket 277 | 278 | logger.info(f"Processing NYISO data request: {args.data_type}") 279 | 280 | client = NYISOClient() 281 | success = False 282 | 283 | try: 284 | # Map string market to enum 285 | market = None 286 | if args.market: 287 | market_map = {"dam": NYISOMarket.DAM, "rtm": NYISOMarket.RTM} 288 | market = market_map.get(args.market.lower()) 289 | 290 | if args.data_type == "lbmp": 291 | if not market: 292 | logger.error("Market type required for LBMP data") 293 | return False 294 | 295 | # Default to zonal if not specified 296 | level = getattr(args, "level", "zonal") 297 | logger.info(f"Downloading NYISO {market.value} {level} LBMP...") 298 | success = client.get_lbmp(market, level, args.start, args.duration) 299 | 300 | elif args.data_type == "load": 301 | # Default to actual load 302 | load_type = getattr(args, "load_type", "actual") 303 | logger.info(f"Downloading NYISO {load_type} load data...") 304 | success = client.get_load_data(load_type, args.start, args.duration) 305 | 306 | elif args.data_type == "fuel-mix": 307 | logger.info("Downloading NYISO fuel mix...") 308 | success = client.get_fuel_mix(args.start, args.duration) 309 | 310 | elif args.data_type == "wind": 311 | logger.info("Downloading NYISO wind generation...") 312 | success = client.get_wind_generation(args.start, args.duration) 313 | 314 | elif args.data_type == "btm-solar": 315 | logger.info("Downloading NYISO BTM solar...") 316 | success = client.get_btm_solar(args.start, args.duration) 317 | 318 | elif args.data_type == "interface-flows": 319 | logger.info("Downloading NYISO interface flows...") 320 | success = client.get_interface_flows(args.start, args.duration) 321 | 322 | elif args.data_type == "as-prices": 323 | if not market: 324 | logger.error("Market type required for AS prices") 325 | return False 326 | 327 | logger.info(f"Downloading NYISO {market.value} AS prices...") 328 | success = client.get_ancillary_services_prices(market, args.start, args.duration) 329 | 330 | else: 331 | logger.error(f"Unknown NYISO data type: {args.data_type}") 332 | logger.info( 333 | "Available types: lbmp, load, fuel-mix, wind, btm-solar, interface-flows, as-prices" 334 | ) 335 | return False 336 | 337 | if success: 338 | logger.info(f"✅ NYISO data downloaded successfully to data/NYISO/") 339 | else: 340 | logger.error("❌ NYISO data download failed") 341 | 342 | return success 343 | 344 | except Exception as e: 345 | logger.error(f"Error downloading NYISO data: {e}", exc_info=True) 346 | return False 347 | 348 | finally: 349 | client.cleanup() 350 | 351 | 352 | def handle_bpa(args): 353 | """Handle BPA-specific data download logic.""" 354 | from lib.iso.bpa import BPAClient, get_bpa_data_availability 355 | 356 | logger.info(f"Processing BPA data request: {args.data_type}") 357 | 358 | # Show BPA limitations 359 | info = get_bpa_data_availability() 360 | logger.warning(f"BPA Data Limitation: {info['temporal_coverage']}") 361 | 362 | client = BPAClient() 363 | success = False 364 | 365 | try: 366 | # Calculate date range 367 | from datetime import date, timedelta 368 | 369 | end_date = calculate_end_date(args.start, args.duration) 370 | 371 | # Route to appropriate method based on data type 372 | if args.data_type == "wind_gen_total_load": 373 | logger.info("Downloading BPA wind, generation and total load data...") 374 | success = client.get_wind_gen_total_load( 375 | args.start.year, start_date=args.start, end_date=end_date 376 | ) 377 | 378 | elif args.data_type == "reserves_deployed": 379 | logger.info("Downloading BPA reserves deployed data...") 380 | success = client.get_reserves_deployed( 381 | args.start.year, start_date=args.start, end_date=end_date 382 | ) 383 | 384 | elif args.data_type == "all": 385 | logger.info("Downloading all BPA data...") 386 | success = client.get_all_data(args.start.year, start_date=args.start, end_date=end_date) 387 | 388 | else: 389 | logger.error(f"Unknown BPA data type: {args.data_type}") 390 | logger.info("Available types: wind_gen_total_load, reserves_deployed, all") 391 | return False 392 | 393 | if success: 394 | logger.info(f"✅ BPA data downloaded successfully to data/BPA/") 395 | print(f"\n✅ BPA data downloaded successfully!") 396 | print(f" Location: data/BPA/") 397 | print(f" Resolution: {info['temporal_resolution']}") 398 | print(f" Coverage: {info['temporal_coverage']}") 399 | else: 400 | logger.error("❌ BPA data download failed") 401 | print(f"\n❌ BPA data download failed. Check logs for details.") 402 | 403 | return success 404 | 405 | except Exception as e: 406 | logger.error(f"Error downloading BPA data: {e}", exc_info=True) 407 | print(f"\n❌ Error downloading BPA data: {e}") 408 | return False 409 | 410 | finally: 411 | client.cleanup() 412 | 413 | 414 | def handle_spp(args): 415 | """Handle SPP-specific data download logic.""" 416 | from lib.iso.spp import SPPClient, SPPMarket 417 | 418 | logger.info(f"Processing SPP data request: {args.data_type}") 419 | 420 | client = SPPClient() 421 | success = False 422 | 423 | try: 424 | end_date = calculate_end_date(args.start, args.duration) 425 | 426 | # Map string market to enum 427 | market = None 428 | if args.market: 429 | market_map = {"dam": SPPMarket.DAM, "rtbm": SPPMarket.RTBM} 430 | market = market_map.get(args.market.lower()) 431 | 432 | if args.data_type == "lmp": 433 | if not market: 434 | logger.error("Market type required for LMP data") 435 | return False 436 | 437 | # Default is by settlement location unless --by-bus is specified 438 | by_location = not getattr(args, "by_bus", False) 439 | 440 | location_type = "by bus" if args.by_bus else "by settlement location" 441 | logger.info(f"Downloading SPP {market.value} LMP data {location_type}...") 442 | success = client.get_lmp(market, args.start, end_date, by_location=by_location) 443 | 444 | elif args.data_type == "mcp": 445 | if not market: 446 | logger.error("Market type required for MCP data") 447 | return False 448 | 449 | logger.info(f"Downloading SPP {market.value} MCP data...") 450 | success = client.get_mcp(market, args.start, end_date) 451 | 452 | elif args.data_type == "operating-reserves": 453 | logger.info("Downloading SPP Operating Reserves data...") 454 | success = client.get_operating_reserves(args.start, end_date) 455 | 456 | elif args.data_type == "binding-constraints": 457 | if not market: 458 | logger.error("Market type required for Binding Constraints data") 459 | return False 460 | 461 | logger.info(f"Downloading SPP {market.value} Binding Constraints data...") 462 | success = client.get_binding_constraints(market, args.start, end_date) 463 | 464 | elif args.data_type == "fuel-on-margin": 465 | logger.info("Downloading SPP Fuel On Margin data...") 466 | success = client.get_fuel_on_margin(args.start, end_date) 467 | 468 | elif args.data_type == "short-term-load-forecast": 469 | logger.info("Downloading SPP short-term load forecast data...") 470 | success = client.get_load_forecast(args.start, end_date, forecast_type="stlf") 471 | 472 | elif args.data_type == "medium-term-load-forecast": 473 | logger.info("Downloading SPP medium-term load forecast data...") 474 | success = client.get_load_forecast(args.start, end_date, forecast_type="mtlf") 475 | 476 | elif args.data_type == "short-term-resource-forecast": 477 | logger.info("Downloading SPP short-term resource (solar + wind) forecast data...") 478 | success = client.get_resource_forecast(args.start, end_date, forecast_type="strf") 479 | 480 | elif args.data_type == "medium-term-resource-forecast": 481 | logger.info("Downloading SPP medium-term resource (solar + wind) forecast data...") 482 | success = client.get_resource_forecast(args.start, end_date, forecast_type="mtrf") 483 | 484 | elif args.data_type == "market-clearing": 485 | logger.info("Downloading SPP Market Clearing data...") 486 | success = client.get_market_clearing(args.start, end_date) 487 | 488 | elif args.data_type == "virtual-clearing": 489 | logger.info("Downloading SPP Virtual Clearing data...") 490 | success = client.get_virtual_clearing(args.start, end_date) 491 | 492 | else: 493 | logger.error(f"Unknown SPP data type: {args.data_type}") 494 | logger.info("Available types: lmp, mcp, operating-reserves") 495 | return False 496 | 497 | if success: 498 | logger.info(f"✅ SPP data downloaded successfully to data/SPP/") 499 | else: 500 | logger.warning("⚠️ SPP data download incomplete") 501 | 502 | return success 503 | 504 | except Exception as e: 505 | logger.error(f"Error downloading SPP data: {e}", exc_info=True) 506 | return False 507 | 508 | finally: 509 | client.cleanup() 510 | 511 | 512 | def handle_weather(args): 513 | """Handle weather data download logic.""" 514 | from lib.weather.client import WeatherClient 515 | 516 | logger.info("Processing weather data request") 517 | 518 | if not args.state: 519 | logger.error("State code required for weather data") 520 | return False 521 | 522 | client = WeatherClient() 523 | success = False 524 | 525 | try: 526 | logger.info(f"Downloading weather data for {args.state}...") 527 | success = client.download_weather_data( 528 | state=args.state, 529 | start_date=args.start, 530 | duration=args.duration, 531 | interactive=False, # Auto-select first station in CLI mode 532 | ) 533 | 534 | if success: 535 | logger.info(f"✅ Weather data downloaded successfully to data/weather/") 536 | 537 | # Optionally download solar data 538 | if getattr(args, "include_solar", False): 539 | year = args.start.year 540 | logger.info(f"Downloading solar data for {year}...") 541 | client.download_solar_data(year=year) 542 | else: 543 | logger.error("❌ Weather data download failed") 544 | 545 | return success 546 | 547 | except Exception as e: 548 | logger.error(f"Error downloading weather data: {e}", exc_info=True) 549 | return False 550 | 551 | 552 | def main(): 553 | parser = argparse.ArgumentParser( 554 | description="ISO-DART v2.0 - Download energy market data from ISOs", 555 | formatter_class=argparse.RawDescriptionHelpFormatter, 556 | epilog=""" 557 | Examples: 558 | # Interactive mode 559 | python isodart.py 560 | 561 | # CAISO Day-Ahead LMP 562 | python isodart.py --iso caiso --data-type lmp --market dam --start 2024-01-01 --duration 7 563 | 564 | # MISO Wind Generation 565 | python isodart.py --iso miso --data-type wind --start 2024-01-01 --duration 30 566 | 567 | # SPP Day-Ahead LMP 568 | python isodart.py --iso spp --data-type lmp --market dam --start 2024-01-01 --duration 7 569 | 570 | # SPP Market Clearing Prices 571 | python isodart.py --iso spp --data-type mcp --market rtbm --start 2024-01-01 --duration 7 572 | 573 | # SPP Operating Reserves 574 | python isodart.py --iso spp --data-type operating-reserves --start 2024-01-01 --duration 7 575 | 576 | # BPA Load Data 577 | python isodart.py --iso bpa --data-type load --start 2024-01-01 --duration 7 578 | 579 | # Weather data 580 | python isodart.py --data-type weather --state CA --start 2024-01-01 --duration 30 581 | """, 582 | ) 583 | 584 | parser.add_argument( 585 | "--iso", 586 | choices=["caiso", "miso", "nyiso", "bpa", "spp"], 587 | help="Independent System Operator", 588 | ) 589 | 590 | parser.add_argument( 591 | "--data-type", 592 | help="Type of data to download (lmp, load, weather, etc.)", 593 | ) 594 | 595 | parser.add_argument( 596 | "--market", 597 | choices=["dam", "rtm", "rtbm", "hasp", "rtpd", "ruc"], 598 | help="Energy market type (rtbm = Real-Time Balancing Market for SPP)", 599 | ) 600 | 601 | parser.add_argument( 602 | "--start", 603 | type=validate_date, 604 | help="Start date (YYYY-MM-DD)", 605 | ) 606 | 607 | parser.add_argument( 608 | "--duration", 609 | type=int, 610 | help="Duration in days", 611 | ) 612 | 613 | parser.add_argument( 614 | "--state", 615 | help="US state 2-letter code (for weather data)", 616 | ) 617 | 618 | parser.add_argument( 619 | "--by-bus", 620 | action="store_true", 621 | help="For SPP LMP: Get data by bus instead of by settlement location (default: settlement location)", 622 | ) 623 | 624 | parser.add_argument( 625 | "--lmp-type", 626 | choices=["da_exante", "da_expost", "rt_exante", "rt_expost"], 627 | default="da_exante", 628 | help="For MISO LMP: type of LMP data to download", 629 | ) 630 | 631 | parser.add_argument( 632 | "--mcp-type", 633 | choices=[ 634 | "asm_da_exante", 635 | "asm_da_expost", 636 | "asm_rt_exante", 637 | "asm_rt_expost", 638 | "asm_rt_summary", 639 | ], 640 | default="asm_da_exante", 641 | help="For MISO MCP: type of MCP data to download", 642 | ) 643 | 644 | parser.add_argument( 645 | "--load-type", 646 | choices=["da_demand", "rt_forecast", "rt_actual", "rt_state_estimator"], 647 | default="rt_actual", 648 | help="For MISO Load: type of load data to download", 649 | ) 650 | 651 | parser.add_argument( 652 | "--gen-type", 653 | choices=[ 654 | "da_cleared_physical", 655 | "da_cleared_virtual", 656 | "da_fuel_type", 657 | "da_offered_ecomax", 658 | "da_offered_ecomin", 659 | "rt_cleared", 660 | "rt_committed_ecomax", 661 | "rt_fuel_margin", 662 | "rt_fuel_type", 663 | "rt_offered_ecomax", 664 | ], 665 | default="rt_fuel_type", 666 | help="For MISO Generation: type of generation data to download", 667 | ) 668 | 669 | parser.add_argument( 670 | "--interchange-type", 671 | choices=["da_net_scheduled", "rt_net_actual", "rt_net_scheduled", "historical"], 672 | default="rt_net_actual", 673 | help="For MISO Interchange: type of interchange data to download", 674 | ) 675 | 676 | parser.add_argument( 677 | "--outage-type", 678 | choices=["forecast", "rt_outage"], 679 | default="rt_outage", 680 | help="For MISO Outage: type of outage data to download", 681 | ) 682 | 683 | parser.add_argument( 684 | "--include-solar", 685 | action="store_true", 686 | help="Include solar data with weather download", 687 | ) 688 | 689 | parser.add_argument( 690 | "--interactive", 691 | action="store_true", 692 | default=False, 693 | help="Run in interactive mode (default if no args provided)", 694 | ) 695 | 696 | parser.add_argument( 697 | "--config", 698 | type=Path, 699 | help="Path to configuration file", 700 | ) 701 | 702 | parser.add_argument( 703 | "--verbose", 704 | action="store_true", 705 | help="Enable verbose logging", 706 | ) 707 | 708 | args = parser.parse_args() 709 | 710 | if args.verbose: 711 | logging.getLogger().setLevel(logging.DEBUG) 712 | 713 | # Setup directories 714 | setup_directories() 715 | 716 | # If no arguments provided, run in interactive mode 717 | if len(sys.argv) == 1 or args.interactive: 718 | from lib.interactive import run_interactive_mode 719 | 720 | run_interactive_mode() 721 | return 722 | 723 | # Validate required arguments for command-line mode 724 | if args.data_type == "weather": 725 | if not args.state or not args.start or not args.duration: 726 | parser.error("Weather data requires --state, --start, and --duration") 727 | return handle_weather(args) 728 | 729 | if not args.iso: 730 | parser.error("--iso is required (or use --interactive mode)") 731 | 732 | if not args.data_type: 733 | parser.error("--data-type is required") 734 | 735 | if not args.start or not args.duration: 736 | parser.error("--start and --duration are required") 737 | 738 | # Route to appropriate handler 739 | handlers = { 740 | "caiso": handle_caiso, 741 | "miso": handle_miso, 742 | "nyiso": handle_nyiso, 743 | "bpa": handle_bpa, 744 | "spp": handle_spp, 745 | } 746 | 747 | handler = handlers.get(args.iso) 748 | if handler: 749 | success = handler(args) 750 | sys.exit(0 if success else 1) 751 | else: 752 | logger.error(f"Unknown ISO: {args.iso}") 753 | sys.exit(1) 754 | 755 | 756 | if __name__ == "__main__": 757 | try: 758 | main() 759 | except KeyboardInterrupt: 760 | logger.info("\nOperation cancelled by user") 761 | sys.exit(0) 762 | except Exception as e: 763 | logger.error(f"Error: {e}", exc_info=True) 764 | sys.exit(1) 765 | --------------------------------------------------------------------------------