├── .editorconfig ├── .github ├── .stale.yml ├── ISSUE_TEMPLATE │ ├── bug_report.md │ ├── config.yml │ ├── feature_request.md │ └── question.md ├── PULL_REQUEST_TEMPLATE.md ├── dependabot.yml ├── release-drafter.yml └── workflows │ ├── build.yml │ ├── greetings.yml │ ├── pre-commit.yml │ └── release-drafter.yml ├── .gitignore ├── .pre-commit-config.yaml ├── LICENSE ├── Makefile ├── README.md ├── pyproject.toml ├── requirements-dev.txt ├── requirements.txt ├── setup.cfg ├── tests └── test_example │ └── test_hello.py └── xarray_fmrc ├── __init__.py ├── accessor.py ├── build_datatree.py ├── constants.py ├── example.py ├── forecast_offsets.py └── forecast_reference_time.py /.editorconfig: -------------------------------------------------------------------------------- 1 | # Check http://editorconfig.org for more information 2 | # This is the main config file for this project: 3 | root = true 4 | 5 | [*] 6 | charset = utf-8 7 | end_of_line = lf 8 | insert_final_newline = true 9 | indent_style = space 10 | indent_size = 2 11 | trim_trailing_whitespace = true 12 | 13 | [*.{py, pyi}] 14 | indent_style = space 15 | indent_size = 4 16 | 17 | [Makefile] 18 | indent_style = tab 19 | 20 | [*.md] 21 | trim_trailing_whitespace = false 22 | 23 | [*.{diff,patch}] 24 | trim_trailing_whitespace = false 25 | -------------------------------------------------------------------------------- /.github/.stale.yml: -------------------------------------------------------------------------------- 1 | # Number of days of inactivity before an issue becomes stale 2 | daysUntilStale: 60 3 | # Number of days of inactivity before a stale issue is closed 4 | daysUntilClose: 7 5 | # Issues with these labels will never be considered stale 6 | exemptLabels: 7 | - pinned 8 | - security 9 | # Label to use when marking an issue as stale 10 | staleLabel: wontfix 11 | # Comment to post when marking an issue as stale. Set to `false` to disable 12 | markComment: > 13 | This issue has been automatically marked as stale because it has not had 14 | recent activity. It will be closed if no further activity occurs. Thank you 15 | for your contributions. 16 | # Comment to post when closing a stale issue. Set to `false` to disable 17 | closeComment: false 18 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: 🐛 Bug report 3 | about: If something isn't working 🔧 4 | title: '' 5 | labels: bug 6 | assignees: 7 | --- 8 | 9 | ## 🐛 Bug Report 10 | 11 | 12 | 13 | ## 🔬 How To Reproduce 14 | 15 | Steps to reproduce the behavior: 16 | 17 | 1. ... 18 | 19 | ### Code sample 20 | 21 | 22 | 23 | ### Environment 24 | 25 | * OS: [e.g. Linux / Windows / macOS] 26 | * Python version, get it with: 27 | 28 | ```bash 29 | python --version 30 | ``` 31 | 32 | ### Screenshots 33 | 34 | 35 | 36 | ## 📈 Expected behavior 37 | 38 | 39 | 40 | ## 📎 Additional context 41 | 42 | 43 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/config.yml: -------------------------------------------------------------------------------- 1 | # Configuration: https://help.github.com/en/github/building-a-strong-community/configuring-issue-templates-for-your-repository 2 | 3 | blank_issues_enabled: false 4 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: 🚀 Feature request 3 | about: Suggest an idea for this project 🏖 4 | title: '' 5 | labels: enhancement 6 | assignees: 7 | --- 8 | 9 | ## 🚀 Feature Request 10 | 11 | 12 | 13 | ## 🔈 Motivation 14 | 15 | 16 | 17 | ## 🛰 Alternatives 18 | 19 | 20 | 21 | ## 📎 Additional context 22 | 23 | 24 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/question.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: ❓ Question 3 | about: Ask a question about this project 🎓 4 | title: '' 5 | labels: question 6 | assignees: 7 | --- 8 | 9 | ## Checklist 10 | 11 | 12 | 13 | - [ ] I've searched the project's [`issues`](https://github.com/abkfenris/xarray-fmrc/issues?q=is%3Aissue). 14 | 15 | ## ❓ Question 16 | 17 | 18 | 19 | How can I [...]? 20 | 21 | Is it possible to [...]? 22 | 23 | ## 📎 Additional context 24 | 25 | 26 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ## Description 2 | 3 | 4 | 5 | ## Related Issue 6 | 7 | 8 | 9 | ## Type of Change 10 | 11 | 12 | 13 | - [ ] 📚 Examples / docs / tutorials / dependencies update 14 | - [ ] 🔧 Bug fix (non-breaking change which fixes an issue) 15 | - [ ] 🥂 Improvement (non-breaking change which improves an existing feature) 16 | - [ ] 🚀 New feature (non-breaking change which adds functionality) 17 | - [ ] 💥 Breaking change (fix or feature that would cause existing functionality to change) 18 | - [ ] 🔐 Security fix 19 | 20 | ## Checklist 21 | 22 | 23 | 24 | - [ ] I've read the [`CODE_OF_CONDUCT.md`](https://github.com/abkfenris/xarray-fmrc/blob/master/CODE_OF_CONDUCT.md) document. 25 | - [ ] I've read the [`CONTRIBUTING.md`](https://github.com/abkfenris/xarray-fmrc/blob/master/CONTRIBUTING.md) guide. 26 | - [ ] I've updated the code style using `make codestyle`. 27 | - [ ] I've written tests for all new methods and classes that I created. 28 | - [ ] I've written the docstring in Google format for all the methods and classes that I used. 29 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # Configuration: https://dependabot.com/docs/config-file/ 2 | # Docs: https://docs.github.com/en/github/administering-a-repository/keeping-your-dependencies-updated-automatically 3 | 4 | version: 2 5 | 6 | updates: 7 | - package-ecosystem: "pip" 8 | directory: "/" 9 | schedule: 10 | interval: "daily" 11 | allow: 12 | - dependency-type: "all" 13 | commit-message: 14 | prefix: ":arrow_up:" 15 | open-pull-requests-limit: 50 16 | 17 | - package-ecosystem: "github-actions" 18 | directory: "/" 19 | schedule: 20 | interval: "daily" 21 | allow: 22 | - dependency-type: "all" 23 | commit-message: 24 | prefix: ":arrow_up:" 25 | open-pull-requests-limit: 50 26 | 27 | - package-ecosystem: "docker" 28 | directory: "/docker" 29 | schedule: 30 | interval: "weekly" 31 | allow: 32 | - dependency-type: "all" 33 | commit-message: 34 | prefix: ":arrow_up:" 35 | open-pull-requests-limit: 50 36 | -------------------------------------------------------------------------------- /.github/release-drafter.yml: -------------------------------------------------------------------------------- 1 | # Release drafter configuration https://github.com/release-drafter/release-drafter#configuration 2 | # Emojis were chosen to match the https://gitmoji.carloscuesta.me/ 3 | 4 | name-template: "v$NEXT_PATCH_VERSION" 5 | tag-template: "v$NEXT_PATCH_VERSION" 6 | 7 | categories: 8 | - title: ":rocket: Features" 9 | labels: [enhancement, feature] 10 | - title: ":wrench: Fixes & Refactoring" 11 | labels: [bug, refactoring, bugfix, fix] 12 | - title: ":package: Build System & CI/CD" 13 | labels: [build, ci, testing] 14 | - title: ":boom: Breaking Changes" 15 | labels: [breaking] 16 | - title: ":pencil: Documentation" 17 | labels: [documentation] 18 | - title: ":arrow_up: Dependencies updates" 19 | labels: [dependencies] 20 | 21 | template: | 22 | ## What’s Changed 23 | 24 | $CHANGES 25 | 26 | ## :busts_in_silhouette: List of contributors 27 | 28 | $CONTRIBUTORS 29 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: Tests 2 | 3 | on: 4 | pull_request: 5 | push: 6 | branches: [main] 7 | 8 | jobs: 9 | build: 10 | runs-on: ${{ matrix.os }} 11 | strategy: 12 | matrix: 13 | python-version: ["3.9", "3.10"] 14 | os: [windows-latest, ubuntu-latest, macos-latest] 15 | 16 | steps: 17 | - uses: actions/checkout@v3 18 | 19 | - name: Setup Micromamba 20 | uses: mamba-org/provision-with-micromamba@main 21 | with: 22 | environment-file: false 23 | 24 | - name: Python ${{ matrix.python-version }} 25 | shell: bash -l {0} 26 | run: > 27 | micromamba create --name TEST python=${{ matrix.python-version }} --file requirements.txt --file requirements-dev.txt --channel conda-forge 28 | && micromamba activate TEST 29 | && pip install -e . --no-deps --force-reinstall 30 | && micromamba info 31 | && micromamba list 32 | 33 | - name: Tests 34 | shell: bash -l {0} 35 | run: > 36 | micromamba activate TEST 37 | && pytest -rxs --cov=xpublish_edr tests 38 | -------------------------------------------------------------------------------- /.github/workflows/greetings.yml: -------------------------------------------------------------------------------- 1 | name: Greetings 2 | 3 | on: [pull_request, issues] 4 | 5 | jobs: 6 | greeting: 7 | runs-on: ubuntu-latest 8 | steps: 9 | - uses: actions/first-interaction@v1 10 | with: 11 | repo-token: ${{ secrets.GITHUB_TOKEN }} 12 | pr-message: 'Hello @${{ github.actor }}, thank you for submitting a PR! We will respond as soon as possible.' 13 | issue-message: | 14 | Hello @${{ github.actor }}, thank you for your interest in our work! 15 | 16 | If this is a bug report, please provide screenshots and **minimum viable code to reproduce your issue**, otherwise we can not help you. 17 | -------------------------------------------------------------------------------- /.github/workflows/pre-commit.yml: -------------------------------------------------------------------------------- 1 | name: pre-commit 2 | 3 | on: 4 | pull_request: 5 | push: 6 | branches: [main] 7 | 8 | jobs: 9 | pre-commit: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@v3 13 | - uses: actions/setup-python@v4.4.0 14 | with: 15 | python-version: 3.9 16 | - uses: pre-commit/action@v3.0.0 17 | -------------------------------------------------------------------------------- /.github/workflows/release-drafter.yml: -------------------------------------------------------------------------------- 1 | name: Release Drafter 2 | 3 | on: 4 | push: 5 | # branches to consider in the event; optional, defaults to all 6 | branches: 7 | - master 8 | 9 | jobs: 10 | update_release_draft: 11 | runs-on: ubuntu-latest 12 | steps: 13 | # Drafts your next Release notes as Pull Requests are merged into "master" 14 | - uses: release-drafter/release-drafter@v5.22.0 15 | env: 16 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 17 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | # Created by https://www.gitignore.io/api/osx,python,pycharm,windows,visualstudio,visualstudiocode 3 | # Edit at https://www.gitignore.io/?templates=osx,python,pycharm,windows,visualstudio,visualstudiocode 4 | 5 | ### OSX ### 6 | # General 7 | .DS_Store 8 | .AppleDouble 9 | .LSOverride 10 | 11 | # Icon must end with two \r 12 | Icon 13 | 14 | # Thumbnails 15 | ._* 16 | 17 | # Files that might appear in the root of a volume 18 | .DocumentRevisions-V100 19 | .fseventsd 20 | .Spotlight-V100 21 | .TemporaryItems 22 | .Trashes 23 | .VolumeIcon.icns 24 | .com.apple.timemachine.donotpresent 25 | 26 | # Directories potentially created on remote AFP share 27 | .AppleDB 28 | .AppleDesktop 29 | Network Trash Folder 30 | Temporary Items 31 | .apdisk 32 | 33 | ### PyCharm ### 34 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and WebStorm 35 | # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 36 | 37 | # User-specific stuff 38 | .idea/**/workspace.xml 39 | .idea/**/tasks.xml 40 | .idea/**/usage.statistics.xml 41 | .idea/**/dictionaries 42 | .idea/**/shelf 43 | 44 | # Generated files 45 | .idea/**/contentModel.xml 46 | 47 | # Sensitive or high-churn files 48 | .idea/**/dataSources/ 49 | .idea/**/dataSources.ids 50 | .idea/**/dataSources.local.xml 51 | .idea/**/sqlDataSources.xml 52 | .idea/**/dynamic.xml 53 | .idea/**/uiDesigner.xml 54 | .idea/**/dbnavigator.xml 55 | 56 | # Gradle 57 | .idea/**/gradle.xml 58 | .idea/**/libraries 59 | 60 | # Gradle and Maven with auto-import 61 | # When using Gradle or Maven with auto-import, you should exclude module files, 62 | # since they will be recreated, and may cause churn. Uncomment if using 63 | # auto-import. 64 | # .idea/modules.xml 65 | # .idea/*.iml 66 | # .idea/modules 67 | # *.iml 68 | # *.ipr 69 | 70 | # CMake 71 | cmake-build-*/ 72 | 73 | # Mongo Explorer plugin 74 | .idea/**/mongoSettings.xml 75 | 76 | # File-based project format 77 | *.iws 78 | 79 | # IntelliJ 80 | out/ 81 | 82 | # mpeltonen/sbt-idea plugin 83 | .idea_modules/ 84 | 85 | # JIRA plugin 86 | atlassian-ide-plugin.xml 87 | 88 | # Cursive Clojure plugin 89 | .idea/replstate.xml 90 | 91 | # Crashlytics plugin (for Android Studio and IntelliJ) 92 | com_crashlytics_export_strings.xml 93 | crashlytics.properties 94 | crashlytics-build.properties 95 | fabric.properties 96 | 97 | # Editor-based Rest Client 98 | .idea/httpRequests 99 | 100 | # Android studio 3.1+ serialized cache file 101 | .idea/caches/build_file_checksums.ser 102 | 103 | ### PyCharm Patch ### 104 | # Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721 105 | 106 | # *.iml 107 | # modules.xml 108 | # .idea/misc.xml 109 | # *.ipr 110 | 111 | # Sonarlint plugin 112 | .idea/**/sonarlint/ 113 | 114 | # SonarQube Plugin 115 | .idea/**/sonarIssues.xml 116 | 117 | # Markdown Navigator plugin 118 | .idea/**/markdown-navigator.xml 119 | .idea/**/markdown-navigator/ 120 | 121 | ### Python ### 122 | # Byte-compiled / optimized / DLL files 123 | __pycache__/ 124 | *.py[cod] 125 | *$py.class 126 | 127 | # C extensions 128 | *.so 129 | 130 | # Distribution / packaging 131 | .Python 132 | build/ 133 | develop-eggs/ 134 | dist/ 135 | downloads/ 136 | eggs/ 137 | .eggs/ 138 | lib/ 139 | lib64/ 140 | parts/ 141 | sdist/ 142 | var/ 143 | wheels/ 144 | pip-wheel-metadata/ 145 | share/python-wheels/ 146 | *.egg-info/ 147 | .installed.cfg 148 | *.egg 149 | MANIFEST 150 | 151 | # PyInstaller 152 | # Usually these files are written by a python script from a template 153 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 154 | *.manifest 155 | *.spec 156 | 157 | # Installer logs 158 | pip-log.txt 159 | pip-delete-this-directory.txt 160 | 161 | # Unit test / coverage reports 162 | htmlcov/ 163 | .tox/ 164 | .nox/ 165 | .coverage 166 | .coverage.* 167 | .cache 168 | nosetests.xml 169 | coverage.xml 170 | *.cover 171 | .hypothesis/ 172 | .pytest_cache/ 173 | 174 | # Translations 175 | *.mo 176 | *.pot 177 | 178 | # Scrapy stuff: 179 | .scrapy 180 | 181 | # Sphinx documentation 182 | docs/_build/ 183 | 184 | # PyBuilder 185 | target/ 186 | 187 | # pyenv 188 | .python-version 189 | 190 | # poetry 191 | .venv 192 | 193 | # pipenv 194 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 195 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 196 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 197 | # install all needed dependencies. 198 | #Pipfile.lock 199 | 200 | # celery beat schedule file 201 | celerybeat-schedule 202 | 203 | # SageMath parsed files 204 | *.sage.py 205 | 206 | # Spyder project settings 207 | .spyderproject 208 | .spyproject 209 | 210 | # Rope project settings 211 | .ropeproject 212 | 213 | # Mr Developer 214 | .mr.developer.cfg 215 | .project 216 | .pydevproject 217 | 218 | # mkdocs documentation 219 | /site 220 | 221 | # mypy 222 | .mypy_cache/ 223 | .dmypy.json 224 | dmypy.json 225 | 226 | # Pyre type checker 227 | .pyre/ 228 | 229 | # Plugins 230 | .secrets.baseline 231 | 232 | ### VisualStudioCode ### 233 | .vscode/* 234 | !.vscode/tasks.json 235 | !.vscode/launch.json 236 | !.vscode/extensions.json 237 | 238 | ### VisualStudioCode Patch ### 239 | # Ignore all local history of files 240 | .history 241 | 242 | ### Windows ### 243 | # Windows thumbnail cache files 244 | Thumbs.db 245 | Thumbs.db:encryptable 246 | ehthumbs.db 247 | ehthumbs_vista.db 248 | 249 | # Dump file 250 | *.stackdump 251 | 252 | # Folder config file 253 | [Dd]esktop.ini 254 | 255 | # Recycle Bin used on file shares 256 | $RECYCLE.BIN/ 257 | 258 | # Windows Installer files 259 | *.cab 260 | *.msi 261 | *.msix 262 | *.msm 263 | *.msp 264 | 265 | # Windows shortcuts 266 | *.lnk 267 | 268 | ### VisualStudio ### 269 | ## Ignore Visual Studio temporary files, build results, and 270 | ## files generated by popular Visual Studio add-ons. 271 | ## 272 | ## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore 273 | 274 | # User-specific files 275 | *.rsuser 276 | *.suo 277 | *.user 278 | *.userosscache 279 | *.sln.docstates 280 | 281 | # User-specific files (MonoDevelop/Xamarin Studio) 282 | *.userprefs 283 | 284 | # Mono auto generated files 285 | mono_crash.* 286 | 287 | # Build results 288 | [Dd]ebug/ 289 | [Dd]ebugPublic/ 290 | [Rr]elease/ 291 | [Rr]eleases/ 292 | x64/ 293 | x86/ 294 | [Aa][Rr][Mm]/ 295 | [Aa][Rr][Mm]64/ 296 | bld/ 297 | [Bb]in/ 298 | [Oo]bj/ 299 | [Ll]og/ 300 | 301 | # Visual Studio 2015/2017 cache/options directory 302 | .vs/ 303 | # Uncomment if you have tasks that create the project's static files in wwwroot 304 | #wwwroot/ 305 | 306 | # Visual Studio 2017 auto generated files 307 | Generated\ Files/ 308 | 309 | # MSTest test Results 310 | [Tt]est[Rr]esult*/ 311 | [Bb]uild[Ll]og.* 312 | 313 | # NUnit 314 | *.VisualState.xml 315 | TestResult.xml 316 | nunit-*.xml 317 | 318 | # Build Results of an ATL Project 319 | [Dd]ebugPS/ 320 | [Rr]eleasePS/ 321 | dlldata.c 322 | 323 | # Benchmark Results 324 | BenchmarkDotNet.Artifacts/ 325 | 326 | # .NET Core 327 | project.lock.json 328 | project.fragment.lock.json 329 | artifacts/ 330 | 331 | # StyleCop 332 | StyleCopReport.xml 333 | 334 | # Files built by Visual Studio 335 | *_i.c 336 | *_p.c 337 | *_h.h 338 | *.ilk 339 | *.obj 340 | *.iobj 341 | *.pch 342 | *.pdb 343 | *.ipdb 344 | *.pgc 345 | *.pgd 346 | *.rsp 347 | *.sbr 348 | *.tlb 349 | *.tli 350 | *.tlh 351 | *.tmp 352 | *.tmp_proj 353 | *_wpftmp.csproj 354 | *.log 355 | *.vspscc 356 | *.vssscc 357 | .builds 358 | *.pidb 359 | *.svclog 360 | *.scc 361 | 362 | # Chutzpah Test files 363 | _Chutzpah* 364 | 365 | # Visual C++ cache files 366 | ipch/ 367 | *.aps 368 | *.ncb 369 | *.opendb 370 | *.opensdf 371 | *.sdf 372 | *.cachefile 373 | *.VC.db 374 | *.VC.VC.opendb 375 | 376 | # Visual Studio profiler 377 | *.psess 378 | *.vsp 379 | *.vspx 380 | *.sap 381 | 382 | # Visual Studio Trace Files 383 | *.e2e 384 | 385 | # TFS 2012 Local Workspace 386 | $tf/ 387 | 388 | # Guidance Automation Toolkit 389 | *.gpState 390 | 391 | # ReSharper is a .NET coding add-in 392 | _ReSharper*/ 393 | *.[Rr]e[Ss]harper 394 | *.DotSettings.user 395 | 396 | # JustCode is a .NET coding add-in 397 | .JustCode 398 | 399 | # TeamCity is a build add-in 400 | _TeamCity* 401 | 402 | # DotCover is a Code Coverage Tool 403 | *.dotCover 404 | 405 | # AxoCover is a Code Coverage Tool 406 | .axoCover/* 407 | !.axoCover/settings.json 408 | 409 | # Visual Studio code coverage results 410 | *.coverage 411 | *.coveragexml 412 | 413 | # NCrunch 414 | _NCrunch_* 415 | .*crunch*.local.xml 416 | nCrunchTemp_* 417 | 418 | # MightyMoose 419 | *.mm.* 420 | AutoTest.Net/ 421 | 422 | # Web workbench (sass) 423 | .sass-cache/ 424 | 425 | # Installshield output folder 426 | [Ee]xpress/ 427 | 428 | # DocProject is a documentation generator add-in 429 | DocProject/buildhelp/ 430 | DocProject/Help/*.HxT 431 | DocProject/Help/*.HxC 432 | DocProject/Help/*.hhc 433 | DocProject/Help/*.hhk 434 | DocProject/Help/*.hhp 435 | DocProject/Help/Html2 436 | DocProject/Help/html 437 | 438 | # Click-Once directory 439 | publish/ 440 | 441 | # Publish Web Output 442 | *.[Pp]ublish.xml 443 | *.azurePubxml 444 | # Note: Comment the next line if you want to checkin your web deploy settings, 445 | # but database connection strings (with potential passwords) will be unencrypted 446 | *.pubxml 447 | *.publishproj 448 | 449 | # Microsoft Azure Web App publish settings. Comment the next line if you want to 450 | # checkin your Azure Web App publish settings, but sensitive information contained 451 | # in these scripts will be unencrypted 452 | PublishScripts/ 453 | 454 | # NuGet Packages 455 | *.nupkg 456 | # NuGet Symbol Packages 457 | *.snupkg 458 | # The packages folder can be ignored because of Package Restore 459 | **/[Pp]ackages/* 460 | # except build/, which is used as an MSBuild target. 461 | !**/[Pp]ackages/build/ 462 | # Uncomment if necessary however generally it will be regenerated when needed 463 | #!**/[Pp]ackages/repositories.config 464 | # NuGet v3's project.json files produces more ignorable files 465 | *.nuget.props 466 | *.nuget.targets 467 | 468 | # Microsoft Azure Build Output 469 | csx/ 470 | *.build.csdef 471 | 472 | # Microsoft Azure Emulator 473 | ecf/ 474 | rcf/ 475 | 476 | # Windows Store app package directories and files 477 | AppPackages/ 478 | BundleArtifacts/ 479 | Package.StoreAssociation.xml 480 | _pkginfo.txt 481 | *.appx 482 | *.appxbundle 483 | *.appxupload 484 | 485 | # Visual Studio cache files 486 | # files ending in .cache can be ignored 487 | *.[Cc]ache 488 | # but keep track of directories ending in .cache 489 | !?*.[Cc]ache/ 490 | 491 | # Others 492 | ClientBin/ 493 | ~$* 494 | *~ 495 | *.dbmdl 496 | *.dbproj.schemaview 497 | *.jfm 498 | *.pfx 499 | *.publishsettings 500 | orleans.codegen.cs 501 | 502 | # Including strong name files can present a security risk 503 | # (https://github.com/github/gitignore/pull/2483#issue-259490424) 504 | #*.snk 505 | 506 | # Since there are multiple workflows, uncomment next line to ignore bower_components 507 | # (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) 508 | #bower_components/ 509 | 510 | # RIA/Silverlight projects 511 | Generated_Code/ 512 | 513 | # Backup & report files from converting an old project file 514 | # to a newer Visual Studio version. Backup files are not needed, 515 | # because we have git ;-) 516 | _UpgradeReport_Files/ 517 | Backup*/ 518 | UpgradeLog*.XML 519 | UpgradeLog*.htm 520 | ServiceFabricBackup/ 521 | *.rptproj.bak 522 | 523 | # SQL Server files 524 | *.mdf 525 | *.ldf 526 | *.ndf 527 | 528 | # Business Intelligence projects 529 | *.rdl.data 530 | *.bim.layout 531 | *.bim_*.settings 532 | *.rptproj.rsuser 533 | *- [Bb]ackup.rdl 534 | *- [Bb]ackup ([0-9]).rdl 535 | *- [Bb]ackup ([0-9][0-9]).rdl 536 | 537 | # Microsoft Fakes 538 | FakesAssemblies/ 539 | 540 | # GhostDoc plugin setting file 541 | *.GhostDoc.xml 542 | 543 | # Node.js Tools for Visual Studio 544 | .ntvs_analysis.dat 545 | node_modules/ 546 | 547 | # Visual Studio 6 build log 548 | *.plg 549 | 550 | # Visual Studio 6 workspace options file 551 | *.opt 552 | 553 | # Visual Studio 6 auto-generated workspace file (contains which files were open etc.) 554 | *.vbw 555 | 556 | # Visual Studio LightSwitch build output 557 | **/*.HTMLClient/GeneratedArtifacts 558 | **/*.DesktopClient/GeneratedArtifacts 559 | **/*.DesktopClient/ModelManifest.xml 560 | **/*.Server/GeneratedArtifacts 561 | **/*.Server/ModelManifest.xml 562 | _Pvt_Extensions 563 | 564 | # Paket dependency manager 565 | .paket/paket.exe 566 | paket-files/ 567 | 568 | # FAKE - F# Make 569 | .fake/ 570 | 571 | # CodeRush personal settings 572 | .cr/personal 573 | 574 | # Python Tools for Visual Studio (PTVS) 575 | *.pyc 576 | 577 | # Cake - Uncomment if you are using it 578 | # tools/** 579 | # !tools/packages.config 580 | 581 | # Tabs Studio 582 | *.tss 583 | 584 | # Telerik's JustMock configuration file 585 | *.jmconfig 586 | 587 | # BizTalk build output 588 | *.btp.cs 589 | *.btm.cs 590 | *.odx.cs 591 | *.xsd.cs 592 | 593 | # OpenCover UI analysis results 594 | OpenCover/ 595 | 596 | # Azure Stream Analytics local run output 597 | ASALocalRun/ 598 | 599 | # MSBuild Binary and Structured Log 600 | *.binlog 601 | 602 | # NVidia Nsight GPU debugger configuration file 603 | *.nvuser 604 | 605 | # MFractors (Xamarin productivity tool) working folder 606 | .mfractor/ 607 | 608 | # Local History for Visual Studio 609 | .localhistory/ 610 | 611 | # BeatPulse healthcheck temp database 612 | healthchecksdb 613 | 614 | # Backup folder for Package Reference Convert tool in Visual Studio 2017 615 | MigrationBackup/ 616 | 617 | # End of https://www.gitignore.io/api/osx,python,pycharm,windows,visualstudio,visualstudiocode 618 | xarray_fmrc/_version.py 619 | *.zarr 620 | *.nc 621 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | default_stages: [commit, push] 2 | 3 | repos: 4 | - repo: https://github.com/pre-commit/pre-commit-hooks 5 | rev: v4.5.0 6 | hooks: 7 | - id: trailing-whitespace 8 | exclude: tests/data 9 | - id: check-ast 10 | - id: debug-statements 11 | - id: end-of-file-fixer 12 | - id: check-docstring-first 13 | - id: check-added-large-files 14 | 15 | - repo: https://github.com/econchick/interrogate 16 | rev: 1.5.0 17 | hooks: 18 | - id: interrogate 19 | exclude: ^(docs|setup.py|tests) 20 | args: [--config=pyproject.toml] 21 | 22 | - repo: https://github.com/keewis/blackdoc 23 | rev: v0.3.8 24 | hooks: 25 | - id: blackdoc 26 | 27 | - repo: https://github.com/pycqa/flake8 28 | rev: 6.1.0 29 | hooks: 30 | - id: flake8 31 | exclude: docs/source/conf.py 32 | args: [--max-line-length=105] 33 | 34 | - repo: https://github.com/pycqa/isort 35 | rev: 5.12.0 36 | hooks: 37 | - id: isort 38 | additional_dependencies: [toml] 39 | args: ["--profile", "black", "--filter-files"] 40 | 41 | - repo: https://github.com/psf/black 42 | rev: 23.9.1 43 | hooks: 44 | - id: black 45 | language_version: python3 46 | 47 | - repo: https://github.com/pre-commit/mirrors-mypy 48 | rev: v1.5.1 49 | hooks: 50 | - id: mypy 51 | exclude: docs/source/conf.py 52 | args: [--ignore-missing-imports] 53 | additional_dependencies: [typing_extensions>=4.2.0, types-setuptools] 54 | 55 | - repo: https://github.com/codespell-project/codespell 56 | rev: v2.2.6 57 | hooks: 58 | - id: codespell 59 | args: 60 | - --quiet-level=2 61 | 62 | - repo: https://github.com/asottile/pyupgrade 63 | rev: v3.15.0 64 | hooks: 65 | - id: pyupgrade 66 | args: 67 | - --py36-plus 68 | 69 | - repo: https://github.com/asottile/add-trailing-comma 70 | rev: v3.1.0 71 | hooks: 72 | - id: add-trailing-comma 73 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | Copyright (c) 2023 abkfenris 3 | 4 | Permission is hereby granted, free of charge, to any person obtaining a copy 5 | of this software and associated documentation files (the "Software"), to deal 6 | in the Software without restriction, including without limitation the rights 7 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | copies of the Software, and to permit persons to whom the Software is 9 | furnished to do so, subject to the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be included in all 12 | copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 15 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 16 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 17 | IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 18 | DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 19 | OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE 20 | OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | #* Variables 2 | SHELL := /usr/bin/env bash 3 | PYTHON := python 4 | PYTHONPATH := `pwd` 5 | 6 | #* Docker variables 7 | IMAGE := xarray_fmrc 8 | VERSION := latest 9 | 10 | #* Poetry 11 | .PHONY: poetry-download 12 | poetry-download: 13 | curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/install-poetry.py | $(PYTHON) - 14 | 15 | .PHONY: poetry-remove 16 | poetry-remove: 17 | curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/install-poetry.py | $(PYTHON) - --uninstall 18 | 19 | #* Installation 20 | .PHONY: install 21 | install: 22 | poetry lock -n && poetry export --without-hashes > requirements.txt 23 | poetry install -n 24 | -poetry run mypy --install-types --non-interactive ./ 25 | 26 | .PHONY: pre-commit-install 27 | pre-commit-install: 28 | poetry run pre-commit install 29 | 30 | #* Formatters 31 | .PHONY: codestyle 32 | codestyle: 33 | poetry run pyupgrade --exit-zero-even-if-changed --py39-plus **/*.py 34 | poetry run isort --settings-path pyproject.toml ./ 35 | poetry run black --config pyproject.toml ./ 36 | 37 | .PHONY: formatting 38 | formatting: codestyle 39 | 40 | #* Linting 41 | .PHONY: test 42 | test: 43 | PYTHONPATH=$(PYTHONPATH) poetry run pytest -c pyproject.toml --cov-report=html --cov=xarray_fmrc tests/ 44 | poetry run coverage-badge -o assets/images/coverage.svg -f 45 | 46 | .PHONY: check-codestyle 47 | check-codestyle: 48 | poetry run isort --diff --check-only --settings-path pyproject.toml ./ 49 | poetry run black --diff --check --config pyproject.toml ./ 50 | poetry run darglint --verbosity 2 xarray_fmrc tests 51 | 52 | .PHONY: mypy 53 | mypy: 54 | poetry run mypy --config-file pyproject.toml ./ 55 | 56 | .PHONY: check-safety 57 | check-safety: 58 | poetry check 59 | poetry run safety check --full-report 60 | poetry run bandit -ll --recursive xarray_fmrc tests 61 | 62 | .PHONY: lint 63 | lint: test check-codestyle mypy check-safety 64 | 65 | .PHONY: update-dev-deps 66 | update-dev-deps: 67 | poetry add -D bandit@latest darglint@latest "isort[colors]@latest" mypy@latest pre-commit@latest pydocstyle@latest pylint@latest pytest@latest pyupgrade@latest safety@latest coverage@latest coverage-badge@latest pytest-html@latest pytest-cov@latest 68 | poetry add -D --allow-prereleases black@latest 69 | 70 | #* Docker 71 | # Example: make docker-build VERSION=latest 72 | # Example: make docker-build IMAGE=some_name VERSION=0.1.0 73 | .PHONY: docker-build 74 | docker-build: 75 | @echo Building docker $(IMAGE):$(VERSION) ... 76 | docker build \ 77 | -t $(IMAGE):$(VERSION) . \ 78 | -f ./docker/Dockerfile --no-cache 79 | 80 | # Example: make docker-remove VERSION=latest 81 | # Example: make docker-remove IMAGE=some_name VERSION=0.1.0 82 | .PHONY: docker-remove 83 | docker-remove: 84 | @echo Removing docker $(IMAGE):$(VERSION) ... 85 | docker rmi -f $(IMAGE):$(VERSION) 86 | 87 | #* Cleaning 88 | .PHONY: pycache-remove 89 | pycache-remove: 90 | find . | grep -E "(__pycache__|\.pyc|\.pyo$$)" | xargs rm -rf 91 | 92 | .PHONY: dsstore-remove 93 | dsstore-remove: 94 | find . | grep -E ".DS_Store" | xargs rm -rf 95 | 96 | .PHONY: mypycache-remove 97 | mypycache-remove: 98 | find . | grep -E ".mypy_cache" | xargs rm -rf 99 | 100 | .PHONY: ipynbcheckpoints-remove 101 | ipynbcheckpoints-remove: 102 | find . | grep -E ".ipynb_checkpoints" | xargs rm -rf 103 | 104 | .PHONY: pytestcache-remove 105 | pytestcache-remove: 106 | find . | grep -E ".pytest_cache" | xargs rm -rf 107 | 108 | .PHONY: build-remove 109 | build-remove: 110 | rm -rf build/ 111 | 112 | .PHONY: cleanup 113 | cleanup: pycache-remove dsstore-remove mypycache-remove ipynbcheckpoints-remove pytestcache-remove 114 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # xarray-fmrc 2 | 3 |
4 | 5 | [![Tests](https://github.com/abkfenris/xarray_fmrc/actions/workflows/build.yml/badge.svg)](https://github.com/abkfenris/xarray_fmrc/actions/workflows/build.yml) 6 | [![Python Version](https://img.shields.io/pypi/pyversions/xarray_fmrc.svg)](https://pypi.org/project/xarray-fmrc/) 7 | [![Dependencies Status](https://img.shields.io/badge/dependencies-up%20to%20date-brightgreen.svg)](https://github.com/abkfenris/xarray-fmrc/pulls?utf8=%E2%9C%93&q=is%3Apr%20author%3Aapp%2Fdependabot) 8 | 9 | [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) 10 | [![Security: bandit](https://img.shields.io/badge/security-bandit-green.svg)](https://github.com/PyCQA/bandit) 11 | [![Pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit&logoColor=white)](https://github.com/abkfenris/xarray-fmrc/blob/master/.pre-commit-config.yaml) 12 | [![Semantic Versions](https://img.shields.io/badge/%20%20%F0%9F%93%A6%F0%9F%9A%80-semantic--versions-e10079.svg)](https://github.com/abkfenris/xarray-fmrc/releases) 13 | [![License](https://img.shields.io/github/license/abkfenris/xarray_fmrc)](https://github.com/abkfenris/xarray_fmrc/blob/master/LICENSE) 14 | 15 | Xarray-FMCR uses Xarray datatrees to provide a standard in-memory and storage representation of [Forecast Model Run Collections](http://www.unidata.ucar.edu/staff/caron/presentations/FmrcPoster.pdf) that can then be access via the various forecast views (best estimate/constant offset/constant time/model run). 16 | 17 |
18 | 19 | ```ipython 20 | In [1]: import xarray as xr 21 | 22 | In [2]: import xarray_fmrc 23 | 24 | In [3]: ds0 = xr.open_dataset("fvcom_gom3_met_2022120118.nc") 25 | 26 | In [4]: ds1 = xr.open_dataset("fvcom_gom3_met_2022121218.nc") 27 | 28 | In [5]: dt = xarray_fmrc.from_model_runs([ds0, ds1]) 29 | 30 | In [6]: dt 31 | Out[6]: 32 | DataTree('None', parent=None) 33 | │ Dimensions: (forecast_reference_time: 2, 34 | │ constant_forecast: 242, constant_offset: 121) 35 | │ Coordinates: 36 | │ * forecast_reference_time (forecast_reference_time) datetime64[ns] 2022-12... 37 | │ * constant_forecast (constant_forecast) datetime64[ns] 2022-12-02 ..... 38 | │ * constant_offset (constant_offset) timedelta64[ns] 06:00:00 ... 5... 39 | │ Data variables: 40 | │ model_run_path (forecast_reference_time) 74 | Dimensions: (longitude: 215, latitude: 220, 75 | forecast_reference_time: 2, time: 2) 76 | Coordinates: 77 | * longitude (longitude) float64 -79.95 -79.86 ... -60.13 -60.04 78 | * latitude (latitude) float64 27.03 27.12 ... 47.32 47.41 79 | * forecast_reference_time (forecast_reference_time) datetime64[ns] 2022-12... 80 | * time (time) datetime64[ns] 2022-12-02T06:00:00 2022-1... 81 | forecast_offset (time) timedelta64[ns] 12:00:00 12:00:00 82 | Data variables: 83 | wind_speed (forecast_reference_time, time, latitude, longitude) float32 ... 84 | wind_from_direction (forecast_reference_time, time, latitude, longitude) float32 ... 85 | Attributes: (12/178) 86 | ... 87 | ``` 88 | 89 | ## Forecast views 90 | 91 | ![Forecast Model Run Collections](https://docs.unidata.ucar.edu/netcdf-java/current/userguide/images/netcdf-java/tutorial/feature_types/fmrc.png) 92 | 93 | The various views are explained in more detail below, but each has a method on the `.fmrc` accessor that returns a dataset. 94 | 95 | - `dt.fmrc.model_run(dt: str | datetime.datetime | pd.Timestamp) -> xr.Dataset` 96 | - `dt.fmrc.constant_offset(offset: str | datetime.timedelta | pd.TimeOffset?) -> xr.Dataset` 97 | - `dt.fmrc.constant_forecast(dt: str | datetime.datetime | pd.Timestamp) -> xr.Dataset` 98 | - `dt.fmrc.best() -> xr.Dataset` 99 | 100 | ## A few ideas 101 | 102 | Here are some things that aren't implemented, but where this library could go. 103 | 104 | ### Kerchunk 105 | 106 | Kerchunk has the ability to break down [chunks into smaller chunks](https://fsspec.github.io/kerchunk/reference.html#kerchunk.utils.subchunk). Xarray-FMRC could provide utilities to take a collection of kerchunk files, break them apart, and rebuild them in the various FMRC views. 107 | 108 | ### Xpublish-FMRC 109 | 110 | Xpublish-FMRC provides new endpoints for xpublish servers to serve forecast model run collections. 111 | 112 | This uses the plugin interface to create a new top level path, and then other dataset plugins to serve various forecast views. For each dataset plugin registered below it, it overrides the `get_dataset` function. 113 | 114 | - `forecasts/gfs/best/edr/position` 115 | - `forecasts/gfs/model_run/20230101/edr/position` 116 | - `forecasts/gfs/constant_forecast/20230101/edr/position` 117 | - `forecasts/gfs/constant_offset/6h/edr/position` 118 | 119 | 120 | ## FMRC Dataset View definitions 121 | 122 | _There may be a better name for these, but my brain is currently comparing them to database views._ 123 | 124 | _Definitions pulled from http://www.unidata.ucar.edu/staff/caron/presentations/FmrcPoster.pdf_ 125 | 126 | ### Model Run Datasets 127 | 128 | The RUC model is run hourly, and 12 runs are show 129 | in this collection; note that different runs contain 130 | forecast hours. The complete results for a single ru 131 | model run dataset. 132 | The selected example here is the run made on 133 | 2006-12-11 06:00 Z, having forecasts at 134 | 0,1,2,3,4,5,6,7,8,9 and 12 hours. 135 | 136 | ### Constant forecast/valid time dataset 137 | 138 | A __constant forecast__ dataset is created from all data that have the same forecast/valid time. Using the 0 hour analysis as the best state estimate, one can use this dataset to evaluate how accurate forecasts are. 139 | 140 | The selected example here is for the forecast time 2006-12-11 12:00 Z, using forecasts from the runs made at 0, 3, 6, 9, 10, 11, and 12 Z. There are a total of 24 such datasets in this collection. 141 | 142 | ### Constant forecast offset datasets 143 | 144 | A __constant offset__ dataset is created from all the data that have the same offset time. This collection has 11 such datasets: the 0, 1, 2, 3, 4, 5, 5, 6, 8, 9, and 12 hour offsets. 145 | 146 | The selected example here is for the 6 hour offset using forecast from the runs made at 0, 3, 6, 9, and 12 Z. 147 | 148 | ### Best estimate dataset 149 | 150 | For each forecast time in the collection, the best estimate for that hour is used to create the __best estimate__ dataset, which covers the entire time range of the collection. 151 | 152 | For this example, the best estimate is the 0 hour analysis from each run, plus all the forecasts from the latest run. 153 | 154 | ## Development 155 | 156 | Using your favorite python environment checkout and install xarray_fmrc 157 | 158 | ```console 159 | git clone git@github.com:abkfenris/xarray_fmrc.git 160 | cd xarray_fmrc 161 | pip install -e . 162 | pytest . 163 | ``` 164 | 165 | TODO: Add more extensive development instructions 166 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools>=61", "wheel", "setuptools_scm[toml]>=3.4"] 3 | build-backend = "setuptools.build_meta" 4 | 5 | [project] 6 | name = "xarray_fmrc" 7 | description = "" 8 | readme = "README.md" 9 | requires-python = ">=3.9" 10 | keywords = [] #! Update me 11 | license = { text = "MIT License" } 12 | 13 | # Pypi classifiers: https://pypi.org/classifiers/ 14 | classifiers = [ #! Update me 15 | "Development Status :: 3 - Alpha", 16 | "Intended Audience :: Developers", 17 | "Operating System :: OS Independent", 18 | "Topic :: Software Development :: Libraries :: Python Modules", 19 | "License :: OSI Approved :: MIT License", 20 | "Programming Language :: Python :: 3", 21 | "Programming Language :: Python :: 3.9", 22 | ] 23 | 24 | dynamic = ["version", "dependencies"] 25 | 26 | [tool.setuptools.dynamic] 27 | dependencies = { file = ["requirements.txt"] } 28 | 29 | [tool.setuptools_scm] 30 | write_to = "xarray_fmrc/_version.py" 31 | 32 | [tool.black] 33 | # https://github.com/psf/black 34 | target-version = ["py39"] 35 | line-length = 88 36 | color = true 37 | 38 | exclude = ''' 39 | /( 40 | \.git 41 | | \.hg 42 | | \.mypy_cache 43 | | \.tox 44 | | \.venv 45 | | _build 46 | | buck-out 47 | | build 48 | | dist 49 | | env 50 | | venv 51 | )/ 52 | ''' 53 | 54 | [tool.isort] 55 | # https://github.com/timothycrosley/isort/ 56 | py_version = 39 57 | line_length = 88 58 | 59 | known_typing = [ 60 | "typing", 61 | "types", 62 | "typing_extensions", 63 | "mypy", 64 | "mypy_extensions", 65 | ] 66 | sections = [ 67 | "FUTURE", 68 | "TYPING", 69 | "STDLIB", 70 | "THIRDPARTY", 71 | "FIRSTPARTY", 72 | "LOCALFOLDER", 73 | ] 74 | include_trailing_comma = true 75 | profile = "black" 76 | multi_line_output = 3 77 | indent = 4 78 | color_output = true 79 | 80 | [tool.mypy] 81 | # https://mypy.readthedocs.io/en/latest/config_file.html#using-a-pyproject-toml-file 82 | python_version = 3.9 83 | pretty = true 84 | show_traceback = true 85 | color_output = true 86 | 87 | allow_redefinition = false 88 | check_untyped_defs = true 89 | disallow_any_generics = true 90 | disallow_incomplete_defs = true 91 | ignore_missing_imports = true 92 | implicit_reexport = false 93 | no_implicit_optional = true 94 | show_column_numbers = true 95 | show_error_codes = true 96 | show_error_context = true 97 | strict_equality = true 98 | strict_optional = true 99 | warn_no_return = true 100 | warn_redundant_casts = true 101 | warn_return_any = true 102 | warn_unreachable = true 103 | warn_unused_configs = true 104 | warn_unused_ignores = true 105 | 106 | 107 | [tool.pytest.ini_options] 108 | # https://docs.pytest.org/en/6.2.x/customize.html#pyproject-toml 109 | # Directories that are not visited by pytest collector: 110 | norecursedirs = [ 111 | "hooks", 112 | "*.egg", 113 | ".eggs", 114 | "dist", 115 | "build", 116 | "docs", 117 | ".tox", 118 | ".git", 119 | "__pycache__", 120 | ] 121 | doctest_optionflags = [ 122 | "NUMBER", 123 | "NORMALIZE_WHITESPACE", 124 | "IGNORE_EXCEPTION_DETAIL", 125 | ] 126 | 127 | # Extra options: 128 | addopts = [ 129 | "--strict-markers", 130 | "--tb=short", 131 | "--doctest-modules", 132 | "--doctest-continue-on-failure", 133 | ] 134 | 135 | [tool.coverage.run] 136 | source = ["tests"] 137 | 138 | [coverage.paths] 139 | source = "xarray-fmrc" 140 | 141 | [coverage.run] 142 | branch = true 143 | 144 | [coverage.report] 145 | fail_under = 50 146 | show_missing = true 147 | 148 | [tool.interrogate] 149 | ignore-init-method = true 150 | ignore-init-module = false 151 | ignore-magic = false 152 | ignore-semiprivate = false 153 | ignore-private = false 154 | ignore-module = false 155 | fail-under = 95 156 | exclude = ["setup.py", "docs", "tests"] 157 | verbose = 1 158 | quiet = false 159 | color = true 160 | -------------------------------------------------------------------------------- /requirements-dev.txt: -------------------------------------------------------------------------------- 1 | black 2 | check-manifest 3 | doctr 4 | flake8 5 | flake8-builtins 6 | flake8-comprehensions 7 | flake8-mutable 8 | flake8-print 9 | interrogate 10 | isort 11 | nbsphinx 12 | netCDF4 13 | pooch 14 | pre-commit 15 | pylint 16 | pytest 17 | pytest-cov 18 | pytest-flake8 19 | pytest-xdist 20 | recommonmark 21 | setuptools_scm 22 | sphinx 23 | twine 24 | wheel 25 | xarray 26 | xpublish 27 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | xarray>=2022.09.0 2 | xarray-datatree>=0.0.10 3 | pandas>=1.4.2 4 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [darglint] 2 | # https://github.com/terrencepreilly/darglint 3 | strictness = long 4 | docstring_style = google 5 | -------------------------------------------------------------------------------- /tests/test_example/test_hello.py: -------------------------------------------------------------------------------- 1 | """Tests for hello function.""" 2 | import pytest 3 | 4 | from xarray_fmrc.example import hello 5 | 6 | 7 | @pytest.mark.parametrize( 8 | ("name", "expected"), 9 | [ 10 | ("Jeanette", "Hello Jeanette!"), 11 | ("Raven", "Hello Raven!"), 12 | ("Maxine", "Hello Maxine!"), 13 | ("Matteo", "Hello Matteo!"), 14 | ("Destinee", "Hello Destinee!"), 15 | ("Alden", "Hello Alden!"), 16 | ("Mariah", "Hello Mariah!"), 17 | ("Anika", "Hello Anika!"), 18 | ("Isabella", "Hello Isabella!"), 19 | ], 20 | ) 21 | def test_hello(name, expected): 22 | """Example test with parametrization.""" 23 | assert hello(name) == expected 24 | -------------------------------------------------------------------------------- /xarray_fmrc/__init__.py: -------------------------------------------------------------------------------- 1 | """A lightweight way to manage forecast datasets using datatrees""" 2 | 3 | from .accessor import FmrcAccessor 4 | from .build_datatree import from_dict 5 | 6 | __all__ = ["from_dict", "FmrcAccessor"] 7 | 8 | 9 | try: 10 | from ._version import __version__ 11 | except ImportError: 12 | __version__ = "unknown" 13 | -------------------------------------------------------------------------------- /xarray_fmrc/accessor.py: -------------------------------------------------------------------------------- 1 | """ 2 | xarray-datatree accessor to provide access to 3 | forecast model run collections 4 | """ 5 | from typing import TYPE_CHECKING, Union 6 | 7 | from datetime import timedelta 8 | 9 | import datatree 10 | import pandas as pd 11 | import xarray as xr 12 | 13 | from . import constants 14 | from .build_datatree import forecast_reference_time_from_path, model_run_path 15 | from .forecast_offsets import with_offsets 16 | 17 | if TYPE_CHECKING: 18 | from pandas.core.tools.datetimes import DatetimeScalar 19 | 20 | 21 | @datatree.register_datatree_accessor("fmrc") 22 | class FmrcAccessor: 23 | """ 24 | xarray-datatree accessor to provide access to 25 | forecast model run collections 26 | """ 27 | 28 | time_coord = constants.DEFAULT_TIME_COORD 29 | group_prefix = constants.DEFAULT_GROUP_PREFIX 30 | time_format = constants.DEFAULT_TIME_FORMAT 31 | forecast_coords = constants.DEFAULT_FORECAST_COORDS 32 | 33 | def __init__(self, datatree_obj: datatree.DataTree): 34 | """Set the internal datatree object, and if any of the top level attrs are set, 35 | override the defaults""" 36 | self._datatree_obj = datatree_obj 37 | 38 | for key, value in constants.DT_ATTRS.items(): 39 | if value in datatree_obj.attrs: 40 | setattr(self, key, datatree_obj.attrs[value]) 41 | 42 | def model_run_path(self, from_dt: "DatetimeScalar") -> str: 43 | """Return the model run path given a datetime-like input""" 44 | return model_run_path(from_dt, self.group_prefix, self.time_format) 45 | 46 | def model_run(self, dt: "DatetimeScalar") -> xr.Dataset: 47 | """Get the dataset for a single model run 48 | 49 | Accepts valid to `pd.to_datetime()` 50 | https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.to_datetime.html 51 | """ 52 | path = self.model_run_path(dt) 53 | 54 | return self._datatree_obj[path].ds 55 | 56 | def forecast_reference_time_from_path(self, path: str) -> pd.Timestamp: 57 | """Return the forecast reference time for a given path""" 58 | return forecast_reference_time_from_path( 59 | path, 60 | self.group_prefix, 61 | self.time_format, 62 | ) 63 | 64 | def forecast_reference_times(self): 65 | """Get the forecast reference times based on the pattern in the path""" 66 | times = {} 67 | 68 | for group in self._datatree_obj.groups: 69 | try: 70 | times[group] = self.forecast_reference_time_from_path(group) 71 | except ValueError: 72 | pass 73 | 74 | return times 75 | 76 | def constant_forecast(self, dt: "DatetimeScalar") -> xr.Dataset: 77 | """ 78 | Returns a dataset for a single time, from all forecast model runs. 79 | 80 | Accepts valid to `pd.to_datetime()` 81 | https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.to_datetime.html 82 | """ 83 | datetime = pd.to_datetime(dt) 84 | filtered_ds = [] 85 | 86 | for t in sorted(self.forecast_reference_times().values()): 87 | ds = self.model_run(t) 88 | 89 | try: 90 | ds = ds.sel({self.time_coord: datetime}) 91 | ds = ( 92 | ds.drop_indexes(self.forecast_coords) 93 | .reset_coords(self.forecast_coords) 94 | .squeeze() 95 | ) 96 | ds["forecast_reference_time"] = t 97 | filtered_ds.append(ds) 98 | except KeyError: 99 | pass 100 | 101 | combined = xr.concat(filtered_ds, "forecast_reference_time") 102 | combined = combined.sortby("forecast_reference_time") 103 | return combined 104 | 105 | def constant_offset(self, offset: Union[str, int, float, timedelta]) -> xr.Dataset: 106 | """ 107 | Returns a dataset with the same offset from the forecasted time. 108 | 109 | Accepts inputs to `pd.to_timedelta()` 110 | https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.to_timedelta.html 111 | """ 112 | timedelta = pd.to_timedelta(offset) 113 | 114 | filtered_ds = [] 115 | 116 | for t in sorted(self.forecast_reference_times().values()): 117 | ds = self.model_run(t) 118 | ds = ( 119 | ds.drop_indexes(self.forecast_coords) 120 | .reset_coords(self.forecast_coords) 121 | .squeeze() 122 | ) 123 | ds["forecast_reference_time"] = t 124 | ds = with_offsets(ds, self.time_coord) 125 | 126 | try: 127 | ds = ds.sel(forecast_offset=timedelta) 128 | 129 | filtered_ds.append(ds) 130 | except KeyError: 131 | pass 132 | 133 | combined = xr.concat(filtered_ds, self.time_coord) 134 | combined = combined.sortby(self.time_coord) 135 | return combined 136 | 137 | def best(self) -> xr.Dataset: 138 | """ 139 | Returns a dataset with the best possible forecast data. 140 | """ 141 | forecast_times = sorted(self.forecast_reference_times().values(), reverse=True) 142 | 143 | dataset: xr.Dataset = None 144 | 145 | for t in forecast_times: 146 | ds = self.model_run(t) 147 | ds = ( 148 | ds.drop_indexes(self.forecast_coords) 149 | .reset_coords(self.forecast_coords) 150 | .squeeze() 151 | ) 152 | ds["forecast_reference_time"] = t 153 | 154 | if dataset is None: 155 | dataset = ds 156 | else: 157 | existing_times = set(dataset[self.time_coord].to_numpy()) 158 | new_times = set(ds[self.time_coord].to_numpy()) 159 | unique_times = new_times.difference(existing_times) 160 | 161 | ds = ds.sel({self.time_coord: sorted(unique_times)}) 162 | 163 | dataset = xr.concat([dataset, ds], self.time_coord) 164 | 165 | dataset = dataset.sortby(self.time_coord) 166 | return dataset 167 | -------------------------------------------------------------------------------- /xarray_fmrc/build_datatree.py: -------------------------------------------------------------------------------- 1 | """Build a forecast datatree from datasets""" 2 | from typing import TYPE_CHECKING, Any, Dict, List, Optional 3 | 4 | import datatree 5 | import pandas as pd 6 | import xarray as xr 7 | 8 | if TYPE_CHECKING: 9 | from pandas.core.tools.datetimes import DatetimeScalar 10 | 11 | from . import constants 12 | 13 | 14 | def model_run_path( 15 | from_dt: "DatetimeScalar", 16 | group_prefix: str = constants.DEFAULT_GROUP_PREFIX, 17 | time_format: str = constants.DEFAULT_TIME_FORMAT, 18 | ) -> str: 19 | """Return the model run path given a datetime-like input of forecast_reference_time""" 20 | dt = pd.to_datetime(from_dt) 21 | 22 | return f"{group_prefix}{dt.strftime(time_format)}" 23 | 24 | 25 | def forecast_reference_time_from_path( 26 | path: str, 27 | group_prefix: str = constants.DEFAULT_GROUP_PREFIX, 28 | time_format: str = constants.DEFAULT_TIME_FORMAT, 29 | ) -> pd.Timestamp: 30 | """Return the datasets forecast reference time based on it's path""" 31 | dt_str = path.removeprefix(f"/{group_prefix}") 32 | return pd.to_datetime(dt_str, format=time_format) 33 | 34 | 35 | def from_dict( 36 | datasets: Dict["DatetimeScalar", xr.Dataset], 37 | group_prefix: str = constants.DEFAULT_GROUP_PREFIX, 38 | time_format: str = constants.DEFAULT_TIME_FORMAT, 39 | time_coord: Optional[str] = None, 40 | forecast_coords: Optional[List[str]] = None, 41 | tree_attrs: Optional[dict[str, Any]] = None, 42 | ) -> datatree.DataTree: 43 | """Build a datatree from a dictionary of forecast reference times to datasets""" 44 | path_dict = {} 45 | for dt, ds in datasets.items(): 46 | path = model_run_path(dt, group_prefix, time_format) 47 | path_dict[path] = ds 48 | 49 | tree = datatree.DataTree.from_dict(path_dict) 50 | 51 | if tree_attrs is None: 52 | tree_attrs = {} 53 | 54 | tree_attrs[constants.DT_ATTR_GROUP_PREFIX] = group_prefix 55 | tree_attrs[constants.DT_ATTR_TIME_FORMAT] = time_format 56 | 57 | if time_coord: 58 | tree_attrs[constants.DT_ATTR_TIME_COORD] = time_coord 59 | 60 | if forecast_coords: 61 | tree_attrs[constants.DT_ATTR_FORECAST_COORDS] = forecast_coords 62 | 63 | tree.attrs = tree_attrs 64 | 65 | return tree 66 | -------------------------------------------------------------------------------- /xarray_fmrc/constants.py: -------------------------------------------------------------------------------- 1 | """ 2 | Constant and default values for attributes and other configuration of 3 | xarray_fmrc. 4 | """ 5 | 6 | DEFAULT_GROUP_PREFIX = "forecast_reference_time/" 7 | DEFAULT_TIME_FORMAT = "%Y-%m-%dT%H:%M:%S" 8 | DEFAULT_TIME_COORD = "time" 9 | DEFAULT_FORECAST_COORDS = ["forecast_reference_time", "forecast_offset"] 10 | 11 | DT_ATTR_GROUP_PREFIX = "xarray_fmrc_group_prefix" 12 | DT_ATTR_TIME_FORMAT = "xarray_fmrc_time_format" 13 | DT_ATTR_TIME_COORD = "xarray_fmrc_time_coord" 14 | DT_ATTR_FORECAST_COORDS = "xarray_fmrc_forecast_coords" 15 | 16 | 17 | DT_ATTRS = { 18 | "group_prefix": DT_ATTR_GROUP_PREFIX, 19 | "time_format": DT_ATTR_TIME_FORMAT, 20 | "time_coord": DT_ATTR_TIME_COORD, 21 | "forecast_coords": DT_ATTR_FORECAST_COORDS, 22 | } 23 | -------------------------------------------------------------------------------- /xarray_fmrc/example.py: -------------------------------------------------------------------------------- 1 | """ 2 | Example module for test structure 3 | """ 4 | 5 | 6 | def hello(name): 7 | """ 8 | Example method for test structure 9 | """ 10 | return f"Hello {name}!" 11 | -------------------------------------------------------------------------------- /xarray_fmrc/forecast_offsets.py: -------------------------------------------------------------------------------- 1 | """ 2 | Forecast offset handling 3 | """ 4 | import pandas as pd 5 | import xarray as xr 6 | 7 | from . import constants 8 | from .forecast_reference_time import forecast_ref_time 9 | 10 | 11 | def calc_offsets( 12 | ds: xr.Dataset, 13 | time_coord: str = constants.DEFAULT_TIME_COORD, 14 | ) -> pd.TimedeltaIndex: 15 | """Calculate forecast offsets in relation to reference time""" 16 | forecast_time = forecast_ref_time(ds) 17 | offsets = pd.to_datetime(ds[time_coord]) - forecast_time 18 | 19 | return offsets 20 | 21 | 22 | def with_offsets( 23 | ds: xr.Dataset, 24 | time_coord: str = constants.DEFAULT_TIME_COORD, 25 | ) -> xr.Dataset: 26 | """Add forecast offset as a dimensionless coordinate""" 27 | offsets = calc_offsets(ds, time_coord=time_coord) 28 | 29 | ds = ds.assign_coords({"forecast_offset": (time_coord, offsets)}) 30 | ds = ds.set_xindex("forecast_offset") 31 | return ds 32 | -------------------------------------------------------------------------------- /xarray_fmrc/forecast_reference_time.py: -------------------------------------------------------------------------------- 1 | """ 2 | Handle forecast reference times 3 | """ 4 | import pandas as pd 5 | import xarray as xr 6 | 7 | 8 | def forecast_ref_time(ds: xr.Dataset) -> pd.Timestamp: 9 | """Get forecast reference time from coordinate or attribute""" 10 | try: 11 | return pd.to_datetime(ds["forecast_reference_time"].item()) 12 | except KeyError: 13 | return pd.to_datetime(ds.attrs["forecast_reference_time"]) 14 | --------------------------------------------------------------------------------