├── .coveragerc ├── .github ├── ISSUE_TEMPLATE │ ├── bug_report.md │ └── feature_request.md └── workflows │ ├── github-pages-deployment.yml │ ├── pull-request-workflow.yml │ ├── release-candidate-workflow.yml │ └── version-update-workflow.yml ├── .gitignore ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── Dockerfile ├── LICENSE ├── LICENSES ├── Apache-2.0.txt ├── CC-BY-2.5.txt └── MPL-2.0.txt ├── PROJECT_GOVERNANCE.md ├── README.md ├── SECURITY.md ├── demos ├── demo_cds_era5sl.py ├── demo_knmi_actuele_waarnemingen.py ├── demo_knmi_daggegevens.py ├── demo_knmi_harmonie_arome.py ├── demo_knmi_pluim.py ├── demo_knmi_uurgegevens.py └── run_weather_api.ipynb ├── doc ├── copyright_profiles │ └── velocity_copyright_text.txt ├── factors.md └── usage.md ├── img ├── gitflow.svg └── gitflow.svg.license ├── poetry.lock ├── pyproject.toml ├── sigrid.yaml ├── sonar-project.properties ├── sphinx-docs ├── Makefile ├── _static │ └── wpas_logo.svg ├── conf.py ├── index.rst ├── introduction.rst ├── modules.rst ├── quick-start.rst ├── weather_provider_access_suite_info_page.rst ├── weather_provider_api.config.rst ├── weather_provider_api.core.exceptions.rst ├── weather_provider_api.core.initializers.rst ├── weather_provider_api.core.rst ├── weather_provider_api.core.utils.rst ├── weather_provider_api.routers.rst ├── weather_provider_api.routers.weather.base_models.rst ├── weather_provider_api.routers.weather.repository.rst ├── weather_provider_api.routers.weather.rst ├── weather_provider_api.routers.weather.sources.cds.client.rst ├── weather_provider_api.routers.weather.sources.cds.models.rst ├── weather_provider_api.routers.weather.sources.cds.rst ├── weather_provider_api.routers.weather.sources.knmi.client.rst ├── weather_provider_api.routers.weather.sources.knmi.models.rst ├── weather_provider_api.routers.weather.sources.knmi.rst ├── weather_provider_api.routers.weather.sources.rst ├── weather_provider_api.routers.weather.sources.weather_alert.rst ├── weather_provider_api.routers.weather.utils.rst ├── weather_provider_api.rst ├── weather_provider_api.scripts.rst └── weather_provider_api.versions.rst ├── tests ├── __init__.py ├── conftest.py ├── pytest.ini ├── test_api_models.py ├── test_api_view_v1.py ├── test_date_helpers.py ├── test_geo_position.py ├── test_grid_helpers.py ├── test_knmi_actuele_waarnemingen.py ├── test_knmi_arome_model.py ├── test_knmi_arome_repository.py ├── test_knmi_daggegevens.py ├── test_knmi_pluim.py ├── test_knmi_uurgegevens.py ├── test_model.py ├── test_serializers.py └── test_weather_alert.py ├── var_maps ├── arome_var_map.json ├── arome_var_map.json.license ├── era5land_var_map.json ├── era5land_var_map.json.license ├── era5sl_var_map.json └── era5sl_var_map.json.license └── weather_provider_api ├── __init__.py ├── __main__.py ├── app_version.py ├── config ├── __init__.py └── config.toml ├── core ├── __init__.py ├── application.py ├── base_model.py ├── exceptions │ ├── __init__.py │ ├── additional_responses.py │ ├── api_models.py │ └── exceptions.py ├── gunicorn_application.py ├── initializers │ ├── __init__.py │ ├── cors.py │ ├── exception_handling.py │ ├── headers.py │ ├── logging_handler.py │ ├── mounting.py │ ├── prometheus.py │ ├── rate_limiter.py │ └── validation.py └── utils │ ├── __init__.py │ └── example_responses.py ├── routers ├── __init__.py └── weather │ ├── __init__.py │ ├── api_models.py │ ├── api_view_v1.py │ ├── api_view_v2.py │ ├── base_models │ ├── __init__.py │ ├── model.py │ └── source.py │ ├── controller.py │ ├── exceptions.py │ ├── repository │ ├── __init__.py │ └── repository.py │ ├── sources │ ├── __init__.py │ ├── cds │ │ ├── __init__.py │ │ ├── cds.py │ │ ├── client │ │ │ ├── __init__.py │ │ │ ├── cds_api_tools.py │ │ │ ├── era5_utils.py │ │ │ ├── era5land_repository.py │ │ │ └── era5sl_repository.py │ │ ├── factors.py │ │ └── models │ │ │ ├── __init__.py │ │ │ ├── era5land.py │ │ │ └── era5sl.py │ ├── knmi │ │ ├── __init__.py │ │ ├── client │ │ │ ├── __init__.py │ │ │ ├── actuele_waarnemingen_register_repository.py │ │ │ ├── arome_repository.py │ │ │ └── knmi_downloader.py │ │ ├── knmi.py │ │ ├── knmi_factors.py │ │ ├── models │ │ │ ├── __init__.py │ │ │ ├── actuele_waarnemingen.py │ │ │ ├── actuele_waarnemingen_register.py │ │ │ ├── daggegevens.py │ │ │ ├── harmonie_arome.py │ │ │ ├── pluim.py │ │ │ └── uurgegevens.py │ │ ├── stations.py │ │ └── utils.py │ └── weather_alert │ │ ├── __init__.py │ │ └── weather_alert.py │ └── utils │ ├── __init__.py │ ├── date_helpers.py │ ├── file_helpers.py │ ├── geo_position.py │ ├── grid_helpers.py │ ├── pandas_helpers.py │ └── serializers.py ├── scripts ├── __init__.py ├── erase_arome_repository.py ├── erase_era5land_repository.py ├── erase_era5sl_repository.py ├── erase_waarnemingen_register.py ├── update_arome_repository.py ├── update_era5land_repository.py ├── update_era5sl_repository.py └── update_waarnemingen_register.py └── versions ├── __init__.py ├── v1.py └── v2.py /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | include = 3 | weather_provider_api/* -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **To Reproduce** 14 | Steps to reproduce the behavior: 15 | 1. Go to '...' 16 | 2. Click on '....' 17 | 3. Scroll down to '....' 18 | 4. See error 19 | 20 | **Expected behavior** 21 | A clear and concise description of what you expected to happen. 22 | 23 | **Screenshots** 24 | If applicable, add screenshots to help explain your problem. 25 | 26 | **Desktop (please complete the following information):** 27 | - OS: [e.g. iOS] 28 | - Browser [e.g. chrome, safari] 29 | - Version [e.g. 22] 30 | 31 | **Smartphone (please complete the following information):** 32 | - Device: [e.g. iPhone6] 33 | - OS: [e.g. iOS8.1] 34 | - Browser [e.g. stock browser, safari] 35 | - Version [e.g. 22] 36 | 37 | **Additional context** 38 | Add any other context about the problem here. 39 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or screenshots about the feature request here. 21 | -------------------------------------------------------------------------------- /.github/workflows/github-pages-deployment.yml: -------------------------------------------------------------------------------- 1 | name: Deployment to GitHub Pages 2 | on: 3 | push: 4 | branches: 5 | - main* 6 | permissions: 7 | id-token: write 8 | pages: write 9 | jobs: 10 | # Build the Sphinx documentation 11 | build: 12 | runs-on: ubuntu-latest 13 | steps: 14 | - uses: actions/checkout@v3 15 | - uses: actions/setup-python@v3 16 | - name: Install Dependencies 17 | run: | 18 | pip install sphinx piccolo-theme myst_parser 19 | - name: Sphinx Build 20 | run: | 21 | sphinx-build ./sphinx-docs/ build_outputs_folder 22 | - name: Upload GitHub Pages artifact 23 | uses: actions/upload-pages-artifact@v3 24 | id: deployment 25 | with: 26 | path: build_outputs_folder/ 27 | # Deploy the Sphinx documentation to GitHub Pages 28 | deploy: 29 | environment: 30 | name: github-pages 31 | url: ${{ steps.deployment.outputs.page_url }} 32 | runs-on: ubuntu-latest 33 | needs: build 34 | steps: 35 | - name: Deploy to GitHub Pages 36 | id: deployment 37 | uses: actions/deploy-pages@v4 -------------------------------------------------------------------------------- /.github/workflows/pull-request-workflow.yml: -------------------------------------------------------------------------------- 1 | name: Pull Request Evaluation 2 | on: 3 | pull_request: 4 | types: [opened, synchronize, reopened] 5 | jobs: 6 | # ---------------------------------------------------------------- # 7 | # | Coverage and SonarCloud.io upload | # 8 | # ---------------------------------------------------------------- # 9 | tests-and-sonarcloud: 10 | strategy: 11 | matrix: 12 | version: ["3.10", "3.11"] # TODO: Get Python versions from the project 13 | name: Testing and updating SonarCloud 14 | runs-on: ubuntu-latest 15 | steps: 16 | # Checkout 17 | - name: Branch checkout 18 | uses: actions/checkout@v3 19 | with: 20 | fetch-depth: 0 21 | # Install Python 22 | - name: Setup Python installation 23 | uses: actions/setup-python@v4 24 | with: 25 | python-version: ${{ matrix.version }} 26 | # Install Poetry 27 | - name: Setup Poetry 28 | run: | 29 | curl -sSL https://install.python-poetry.org | python3 - 30 | poetry self update --preview 31 | echo "$HOME/.local/bin" >> $GITHUB_PATH 32 | # Install required dependencies 33 | - name: Install dependencies 34 | run: poetry install -v --with dev 35 | # Run Coverage 36 | - name: Run Coverage 37 | env: 38 | CDSAPI_URL: ${{ secrets.CDSAPI_URL }} 39 | CDSAPI_KEY: ${{ secrets.CDSAPI_KEY }} 40 | CDSAPI_VERIFY: 1 41 | run: | 42 | poetry run coverage run -m pytest 43 | poetry run coverage report 44 | # Generate XML Coverage result file 45 | - if: matrix.version == '3.10' 46 | name: Generate Coverage XML file 47 | run: poetry run coverage xml 48 | # Upload coverage data to SonarCloud.io 49 | - if: matrix.version == '3.10' 50 | name: Update SonarCloud.io 51 | uses: SonarSource/sonarcloud-github-action@master 52 | env: 53 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any 54 | SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} 55 | -------------------------------------------------------------------------------- /.github/workflows/release-candidate-workflow.yml: -------------------------------------------------------------------------------- 1 | name: Release Candidate evaluation 2 | on: 3 | push: 4 | branches: 5 | release-* 6 | jobs: 7 | # ---------------------------------------------------------------- # 8 | # | Coverage and SonarCloud.io upload | # 9 | # ---------------------------------------------------------------- # 10 | tests-and-sonarcloud: 11 | strategy: 12 | matrix: 13 | version: ["3.10", "3.11"] # TODO: Get Python versions from the project 14 | name: Testing and updating SonarCloud 15 | runs-on: ubuntu-latest 16 | steps: 17 | # Checkout 18 | - name: Branch checkout 19 | uses: actions/checkout@v3 20 | with: 21 | fetch-depth: 0 22 | # Install Python 23 | - name: Setup Python installation 24 | uses: actions/setup-python@v4 25 | with: 26 | python-version: ${{ matrix.version }} 27 | # Install Poetry 28 | - name: Setup Poetry 29 | run: | 30 | curl -sSL https://install.python-poetry.org | python3 - 31 | poetry self update --preview 32 | echo "$HOME/.local/bin" >> $GITHUB_PATH 33 | # Install required dependencies 34 | - name: Install dependencies 35 | run: poetry install -v --with dev 36 | # Run Coverage 37 | - name: Run Coverage 38 | env: 39 | CDSAPI_URL: ${{ secrets.CDSAPI_URL }} 40 | CDSAPI_KEY: ${{ secrets.CDSAPI_KEY }} 41 | CDSAPI_VERIFY: 1 42 | run: | 43 | poetry run coverage run -m pytest 44 | poetry run coverage report 45 | # Generate XML Coverage result file 46 | - if: matrix.version == '3.10' 47 | name: Generate Coverage XML file 48 | run: poetry run coverage xml 49 | # Upload coverage data to SonarCloud.io 50 | - if: matrix.version == '3.10' 51 | name: Update SonarCloud.io 52 | uses: SonarSource/sonarcloud-github-action@master 53 | env: 54 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any 55 | SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} 56 | # ---------------------------------------------------------------- # 57 | # | Build image and upload it to AWS and DockerHub | # 58 | # ---------------------------------------------------------------- # 59 | create-and-upload-image: 60 | name: Create and upload image 61 | needs: tests-and-sonarcloud 62 | runs-on: ubuntu-latest 63 | permissions: 64 | id-token: write 65 | contents: read 66 | environment: 67 | name: ci 68 | steps: 69 | # Checkout 70 | - name: Check out source code 71 | uses: actions/checkout@v3 72 | # Configure AWS 73 | - name: Configure AWS Credentials 74 | uses: aws-actions/configure-aws-credentials@master 75 | with: 76 | role-to-assume: arn:aws:iam::325973380531:role/dnb-inno-repository-access-role 77 | aws-region: eu-central-1 78 | # Install Poetry (needed for version) 79 | - name: Setup Poetry 80 | run: | 81 | curl -sSL https://install.python-poetry.org | python3 - 82 | poetry self update --preview 83 | echo "$HOME/.local/bin" >> $GITHUB_PATH 84 | # Login to Amazon ECR 85 | - name: Login to Amazon ECR 86 | id: login-ecr 87 | uses: aws-actions/amazon-ecr-login@v1 88 | - name: Determine current version 89 | run: | 90 | version="$(poetry version --short)rc${{ github.run_number }}" 91 | echo "Package version: $version" 92 | echo "package_version=$version" >> $GITHUB_ENV 93 | # Create the image and upload it to Amazon ECR 94 | - name: Build, tag, and push image to Amazon ECR 95 | env: 96 | ECR_REGISTRY: 325973380531.dkr.ecr.eu-central-1.amazonaws.com 97 | ECR_REPOSITORY: dnb-inno-image-repository 98 | run: | 99 | docker build --target gunicorn-image -t ${{ secrets.ECR_REGISTRY }}/${{ secrets.ECR_REPOSITORY_RC }}:${{ env.package_version }} . 100 | docker push ${{ secrets.ECR_REGISTRY }}/${{ secrets.ECR_REPOSITORY_RC }}:${{ env.package_version }} 101 | docker build -t ${{ secrets.ECR_REGISTRY }}/${{ secrets.ECR_REPOSITORY_RC }} . 102 | docker push ${{ secrets.ECR_REGISTRY }}/${{ secrets.ECR_REPOSITORY_RC }} 103 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | 5 | 6 | # Code of Conduct 7 | 8 | ## Our Pledge 9 | 10 | In the interest of fostering an open and welcoming environment, we as 11 | contributors and maintainers pledge to making participation in our project and 12 | our community a harassment-free experience for everyone, regardless of age, body 13 | size, disability, ethnicity, gender identity and expression, level of 14 | experience, education, socio-economic status, nationality, personal appearance, 15 | race, religion, or sexual identity and orientation. 16 | 17 | ## Our Standards 18 | 19 | Examples of behavior that contributes to creating a positive environment 20 | include: 21 | 22 | * Using welcoming and inclusive language 23 | * Being respectful of differing viewpoints and experiences 24 | * Gracefully accepting constructive criticism 25 | * Focusing on what is best for the community 26 | * Showing empathy towards other community members 27 | 28 | Examples of unacceptable behavior by participants include: 29 | 30 | * The use of sexualized language or imagery and unwelcome sexual attention or 31 | advances 32 | * Trolling, insulting/derogatory comments, and personal or political attacks 33 | * Public or private harassment 34 | * Publishing others' private information, such as a physical or electronic 35 | address, without explicit permission 36 | * Other conduct which could reasonably be considered inappropriate in a 37 | professional setting 38 | 39 | ## Our Responsibilities 40 | 41 | Project maintainers are responsible for clarifying the standards of acceptable 42 | behavior and are expected to take appropriate and fair corrective action in 43 | response to any instances of unacceptable behavior. 44 | 45 | Project maintainers have the right and responsibility to remove, edit, or reject 46 | comments, commits, code, wiki edits, issues, and other contributions that are 47 | not aligned to this Code of Conduct, or to ban temporarily or permanently any 48 | contributor for other behaviors that they deem inappropriate, threatening, 49 | offensive, or harmful. 50 | 51 | ## Scope 52 | 53 | This Code of Conduct applies both within project spaces and in public spaces 54 | when an individual is representing the project or its community. Examples of 55 | representing a project or community include using an official project e-mail 56 | address, posting via an official social media account, or acting as an appointed 57 | representative at an online or offline event. Representation of a project may be 58 | further defined and clarified by project maintainers. 59 | 60 | 61 | ## Conflict Resolution 62 | 63 | Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at OSPO@alliander. All complaints will be reviewed and investigated and will result in a response that is deemed necessary and appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately. 64 | 65 | Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project’s leadership. 66 | 67 | ## Attribution 68 | 69 | This Code of Conduct is adapted from the Contributor Covenant, version 1.4, 70 | available at 71 | https://www.contributor-covenant.org/version/1/4/code-of-conduct.html 72 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | 5 | 6 | # How to Contribute 7 | 8 | We'd love to accept your patches and contributions to this project. There are 9 | just a few small guidelines you need to follow. 10 | 11 | 12 | ## Filing bugs, change requests and questions 13 | 14 | You can file bugs against, change requests for and questions about the project via github issues. Consult [GitHub Help](https://docs.github.com/en/free-pro-team@latest/github/managing-your-work-on-github/creating-an-issue) for more 15 | information on using github issues. 16 | 17 | ## Community Guidelines 18 | 19 | This project follows the following [Code of Conduct](CODE_OF_CONDUCT.md). 20 | 21 | ## Source Code Headers 22 | 23 | Every file containing source code must include copyright and license 24 | information. This includes any JS/CSS files that you might be serving out to 25 | browsers. (This is to help well-intentioned people avoid accidental copying that 26 | doesn't comply with the license.) 27 | 28 | Mozilla header: 29 | 30 | SPDX-FileCopyrightText: 2021 Alliander N.V. 31 | SPDX-License-Identifier: MPL-2.0 32 | 33 | ## Git branching 34 | 35 | This project uses the [Gitflow Workflow](https://www.atlassian.com/git/tutorials/comparing-workflows/gitflow-workflow) and branching model. The `master` branch always contains the latest release, after a release is made new feature branches are branched of `develop`. When a feature is finished it is merged back into `develop`. At the end of a sprint `develop` is merged back into `master` or (optional) into a `release` branch first before it is merged into `master`. 36 | 37 | ![Gitflow](img/gitflow.svg) 38 | 39 | ## Code reviews 40 | 41 | All patches and contributions, including patches and contributions by project members, require review by one of the maintainers of the project. We 42 | use GitHub pull requests for this purpose. Consult 43 | [GitHub Help](https://help.github.com/articles/about-pull-requests/) for more 44 | information on using pull requests. 45 | 46 | ## Pull Request Process 47 | Contributions should be submitted as Github pull requests. See [Creating a pull request](https://docs.github.com/en/github/collaborating-with-issues-and-pull-requests/creating-a-pull-request) if you're unfamiliar with this concept. 48 | 49 | The process for a code change and pull request you should follow: 50 | 51 | 1. Create a topic branch in your local repository, following the naming format 52 | "feature-[description]". For more information see the Git branching guideline. 53 | 1. Make changes, compile, and test thoroughly. Ensure any install or build dependencies are removed before the end of the layer when doing a build. Code style should match existing style and conventions, and changes should be focused on the topic the pull request will be addressed. For more information see the style guide. 54 | 1. Push commits to your fork. 55 | 1. Create a Github pull request from your topic branch. 56 | 1. Pull requests will be reviewed by one of the maintainers who may discuss, offer constructive feedback, request changes, or approve 57 | the work. For more information see the Code review guideline. 58 | 1. Upon receiving the sign-off of one of the maintainers you may merge your changes, or if you 59 | do not have permission to do that, you may request a maintainer to merge it for you. 60 | 61 | ## Attribution 62 | 63 | This Contributing.md is adapted from Google 64 | available at 65 | https://github.com/google/new-project/blob/master/docs/contributing.md 66 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # 2 | # SPDX-FileCopyrightText: 2019-2022 Alliander N.V. 3 | # SPDX-License-Identifier: MPL-2.0 4 | # 5 | 6 | FROM python:3.10.16-bullseye AS base-image 7 | 8 | RUN apt-get update && \ 9 | apt-get -y install libeccodes-dev && \ 10 | apt-get -y install libeccodes-tools && \ 11 | apt-get clean 12 | 13 | ENV ECCODES_DIR=/usr/src/eccodes 14 | ENV ECMWFLIBS_ECCODES_DEFINITION_PATH=/usr/src/eccodes/share/eccodes/definitions 15 | 16 | ARG APP_HOME=/app 17 | RUN pip install poetry 18 | 19 | 20 | # Setup WPLA user and switch to WPLA user 21 | ARG APP_USER=wpla-user 22 | 23 | RUN groupadd --system "$APP_USER" && \ 24 | useradd --system --gid "$APP_USER" --create-home --home "$APP_HOME" "$APP_USER" 25 | 26 | WORKDIR $APP_HOME 27 | 28 | USER $APP_USER 29 | 30 | COPY --chown=$APP_USER:$APP_USER ./pyproject.toml ./pyproject.toml 31 | COPY --chown=$APP_USER:$APP_USER ./weather_provider_api ./weather_provider_api 32 | COPY --chown=$APP_USER:$APP_USER ./var_maps ./var_maps 33 | 34 | RUN poetry config virtualenvs.in-project true && \ 35 | poetry install --no-interaction --no-ansi -v --no-root 36 | 37 | ENV PATH="$APP_HOME/.venv/bin:$PATH" 38 | 39 | # --- DEV image -- 40 | FROM base-image AS dev-image 41 | # TODO: Hookup SSH interface 42 | 43 | USER $APP_USER 44 | CMD ["ls", "-l"] 45 | 46 | # --- UVICORN image -- 47 | FROM base-image AS uvicorn-image 48 | 49 | USER $APP_USER 50 | EXPOSE 8000 51 | CMD ["uvicorn", "--reload", "--host", "0.0.0.0", "--port", "8000", "weather_provider_api.core.application:WPLA_APPLICATION" ] 52 | 53 | # --- GUNICORN image -- 54 | FROM base-image AS gunicorn-image 55 | 56 | USER $APP_USER 57 | EXPOSE 8000 58 | CMD ["gunicorn", "-k", "uvicorn.workers.UvicornWorker", "-b", "0.0.0.0:8000", "weather_provider_api.core.application:WPLA_APPLICATION", "--timeout", "180"] 59 | -------------------------------------------------------------------------------- /PROJECT_GOVERNANCE.md: -------------------------------------------------------------------------------- 1 | 5 | 6 | # Project Governance 7 | 8 | The basic principle is that decisions are based on consensus. If this decision-making process takes too long or a decision is required, the project committee has the authority to make a decision. 9 | 10 | ## Technical Steering Committee 11 | 12 | The Technical Steering Committee (TSC) is responsible for: 13 | 14 | 1. General ambitions, objectives and goals of this project 15 | 2. Guidelines and procedures and tool selection 16 | 3. Guidelines and procedures and tool selection 17 | 4. Architectural and (development) infrastructure choices 18 | 5. Raise subjects/issues that are important for the direction/development of this project 19 | 20 | The community council consists of the following members: 21 | 22 | * **Tim Weelinck** - *API development* 23 | * **Raoul Linnenbank** - *Active API Development, Geo positioning, CDS ERA5, caching, remodeling, Harmonie Arome and optimisation* 24 | * **Joan Ressing** - *Infrastructure and deployment* 25 | 26 | **Tim Weelinck** will chair the TSC. 27 | 28 | ## Maintainers 29 | 30 | Maintainers are responsible for maintaining parts of the code-base. Maintainers have the following responsibilities 31 | 32 | 1. Coordinate development activity 33 | 2. Make sure code/documentation reviews are being done 34 | 3. Coordinate pull-requests 35 | 4. Coordinate bug follow-ups 36 | 5. Coordinate questions 37 | 6. In case of long discussions or arguments, maintainers or other can request a community council decision. 38 | 39 | The current maintainers of this project are: 40 | * **Raoul Linnenbank** - *Active API Development, debugging, repository management* 41 | * **Joan Ressing** - *Infrastructure and deployment, back-up for Raoul Linnenbank* 42 | 43 | ## Contributors 44 | 45 | Contributors include anyone in the technical community that contributes code, documentation, or other technical artifacts to the project. 46 | 47 | Anyone can become a contributor. There is no expectation of commitment to the project, no specific skill requirements and no selection process. To become a contributor, a community member simply has to perform one or more actions that are beneficial to the project. 48 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | 5 | # Security Policy 6 | 7 | ## Supported Versions 8 | 9 | The following versions of the API and its libraries are currently being supported: 10 | 11 | | VERSION | SUPPORTED | 12 | | ------- | ---------------------- | 13 | | 3.0.x | :x: *(in development)* | 14 | | 2.x | :white_check_mark: | 15 | | < 2.0 | :x: | 16 | 17 | ## Reporting a Vulnerability 18 | 19 | To report a vulnerability, please directly contact us at: weather.provider@alliander.com 20 | 21 | Upon receiving information on this vulnerability, we will aim to either immediately repair it, or issue a warning via the discussion board, depending on the ease with which it can be fixed. In both cases we will aim to solve the vulnerability as fast as possible. 22 | 23 | Should a vulnerability pose an extremely high risk, we may decide to flag releases as "High Risk", and we may issue temporary releases with limited functionality, to prevent our users from being unnecessarily at risk. These events will also be advertised on the Discussions board, and these "High Risk" and "Limited Functionality" releases will be tagged as such. 24 | -------------------------------------------------------------------------------- /demos/demo_cds_era5sl.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # SPDX-FileCopyrightText: 2019-2022 Alliander N.V. 5 | # SPDX-License-Identifier: MPL-2.0 6 | 7 | from datetime import datetime 8 | 9 | import numpy as np 10 | import xarray as xr 11 | 12 | from weather_provider_api.routers.weather.api_models import ( 13 | OutputUnit, 14 | ResponseFormat, 15 | WeatherContentRequestQuery, 16 | ) 17 | from weather_provider_api.routers.weather.api_view_v2 import controller 18 | from weather_provider_api.routers.weather.utils import serializers 19 | 20 | """ 21 | This is a DEMO for a specific weather model: ERA5 Single Levels 22 | 23 | The purpose of this demo is to show users a method of how they can use specific models from the Weather Provider API 24 | to request specific output. 25 | """ 26 | 27 | if __name__ == "__main__": 28 | coords = [ 29 | [(51.873419, 5.705929), (51.873419, 5.71), (51.88, 5.71)], 30 | [(52.121976, 4.782497)], 31 | [(52.424793, 4.776927)], 32 | ] 33 | 34 | ds_hist_day: xr.Dataset = controller.get_weather( 35 | source_id="cds", 36 | model_id="era5sl", 37 | fetch_async=False, 38 | coords=coords, 39 | begin=datetime(2018, 2, 1, 0, 0), 40 | end=datetime(2018, 2, 28, 23, 59), 41 | ) 42 | 43 | response_format = ResponseFormat.netcdf4 44 | 45 | converted_weather_data = controller.convert_names_and_units("cds", "era5sl", False, ds_hist_day, OutputUnit.si) 46 | 47 | ret_args = WeatherContentRequestQuery( 48 | begin="2018-01-01 00:00", 49 | end="2018-01-31 23:59", 50 | lat=52.1, 51 | lon=5.18, 52 | factors=None, 53 | ) 54 | 55 | coords = [ 56 | ( 57 | float(np.mean([single_point[0] for single_point in single_polygon])), 58 | float(np.mean([single_point[1] for single_point in single_polygon])), 59 | ) 60 | for single_polygon in coords 61 | ] # means to match the used coordinates for the request 62 | response, optional_file_path = serializers.file_or_text_response( 63 | converted_weather_data, response_format, "cds", "era5sl", ret_args, coords 64 | ) 65 | 66 | print(response, optional_file_path) 67 | -------------------------------------------------------------------------------- /demos/demo_knmi_actuele_waarnemingen.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # SPDX-FileCopyrightText: 2019-2022 Alliander N.V. 5 | # SPDX-License-Identifier: MPL-2.0 6 | 7 | from datetime import datetime 8 | 9 | import xarray as xr 10 | 11 | from weather_provider_api.routers.weather.api_models import ( 12 | OutputUnit, 13 | ResponseFormat, 14 | WeatherContentRequestQuery, 15 | ) 16 | from weather_provider_api.routers.weather.api_view_v2 import controller 17 | from weather_provider_api.routers.weather.utils import serializers 18 | 19 | """ 20 | This is a DEMO for a specific weather model: KNMI Actuele Waarnemingen 21 | 22 | The purpose of this demo is to show users a method of how they can use specific models from the Weather Provider API 23 | to request specific output. 24 | """ 25 | 26 | if __name__ == "__main__": 27 | coords = [ 28 | [(51.873419, 5.705929)], 29 | [(52.121976, 4.782497)], 30 | [(52.424793, 4.776927)], 31 | [(51.873777, 5.705111)], 32 | ] 33 | 34 | ds_hist_day: xr.Dataset = controller.get_weather( 35 | source_id="knmi", 36 | model_id="waarnemingen", 37 | fetch_async=False, 38 | coords=coords, 39 | begin=datetime(year=2018, month=1, day=1), 40 | end=datetime(year=2018, month=1, day=31), 41 | ) 42 | 43 | response_format = ResponseFormat.netcdf4 44 | 45 | converted_weather_data = controller.convert_names_and_units( 46 | "knmi", "waarnemingen", False, ds_hist_day, OutputUnit.si 47 | ) 48 | 49 | ret_args = WeatherContentRequestQuery( 50 | begin="2018-01-01 00:00", 51 | end="2018-01-31 23:59", 52 | lat=52.1, 53 | lon=5.18, 54 | factors=None, 55 | ) 56 | 57 | response, optional_file_path = serializers.file_or_text_response( 58 | converted_weather_data, 59 | response_format, 60 | "knmi", 61 | "waarnemingen", 62 | ret_args, 63 | coords, 64 | ) 65 | 66 | print(response) 67 | print(optional_file_path) 68 | -------------------------------------------------------------------------------- /demos/demo_knmi_daggegevens.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # SPDX-FileCopyrightText: 2019-2022 Alliander N.V. 5 | # SPDX-License-Identifier: MPL-2.0 6 | 7 | from datetime import datetime 8 | 9 | import xarray as xr 10 | 11 | from weather_provider_api.routers.weather.api_models import ( 12 | OutputUnit, 13 | ResponseFormat, 14 | WeatherContentRequestQuery, 15 | ) 16 | from weather_provider_api.routers.weather.controller import WeatherController 17 | from weather_provider_api.routers.weather.utils import serializers 18 | 19 | """ 20 | This is a DEMO for a specific weather model: KNMI Daggegevens 21 | 22 | The purpose of this demo is to show users a method of how they can use specific models from the Weather Provider API 23 | to request specific output. 24 | """ 25 | 26 | if __name__ == "__main__": 27 | coords = [ 28 | [(51.873419, 5.705929)], 29 | [(52.121976, 4.782497)], 30 | [(52.424793, 4.776927)], 31 | [(51.873777, 5.705111)], 32 | ] 33 | 34 | controller = WeatherController() 35 | 36 | ds_hist_day: xr.Dataset = controller.get_weather( 37 | source_id="knmi", 38 | model_id="daggegevens", 39 | fetch_async=False, 40 | coords=coords, 41 | begin=datetime(year=2018, month=1, day=1), 42 | end=datetime(year=2018, month=1, day=31), 43 | ) 44 | 45 | response_format = ResponseFormat.netcdf4 46 | 47 | converted_weather_data = controller.convert_names_and_units( 48 | "knmi", "daggegevens", False, ds_hist_day, OutputUnit.si 49 | ) 50 | 51 | ret_args = WeatherContentRequestQuery( 52 | begin="2018-01-01 00:00", 53 | end="2018-01-31 23:59", 54 | lat=52.1, 55 | lon=5.18, 56 | factors=None, 57 | ) 58 | 59 | response, optional_file_path = serializers.file_or_text_response( 60 | converted_weather_data, response_format, "knmi", "daggegevens", ret_args, coords 61 | ) 62 | 63 | print(response) 64 | print(optional_file_path) 65 | -------------------------------------------------------------------------------- /demos/demo_knmi_harmonie_arome.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # SPDX-FileCopyrightText: 2019-2022 Alliander N.V. 5 | # SPDX-License-Identifier: MPL-2.0 6 | 7 | from datetime import datetime 8 | 9 | import xarray as xr 10 | 11 | from weather_provider_api.routers.weather.api_models import ( 12 | OutputUnit, 13 | ResponseFormat, 14 | WeatherContentRequestQuery, 15 | ) 16 | from weather_provider_api.routers.weather.api_view_v2 import controller 17 | from weather_provider_api.routers.weather.utils import serializers 18 | 19 | """ 20 | This is a DEMO for a specific weather model: KNMI Harmonie Arome 21 | 22 | The purpose of this demo is to show users a method of how they can use specific models from the Weather Provider API 23 | to request specific output. 24 | """ 25 | 26 | if __name__ == "__main__": 27 | coords = [ 28 | [(51.873419, 5.705929)], 29 | [(52.121976, 4.782497)], 30 | [(52.424793, 4.776927)], 31 | [(51.873777, 5.705111)], 32 | ] 33 | 34 | ds_hist_day: xr.Dataset = controller.get_weather( 35 | source_id="knmi", 36 | model_id="arome", 37 | fetch_async=False, 38 | coords=coords, 39 | begin=datetime(year=2020, month=9, day=9), 40 | end=datetime(year=2020, month=9, day=9), 41 | ) 42 | 43 | response_format = ResponseFormat.netcdf4 44 | 45 | converted_weather_data = controller.convert_names_and_units("knmi", "arome", False, ds_hist_day, OutputUnit.si) 46 | 47 | ret_args = WeatherContentRequestQuery( 48 | begin="2018-01-01 00:00", 49 | end="2018-01-31 23:59", 50 | lat=52.1, 51 | lon=5.18, 52 | factors=None, 53 | ) 54 | 55 | response, optional_file_path = serializers.file_or_text_response( 56 | converted_weather_data, response_format, "knmi", "arome", ret_args, coords 57 | ) 58 | 59 | print(response) 60 | print(optional_file_path) 61 | -------------------------------------------------------------------------------- /demos/demo_knmi_pluim.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # SPDX-FileCopyrightText: 2019-2022 Alliander N.V. 5 | # SPDX-License-Identifier: MPL-2.0 6 | 7 | from datetime import datetime 8 | 9 | import xarray as xr 10 | 11 | from weather_provider_api.routers.weather.api_models import ( 12 | OutputUnit, 13 | ResponseFormat, 14 | WeatherContentRequestQuery, 15 | ) 16 | from weather_provider_api.routers.weather.api_view_v2 import controller 17 | from weather_provider_api.routers.weather.utils import serializers 18 | 19 | """ 20 | This is a DEMO for a specific weather model: KNMI Pluim 21 | 22 | The purpose of this demo is to show users a method of how they can use specific models from the Weather Provider API 23 | to request specific output. 24 | """ 25 | 26 | if __name__ == "__main__": 27 | coords = [ 28 | [(51.873419, 5.705929)], 29 | [(52.121976, 4.782497)], 30 | [(52.424793, 4.776927)], 31 | [(51.873777, 5.705111)], 32 | ] 33 | 34 | ds_hist_day: xr.Dataset = controller.get_weather( 35 | source_id="knmi", 36 | model_id="pluim", 37 | fetch_async=False, 38 | coords=coords, 39 | begin=datetime(year=2018, month=1, day=1), 40 | end=datetime(year=2018, month=1, day=31), 41 | ) 42 | 43 | response_format = ResponseFormat.netcdf4 44 | 45 | converted_weather_data = controller.convert_names_and_units("knmi", "pluim", False, ds_hist_day, OutputUnit.si) 46 | 47 | ret_args = WeatherContentRequestQuery( 48 | begin="2018-01-01 00:00", 49 | end="2018-01-31 23:59", 50 | lat=52.1, 51 | lon=5.18, 52 | factors=None, 53 | ) 54 | 55 | response, optional_file_path = serializers.file_or_text_response( 56 | converted_weather_data, response_format, "knmi", "pluim", ret_args, coords 57 | ) 58 | 59 | print(response) 60 | print(optional_file_path) 61 | -------------------------------------------------------------------------------- /demos/demo_knmi_uurgegevens.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # SPDX-FileCopyrightText: 2019-2022 Alliander N.V. 5 | # SPDX-License-Identifier: MPL-2.0 6 | 7 | from datetime import datetime 8 | 9 | import xarray as xr 10 | 11 | from weather_provider_api.routers.weather.api_models import ( 12 | OutputUnit, 13 | ResponseFormat, 14 | WeatherContentRequestQuery, 15 | ) 16 | from weather_provider_api.routers.weather.api_view_v2 import controller 17 | from weather_provider_api.routers.weather.utils import serializers 18 | 19 | """ 20 | This is a DEMO for a specific weather model: KNMI Uurgegevens 21 | 22 | The purpose of this demo is to show users a method of how they can use specific models from the Weather Provider API 23 | to request specific output. 24 | """ 25 | 26 | if __name__ == "__main__": 27 | coords = [ 28 | [(51.873419, 5.705929)], 29 | [(52.121976, 4.782497)], 30 | [(52.424793, 4.776927)], 31 | [(51.873777, 5.705111)], 32 | ] 33 | 34 | ds_hist_day: xr.Dataset = controller.get_weather( 35 | source_id="knmi", 36 | model_id="uurgegevens", 37 | fetch_async=False, 38 | coords=coords, 39 | begin=datetime(year=2018, month=1, day=1), 40 | end=datetime(year=2018, month=1, day=31), 41 | ) 42 | 43 | response_format = ResponseFormat.netcdf4 44 | 45 | converted_weather_data = controller.convert_names_and_units( 46 | "knmi", "uurgegevens", False, ds_hist_day, OutputUnit.si 47 | ) 48 | 49 | ret_args = WeatherContentRequestQuery( 50 | begin="2018-01-01 00:00", 51 | end="2018-01-31 23:59", 52 | lat=[51.873419], 53 | lon=[5.705929], 54 | factors=None, 55 | ) 56 | 57 | response, optional_file_path = serializers.file_or_text_response( 58 | converted_weather_data, response_format, "knmi", "uurgegevens", ret_args, coords 59 | ) 60 | 61 | print(response) 62 | print(optional_file_path) 63 | -------------------------------------------------------------------------------- /doc/copyright_profiles/velocity_copyright_text.txt: -------------------------------------------------------------------------------- 1 | SPDX-FileCopyrightText: 2019-$today.year Alliander N.V. 2 | SPDX-License-Identifier: MPL-2.0 -------------------------------------------------------------------------------- /img/gitflow.svg.license: -------------------------------------------------------------------------------- 1 | SPDX-FileCopyrightText: 2019-2022 Alliander N.V. 2 | SPDX-License-Identifier: MPL-2.0 -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "weather_provider_api" 3 | version = "2.72.0" 4 | description = "Weather Provider Libraries and API" 5 | authors = ["Verbindingsteam", "Raoul Linnenbank <58594297+rflinnenbank@users.noreply.github.com>"] 6 | license = "MPL-2.0" 7 | readme = "README.md" 8 | repository="https://github.com/alliander-opensource/wpla/" 9 | include = [ 10 | {path = "var_maps/*.json", format = "wheel"} 11 | ] 12 | 13 | [tool.poetry.dependencies] 14 | python = ">=3.10,<3.12" 15 | fastapi = "^0.115.12" 16 | requests = "^2.32.3" 17 | geopy = "^2.4.1" 18 | numpy = "^2.2.4" 19 | gunicorn = "^23.0.0" 20 | lxml = "^5.3.1" 21 | starlette-prometheus = "^0.10.0" 22 | pandas = "^2.2.3" 23 | xarray = "^2025.3.0" 24 | cfgrib = "^0.9.15.0" 25 | uvicorn = "^0.34.0" 26 | slowapi = "^0.1.9" 27 | loguru = "^0.7.3" 28 | email-validator = "^2.2.0" 29 | eccodes = "^2.40.1" 30 | ecmwflibs = "0.6.3" 31 | accept-types = "^0.4.1" 32 | cdsapi = "^0.7.5" 33 | beautifulsoup4 = "^4.13.3" 34 | netcdf4 = "^1.7.2" 35 | tomli = "^2.2.1" 36 | dask = "^2025.3.0" 37 | 38 | [tool.poetry.group.dev] 39 | optional = true 40 | 41 | [tool.poetry.group.dev.dependencies] 42 | coverage = "^7.7.1" 43 | pytest-cov = "^6.0.0" 44 | pylint = "^3.3.6" 45 | ruff = {version = "^0.4.1", source = "pypi"} 46 | pytest = "^8.3.5" 47 | sphinx = "^8.1.3" 48 | myst-parser = "^4.0.1" 49 | piccolo-theme = "^0.24.0" 50 | 51 | [tool.poetry.scripts] 52 | wpla_update_era5sl = "weather_provider_api.scripts.update_era5sl_repository:main" 53 | wpla_update_era5land = "weather_provider_api.scripts.update_era5land_repository:main" 54 | wpla_update_arome = "weather_provider_api.scripts.update_arome_repository:main" 55 | wpla_update_waarnemingen = "weather_provider_api.scripts.update_waarnemingen_register:main" 56 | wpla_clear_era5sl = "weather_provider_api.scripts.erase_era5sl_repository:main" 57 | wpla_clear_era5land = "weather_provider_api.scripts.erase_era5land_repository:main" 58 | wpla_clear_arome = "weather_provider_api.scripts.erase_arome_repository:main" 59 | wpla_clear_waarnemingen = "weather_provider_api.scripts.erase_waarnemingen_register:main" 60 | wpla_run_api = "weather_provider_api.main:main" 61 | 62 | [[tool.poetry.source]] 63 | name = "PyPI" 64 | priority = "primary" 65 | 66 | [tool.pylint] 67 | max-line-length = 120 68 | 69 | [build-system] 70 | requires = ["poetry-core"] 71 | build-backend = "poetry.core.masonry.api" 72 | 73 | [tool.ruff] 74 | line-length = 120 75 | 76 | [tool.ruff.lint] 77 | select = [ 78 | "ANN", # flake8-annotations 79 | "E", # flake8 80 | "F", # flake8 81 | "I", # isort 82 | "D", # pydocstyle 83 | "S", # flake8-bandit 84 | "NPY", # numpy-specific rules 85 | "RUF", # ruff specific rules 86 | ] 87 | ignore = [ 88 | "E501", 89 | "E712", 90 | 91 | "ANN101", # Missing type annotation for `self` in method 92 | "ANN202", # Missing return type annotation for private function 93 | "ANN204", # Missing return type annotation for special function 94 | "ANN401", # Dynamically typed expressions (typing.Any) are disallowed 95 | 96 | # pydocstyle 97 | "D100", # Missing docstring in public module 98 | "D104", # Missing docstring in public package 99 | "D106", # Missing docstring in public nested class 100 | ] 101 | 102 | [tool.ruff.lint.per-file-ignores] 103 | "__init__.py" = ["F401"] 104 | "tests/**" = ["S", "ANN"] 105 | 106 | [tool.ruff.lint.pydocstyle] 107 | convention = "google" -------------------------------------------------------------------------------- /sigrid.yaml: -------------------------------------------------------------------------------- 1 | component_depth: 2 2 | exclude: 3 | - "/demos/.*" 4 | - "/doc/.*" 5 | - "/LICENSES/.*" 6 | - "/img/.*" 7 | - "weather_provider_api/scripts/.*" 8 | - ".*/__init__.py" 9 | languages: 10 | - name: python 11 | - name: docker 12 | - name: json 13 | component_base_dirs: 14 | - "weather_provider_api" 15 | - "var_maps" 16 | 17 | -------------------------------------------------------------------------------- /sonar-project.properties: -------------------------------------------------------------------------------- 1 | # 2 | # SPDX-FileCopyrightText: 2019-2022 Alliander N.V. 3 | # SPDX-License-Identifier: MPL-2.0 4 | # 5 | 6 | 7 | # Required metadata 8 | sonar.projectKey=alliander-opensource_weather-provider-api 9 | sonar.organization=alliander-opensource 10 | sonar.projectName=Weather Provider Libraries and API 11 | sonar.projectVersion=2.14.0 12 | sonar.sourceEncoding=UTF-8 13 | 14 | # Project folders 15 | sonar.sources=weather_provider_api 16 | sonar.tests=tests 17 | 18 | # Language 19 | sonar.language=py 20 | 21 | # Python versions 22 | sonar.python.version=3.8,3.9,3.10 23 | 24 | sonar.links.homepage=https://github.com/alliander-opensource/Weather-Provider-API/ 25 | sonar.links.ci=https://github.com/alliander-opensource/Weather-Provider-API/img/gitflow.svg 26 | sonar.links.scm=https://github.com/alliander-opensource/Weather-Provider-API/ 27 | sonar.links.issue=https://github.com/alliander-opensource/Weather-Provider-API/issues 28 | 29 | # Coverage 30 | sonar.python.coverage.reportPaths=coverage.xml 31 | -------------------------------------------------------------------------------- /sphinx-docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line, and also 5 | # from the environment for the first two. 6 | SPHINXOPTS ?= 7 | SPHINXBUILD ?= sphinx-build 8 | SOURCEDIR = . 9 | BUILDDIR = _build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | -------------------------------------------------------------------------------- /sphinx-docs/conf.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # SPDX-FileCopyrightText: 2019-2024 Alliander N.V. 4 | # SPDX-License-Identifier: MPL-2.0 5 | 6 | import os 7 | import sys 8 | 9 | # Configuration file for the Sphinx documentation builder. 10 | # 11 | # For the full list of built-in configuration values, see the documentation: 12 | # https://www.sphinx-doc.org/en/master/usage/configuration.html 13 | 14 | # -- Project information ----------------------------------------------------- 15 | # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information 16 | sys.path.insert(0, os.path.abspath("..")) 17 | 18 | project = "Weather Provider API" 19 | copyright = "2024, Alliander" 20 | author = "Raoul Linnenbank" 21 | release = "2.5.5" 22 | 23 | highlight_language = "python3" 24 | 25 | # -- General configuration --------------------------------------------------- 26 | # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration 27 | 28 | extensions = ["myst_parser", "sphinx.ext.autodoc"] 29 | 30 | templates_path = ["_templates"] 31 | exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] 32 | 33 | 34 | # -- Options for HTML output ------------------------------------------------- 35 | # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output 36 | 37 | 38 | html_theme = "piccolo_theme" 39 | html_static_path = ["_static"] 40 | 41 | # Automatically extract typehints when specified and place them in 42 | # descriptions of the relevant function/method. 43 | autodoc_typehints = "description" 44 | 45 | # Don't show class signature with the class' name. 46 | autodoc_class_signature = "separated" 47 | 48 | add_module_names = False 49 | show_authors = True 50 | toc_object_entries_show_parents = "hide" 51 | -------------------------------------------------------------------------------- /sphinx-docs/index.rst: -------------------------------------------------------------------------------- 1 | .. coding=utf-8 2 | .. Weather Provider Libraries documentation master file, created by 3 | sphinx-quickstart on Wed Oct 4 21:03:43 2023. 4 | You can adapt this file completely to your liking, but it should at least 5 | contain the root `toctree` directive. 6 | 7 | ############################################################## 8 | Welcome to the Weather Provider API (WPA) documentation! 9 | ############################################################## 10 | 11 | .. image:: /_static/wpas_logo.svg 12 | :alt: Weather Provider Access Suite 13 | :align: center 14 | :width: 480px 15 | 16 | .. include:: introduction.rst 17 | 18 | .. toctree:: 19 | :maxdepth: 1 20 | 21 | self 22 | 23 | .. toctree:: 24 | :caption: Basic Information 25 | :maxdepth: 3 26 | 27 | introduction 28 | quick-start 29 | weather_provider_access_suite_info_page 30 | 31 | .. toctree:: 32 | :caption: Modular information 33 | :maxdepth: 3 34 | :hidden: 35 | 36 | modules 37 | modindex 38 | -------------------------------------------------------------------------------- /sphinx-docs/introduction.rst: -------------------------------------------------------------------------------- 1 | .. coding=utf-8 2 | .. SPDX-FileCopyrightText: 2019-2023 Alliander N.V. 3 | .. SPDX-License-Identifier: MPL-2.0 4 | 5 | ********************************************** 6 | An introduction to the Weather Provider API V2 7 | ********************************************** 8 | 9 | "NOTE: The follow-up project to the Weather Provider APi V2 is currently under development. 10 | For more information on the new project, please check the "Weather Provider Access Suite" project information at: 11 | :ref:`Weather Provider Access Suite Info Page`" 12 | 13 | ============================================= 14 | What is the Weather Provider API (WPA)? 15 | ============================================= 16 | 17 | ---------------- 18 | **Project Goal** 19 | ---------------- 20 | The Weather Provider Libraries project is a project with a singular goal in mind:: 21 | 22 | Easily accessing any data for a multitude of meteorological datasets and meteorological site-pages 23 | without any prior knowledge of those datasets or even their specific content. 24 | 25 | That is right. Without any prior knowledge of a supported dataset itself, we want you to be able to achieve the 26 | following: 27 | 28 | * *Make requests for specific periods and meteorological factors.* 29 | * *Transform the received data into one of several supported uniform formats, allowing for comparison of data between 30 | datasets if the fields are identical in nature.* 31 | * *Transform the output for those requests into a wide number of commonly used file formats, flattening the output from 32 | multi-dimensional data as needed.* 33 | * *Translate existing dataset output directly into the aforementioned supported uniform data and allow for outputting 34 | that result in the supported output file formats as well.* 35 | 36 | **As a secondary goal, we also wish to achieve the following:** 37 | 38 | * For motivated people that have knowledge or affinity with unsupported datasets to build their own compatible model(s) 39 | and source(s) without prior knowledge of the WPL system by being guided by the base classes themselves. 40 | This will follow in the upcoming format of the Weather Provider API, the Weather Provider Access Suite Project. 41 | 42 | * Allow for the easy access and plugging of sources and models as desired. You can access the data you want to use 43 | without any need for installing more than just a singular source and calling more than just the model you need if 44 | that is what you need, while also retaining the possibility to upscale to a multitude of sources and models and 45 | even connect those to the `Weather Provider API`_ project for a fully functional API based on your wishes. 46 | 47 | =============== 48 | Project Origins 49 | =============== 50 | 51 | The Weather Provider Libraries Project, or WPL, as it will be abbreviated a lot in the documentation of this project, 52 | is a project based on the original "**weather_provider_api**" project found at: 53 | 54 | `https://github.com/alliander-opensource/weather-provider-api/ `_ 55 | 56 | Until version 3.0 of this project, every component thereof was considered a part of a singular whole, but to allow for 57 | easier usage and the easier building of new models and sources, the project was split up into three components: 58 | 59 | --------------------------------- 60 | **1. Weather Provider Libraries** 61 | --------------------------------- 62 | 63 | This project and the part that holds all of the common components and tools responsible for formatting, processing 64 | and transforming meteorological data, as well as all of the base classes for creating Models and Sources for the 65 | project. Finally the project also houses the Controller which allows for easy configuration and acquisition of data 66 | over multiple sources and models. 67 | 68 | --------------------------- 69 | **2. Weather Provider API** 70 | --------------------------- 71 | 72 | This project houses the API implementation of this project. It uses the Weather Provider Libraries project to 73 | transform any connected source and model into appropriate endpoints. This fully functional FastAPI implementation is 74 | fully supportive of the OpenAPI standard and can easily be scaled according to your wishes. The project repository 75 | even comes with a number of example deployment folders. The project can be used via custom deployment through its 76 | package or deployment using the readily available Docker images. 77 | For more information on this project please check the Project's repository page at: `Weather Provider API`_ 78 | 79 | ------------------------------- 80 | **3. Weather Provider Sources** 81 | ------------------------------- 82 | 83 | This project actually consists of multiple repositories. Each repository houses one or multiple Sources that can be 84 | installed as packages used separately or from a Weather Provider Libraries system. Each Source can house one or 85 | multiple Models, each representing a specific meteorological dataset, site-page with meteorological data, or fusion 86 | thereof. 87 | 88 | .. _Weather Provider API: https://github.com/alliander-opensource/weather-provider-api 89 | -------------------------------------------------------------------------------- /sphinx-docs/modules.rst: -------------------------------------------------------------------------------- 1 | weather_provider_api 2 | ==================== 3 | 4 | .. toctree:: 5 | :maxdepth: 4 6 | 7 | weather_provider_api 8 | -------------------------------------------------------------------------------- /sphinx-docs/quick-start.rst: -------------------------------------------------------------------------------- 1 | .. coding=utf-8 2 | 3 | *************************************************** 4 | Getting Started with the Weather Provider Libraries 5 | *************************************************** 6 | 7 | ==================== 8 | Package installation 9 | ==================== 10 | To install the Weather Provider Libraries package you only need to install it using PyPi:: 11 | 12 | > pip install weather-provider-libraries 13 | 14 | *Please note that when building your own model(s) and source(s) you will need to use the base classes declared 15 | within the package to allow them to be recognized and used by the API.* 16 | 17 | ========================== 18 | Default Sources and Models 19 | ========================== 20 | The Weather Provider Libraries (WPL) package will automatically install a number of sources and models natively 21 | supported by the project. For a information on these sources, their models and their specific usage, please read the 22 | documentation available at: 23 | 24 | `https://github.com/alliander-opensource/ `_ 25 | 26 | =========================================== 27 | Your first request using the WPL Controller 28 | =========================================== 29 | When using WPL directly, rather than through the API, you'll be able to contact all configured and set models by using 30 | the Controller. The controller can be either instantiated by using Python or called upon via one of the available 31 | scripts. 32 | 33 | .. code-block:: 34 | :caption: **Python instantiation of the Controller:** 35 | 36 | from weather_provider_libraries.wpl_controller import WPLController 37 | 38 | controller = WPLController() 39 | 40 | *Usually your IDE should be able to pick up on the possible requests and options available to this object. If unsure or 41 | if you wish for more information, please check the module documentation as well.* 42 | 43 | 44 | .. code-block:: 45 | :caption: **Script activation of the controller:** 46 | 47 | python run wpl_controller_commands 48 | 49 | *This script should retrieve a list of known WPL Controller commands that can be executed directly from the command line 50 | and their accepted parameters* 51 | -------------------------------------------------------------------------------- /sphinx-docs/weather_provider_access_suite_info_page.rst: -------------------------------------------------------------------------------- 1 | .. coding=utf-8 2 | .. SPDX-FileCopyrightText: 2019-2023 Alliander N.V. 3 | .. SPDX-License-Identifier: MPL-2.0 4 | 5 | .. _Weather Provider Access Suite Info Page: 6 | 7 | ************************************************ 8 | Information on the Weather Provider Access Suite 9 | ************************************************ 10 | 11 | "NOTE: The WPAS project is the successor to the Weather Provider API V2 project, and currently under development. 12 | Upon release to the public, the WPAS project components will all start versioning at 3.0.0 to reflect this." 13 | 14 | 15 | ====================================================================== 16 | The current status of development on the Weather Provider Access Suite 17 | ====================================================================== 18 | 19 | .. list-table:: Weather Provider Access Suite Development Status 20 | :widths: 10 20 21 | :header-rows: 1 22 | 23 | * - Project component 24 | - Development status 25 | * - weather-provider-api 26 | - Mostly complete, awaiting release of the libraries component for integration 27 | * - weather-provider-libraries 28 | - Core components complete and being re-integrated, ZARR file formatting and controller class redesign in progress. 29 | Data transformation and output formatting components are mostly complete. 30 | * - weather-provider-sources 31 | - Awaiting completion of the libraries component for integration. The sources component will be the last to be 32 | integrated into the WPAS project. Backwards compatibility with the Weather Provider API V2 project will be 33 | generated via the rebuilding of the old weather sources and models that existed there. These models will be 34 | callable from both their old V2 and new V3 endpoints, with the newer endpoints providing more functionality. 35 | 36 | 37 | =================================================================================== 38 | How does the Weather Provider Access Suite differ from the Weather Provider API V2? 39 | =================================================================================== 40 | 41 | The Weather Provider Access Suite (or WPAS) is a project that is built from scratch, based on the finding from users of 42 | the original Weather Provider API V2 project, and a reevaluation of the project goals. 43 | 44 | In its new form the WPAS project will be built with the same goals als the original project, but with a few key 45 | differences in its execution: 46 | 47 | * **The WPAS project will be built with a more transparent modular structure and far better code.** 48 | 49 | This will allow for easier integration of new sources and models, and will allow for the easy addition of new 50 | functionality to the project. Because the project is rebuilt from scratch with the "one function, one purpose" 51 | principle in mind, the code will be far easier to read and understand. 52 | 53 | This will also allow for easier debugging and testing of the project, while allowing external developers to more 54 | easily contribute to the project, either by adding new sources and models, or by adding new functionality to the 55 | project. 56 | 57 | * **The WPAS project will be built with a more transparent and user-friendly API.** 58 | 59 | The API will be built with the same goals as the original project, but with a more user-friendly interface that will 60 | allow for easier access to the data that the project provides. 61 | 62 | Several new endpoint groups will also be added to the API with the purpose of allowing for easier access to stored 63 | data, and if activated, an endpoint group with the sole purpose of self-evaluation can also be activated. 64 | 65 | 66 | * **The WPAS project will be built with a more transparent and user-friendly documentation.** 67 | 68 | The documentation will be built with the same goals as the original project, but with a more user-friendly interface 69 | that will allow for easier access to the data that the project provides. 70 | 71 | Several new articles will also be added to allow external developers to easily understand the project, and to allow 72 | them to more easily contribute to the project. 73 | 74 | * **The WPAS project will be built with more efficiency in mind.** 75 | 76 | The project will be built with a more efficient codebase that will allow for easier integration of new sources and 77 | models, and will allow for the easy addition of new functionality to the project. Base reusable code and classes will 78 | also be developed with an eye on speed and memory efficiency. 79 | 80 | This will also allow external developers to contribute to the project more easily, as they wont need to guard memory 81 | usage or processing speed as much as they would have to with the original project. 82 | -------------------------------------------------------------------------------- /sphinx-docs/weather_provider_api.config.rst: -------------------------------------------------------------------------------- 1 | weather\_provider\_api.config package 2 | ===================================== 3 | 4 | Module contents 5 | --------------- 6 | 7 | .. automodule:: weather_provider_api.config 8 | :members: 9 | :undoc-members: 10 | :show-inheritance: 11 | -------------------------------------------------------------------------------- /sphinx-docs/weather_provider_api.core.exceptions.rst: -------------------------------------------------------------------------------- 1 | weather\_provider\_api.core.exceptions package 2 | ============================================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | weather\_provider\_api.core.exceptions.additional\_responses module 8 | ------------------------------------------------------------------- 9 | 10 | .. automodule:: weather_provider_api.core.exceptions.additional_responses 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | weather\_provider\_api.core.exceptions.api\_models module 16 | --------------------------------------------------------- 17 | 18 | .. automodule:: weather_provider_api.core.exceptions.api_models 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | weather\_provider\_api.core.exceptions.exceptions module 24 | -------------------------------------------------------- 25 | 26 | .. automodule:: weather_provider_api.core.exceptions.exceptions 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | Module contents 32 | --------------- 33 | 34 | .. automodule:: weather_provider_api.core.exceptions 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | -------------------------------------------------------------------------------- /sphinx-docs/weather_provider_api.core.initializers.rst: -------------------------------------------------------------------------------- 1 | weather\_provider\_api.core.initializers package 2 | ================================================ 3 | 4 | Submodules 5 | ---------- 6 | 7 | weather\_provider\_api.core.initializers.cors module 8 | ---------------------------------------------------- 9 | 10 | .. automodule:: weather_provider_api.core.initializers.cors 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | weather\_provider\_api.core.initializers.exception\_handling module 16 | ------------------------------------------------------------------- 17 | 18 | .. automodule:: weather_provider_api.core.initializers.exception_handling 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | weather\_provider\_api.core.initializers.headers module 24 | ------------------------------------------------------- 25 | 26 | .. automodule:: weather_provider_api.core.initializers.headers 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | weather\_provider\_api.core.initializers.logging\_handler module 32 | ---------------------------------------------------------------- 33 | 34 | .. automodule:: weather_provider_api.core.initializers.logging_handler 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | weather\_provider\_api.core.initializers.mounting module 40 | -------------------------------------------------------- 41 | 42 | .. automodule:: weather_provider_api.core.initializers.mounting 43 | :members: 44 | :undoc-members: 45 | :show-inheritance: 46 | 47 | weather\_provider\_api.core.initializers.prometheus module 48 | ---------------------------------------------------------- 49 | 50 | .. automodule:: weather_provider_api.core.initializers.prometheus 51 | :members: 52 | :undoc-members: 53 | :show-inheritance: 54 | 55 | weather\_provider\_api.core.initializers.rate\_limiter module 56 | ------------------------------------------------------------- 57 | 58 | .. automodule:: weather_provider_api.core.initializers.rate_limiter 59 | :members: 60 | :undoc-members: 61 | :show-inheritance: 62 | 63 | weather\_provider\_api.core.initializers.validation module 64 | ---------------------------------------------------------- 65 | 66 | .. automodule:: weather_provider_api.core.initializers.validation 67 | :members: 68 | :undoc-members: 69 | :show-inheritance: 70 | 71 | Module contents 72 | --------------- 73 | 74 | .. automodule:: weather_provider_api.core.initializers 75 | :members: 76 | :undoc-members: 77 | :show-inheritance: 78 | -------------------------------------------------------------------------------- /sphinx-docs/weather_provider_api.core.rst: -------------------------------------------------------------------------------- 1 | weather\_provider\_api.core package 2 | =================================== 3 | 4 | Subpackages 5 | ----------- 6 | 7 | .. toctree:: 8 | :maxdepth: 4 9 | 10 | weather_provider_api.core.exceptions 11 | weather_provider_api.core.initializers 12 | weather_provider_api.core.utils 13 | 14 | Submodules 15 | ---------- 16 | 17 | weather\_provider\_api.core.application module 18 | ---------------------------------------------- 19 | 20 | .. automodule:: weather_provider_api.core.application 21 | :members: 22 | :undoc-members: 23 | :show-inheritance: 24 | 25 | weather\_provider\_api.core.base\_model module 26 | ---------------------------------------------- 27 | 28 | .. automodule:: weather_provider_api.core.base_model 29 | :members: 30 | :undoc-members: 31 | :show-inheritance: 32 | 33 | weather\_provider\_api.core.gunicorn\_application module 34 | -------------------------------------------------------- 35 | 36 | .. automodule:: weather_provider_api.core.gunicorn_application 37 | :members: 38 | :undoc-members: 39 | :show-inheritance: 40 | 41 | Module contents 42 | --------------- 43 | 44 | .. automodule:: weather_provider_api.core 45 | :members: 46 | :undoc-members: 47 | :show-inheritance: 48 | -------------------------------------------------------------------------------- /sphinx-docs/weather_provider_api.core.utils.rst: -------------------------------------------------------------------------------- 1 | weather\_provider\_api.core.utils package 2 | ========================================= 3 | 4 | Submodules 5 | ---------- 6 | 7 | weather\_provider\_api.core.utils.example\_responses module 8 | ----------------------------------------------------------- 9 | 10 | .. automodule:: weather_provider_api.core.utils.example_responses 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | Module contents 16 | --------------- 17 | 18 | .. automodule:: weather_provider_api.core.utils 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | -------------------------------------------------------------------------------- /sphinx-docs/weather_provider_api.routers.rst: -------------------------------------------------------------------------------- 1 | weather\_provider\_api.routers package 2 | ====================================== 3 | 4 | Subpackages 5 | ----------- 6 | 7 | .. toctree:: 8 | :maxdepth: 4 9 | 10 | weather_provider_api.routers.weather 11 | 12 | Module contents 13 | --------------- 14 | 15 | .. automodule:: weather_provider_api.routers 16 | :members: 17 | :undoc-members: 18 | :show-inheritance: 19 | -------------------------------------------------------------------------------- /sphinx-docs/weather_provider_api.routers.weather.base_models.rst: -------------------------------------------------------------------------------- 1 | weather\_provider\_api.routers.weather.base\_models package 2 | =========================================================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | weather\_provider\_api.routers.weather.base\_models.model module 8 | ---------------------------------------------------------------- 9 | 10 | .. automodule:: weather_provider_api.routers.weather.base_models.model 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | weather\_provider\_api.routers.weather.base\_models.source module 16 | ----------------------------------------------------------------- 17 | 18 | .. automodule:: weather_provider_api.routers.weather.base_models.source 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | Module contents 24 | --------------- 25 | 26 | .. automodule:: weather_provider_api.routers.weather.base_models 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | -------------------------------------------------------------------------------- /sphinx-docs/weather_provider_api.routers.weather.repository.rst: -------------------------------------------------------------------------------- 1 | weather\_provider\_api.routers.weather.repository package 2 | ========================================================= 3 | 4 | Submodules 5 | ---------- 6 | 7 | weather\_provider\_api.routers.weather.repository.repository module 8 | ------------------------------------------------------------------- 9 | 10 | .. automodule:: weather_provider_api.routers.weather.repository.repository 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | Module contents 16 | --------------- 17 | 18 | .. automodule:: weather_provider_api.routers.weather.repository 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | -------------------------------------------------------------------------------- /sphinx-docs/weather_provider_api.routers.weather.rst: -------------------------------------------------------------------------------- 1 | weather\_provider\_api.routers.weather package 2 | ============================================== 3 | 4 | Subpackages 5 | ----------- 6 | 7 | .. toctree:: 8 | :maxdepth: 4 9 | 10 | weather_provider_api.routers.weather.base_models 11 | weather_provider_api.routers.weather.repository 12 | weather_provider_api.routers.weather.sources 13 | weather_provider_api.routers.weather.utils 14 | 15 | Submodules 16 | ---------- 17 | 18 | weather\_provider\_api.routers.weather.api\_models module 19 | --------------------------------------------------------- 20 | 21 | .. automodule:: weather_provider_api.routers.weather.api_models 22 | :members: 23 | :undoc-members: 24 | :show-inheritance: 25 | 26 | weather\_provider\_api.routers.weather.api\_view\_v1 module 27 | ----------------------------------------------------------- 28 | 29 | .. automodule:: weather_provider_api.routers.weather.api_view_v1 30 | :members: 31 | :undoc-members: 32 | :show-inheritance: 33 | 34 | weather\_provider\_api.routers.weather.api\_view\_v2 module 35 | ----------------------------------------------------------- 36 | 37 | .. automodule:: weather_provider_api.routers.weather.api_view_v2 38 | :members: 39 | :undoc-members: 40 | :show-inheritance: 41 | 42 | weather\_provider\_api.routers.weather.controller module 43 | -------------------------------------------------------- 44 | 45 | .. automodule:: weather_provider_api.routers.weather.controller 46 | :members: 47 | :undoc-members: 48 | :show-inheritance: 49 | 50 | weather\_provider\_api.routers.weather.exceptions module 51 | -------------------------------------------------------- 52 | 53 | .. automodule:: weather_provider_api.routers.weather.exceptions 54 | :members: 55 | :undoc-members: 56 | :show-inheritance: 57 | 58 | Module contents 59 | --------------- 60 | 61 | .. automodule:: weather_provider_api.routers.weather 62 | :members: 63 | :undoc-members: 64 | :show-inheritance: 65 | -------------------------------------------------------------------------------- /sphinx-docs/weather_provider_api.routers.weather.sources.cds.client.rst: -------------------------------------------------------------------------------- 1 | weather\_provider\_api.routers.weather.sources.cds.client package 2 | ================================================================= 3 | 4 | Submodules 5 | ---------- 6 | 7 | weather\_provider\_api.routers.weather.sources.cds.client.downloader module 8 | --------------------------------------------------------------------------- 9 | 10 | .. automodule:: weather_provider_api.routers.weather.sources.cds.client.downloader 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | weather\_provider\_api.routers.weather.sources.cds.client.era5land\_repository module 16 | ------------------------------------------------------------------------------------- 17 | 18 | .. automodule:: weather_provider_api.routers.weather.sources.cds.client.era5land_repository 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | weather\_provider\_api.routers.weather.sources.cds.client.era5sl\_repository module 24 | ----------------------------------------------------------------------------------- 25 | 26 | .. automodule:: weather_provider_api.routers.weather.sources.cds.client.era5sl_repository 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | weather\_provider\_api.routers.weather.sources.cds.client.utils\_era5 module 32 | ---------------------------------------------------------------------------- 33 | 34 | .. automodule:: weather_provider_api.routers.weather.sources.cds.client.utils_era5 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | Module contents 40 | --------------- 41 | 42 | .. automodule:: weather_provider_api.routers.weather.sources.cds.client 43 | :members: 44 | :undoc-members: 45 | :show-inheritance: 46 | -------------------------------------------------------------------------------- /sphinx-docs/weather_provider_api.routers.weather.sources.cds.models.rst: -------------------------------------------------------------------------------- 1 | weather\_provider\_api.routers.weather.sources.cds.models package 2 | ================================================================= 3 | 4 | Submodules 5 | ---------- 6 | 7 | weather\_provider\_api.routers.weather.sources.cds.models.era5land module 8 | ------------------------------------------------------------------------- 9 | 10 | .. automodule:: weather_provider_api.routers.weather.sources.cds.models.era5land 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | weather\_provider\_api.routers.weather.sources.cds.models.era5sl module 16 | ----------------------------------------------------------------------- 17 | 18 | .. automodule:: weather_provider_api.routers.weather.sources.cds.models.era5sl 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | Module contents 24 | --------------- 25 | 26 | .. automodule:: weather_provider_api.routers.weather.sources.cds.models 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | -------------------------------------------------------------------------------- /sphinx-docs/weather_provider_api.routers.weather.sources.cds.rst: -------------------------------------------------------------------------------- 1 | weather\_provider\_api.routers.weather.sources.cds package 2 | ========================================================== 3 | 4 | Subpackages 5 | ----------- 6 | 7 | .. toctree:: 8 | :maxdepth: 4 9 | 10 | weather_provider_api.routers.weather.sources.cds.client 11 | weather_provider_api.routers.weather.sources.cds.models 12 | 13 | Submodules 14 | ---------- 15 | 16 | weather\_provider\_api.routers.weather.sources.cds.cds module 17 | ------------------------------------------------------------- 18 | 19 | .. automodule:: weather_provider_api.routers.weather.sources.cds.cds 20 | :members: 21 | :undoc-members: 22 | :show-inheritance: 23 | 24 | weather\_provider\_api.routers.weather.sources.cds.factors module 25 | ----------------------------------------------------------------- 26 | 27 | .. automodule:: weather_provider_api.routers.weather.sources.cds.factors 28 | :members: 29 | :undoc-members: 30 | :show-inheritance: 31 | 32 | Module contents 33 | --------------- 34 | 35 | .. automodule:: weather_provider_api.routers.weather.sources.cds 36 | :members: 37 | :undoc-members: 38 | :show-inheritance: 39 | -------------------------------------------------------------------------------- /sphinx-docs/weather_provider_api.routers.weather.sources.knmi.client.rst: -------------------------------------------------------------------------------- 1 | weather\_provider\_api.routers.weather.sources.knmi.client package 2 | ================================================================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | weather\_provider\_api.routers.weather.sources.knmi.client.actuele\_waarnemingen\_register\_repository module 8 | ------------------------------------------------------------------------------------------------------------- 9 | 10 | .. automodule:: weather_provider_api.routers.weather.sources.knmi.client.actuele_waarnemingen_register_repository 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | weather\_provider\_api.routers.weather.sources.knmi.client.arome\_repository module 16 | ----------------------------------------------------------------------------------- 17 | 18 | .. automodule:: weather_provider_api.routers.weather.sources.knmi.client.arome_repository 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | weather\_provider\_api.routers.weather.sources.knmi.client.knmi\_downloader module 24 | ---------------------------------------------------------------------------------- 25 | 26 | .. automodule:: weather_provider_api.routers.weather.sources.knmi.client.knmi_downloader 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | Module contents 32 | --------------- 33 | 34 | .. automodule:: weather_provider_api.routers.weather.sources.knmi.client 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | -------------------------------------------------------------------------------- /sphinx-docs/weather_provider_api.routers.weather.sources.knmi.models.rst: -------------------------------------------------------------------------------- 1 | weather\_provider\_api.routers.weather.sources.knmi.models package 2 | ================================================================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | weather\_provider\_api.routers.weather.sources.knmi.models.actuele\_waarnemingen module 8 | --------------------------------------------------------------------------------------- 9 | 10 | .. automodule:: weather_provider_api.routers.weather.sources.knmi.models.actuele_waarnemingen 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | weather\_provider\_api.routers.weather.sources.knmi.models.actuele\_waarnemingen\_register module 16 | ------------------------------------------------------------------------------------------------- 17 | 18 | .. automodule:: weather_provider_api.routers.weather.sources.knmi.models.actuele_waarnemingen_register 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | weather\_provider\_api.routers.weather.sources.knmi.models.daggegevens module 24 | ----------------------------------------------------------------------------- 25 | 26 | .. automodule:: weather_provider_api.routers.weather.sources.knmi.models.daggegevens 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | weather\_provider\_api.routers.weather.sources.knmi.models.harmonie\_arome module 32 | --------------------------------------------------------------------------------- 33 | 34 | .. automodule:: weather_provider_api.routers.weather.sources.knmi.models.harmonie_arome 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | weather\_provider\_api.routers.weather.sources.knmi.models.pluim module 40 | ----------------------------------------------------------------------- 41 | 42 | .. automodule:: weather_provider_api.routers.weather.sources.knmi.models.pluim 43 | :members: 44 | :undoc-members: 45 | :show-inheritance: 46 | 47 | weather\_provider\_api.routers.weather.sources.knmi.models.uurgegevens module 48 | ----------------------------------------------------------------------------- 49 | 50 | .. automodule:: weather_provider_api.routers.weather.sources.knmi.models.uurgegevens 51 | :members: 52 | :undoc-members: 53 | :show-inheritance: 54 | 55 | Module contents 56 | --------------- 57 | 58 | .. automodule:: weather_provider_api.routers.weather.sources.knmi.models 59 | :members: 60 | :undoc-members: 61 | :show-inheritance: 62 | -------------------------------------------------------------------------------- /sphinx-docs/weather_provider_api.routers.weather.sources.knmi.rst: -------------------------------------------------------------------------------- 1 | weather\_provider\_api.routers.weather.sources.knmi package 2 | =========================================================== 3 | 4 | Subpackages 5 | ----------- 6 | 7 | .. toctree:: 8 | :maxdepth: 4 9 | 10 | weather_provider_api.routers.weather.sources.knmi.client 11 | weather_provider_api.routers.weather.sources.knmi.models 12 | 13 | Submodules 14 | ---------- 15 | 16 | weather\_provider\_api.routers.weather.sources.knmi.knmi module 17 | --------------------------------------------------------------- 18 | 19 | .. automodule:: weather_provider_api.routers.weather.sources.knmi.knmi 20 | :members: 21 | :undoc-members: 22 | :show-inheritance: 23 | 24 | weather\_provider\_api.routers.weather.sources.knmi.knmi\_factors module 25 | ------------------------------------------------------------------------ 26 | 27 | .. automodule:: weather_provider_api.routers.weather.sources.knmi.knmi_factors 28 | :members: 29 | :undoc-members: 30 | :show-inheritance: 31 | 32 | weather\_provider\_api.routers.weather.sources.knmi.stations module 33 | ------------------------------------------------------------------- 34 | 35 | .. automodule:: weather_provider_api.routers.weather.sources.knmi.stations 36 | :members: 37 | :undoc-members: 38 | :show-inheritance: 39 | 40 | weather\_provider\_api.routers.weather.sources.knmi.utils module 41 | ---------------------------------------------------------------- 42 | 43 | .. automodule:: weather_provider_api.routers.weather.sources.knmi.utils 44 | :members: 45 | :undoc-members: 46 | :show-inheritance: 47 | 48 | Module contents 49 | --------------- 50 | 51 | .. automodule:: weather_provider_api.routers.weather.sources.knmi 52 | :members: 53 | :undoc-members: 54 | :show-inheritance: 55 | -------------------------------------------------------------------------------- /sphinx-docs/weather_provider_api.routers.weather.sources.rst: -------------------------------------------------------------------------------- 1 | weather\_provider\_api.routers.weather.sources package 2 | ====================================================== 3 | 4 | Subpackages 5 | ----------- 6 | 7 | .. toctree:: 8 | :maxdepth: 4 9 | 10 | weather_provider_api.routers.weather.sources.cds 11 | weather_provider_api.routers.weather.sources.knmi 12 | weather_provider_api.routers.weather.sources.weather_alert 13 | 14 | Module contents 15 | --------------- 16 | 17 | .. automodule:: weather_provider_api.routers.weather.sources 18 | :members: 19 | :undoc-members: 20 | :show-inheritance: 21 | -------------------------------------------------------------------------------- /sphinx-docs/weather_provider_api.routers.weather.sources.weather_alert.rst: -------------------------------------------------------------------------------- 1 | weather\_provider\_api.routers.weather.sources.weather\_alert package 2 | ===================================================================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | weather\_provider\_api.routers.weather.sources.weather\_alert.weather\_alert module 8 | ----------------------------------------------------------------------------------- 9 | 10 | .. automodule:: weather_provider_api.routers.weather.sources.weather_alert.weather_alert 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | Module contents 16 | --------------- 17 | 18 | .. automodule:: weather_provider_api.routers.weather.sources.weather_alert 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | -------------------------------------------------------------------------------- /sphinx-docs/weather_provider_api.routers.weather.utils.rst: -------------------------------------------------------------------------------- 1 | weather\_provider\_api.routers.weather.utils package 2 | ==================================================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | weather\_provider\_api.routers.weather.utils.date\_helpers module 8 | ----------------------------------------------------------------- 9 | 10 | .. automodule:: weather_provider_api.routers.weather.utils.date_helpers 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | weather\_provider\_api.routers.weather.utils.file\_helpers module 16 | ----------------------------------------------------------------- 17 | 18 | .. automodule:: weather_provider_api.routers.weather.utils.file_helpers 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | weather\_provider\_api.routers.weather.utils.geo\_position module 24 | ----------------------------------------------------------------- 25 | 26 | .. automodule:: weather_provider_api.routers.weather.utils.geo_position 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | weather\_provider\_api.routers.weather.utils.grid\_helpers module 32 | ----------------------------------------------------------------- 33 | 34 | .. automodule:: weather_provider_api.routers.weather.utils.grid_helpers 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | weather\_provider\_api.routers.weather.utils.pandas\_helpers module 40 | ------------------------------------------------------------------- 41 | 42 | .. automodule:: weather_provider_api.routers.weather.utils.pandas_helpers 43 | :members: 44 | :undoc-members: 45 | :show-inheritance: 46 | 47 | weather\_provider\_api.routers.weather.utils.serializers module 48 | --------------------------------------------------------------- 49 | 50 | .. automodule:: weather_provider_api.routers.weather.utils.serializers 51 | :members: 52 | :undoc-members: 53 | :show-inheritance: 54 | 55 | Module contents 56 | --------------- 57 | 58 | .. automodule:: weather_provider_api.routers.weather.utils 59 | :members: 60 | :undoc-members: 61 | :show-inheritance: 62 | -------------------------------------------------------------------------------- /sphinx-docs/weather_provider_api.rst: -------------------------------------------------------------------------------- 1 | weather\_provider\_api package 2 | ============================== 3 | 4 | Subpackages 5 | ----------- 6 | 7 | .. toctree:: 8 | :maxdepth: 4 9 | 10 | weather_provider_api.config 11 | weather_provider_api.core 12 | weather_provider_api.routers 13 | weather_provider_api.scripts 14 | weather_provider_api.versions 15 | 16 | Submodules 17 | ---------- 18 | 19 | weather\_provider\_api.app\_version module 20 | ------------------------------------------ 21 | 22 | .. automodule:: weather_provider_api.app_version 23 | :members: 24 | :undoc-members: 25 | :show-inheritance: 26 | 27 | Module contents 28 | --------------- 29 | 30 | .. automodule:: weather_provider_api 31 | :members: 32 | :undoc-members: 33 | :show-inheritance: 34 | -------------------------------------------------------------------------------- /sphinx-docs/weather_provider_api.scripts.rst: -------------------------------------------------------------------------------- 1 | weather\_provider\_api.scripts package 2 | ====================================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | weather\_provider\_api.scripts.erase\_arome\_repository module 8 | -------------------------------------------------------------- 9 | 10 | .. automodule:: weather_provider_api.scripts.erase_arome_repository 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | weather\_provider\_api.scripts.erase\_era5land\_repository module 16 | ----------------------------------------------------------------- 17 | 18 | .. automodule:: weather_provider_api.scripts.erase_era5land_repository 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | weather\_provider\_api.scripts.erase\_era5sl\_repository module 24 | --------------------------------------------------------------- 25 | 26 | .. automodule:: weather_provider_api.scripts.erase_era5sl_repository 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | weather\_provider\_api.scripts.erase\_waarnemingen\_register module 32 | ------------------------------------------------------------------- 33 | 34 | .. automodule:: weather_provider_api.scripts.erase_waarnemingen_register 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | weather\_provider\_api.scripts.update\_arome\_repository module 40 | --------------------------------------------------------------- 41 | 42 | .. automodule:: weather_provider_api.scripts.update_arome_repository 43 | :members: 44 | :undoc-members: 45 | :show-inheritance: 46 | 47 | weather\_provider\_api.scripts.update\_era5land\_repository module 48 | ------------------------------------------------------------------ 49 | 50 | .. automodule:: weather_provider_api.scripts.update_era5land_repository 51 | :members: 52 | :undoc-members: 53 | :show-inheritance: 54 | 55 | weather\_provider\_api.scripts.update\_era5sl\_repository module 56 | ---------------------------------------------------------------- 57 | 58 | .. automodule:: weather_provider_api.scripts.update_era5sl_repository 59 | :members: 60 | :undoc-members: 61 | :show-inheritance: 62 | 63 | weather\_provider\_api.scripts.update\_waarnemingen\_register module 64 | -------------------------------------------------------------------- 65 | 66 | .. automodule:: weather_provider_api.scripts.update_waarnemingen_register 67 | :members: 68 | :undoc-members: 69 | :show-inheritance: 70 | 71 | Module contents 72 | --------------- 73 | 74 | .. automodule:: weather_provider_api.scripts 75 | :members: 76 | :undoc-members: 77 | :show-inheritance: 78 | -------------------------------------------------------------------------------- /sphinx-docs/weather_provider_api.versions.rst: -------------------------------------------------------------------------------- 1 | weather\_provider\_api.versions package 2 | ======================================= 3 | 4 | Submodules 5 | ---------- 6 | 7 | weather\_provider\_api.versions.v1 module 8 | ----------------------------------------- 9 | 10 | .. automodule:: weather_provider_api.versions.v1 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | weather\_provider\_api.versions.v2 module 16 | ----------------------------------------- 17 | 18 | .. automodule:: weather_provider_api.versions.v2 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | Module contents 24 | --------------- 25 | 26 | .. automodule:: weather_provider_api.versions 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # SPDX-FileCopyrightText: 2019-2022 Alliander N.V. 5 | # SPDX-License-Identifier: MPL-2.0 6 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # SPDX-FileCopyrightText: 2019-2022 Alliander N.V. 5 | # SPDX-License-Identifier: MPL-2.0 6 | 7 | import tempfile 8 | from datetime import datetime 9 | from pathlib import Path 10 | 11 | import numpy as np 12 | import pandas as pd 13 | import pytest 14 | import xarray as xr 15 | from dateutil.relativedelta import relativedelta 16 | 17 | from weather_provider_api.routers.weather.utils.geo_position import GeoPosition 18 | from weather_provider_api.routers.weather.utils.pandas_helpers import coords_to_pd_index 19 | 20 | 21 | @pytest.fixture(scope="session") 22 | def _get_mock_repository_dir(): 23 | return Path(tempfile.gettempdir()).joinpath("PyTest_REPO") 24 | 25 | 26 | @pytest.fixture(scope="session") 27 | def mock_coordinates(): 28 | return [(51.873419, 5.705929), (53.2194, 6.5665)] 29 | 30 | 31 | @pytest.fixture(scope="session") 32 | def mock_factors(): 33 | return [ 34 | "fake_factor_1", 35 | "fake_factor_2", 36 | "fake_factor_3", 37 | "fake_factor_4", 38 | "fake_factor_5", 39 | ] 40 | 41 | 42 | @pytest.fixture(scope="session") 43 | def mock_dataset(mock_coordinates, mock_factors): 44 | """ 45 | returns a mock Xarray Dataset for 46 | Args: 47 | mock_coordinates: 48 | mock_factors: 49 | 50 | Returns: 51 | 52 | """ 53 | timeline = pd.date_range(end=datetime.utcnow(), periods=96, freq="1H", inclusive="left") 54 | coord_indices = coords_to_pd_index([GeoPosition(51.873419, 5.705929), GeoPosition(53.2194, 6.5665)]) 55 | weather_factors = mock_factors 56 | data_dict = { 57 | weather_factor: ( 58 | ["time", "coord"], 59 | np.zeros(shape=(len(timeline), len(coord_indices)), dtype=np.float64), 60 | ) 61 | for weather_factor in weather_factors 62 | } 63 | ds = xr.Dataset(data_vars=data_dict, coords={"time": timeline, "coord": coord_indices}) 64 | ds = ds.unstack("coord") 65 | return ds 66 | 67 | 68 | @pytest.fixture(scope="session") 69 | def mock_dataset_era5(mock_coordinates, mock_factors): 70 | """ 71 | returns a mock Xarray Dataset for 72 | Args: 73 | mock_coordinates: 74 | mock_factors: 75 | 76 | Returns: 77 | 78 | """ 79 | timeline = pd.date_range( 80 | end=(datetime.utcnow() - relativedelta(days=61)), 81 | periods=96, 82 | freq="1H", 83 | inclusive="left", 84 | ) 85 | coord_indices = coords_to_pd_index([GeoPosition(51.873419, 5.705929), GeoPosition(53.2194, 6.5665)]) 86 | weather_factors = mock_factors 87 | data_dict = { 88 | weather_factor: ( 89 | ["time", "coord"], 90 | np.zeros(shape=(len(timeline), len(coord_indices)), dtype=np.float64), 91 | ) 92 | for weather_factor in weather_factors 93 | } 94 | ds = xr.Dataset(data_vars=data_dict, coords={"time": timeline, "coord": coord_indices}) 95 | ds = ds.unstack("coord") 96 | return ds 97 | 98 | 99 | @pytest.fixture(scope="session") 100 | def mock_dataset_arome(mock_coordinates, mock_factors): 101 | """ 102 | returns a mock Xarray Dataset for 103 | Args: 104 | mock_coordinates: 105 | mock_factors: 106 | 107 | Returns: 108 | 109 | """ 110 | timeline = pd.date_range( 111 | end=(datetime.utcnow() - relativedelta(days=6)), 112 | periods=96, 113 | freq="1H", 114 | inclusive="left", 115 | ) 116 | coord_indices = coords_to_pd_index([GeoPosition(51.873419, 5.705929), GeoPosition(53.2194, 6.5665)]) 117 | weather_factors = mock_factors 118 | data_dict = { 119 | weather_factor: ( 120 | ["prediction_moment", "time", "coord"], 121 | np.zeros(shape=(48, len(timeline), len(coord_indices)), dtype=np.float64), 122 | ) 123 | for weather_factor in weather_factors 124 | } 125 | 126 | ds = xr.Dataset( 127 | data_vars=data_dict, 128 | coords={ 129 | "prediction_moment": timeline[0:48], 130 | "time": timeline, 131 | "coord": coord_indices, 132 | }, 133 | ) 134 | ds = ds.unstack("coord") 135 | return ds 136 | -------------------------------------------------------------------------------- /tests/pytest.ini: -------------------------------------------------------------------------------- 1 | ; SPDX-FileCopyrightText: 2019-2022 Alliander N.V. 2 | ; SPDX-License-Identifier: MPL-2.0 3 | 4 | [pytest] 5 | junit_family=xunit1junit_family=xunit1 6 | -------------------------------------------------------------------------------- /tests/test_api_models.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # SPDX-FileCopyrightText: 2019-2022 Alliander N.V. 5 | # SPDX-License-Identifier: MPL-2.0 6 | 7 | from datetime import datetime 8 | 9 | from weather_provider_api.routers.weather.api_models import ScientificJSONResponse 10 | 11 | 12 | # The custom float encoder is the sole function inside api_models.py and is therefore the only thing that needs testing. 13 | def test_custom_json_float_encoder(): 14 | # Setting up generic weather output format, ready for value injection 15 | mock_response = ScientificJSONResponse( 16 | { 17 | "coords": { 18 | "time": { 19 | "dims": ("time",), 20 | "attrs": {"long_name": "time"}, 21 | "data": [datetime.utcnow()], 22 | "coord": {"dims": ("coord",), "attrs": {}, "data": [(5.25, 52.0)]}, 23 | }, 24 | "attrs": {}, 25 | "dims": {"coord": 1, "time": 744}, 26 | "data_vars": { 27 | "2m_dewpoint_temperature": { 28 | "dims": ("time", "coord"), 29 | "attrs": { 30 | "units": "K", 31 | "long_name": "2 metre dewpoint temperature", 32 | }, 33 | "data": [[279.6545715332031]], 34 | } 35 | }, 36 | } 37 | } 38 | ) 39 | 40 | # Test NaN 41 | assert mock_response.render(float("nan")) == b"null" 42 | 43 | # Test infinity 44 | assert mock_response.render(float("inf")) == b"Infinity" 45 | 46 | # Test negative infinity 47 | assert mock_response.render(-float("inf")) == b"-Infinity" 48 | -------------------------------------------------------------------------------- /tests/test_api_view_v1.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # SPDX-FileCopyrightText: 2019-2022 Alliander N.V. 5 | # SPDX-License-Identifier: MPL-2.0 6 | 7 | from weather_provider_api.routers.weather.api_view_v2 import header_accept_type 8 | 9 | 10 | def test_header_accept_type(): 11 | assert header_accept_type(accept=str("application/netcdf")) == "netcdf4" 12 | assert header_accept_type(accept=str("application/netcdf3")) == "netcdf3" 13 | assert header_accept_type(accept=str("application/json")) == "json" 14 | assert header_accept_type(accept=str("application/json-dataset")) == "json_dataset" 15 | assert header_accept_type(accept=str("text/csv")) == "csv" 16 | 17 | 18 | # The get_source, get_sources and get_sync_models only pass on requests to other functions and do not need to be tested 19 | 20 | # All the code inside get_sync_weather consists of either externally called functions that are already covered in 21 | # their respective modules, or either Python system calls or calls to external libraries outside the testing scope. 22 | # Full coverage is therefore implied and assumed 23 | -------------------------------------------------------------------------------- /tests/test_date_helpers.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # SPDX-FileCopyrightText: 2019-2022 Alliander N.V. 5 | # SPDX-License-Identifier: MPL-2.0 6 | 7 | from datetime import datetime 8 | 9 | import pytest 10 | from fastapi import HTTPException 11 | 12 | import weather_provider_api.routers.weather.utils.date_helpers as dh 13 | 14 | 15 | # The function time_unknown() isn't tested as it only verifies that no time may have been set in the datetime conversion 16 | # and that the datetime string didn't contain a colon. 17 | 18 | 19 | # The parse_datetime() function converts a string to a datetime, returning the exact date and time passed as a string. 20 | # If no time was given, however: 21 | # - It should return 00:00 if neither time or date were rounded up 22 | # - It should return 23:59:59 for the given day if the time is rounded up 23 | # - It should return 00:00 for the next day if the date is rounded up 24 | # (if both were rounded up, only the day roundup is used) 25 | # Also, if an invalid string is passed, the result should be: 26 | # - None, if the raise_errors parameter was False 27 | # - A FastAPI HTTPException (status code 422), also containing any given value for loc, 28 | # if the raise_errors parameter was True 29 | # If no value was given at all, None should be returned 30 | @pytest.mark.parametrize( 31 | "test_datetime,test_round_time,test_round_days,test_raise_errors,test_loc,test_result", 32 | [ 33 | ("2019-01-01 00:00", False, False, False, None, datetime(2019, 1, 1, 0, 0)), 34 | ("2019-01-01 23:59", True, True, False, None, datetime(2019, 1, 1, 23, 59)), 35 | ("2019-01-01", True, False, False, None, datetime(2019, 1, 1, 23, 59, 59)), 36 | ("2019-01-01", False, True, False, None, datetime(2019, 1, 2, 0, 0)), 37 | ("2019-01-01", True, True, False, None, datetime(2019, 1, 2, 0, 0)), 38 | ], 39 | ) 40 | def test_parse_datetime( 41 | test_datetime, 42 | test_round_time, 43 | test_round_days, 44 | test_raise_errors, 45 | test_loc, 46 | test_result, 47 | ): 48 | assert ( 49 | dh.parse_datetime( 50 | datetime_string=test_datetime, 51 | round_missing_time_up=test_round_time, 52 | round_to_days=test_round_days, 53 | raise_errors=test_raise_errors, 54 | loc=test_loc, 55 | ) 56 | == test_result 57 | ) 58 | 59 | 60 | def test_parse_datetime_error_handling(): 61 | with pytest.raises(HTTPException) as e: 62 | assert dh.parse_datetime( 63 | datetime_string="2019-01-0Z", 64 | round_missing_time_up=False, 65 | round_to_days=False, 66 | raise_errors=True, 67 | loc="Oh no", 68 | ) 69 | assert e is not None 70 | 71 | assert ( 72 | dh.parse_datetime( 73 | datetime_string="2019-01-0Z", 74 | round_missing_time_up=False, 75 | round_to_days=False, 76 | raise_errors=False, 77 | loc="Oh no", 78 | ) 79 | is None 80 | ) 81 | 82 | assert ( 83 | dh.parse_datetime( 84 | datetime_string=None, 85 | round_missing_time_up=False, 86 | round_to_days=False, 87 | raise_errors=False, 88 | loc="Oh no", 89 | ) 90 | is None 91 | ) 92 | 93 | 94 | """ 95 | @pytest.mark.parametrize("starting_date, ending_date, result", [ 96 | # VALIDATION TEST 1: Date range lies within scope. Expected results: identical to input 97 | (datetime(2019, 4, 13), datetime(2019, 4, 18), (datetime(2019, 4, 13), datetime(2019, 4, 18))), 98 | # VALIDATION TEST 2: Starting time before scope. Expected results: move start to repo start 99 | (datetime(2019, 3, 1), datetime(2019, 3, 10), (datetime(2019, 3, 3), datetime(2019, 3, 10))), 100 | # VALIDATION TEST 3: Ending time after scope. Expected results: move end to repo end 101 | (datetime(2020, 3, 11), datetime(2020, 8, 12), (datetime(2020, 3, 11), datetime(2020, 8, 8))), 102 | # VALIDATION TEST 4: Range before scope. Expected results: end at 7 days after starting time, start at starting time 103 | (datetime(2019, 1, 1), datetime(2019, 1, 12), (datetime(2019, 3, 3), datetime(2019, 3, 10))), 104 | # VALIDATION TEST 5: Range after scope. Expected results: end at ending time, start seven days before that 105 | (datetime(2021, 1, 1), datetime(2021, 1, 12), (datetime(2020, 8, 1), datetime(2020, 8, 8))), 106 | # VALIDATION TEST 6: No starting time. Expected results: starting time at 7 days before ending time 107 | (None, datetime(2019, 4, 18), (datetime(2019, 4, 11), datetime(2019, 4, 18))), 108 | # VALIDATION TEST 7: No ending time. Expected results: ending time at repo ending time 109 | (datetime(2019, 4, 13), None, (datetime(2019, 4, 13), datetime(2020, 8, 8))), 110 | # VALIDATION TEST 8: No times at all. Expected results: ending time at repo ending, starting time 7 days before that 111 | (None, None, (datetime(2020, 8, 1), datetime(2020, 8, 8))), 112 | ]) 113 | def test_validate_begin_and_end(starting_date, ending_date, result): 114 | # Testing assumes a repo starting date of 2019-03-03 and an ending date of 2020-08-08 115 | assert dh.validate_begin_and_end(starting_date, ending_date, datetime(2019, 3, 3), datetime(2020, 8, 8)) == result 116 | """ # Temporarily disabled due to fixes for this function 117 | -------------------------------------------------------------------------------- /tests/test_geo_position.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # SPDX-FileCopyrightText: 2019-2022 Alliander N.V. 5 | # SPDX-License-Identifier: MPL-2.0 6 | 7 | import pytest 8 | 9 | from weather_provider_api.routers.weather.utils.geo_position import GeoPosition 10 | 11 | 12 | @pytest.fixture 13 | def input_value(): 14 | return 5 15 | 16 | 17 | def test_location_type_detection(): 18 | # Tests to verify proper type detection for regular and extreme coordinate values 19 | 20 | # Test 1: Out of bounds 'coordinate' for both WGS84 and RD, using auto-detection 21 | with pytest.raises(ValueError) as e: 22 | assert GeoPosition(52.2, 182) 23 | assert str(e.value.args[0]) == "No valid coordinate system could be determined from the coordinates given.." 24 | 25 | # Test 2: Value in bounds as WSG84, but specified as RD and out of bounds for RD. 26 | with pytest.raises(ValueError) as e: 27 | assert GeoPosition(52.2, 90, "RD") 28 | assert str(e.value.args[0]) == "Invalid coordinates for type were used" 29 | 30 | # Test 3: Value in bounds as RD, but specified as WGS84 and out of bounds for WGS84. 31 | with pytest.raises(ValueError) as e: 32 | assert GeoPosition(155000, 463000, "WGS84") 33 | assert str(e.value.args[0]) == "Invalid coordinates for type were used" 34 | 35 | # Test 4: Highest possible and lowest possible values for WGS84, autodetect. 36 | assert GeoPosition(180, 90) 37 | assert GeoPosition(-180, -90) 38 | 39 | # Test 5: Highest possible and lowest possible values for RD, autodetect. 40 | assert GeoPosition(7000, 289000) 41 | assert GeoPosition(300000, 629000) 42 | 43 | # Test 6: Unknown coordinate system is passed for a coordinate. Coordinate resolves (to RD) though. 44 | assert ( 45 | GeoPosition(155000, 463000, "MARSHMALLOW").system == "RD" 46 | ), "GeoPosition should determine RD to be the correctformat" 47 | 48 | # Test 7: Unknown coordinate system is passed for a coordinate. Coordinate does not resolve. 49 | with pytest.raises(ValueError) as e: 50 | GeoPosition(-1000, -2000, "MARSHMALLOW") 51 | assert str(e.value.args[0]) == "No valid coordinate system could be determined from the coordinates given.." 52 | 53 | 54 | def test_coordinate_live_locations(): 55 | """ 56 | Tests using existing locations in the Netherlands to verify proper conversion between RD and WGS84 57 | Location positions were verified using reliable online registers containing RD coordinates for streets, matching 58 | them to their Google WGS84 locations, and using a known reliable tertiary conversion tool to confirm everything. 59 | """ 60 | 61 | # Test 1: "Onze Lieve Vrouwe Toren" - Amersfoort (building and the center point of the RD coordinate system) 62 | geo_pos = GeoPosition(155000, 463000, "RD") 63 | 64 | assert geo_pos.get_RD() == ( 65 | 155000, 66 | 463000, 67 | ), "Error - Value not properly saved as RD-value" 68 | assert geo_pos.get_WGS84() == ( 69 | 52.15517440, 70 | 5.38720621, 71 | ), "Error - WGS84 value not within allowed constraints" 72 | 73 | # Test 2: Wijnbergseweg - Braamt (street on the middle east side of the country) 74 | geo_pos = GeoPosition(215803, 438150) 75 | 76 | assert geo_pos.get_RD() == ( 77 | 215803, 78 | 438150, 79 | ), "Error - Value not properly identified or saved as RD" 80 | assert geo_pos.get_WGS84() == pytest.approx( 81 | (51.92849584, 6.27121733), rel=1e-5 82 | ), "Error - WGS84 value not within allowed constraints" 83 | 84 | # Test 3: Admiralengracht - Amsterdam (street on the middle west side of the country) 85 | geo_pos = GeoPosition(52.36954423, 4.85667541) 86 | 87 | assert geo_pos.get_RD() == ( 88 | 118868, 89 | 486984, 90 | ), "Error - RD value not within allowed constraints" 91 | assert geo_pos.get_WGS84() == ( 92 | 52.36954423, 93 | 4.85667541, 94 | ), "Error - Value not properly identified or saved as WGS84" 95 | 96 | # Test 4: Gasthuisstraat - Dokkum (street on the middle north side of the country) 97 | geo_pos = GeoPosition(195703, 593452) 98 | 99 | assert geo_pos.get_RD() == ( 100 | 195703, 101 | 593452, 102 | ), "Error - Value not properly identified or saved as RD" 103 | assert geo_pos.get_WGS84() == pytest.approx( 104 | (53.3259597, 5.9980788), rel=1e-5 105 | ), "Error - WGS84 value not within allowed constraints" 106 | 107 | # Test 5: Burgemeester de Grauwstraat - Baarle Nassau (street on the middle south side of the country) 108 | geo_pos = GeoPosition(51.4450399, 4.9284643) 109 | 110 | assert geo_pos.get_RD() == ( 111 | 123108, 112 | 384094, 113 | ), "Error - RD value not within allowed constraints" 114 | assert geo_pos.get_WGS84() == ( 115 | 51.4450399, 116 | 4.9284643, 117 | ), "Error - Value not properly identified or saved as WGS84" 118 | -------------------------------------------------------------------------------- /tests/test_grid_helpers.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # SPDX-FileCopyrightText: 2019-2022 Alliander N.V. 5 | # SPDX-License-Identifier: MPL-2.0 6 | 7 | import pytest 8 | 9 | from weather_provider_api.routers.weather.utils.geo_position import ( 10 | CoordinateSystem, 11 | GeoPosition, 12 | ) 13 | from weather_provider_api.routers.weather.utils.grid_helpers import ( 14 | round_coordinates_to_wgs84_grid, 15 | ) 16 | 17 | 18 | @pytest.mark.parametrize( 19 | "coordinates, grid_resolution_lat_lon, starting_points_lat_lon, expected_results", 20 | [ 21 | ( 22 | [GeoPosition(6.0, 9.0, CoordinateSystem.wgs84)], 23 | (0.25, 0.25), 24 | (5.0, 10.0), 25 | [GeoPosition(6.0, 9.0, CoordinateSystem.wgs84)], 26 | ), # Already properly formatted 27 | ( 28 | [GeoPosition(6.11, 9.22, CoordinateSystem.wgs84)], 29 | (0.25, 0.25), 30 | (5.0, 10.0), 31 | [GeoPosition(6.0, 9.25, CoordinateSystem.wgs84)], 32 | ), # Rounding simple 33 | ( 34 | [GeoPosition(-6, -9, CoordinateSystem.wgs84)], 35 | (0.25, 0.25), 36 | (5.0, 10.0), 37 | [GeoPosition(-6, -9, CoordinateSystem.wgs84)], 38 | ), # Negative modifiers and before "starting point of grid" 39 | ( 40 | [GeoPosition(7.112233444, 9.3423421233, CoordinateSystem.wgs84)], 41 | (0.25, 0.25), 42 | (5.0, 10.0), 43 | [GeoPosition(7, 9.25, CoordinateSystem.wgs84)], 44 | ), # Extreme coordinate values 45 | ( 46 | [GeoPosition(-6.92, -9.2, CoordinateSystem.wgs84)], 47 | (0.125, 0.75), 48 | (5.0, 10.0), 49 | [GeoPosition(-6.875, -9.5, CoordinateSystem.wgs84)], 50 | ), # Negative modifiers and before "starting point of grid" 51 | ], 52 | ) 53 | def test_round_to_grid(coordinates, grid_resolution_lat_lon, starting_points_lat_lon, expected_results): 54 | results = round_coordinates_to_wgs84_grid(coordinates, grid_resolution_lat_lon, starting_points_lat_lon) 55 | 56 | for result_coordinate, expected_result_coordinate in zip(results, expected_results): 57 | assert result_coordinate.get_WGS84() == expected_result_coordinate.get_WGS84() 58 | -------------------------------------------------------------------------------- /tests/test_knmi_actuele_waarnemingen.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # SPDX-FileCopyrightText: 2019-2022 Alliander N.V. 5 | # SPDX-License-Identifier: MPL-2.0 6 | 7 | import locale 8 | from datetime import datetime 9 | 10 | import numpy as np 11 | import pytest 12 | import requests 13 | import xarray as xr 14 | 15 | from weather_provider_api.routers.weather.sources.knmi.models.actuele_waarnemingen import ( 16 | ActueleWaarnemingenModel, 17 | ) 18 | from weather_provider_api.routers.weather.sources.knmi.utils import ( 19 | _retrieve_observation_moment, 20 | download_actuele_waarnemingen_weather, 21 | ) 22 | from weather_provider_api.routers.weather.utils.geo_position import GeoPosition 23 | 24 | 25 | @pytest.fixture() 26 | def start(): 27 | return np.datetime64("2018-01-01") 28 | 29 | 30 | @pytest.fixture() 31 | def end(): 32 | return np.datetime64("2018-01-31") 33 | 34 | 35 | def test_get_weather(mock_coordinates, start, end): 36 | mock_geo_coordinates = [GeoPosition(coordinate[0], coordinate[1]) for coordinate in mock_coordinates] 37 | aw_model = ActueleWaarnemingenModel() 38 | 39 | # TODO: Monkeypatch the download call to test without connection 40 | ds = aw_model.get_weather(coords=mock_geo_coordinates, begin=start, end=end) 41 | 42 | assert ds is not None 43 | assert "temperature" in ds 44 | assert len(ds["temperature"]) == 1 45 | assert isinstance(ds, xr.Dataset) 46 | 47 | 48 | @pytest.mark.skip(reason="Test currently not working via Tox on GitHub Actions") 49 | def test__retrieve_observation_date(): 50 | # Test to verify error handling 51 | current_locale = locale.getlocale(locale.LC_TIME) 52 | locale.setlocale(locale.LC_TIME, "dutch") 53 | assert _retrieve_observation_moment(None).date() == datetime.now().date() # System now 54 | locale.setlocale(locale.LC_TIME, current_locale) 55 | 56 | 57 | def test__download_weather(monkeypatch): 58 | # Test to verify error handling 59 | def mock_request_get(_, *args, **kwargs): 60 | raise requests.exceptions.BaseHTTPError("Fake BaseHTTP Error!") 61 | 62 | monkeypatch.setattr(requests, "get", mock_request_get) 63 | assert download_actuele_waarnemingen_weather() is None 64 | -------------------------------------------------------------------------------- /tests/test_knmi_arome_model.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # SPDX-FileCopyrightText: 2019-2022 Alliander N.V. 5 | # SPDX-License-Identifier: MPL-2.0 6 | 7 | import random 8 | from datetime import datetime, timedelta 9 | 10 | import xarray as xr 11 | from dateutil.relativedelta import relativedelta 12 | 13 | from weather_provider_api.routers.weather.sources.cds.factors import era5sl_factors 14 | from weather_provider_api.routers.weather.sources.cds.models.era5sl import ERA5SLModel 15 | from weather_provider_api.routers.weather.sources.knmi.client.arome_repository import ( 16 | HarmonieAromeRepository, 17 | ) 18 | from weather_provider_api.routers.weather.sources.knmi.models.harmonie_arome import ( 19 | HarmonieAromeModel, 20 | ) 21 | from weather_provider_api.routers.weather.utils.geo_position import GeoPosition 22 | 23 | 24 | # The get_weather() function is not part of the test, as it consists purely of function calls to other functions that 25 | # are already part of the testing regiment. 26 | # The _get_list_of_factors_to_drop() function is not part of the test as it only compares the passed list to a list of 27 | # ERA5SL factors and returns those ERA5SL factors that weren't on the list. Testing would be more error-prone and larger 28 | # than the actual function. 29 | # The _get_list_of_months function isn't tested as it only takes the first and last month of the passed time grid and 30 | # uses the Numpy 'arange' function to turn those into a range of months. 31 | 32 | 33 | # Valid periods should be between three years ago (rounding down the month), and 5 days before today 34 | def test__validate_weather_factors(): 35 | arome_model = ERA5SLModel() 36 | 37 | # TEST 1: No factors are passed. The full standard list of factors should be returned. 38 | assert arome_model._validate_weather_factors(None) == list(era5sl_factors.values()) 39 | 40 | # TEST 2: Only valid factors are passed. The same list should be returned. 41 | list_of_factors = list(era5sl_factors.keys()) 42 | random_list = [ 43 | random.choice(list_of_factors), 44 | random.choice(list_of_factors), 45 | random.choice(list_of_factors), 46 | ] 47 | expected_returns = [ 48 | era5sl_factors[random_list[0]], 49 | era5sl_factors[random_list[1]], 50 | era5sl_factors[random_list[2]], 51 | ] 52 | assert arome_model._validate_weather_factors(random_list) == expected_returns 53 | 54 | # TEST 3: Valid and invalid factors are passed. A KeyError should occur. 55 | list_of_factors = list(era5sl_factors.keys()) 56 | random_list = [ 57 | random.choice(list_of_factors), 58 | random.choice(list_of_factors), 59 | random.choice(list_of_factors), 60 | "mock_factor", 61 | ] 62 | assert arome_model._validate_weather_factors(random_list) 63 | assert "mock_factor" not in arome_model._validate_weather_factors(random_list) 64 | 65 | 66 | def test_retrieve_weather(monkeypatch, mock_dataset_arome: xr.Dataset, _get_mock_repository_dir): 67 | five_days_ago = datetime.utcnow() - timedelta(days=5) 68 | one_month_ago = five_days_ago - relativedelta(months=1) 69 | 70 | # Instead of returning the regular data 71 | def mock_fill_dataset_with_data(self, begin, end, coordinates): 72 | return mock_dataset_arome 73 | 74 | monkeypatch.setattr(HarmonieAromeRepository, "gather_period", mock_fill_dataset_with_data) 75 | 76 | arome_model = HarmonieAromeModel() 77 | # The coordinates requested are those of Amsterdam and Arnhem 78 | ds = arome_model.get_weather( 79 | coords=[GeoPosition(52.3667, 4.8945), GeoPosition(51.9851, 5.8987)], 80 | begin=one_month_ago, 81 | end=five_days_ago, 82 | ) 83 | 84 | # TODO: len doesn't match expectations for the field. Find out why 85 | # TODO: The mock-format needs to be enhanced to properly pass the formatting round. This already happens when not 86 | # mocking, but needs to be supported by the mock-result as well, to get this to properly work offline.. 87 | assert ds is not None 88 | assert "fake_factor_1" in ds 89 | assert len(ds["fake_factor_1"]) == 48 # 49 prediction moments per time in the mock dataset 90 | assert len(ds["fake_factor_1"][0]) == 95 # 96 periods in the mock dataset 91 | assert isinstance(ds, xr.Dataset) 92 | -------------------------------------------------------------------------------- /tests/test_knmi_daggegevens.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # SPDX-FileCopyrightText: 2019-2022 Alliander N.V. 5 | # SPDX-License-Identifier: MPL-2.0 6 | 7 | from datetime import datetime 8 | 9 | import pytest 10 | import requests 11 | import xarray as xr 12 | 13 | from weather_provider_api.routers.weather.sources.knmi.models.daggegevens import ( 14 | DagGegevensModel, 15 | ) 16 | from weather_provider_api.routers.weather.utils.geo_position import GeoPosition 17 | 18 | inseason_options = {True, False} 19 | 20 | 21 | @pytest.fixture() 22 | def start(): 23 | return datetime(year=2019, month=1, day=1) 24 | 25 | 26 | @pytest.fixture() 27 | def end(): 28 | return datetime(year=2019, month=1, day=31) 29 | 30 | 31 | @pytest.mark.parametrize("inseason", inseason_options) 32 | def test_retrieve_weather(mock_coordinates, start, end, inseason): 33 | mock_geoposition_coordinates = [GeoPosition(coordinate[0], coordinate[1]) for coordinate in mock_coordinates] 34 | # TODO: Monkeypatch the download call to test without connection 35 | daggegevens_model = DagGegevensModel() 36 | ds = daggegevens_model.get_weather(coords=mock_geoposition_coordinates, begin=start, end=end, inseason=inseason) 37 | 38 | print(ds["time"]) 39 | assert ds is not None 40 | assert "TN" in ds 41 | assert len(ds["TN"]) == 31 # TODO: Evaluate changes in Daggegevens to handle end date differently 42 | assert isinstance(ds, xr.Dataset) 43 | 44 | 45 | def test__download_weather(monkeypatch): 46 | # Test for HTTP exceptions 47 | dag_model = DagGegevensModel() 48 | 49 | class MockResponse: 50 | def __init__(self, json_data, status_code): 51 | self.json_data = json_data 52 | self.status_code = status_code 53 | 54 | def json(self): 55 | return self.json_data 56 | 57 | def mock_request_post(*args, **kwargs): 58 | return MockResponse({"dummy": "value"}, 404) 59 | 60 | monkeypatch.setattr(requests, "post", mock_request_post) 61 | 62 | with pytest.raises(requests.HTTPError) as e: 63 | assert dag_model._download_weather( 64 | [1, 2], 65 | datetime(year=2020, month=3, day=1), 66 | datetime(year=2020, month=3, day=2), 67 | False, 68 | None, 69 | ) 70 | assert str(e.value.args[0]) == "Failed to retrieve data from the KNMI website" 71 | -------------------------------------------------------------------------------- /tests/test_knmi_pluim.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # SPDX-FileCopyrightText: 2019-2022 Alliander N.V. 5 | # SPDX-License-Identifier: MPL-2.0 6 | 7 | from datetime import datetime 8 | 9 | import pytest 10 | import requests 11 | import xarray as xr 12 | from dateutil.relativedelta import relativedelta 13 | 14 | from weather_provider_api.routers.weather.sources.knmi.models.pluim import PluimModel 15 | from weather_provider_api.routers.weather.utils.geo_position import GeoPosition 16 | 17 | 18 | @pytest.fixture() 19 | def start(): 20 | return datetime.utcnow() 21 | 22 | 23 | @pytest.fixture() 24 | def end(): 25 | return datetime.utcnow() + relativedelta(days=15) 26 | 27 | 28 | def test_retrieve_weather(monkeypatch, mock_coordinates, start, end): 29 | mock_geoposition_coordinates = [GeoPosition(coordinate[0], coordinate[1]) for coordinate in mock_coordinates] 30 | # TODO: Monkeypatch the download call to test without connection 31 | 32 | # TEST 1: Regular usage, with a non-existing factor 33 | pluim_model = PluimModel() 34 | # Factors contain both existing and non-existing factors. Non-existing factors should just be ignored.. 35 | mock_factors = [ 36 | "fake_factor_1", 37 | "wind_speed", 38 | "wind_direction", 39 | "short_time_wind_speed", 40 | "temperature", 41 | "precipitation", 42 | "cape", 43 | ] 44 | ds = pluim_model.get_weather( 45 | coords=mock_geoposition_coordinates, 46 | begin=start, 47 | end=end, 48 | weather_factors=mock_factors, 49 | ) 50 | 51 | assert ds is not None 52 | assert "wind_speed" in ds 53 | assert "precipitation_sum" not in ds 54 | assert "fake_factor_1" not in ds 55 | assert isinstance(ds, xr.Dataset) 56 | 57 | # TEST 2: Empty list of weather factors should get the full set 58 | mock_factors = None 59 | ds = pluim_model.get_weather( 60 | coords=mock_geoposition_coordinates, 61 | begin=start, 62 | end=end, 63 | weather_factors=mock_factors, 64 | ) 65 | 66 | assert ds is not None 67 | assert "wind_speed" in ds 68 | assert "precipitation_sum" in ds 69 | assert isinstance(ds, xr.Dataset) 70 | 71 | # TEST 3: Test for HTTPError handling of non-200 status codes 72 | class MockResponse: 73 | def __init__(self, json_data, status_code): 74 | self.json_data = json_data 75 | self.status_code = status_code 76 | 77 | def json(self): 78 | return self.json_data 79 | 80 | def mock_request_get(self, *args, **kwargs): 81 | return MockResponse({"dummy": "value"}, 404) 82 | 83 | monkeypatch.setattr(requests, "get", mock_request_get) 84 | 85 | with pytest.raises(requests.exceptions.HTTPError) as e: 86 | pluim_model.get_weather( 87 | coords=mock_geoposition_coordinates, 88 | begin=start, 89 | end=end, 90 | weather_factors=mock_factors, 91 | ) 92 | 93 | assert str(e.value.args[0])[:53] == "Failed to retrieve data from the KNMI website" 94 | -------------------------------------------------------------------------------- /tests/test_knmi_uurgegevens.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # SPDX-FileCopyrightText: 2019-2022 Alliander N.V. 5 | # SPDX-License-Identifier: MPL-2.0 6 | 7 | from datetime import datetime 8 | 9 | import pytest 10 | import requests 11 | import xarray as xr 12 | 13 | from weather_provider_api.routers.weather.sources.knmi.models.uurgegevens import ( 14 | UurgegevensModel, 15 | ) 16 | from weather_provider_api.routers.weather.utils.geo_position import GeoPosition 17 | 18 | 19 | @pytest.fixture() 20 | def start(): 21 | today = datetime.today() 22 | year_to_use = today.year if today.month != 1 else (today.year - 1) # This year if not January, else previous year 23 | return datetime(year_to_use, 1, 1) # The start of the current year 24 | 25 | 26 | @pytest.fixture() 27 | def end(): 28 | today = datetime.today() 29 | year_to_use = today.year if today.month != 1 else (today.year - 1) # This year if not January, else previous year 30 | return datetime(year_to_use, 1, 31) # The end of first month of the current year 31 | 32 | 33 | def test_retrieve_weather(monkeypatch, mock_coordinates, start, end): 34 | mock_geoposition_coordinates = [GeoPosition(coordinate[0], coordinate[1]) for coordinate in mock_coordinates] 35 | # Version 3.x will be tested without an actual connection. 36 | uurgegevens_model = UurgegevensModel() 37 | ds = uurgegevens_model.get_weather(coords=mock_geoposition_coordinates, begin=start, end=end) 38 | 39 | assert ds is not None 40 | assert "TD" in ds 41 | assert isinstance(ds, xr.Dataset) 42 | 43 | # TEST 3: Test for HTTPError handling of non-200 status codes 44 | class MockResponse: 45 | def __init__(self, json_data, status_code): 46 | self.json_data = json_data 47 | self.status_code = status_code 48 | 49 | def json(self): 50 | return self.json_data 51 | 52 | def mock_request_post(*args, **kwargs): 53 | return MockResponse({"dummy": "value"}, 404) 54 | 55 | monkeypatch.setattr(requests, "post", mock_request_post) 56 | 57 | with pytest.raises(requests.exceptions.HTTPError) as e: 58 | uurgegevens_model.get_weather( 59 | coords=mock_geoposition_coordinates, 60 | begin=start, 61 | end=end, 62 | weather_factors=None, 63 | ) 64 | 65 | assert str(e.value.args[0]) == "Failed to retrieve data from the KNMI website" 66 | 67 | 68 | def test__create_request_params(): 69 | # If no weather factors are passed, the _create_request_params() function should return ["ALL"] 70 | # as the list of weather factors 71 | dag_model = UurgegevensModel() 72 | params_result = dag_model._create_request_params( 73 | datetime(2019, 4, 13), datetime(2019, 4, 18), ["DUMMYSTATION"], None 74 | ) 75 | 76 | assert params_result["vars"] == "ALL" 77 | -------------------------------------------------------------------------------- /tests/test_serializers.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # SPDX-FileCopyrightText: 2019-2022 Alliander N.V. 5 | # SPDX-License-Identifier: MPL-2.0 6 | 7 | from enum import Enum 8 | 9 | import pytest 10 | 11 | from weather_provider_api.routers.weather import api_models 12 | from weather_provider_api.routers.weather.api_models import ( 13 | ResponseFormat, 14 | WeatherContentRequestQuery, 15 | ) 16 | from weather_provider_api.routers.weather.utils.serializers import ( 17 | file_or_text_response, 18 | file_response, 19 | ) 20 | 21 | 22 | class MockResponseFormat(str, Enum): 23 | mock_format = "mock_format" 24 | 25 | 26 | @pytest.fixture() 27 | def mock_response_query(mock_factors): 28 | result = WeatherContentRequestQuery("2020-01-01", "2020-02-02", 51.873419, 5.705929, mock_factors) 29 | return result 30 | 31 | 32 | @pytest.mark.parametrize("response_format", [response.value for response in ResponseFormat]) 33 | def test_file_or_text_response_regular(response_format, mock_coordinates, mock_dataset, mock_response_query): 34 | assert file_or_text_response( 35 | mock_dataset, 36 | ResponseFormat(response_format), 37 | "knmi", 38 | "pluim", 39 | mock_response_query, 40 | mock_coordinates, 41 | ) 42 | 43 | 44 | def test_file_or_text_response_forged_response_format(monkeypatch, mock_coordinates, mock_dataset, mock_response_query): 45 | # TEST 1: Non-existing ResponseFormat is intercepted by Class 46 | with pytest.raises(ValueError) as e: 47 | assert file_or_text_response( 48 | mock_dataset, 49 | ResponseFormat("mock_format"), 50 | "knmi", 51 | "pluim", 52 | mock_response_query, 53 | [mock_coordinates], 54 | ) 55 | assert str(e.value.args[0]) == "'mock_format' is not a valid ResponseFormat" 56 | 57 | # TEST 2: Forged non-existing ResponseFormat should be intercepted by file_response 58 | monkeypatch.setattr(api_models, "ResponseFormat", MockResponseFormat) 59 | with pytest.raises(NotImplementedError) as e: 60 | assert file_response( 61 | mock_dataset, 62 | api_models.ResponseFormat.mock_format, 63 | "knmi", 64 | "pluim", 65 | mock_response_query, 66 | [mock_coordinates], 67 | ) 68 | assert str(e.value.args[0]) == f"Cannot create file response for the mock_format response format" 69 | -------------------------------------------------------------------------------- /tests/test_weather_alert.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # SPDX-FileCopyrightText: 2019-2022 Alliander N.V. 5 | # SPDX-License-Identifier: MPL-2.0 6 | 7 | import requests 8 | from requests.exceptions import ProxyError 9 | 10 | from weather_provider_api.routers.weather.sources.weather_alert.weather_alert import ( 11 | WeatherAlert, 12 | ) 13 | 14 | 15 | def test_weather_alert_(): 16 | wa = WeatherAlert() 17 | output = wa.get_alarm() 18 | assert len(output) == 12 # 1 response per province 19 | assert output[0][0] in ( 20 | "drenthe", 21 | "friesland", 22 | "gelderland", 23 | "groningen", 24 | "flevoland", 25 | "limburg", 26 | "noord-brabant", 27 | "noord-holland", 28 | "overijssel", 29 | "utrecht", 30 | "zeeland", 31 | "zuid-holland", 32 | ) 33 | assert output[0][1] in ( 34 | "green", 35 | "yellow", 36 | "red", 37 | "page didn't match", 38 | "page was inaccessible", 39 | ) 40 | 41 | 42 | def test_weather_alert_errors(monkeypatch): 43 | wa = WeatherAlert() 44 | 45 | # Generating a fake ProxyError for the "_requests_retry_session.get" function 46 | class ProxyErrorSessionMock: 47 | def get(self, *args, **kwargs): 48 | raise ProxyError("Fake Proxy Error!") 49 | 50 | class TimeoutSessionMock: 51 | def get(self, *args, **kwargs): 52 | raise requests.Timeout("Fake Timeout Error!") 53 | 54 | class TooManyRedirectsSessionMock: 55 | def get(self, *args, **kwargs): 56 | raise requests.TooManyRedirects("Fake Too Many Redirects Error!") 57 | 58 | # Testing Proxy Response 59 | monkeypatch.setattr(WeatherAlert, "_requests_retry_session", ProxyErrorSessionMock) 60 | 61 | output = wa.get_alarm() 62 | assert len(output) == 12 # Still 12 responses, but with proper error description inside.. 63 | assert output[0][1] == "proxy error on loading page" 64 | 65 | # Testing Timeout Response 66 | monkeypatch.setattr(WeatherAlert, "_requests_retry_session", TimeoutSessionMock) 67 | 68 | output = wa.get_alarm() 69 | assert len(output) == 12 # Still 12 responses, but with proper error description inside.. 70 | assert output[0][1] == "time out op loading page" 71 | 72 | # Testing TooManyRedirects Response 73 | monkeypatch.setattr(WeatherAlert, "_requests_retry_session", TooManyRedirectsSessionMock) 74 | 75 | output = wa.get_alarm() 76 | assert len(output) == 12 # Still 12 responses, but with proper error description inside.. 77 | assert output[0][1] == "page proved inaccessible" 78 | 79 | 80 | # @pytest.mark.skip(reason="Monkeypatch for Response content not working properly. ") # TODO: FIX 81 | def test_weather_alert_wrongly_formatted_page(monkeypatch): 82 | wa = WeatherAlert() 83 | 84 | def mock_request_response(_, nope): 85 | class FakeResponse: 86 | status_code = 200 87 | text = "
Nothing Here!
" 88 | 89 | return FakeResponse() 90 | 91 | monkeypatch.setattr(requests.Session, "get", mock_request_response) # Intercepting request response 92 | output = wa.get_alarm() 93 | 94 | assert len(output) == 12 # Still 12 responses, but with proper error description inside.. 95 | assert output[0][1] == "could not find expected data on page" 96 | -------------------------------------------------------------------------------- /var_maps/arome_var_map.json: -------------------------------------------------------------------------------- 1 | { 2 | "1": "mean_sea_level_pressure", 3 | "6": "geopotential", 4 | "11": "temperature", 5 | "33": "u_component_of_wind", 6 | "34": "v_component_of_wind", 7 | "52": "relative_humidity", 8 | "66": "snow_cover", 9 | "67": "boundary_layer_height", 10 | "71": "cloud_cover", 11 | "73": "low_cloud_cover", 12 | "74": "medium_cloud_cover", 13 | "75": "high_cloud_cover", 14 | "111": "net_short_wave_radiation", 15 | "112": "net_long_wave_radiation", 16 | "117": "global_radiation", 17 | "122": "sensible_heat_flux", 18 | "132": "latent_heat_flux", 19 | "162": "u_component_max_squall", 20 | "163": "v_component_max_squall", 21 | "181": "_rain_water", 22 | "184": "_snow_water", 23 | "186": "cloud_base", 24 | "201": "_graupel", 25 | "SD Snow depth m": "snow_depth", 26 | "T Temperature K": "temperature - Extra copy" 27 | } -------------------------------------------------------------------------------- /var_maps/arome_var_map.json.license: -------------------------------------------------------------------------------- 1 | SPDX-FileCopyrightText: 2019-2022 Alliander N.V. 2 | SPDX-License-Identifier: MPL-2.0 -------------------------------------------------------------------------------- /var_maps/era5land_var_map.json: -------------------------------------------------------------------------------- 1 | { 2 | "stl1": "soil_temperature_level_1", 3 | "stl2": "soil_temperature_level_2", 4 | "stl3": "soil_temperature_level_3", 5 | "stl4": "soil_temperature_level_4", 6 | "swvl1": "volumetric_soil_water_layer_1", 7 | "swvl2": "volumetric_soil_water_layer_2", 8 | "swvl3": "volumetric_soil_water_layer_3", 9 | "swvl4": "volumetric_soil_water_layer_4" 10 | } -------------------------------------------------------------------------------- /var_maps/era5land_var_map.json.license: -------------------------------------------------------------------------------- 1 | SPDX-FileCopyrightText: 2019-2022 Alliander N.V. 2 | SPDX-License-Identifier: MPL-2.0 3 | -------------------------------------------------------------------------------- /var_maps/era5sl_var_map.json: -------------------------------------------------------------------------------- 1 | { 2 | "u100": "100m_u_component_of_wind", 3 | "v100": "100m_v_component_of_wind", 4 | "d2m": "2m_dewpoint_temperature", 5 | "sp": "surface_pressure", 6 | "stl1": "soil_temperature_level_1", 7 | "stl2": "soil_temperature_level_2", 8 | "stl3": "soil_temperature_level_3", 9 | "stl4": "soil_temperature_level_4", 10 | "ssrd": "surface_solar_radiation_downwards", 11 | "ssrdc": "surface_solar_radiation_downward_clear_sky", 12 | "swvl1": "volumetric_soil_water_layer_1", 13 | "swvl2": "volumetric_soil_water_layer_2", 14 | "swvl3": "volumetric_soil_water_layer_3", 15 | "swvl4": "volumetric_soil_water_layer_4", 16 | "t2m": "2m_temperature", 17 | "tp": "total_precipitation", 18 | "u10": "10m_u_component_of_wind", 19 | "v10": "10m_v_component_of_wind" 20 | } 21 | -------------------------------------------------------------------------------- /var_maps/era5sl_var_map.json.license: -------------------------------------------------------------------------------- 1 | SPDX-FileCopyrightText: 2019-2022 Alliander N.V. 2 | SPDX-License-Identifier: MPL-2.0 -------------------------------------------------------------------------------- /weather_provider_api/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # SPDX-FileCopyrightText: 2019-2022 Alliander N.V. 5 | # SPDX-License-Identifier: MPL-2.0 6 | -------------------------------------------------------------------------------- /weather_provider_api/__main__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # SPDX-FileCopyrightText: 2019-2023 Alliander N.V. 5 | # SPDX-License-Identifier: MPL-2.0 6 | 7 | """Main executable module.""" 8 | 9 | import uvicorn 10 | from loguru import logger 11 | 12 | from weather_provider_api.core.initializers.logging_handler import initialize_logging 13 | 14 | # Logging is initialized before the importing of APP_CONFIG, to ensure custom logging for APP_CONFIG initialisation. 15 | initialize_logging() 16 | 17 | # Import application configuration settings 18 | from weather_provider_api.config import APP_CONFIG 19 | 20 | 21 | def launch_api(run_mode: str = "uvicorn", host: str = "127.0.0.1", port: int = 8080): 22 | """The main method for running this application directly. 23 | 24 | (The Dockerfile uses the WPLA_APPLICATION object in [vbd_memo_api.core.application].) 25 | 26 | Args: 27 | run_mode (str): The run mode for the application. Accepted values are 'uvicorn' and 'gunicorn'. 28 | host (str): The host id to run this application on. Usually 'localhost', '127.0.0.1' or '0.0.0.0' in this 29 | context. 30 | port (str): The port to broadcast the application at. 31 | 32 | Returns: 33 | Nothing. Either runs successfully until stopped, or breaks from an Exception. 34 | 35 | Notes: 36 | As Gunicorn only works within the Linux OS, the 'gunicorn' run_mode setting will not work from any other OS. 37 | 38 | """ 39 | project_title = ( 40 | APP_CONFIG["base"]["full_title"] if APP_CONFIG["base"]["full_title"] else APP_CONFIG["base"]["title"] 41 | ) 42 | launch_string = f"Launching: {project_title}..." 43 | logger.info("-" * len(launch_string)) 44 | logger.info(launch_string) 45 | logger.info("-" * len(launch_string)) 46 | 47 | from weather_provider_api.core.application import WPLA_APPLICATION 48 | 49 | # start application based on parameters 50 | if run_mode.upper() == "UVICORN": 51 | uvicorn.run(WPLA_APPLICATION, host=host, port=port) 52 | elif run_mode.upper() == "GUNICORN": 53 | # Error handling for problems with the Gunicorn application get handled by Gunicorn itself. 54 | from weather_provider_api.core.gunicorn_application import GunicornApplication 55 | 56 | gunicorn_app = GunicornApplication(options={"bind": f"{host}:{port}"}, fastapi_application=WPLA_APPLICATION) 57 | gunicorn_app.run() 58 | else: 59 | raise ValueError(f"Invalid run-mode selected: {run_mode}") 60 | 61 | shutting_down_string = f"Shutting down: {project_title}" 62 | logger.info("-" * len(shutting_down_string)) 63 | logger.info(shutting_down_string) 64 | logger.info("-" * len(shutting_down_string)) 65 | 66 | 67 | # The main function for easy local execution 68 | if __name__ == "__main__": 69 | # If run from main start using the defaults 70 | launch_api() 71 | -------------------------------------------------------------------------------- /weather_provider_api/app_version.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # SPDX-FileCopyrightText: 2019-2023 Alliander N.V. 5 | # SPDX-License-Identifier: MPL-2.0 6 | 7 | """Version detection module.""" 8 | 9 | from importlib import metadata 10 | 11 | import tomli 12 | from loguru import logger 13 | 14 | 15 | def _get_app_version() -> str: 16 | """This method tries to identify the API's main version identifier and return it. 17 | 18 | Returns: 19 | str: The found version identifier, if found. If no version identifier was found, a warning value is returned 20 | instead. 21 | 22 | """ 23 | # First attempt: Get the version number by looking for a pyproject.toml file in the working directory. 24 | # Please note that this assumes that this function was called from a context that has the Project's main folder as 25 | # the working directory. 26 | try: 27 | with open("./pyproject.toml", mode="rb") as project_file: 28 | version = tomli.load(project_file)["tool"]["poetry"]["version"] 29 | logger.info(f"Retrieved the project version from the pyproject.toml file: {version}") 30 | return version 31 | except FileNotFoundError as fnf_error: 32 | logger.debug(f"Could not retrieve the active version from the pyproject.toml file: {fnf_error}") 33 | 34 | # Second attempt: Get the version number from the package that was used to install this component, if applicable. 35 | try: 36 | version = metadata.version(__package__) 37 | logger.info(f"Retrieved the project version from package data: {version}") 38 | return version 39 | except metadata.PackageNotFoundError as pnf_error: 40 | logger.debug(f"Could not retrieve the active version from package data: {pnf_error}") 41 | 42 | # No version could be determined 43 | logger.warning("No version could be found for the project!") 44 | return "<< version could not be determined >>" 45 | 46 | 47 | APP_VERSION = _get_app_version() 48 | -------------------------------------------------------------------------------- /weather_provider_api/config/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # SPDX-FileCopyrightText: 2019-2023 Alliander N.V. 5 | # SPDX-License-Identifier: MPL-2.0 6 | 7 | """Configuration folder""" 8 | 9 | import os 10 | import tempfile 11 | from pathlib import Path 12 | 13 | from tomli import load 14 | 15 | config_file_path = Path(__file__).parent.joinpath("config.toml") 16 | with config_file_path.open(mode="rb") as file_processor: 17 | # TODO: Check if config.toml overwrite is set in the environment and replace accordingly!!! 18 | APP_CONFIG = load(file_processor) 19 | 20 | 21 | # Settings taken from the environment if available. 22 | APP_DEBUGGING = os.environ.get("WPLA_DEBUG", "False").lower() in ( 23 | "true", 24 | "1", 25 | "y", 26 | "yes", 27 | ) 28 | APP_DEPLOYED = os.environ.get("WPLA_DEPLOYED", "False").lower() in ( 29 | "true", 30 | "1", 31 | "y", 32 | "yes", 33 | ) 34 | APP_STORAGE_FOLDER = os.environ.get("WPLA_STORAGE_FOLDER", f"{tempfile.gettempdir()}/Weather_Repository") 35 | APP_LOG_LEVEL = os.environ.get("WPLA_LOG_LEVEL", APP_CONFIG["logging"]["log_level"]).upper() 36 | APP_SERVER = os.environ.get("WPLA_SERVER_URL", "http://127.0.0.1:8080") # 127.0.0.1:8080 for debugging is the default 37 | -------------------------------------------------------------------------------- /weather_provider_api/config/config.toml: -------------------------------------------------------------------------------- 1 | [base] 2 | title = "WPLA" 3 | full_title = "Weather Provider Libraries and API" 4 | description = """ 5 | The Weather Provider Libraries and API (WPLA) is an Open Source Project intended to present an easy 6 | and uniform way to request meteorological data from a extensive numer of sources in a plethora of formats and 7 | styles. 8 | """ 9 | expiration_checking = true 10 | expiration_date = "2025-12-31" 11 | 12 | [maintainer] 13 | show_info = true 14 | name = "Verbindingsteam" 15 | email_address = "weather.provider@alliander.com" 16 | 17 | [networking] 18 | default_interface_ip = "127.0.0.1" 19 | default_port = 8080 20 | 21 | [logging] 22 | log_level = "INFO" 23 | use_stdout_logger = true 24 | use_file_loggers = false 25 | file_logger_folder = "/tmp/wpla_logs" 26 | 27 | [components] 28 | prometheus = true 29 | cors = true 30 | cors_allowed_origins = ['*'] 31 | cors_allowed_origins_regex = [] 32 | 33 | [api_v1] 34 | implementation = "2.2.0" 35 | expiration_date = "2025-12-31" 36 | 37 | [api_v2] 38 | implementation = "2.2.0" 39 | expiration_date = "2025-12-31" 40 | -------------------------------------------------------------------------------- /weather_provider_api/core/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # SPDX-FileCopyrightText: 2019-2022 Alliander N.V. 5 | # SPDX-License-Identifier: MPL-2.0 6 | -------------------------------------------------------------------------------- /weather_provider_api/core/application.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # SPDX-FileCopyrightText: 2019-2023 Alliander N.V. 5 | # SPDX-License-Identifier: MPL-2.0 6 | 7 | """Main Application.""" 8 | 9 | from fastapi import FastAPI 10 | from starlette.responses import RedirectResponse 11 | 12 | from weather_provider_api.app_version import APP_VERSION 13 | from weather_provider_api.config import APP_CONFIG 14 | from weather_provider_api.core.initializers.cors import initialize_cors_middleware 15 | from weather_provider_api.core.initializers.exception_handling import ( 16 | initialize_exception_handler, 17 | ) 18 | from weather_provider_api.core.initializers.headers import initialize_header_metadata 19 | from weather_provider_api.core.initializers.logging_handler import initialize_logging 20 | from weather_provider_api.core.initializers.mounting import mount_api_version 21 | from weather_provider_api.core.initializers.prometheus import ( 22 | initialize_prometheus_interface, 23 | ) 24 | from weather_provider_api.core.initializers.validation import initialize_api_validation 25 | from weather_provider_api.versions.v1 import app as v1 26 | from weather_provider_api.versions.v2 import app as v2 27 | 28 | 29 | def _build_api_application() -> FastAPI: 30 | """The main method for building a standardized FastAPI application for use with for instance Uvicorn. 31 | 32 | Returns: 33 | FastAPI: A FastAPI application with the full API hooked up and properly initialized. 34 | 35 | Notes: 36 | This method can also serve as an example for setting up your own version of this API with different settings. 37 | 38 | """ 39 | app_title = APP_CONFIG["base"]["title"] 40 | app_description = APP_CONFIG["base"]["description"] 41 | 42 | # Setting up the base application 43 | application = FastAPI( 44 | version=APP_VERSION, 45 | title=app_title, 46 | summary=app_description, 47 | description=app_description, 48 | contact={ 49 | "name": APP_CONFIG["maintainer"]["name"], 50 | "email": APP_CONFIG["maintainer"]["email_address"], 51 | }, 52 | ) 53 | application.openapi_version = "3.0.2" 54 | 55 | # Attach logging 56 | application.add_event_handler("startup", initialize_logging) 57 | 58 | # Attach selected middleware 59 | initialize_exception_handler(application) 60 | initialize_header_metadata(application) 61 | if APP_CONFIG["base"]["expiration_checking"]: 62 | initialize_api_validation(application) 63 | if APP_CONFIG["components"]["prometheus"]: 64 | initialize_prometheus_interface(application) 65 | if APP_CONFIG["components"]["cors"]: 66 | initialize_cors_middleware(application) 67 | 68 | # Create and connect to end-points 69 | mount_api_version(application, v1) 70 | mount_api_version(application, v2) 71 | 72 | # Adding a redirect from the root of the application to our default view 73 | @application.get("/") 74 | def redirect_to_docs(): 75 | """This function redirects the visitors to the default view from the application's base URL.""" 76 | return RedirectResponse(url="/api/v2/docs") 77 | 78 | application.openapi() 79 | 80 | return application 81 | 82 | 83 | # Application constant available for re-use, rather than re-initialization. 84 | WPLA_APPLICATION = _build_api_application() 85 | -------------------------------------------------------------------------------- /weather_provider_api/core/base_model.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # SPDX-FileCopyrightText: 2019-2022 Alliander N.V. 5 | # SPDX-License-Identifier: MPL-2.0 6 | 7 | """Base Model class. 8 | 9 | An expansion of the PydanticBaseModel used to build each Weather Provider Base class. 10 | """ 11 | 12 | from pydantic import BaseModel as PydanticBaseModel 13 | 14 | 15 | class BaseModel(PydanticBaseModel): 16 | """Minimal PydanticBaseMode expansion used in all base classes.""" 17 | 18 | class Config: 19 | from_attributes = True 20 | -------------------------------------------------------------------------------- /weather_provider_api/core/exceptions/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # SPDX-FileCopyrightText: 2019-2022 Alliander N.V. 5 | # SPDX-License-Identifier: MPL-2.0 6 | -------------------------------------------------------------------------------- /weather_provider_api/core/exceptions/additional_responses.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # SPDX-FileCopyrightText: 2019-2022 Alliander N.V. 5 | # SPDX-License-Identifier: MPL-2.0 6 | 7 | from weather_provider_api.core.exceptions.api_models import ErrorResponseModel 8 | 9 | ERROR_RESPONSE = {"model": ErrorResponseModel} 10 | -------------------------------------------------------------------------------- /weather_provider_api/core/exceptions/api_models.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # SPDX-FileCopyrightText: 2019-2022 Alliander N.V. 5 | # SPDX-License-Identifier: MPL-2.0 6 | 7 | from weather_provider_api.core.base_model import BaseModel 8 | 9 | 10 | class ErrorResponseModel(BaseModel): 11 | detail: str # error message 12 | -------------------------------------------------------------------------------- /weather_provider_api/core/exceptions/exceptions.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # SPDX-FileCopyrightText: 2019-2022 Alliander N.V. 5 | # SPDX-License-Identifier: MPL-2.0 6 | 7 | from typing import Any 8 | 9 | from fastapi import HTTPException 10 | from pydantic import BaseModel 11 | 12 | 13 | class ExceptionResponseModel(BaseModel): 14 | """Class only used to relay the output for the HTTP Exception classes to the OpenAPI specification and 15 | Swagger UI. 16 | """ 17 | 18 | detail: str 19 | 20 | 21 | class APIExpiredException(HTTPException): 22 | def __init__(self, detail: Any = None): 23 | self.detail = ( 24 | detail 25 | or "This API has passed it's expiry date and should be revalidated. Please contact the API maintainer." 26 | ) 27 | self.status_code = 404 28 | -------------------------------------------------------------------------------- /weather_provider_api/core/gunicorn_application.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | from typing import Dict 4 | 5 | from fastapi import FastAPI 6 | from gunicorn.app.base import Application 7 | from loguru import logger 8 | 9 | 10 | class GunicornApplication(Application): 11 | """Gunicorn Application class. 12 | 13 | This class extends the base Gunicorn application class with the settings and methods needed to directly initialize 14 | the API application. 15 | 16 | Notes: 17 | This class can also serve as an example for deploying your own version of this API via Gunicorn. 18 | 19 | """ 20 | 21 | def __init__(self, fastapi_application: FastAPI, options: Dict): 22 | """Overwrite of the base method with the purpose of automatically loading a FastAPI application and deployment 23 | options passed. 24 | 25 | Args: 26 | fastapi_application: The FastAPI application to deploy 27 | options: The deployment options to use 28 | """ 29 | super().__init__() 30 | self.usage = None 31 | self.callable = None 32 | self.options = options 33 | self.do_load_config() # Loads the default config and requests extra config settings via the init function 34 | self.fastapi_application = fastapi_application 35 | logger.info("Gunicorn application initialized successfully...") 36 | 37 | def init(self, *args): 38 | """Method overwrite of the base method. 39 | 40 | If effectively loaded upon configuration loading. The returned dictionary holds configuration settings which 41 | are used to overwrite and/or append self.cfg settings. 42 | 43 | Args: 44 | *args: Not actually used. 45 | 46 | Returns: 47 | (dict): A dictionary holding the configuration settings overwrite or append in self.cfg 48 | 49 | """ 50 | config = {} 51 | for key, value in self.options.items(): 52 | # Match list to actual settings in self.cfg and add those validated with proper capitalization to the 53 | # return value 54 | if key.lower() in self.cfg.settings and value is not None: 55 | config[key.lower()] = value 56 | 57 | return config 58 | 59 | def load(self): 60 | """Overwrite of the base method. 61 | 62 | Used to load the application for use. 63 | 64 | Returns: 65 | FastAPI: The FastAPI application to load. 66 | 67 | """ 68 | return self.fastapi_application 69 | -------------------------------------------------------------------------------- /weather_provider_api/core/initializers/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # SPDX-FileCopyrightText: 2019-2022 Alliander N.V. 5 | # SPDX-License-Identifier: MPL-2.0 6 | -------------------------------------------------------------------------------- /weather_provider_api/core/initializers/cors.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # SPDX-FileCopyrightText: 2019-2023 Alliander N.V. 5 | # SPDX-License-Identifier: MPL-2.0 6 | 7 | """CORS support""" 8 | 9 | from fastapi import FastAPI 10 | from fastapi.middleware.cors import CORSMiddleware 11 | from loguru import logger 12 | 13 | from weather_provider_api.config import APP_CONFIG 14 | 15 | 16 | def initialize_cors_middleware(app: FastAPI): 17 | """Initializes the CORS middleware. 18 | 19 | Enables CORS handling for the allowed origins set in the config setting `CORS_ALLOWED_ORIGINS`, a list of strings 20 | each corresponding to a host. The FastAPI CORSMiddleware is used to enable CORS handling. 21 | 22 | Args: 23 | app (FastAPI): The FastAPI application to attach the CORS middleware to. 24 | 25 | Returns: 26 | Nothing. The application itself is updated. 27 | 28 | """ 29 | origins = APP_CONFIG["components"]["cors_allowed_origins"] 30 | origins_regex = APP_CONFIG["components"]["cors_allowed_origins_regex"] 31 | 32 | if origins and len(origins) == 0: 33 | origins = None 34 | if origins_regex and len(origins_regex) == 0: 35 | origins_regex = None 36 | 37 | if origins and origins_regex: 38 | app.add_middleware( 39 | CORSMiddleware, 40 | allow_origins=origins, 41 | allow_credentials=True, 42 | allow_methods=["*"], 43 | allow_headers=["*"], 44 | allow_origins_regex=origins_regex, 45 | ) 46 | logger.info(f"Attached CORS middleware enabled for the following origins: {origins}, {origins_regex}") 47 | elif origins: 48 | app.add_middleware( 49 | CORSMiddleware, 50 | allow_origins=origins, 51 | allow_credentials=True, 52 | allow_methods=["*"], 53 | allow_headers=["*"], 54 | ) 55 | logger.info(f"Attached CORS middleware enabled for the following origins: {origins}") 56 | elif origins_regex: 57 | app.add_middleware( 58 | CORSMiddleware, 59 | allow_origins_regex=origins_regex, 60 | allow_credentials=True, 61 | allow_methods=["*"], 62 | allow_headers=["*"], 63 | ) 64 | logger.info(f"Attached CORS middleware enabled for the following origins: {origins_regex}") 65 | else: 66 | logger.warning("CORS middleware enabled but no allowed origins are set.") 67 | -------------------------------------------------------------------------------- /weather_provider_api/core/initializers/exception_handling.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # SPDX-FileCopyrightText: 2019-2022 Alliander N.V. 5 | # SPDX-License-Identifier: MPL-2.0 6 | 7 | """Exception Handling 8 | 9 | This module holds the API's exception handler. 10 | """ 11 | 12 | from fastapi import FastAPI 13 | from loguru import logger 14 | from starlette.exceptions import HTTPException as StarletteHTTPException 15 | from starlette.requests import Request 16 | from starlette.responses import JSONResponse 17 | 18 | from weather_provider_api.config import APP_CONFIG 19 | 20 | 21 | async def handle_http_exception(request: Request, exc: StarletteHTTPException) -> JSONResponse: 22 | headers = getattr(exc, "headers", None) 23 | 24 | body = {"detail": exc.detail, "request": str(request.url)} 25 | if APP_CONFIG["maintainer"]["show_info"]: 26 | body["maintainer"] = APP_CONFIG["maintainer"]["name"] 27 | body["maintainer_email"] = APP_CONFIG["maintainer"]["email_address"] 28 | 29 | return JSONResponse(body, status_code=exc.status_code, headers=headers) 30 | 31 | 32 | def initialize_exception_handler(application: FastAPI): 33 | """The method that attaches the customized exception handling method to a FastAPI application. 34 | 35 | Args: 36 | application: The FastAPI application to attach the custom exception handler to. 37 | 38 | Returns: 39 | Nothing. The FastAPI application itself is updated. 40 | 41 | Notes: 42 | This method assumes that the FastAPI application given has a [title] set. 43 | 44 | Todo: 45 | Evaluate the dependency on [title] and [root_path] parameters being set for FastAPI applications in ALL 46 | initializers. By either extending the base class to enforce these or improving the code to not be dependent on 47 | these parameters, we can eradiate code smell and chances at Exceptions caused by not having these parameters. 48 | """ 49 | application.add_exception_handler(StarletteHTTPException, handler=handle_http_exception) 50 | logger.info(f"Attached the Exception Handler to the application ({application.title})...") 51 | 52 | 53 | NOT_IMPLEMENTED_ERROR = "This method is abstract and should be overridden." 54 | -------------------------------------------------------------------------------- /weather_provider_api/core/initializers/headers.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # SPDX-FileCopyrightText: 2019-2022 Alliander N.V. 5 | # SPDX-License-Identifier: MPL-2.0 6 | 7 | 8 | from fastapi import FastAPI 9 | from starlette.middleware.base import BaseHTTPMiddleware 10 | from starlette.requests import Request 11 | 12 | from weather_provider_api.app_version import APP_VERSION 13 | from weather_provider_api.config import APP_CONFIG 14 | 15 | 16 | def initialize_header_metadata(application: FastAPI): 17 | """Method that attaches the customized Metadata Header method that adds extra metadata. 18 | 19 | Args: 20 | application: The FastAPI application to attach the custom method to. 21 | 22 | Returns: 23 | Nothing. The FastAPI application itself is updated. 24 | """ 25 | 26 | async def add_metadata_headers(request: Request, call_next): 27 | response = await call_next(request) 28 | response.headers["X-App-Version"] = APP_VERSION 29 | response.headers["X-App-Valid-Till"] = APP_CONFIG["base"]["expiration_date"] 30 | 31 | if APP_CONFIG["maintainer"]["show_info"]: 32 | response.headers["X-Maintainer"] = APP_CONFIG["maintainer"]["name"] 33 | response.headers["X-Maintainer-Email"] = APP_CONFIG["maintainer"]["email_address"] 34 | 35 | return response 36 | 37 | application.add_middleware(BaseHTTPMiddleware, dispatch=add_metadata_headers) 38 | -------------------------------------------------------------------------------- /weather_provider_api/core/initializers/mounting.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # SPDX-FileCopyrightText: 2019-2023 Alliander N.V. 5 | # SPDX-License-Identifier: MPL-2.0 6 | 7 | """(sub)-API Mounting 8 | 9 | This module handles the mounting of sub-API's onto a main API. 10 | """ 11 | 12 | from fastapi import FastAPI 13 | from loguru import logger 14 | 15 | from weather_provider_api.core.initializers.exception_handling import ( 16 | initialize_exception_handler, 17 | ) 18 | 19 | 20 | def mount_api_version(base_application: FastAPI, api_to_mount: FastAPI): 21 | """The method that mounts a FastAPI object as a child into another. 22 | 23 | Args: 24 | base_application: The FastAPI application to be used as the parent application. 25 | api_to_mount: The FastAPI application to be used as the child application 26 | Returns: 27 | Nothing. The parent object itself is updated after this method. 28 | 29 | Notes: 30 | This implementation assumes that the child application at least has a [title] and [root_path] set, and that the 31 | parent application at least has a title set. 32 | """ 33 | initialize_exception_handler(api_to_mount) # Hookup the exception handler to the new sub-API 34 | base_application.mount(api_to_mount.root_path, api_to_mount) 35 | logger.info(f"Mounted API [{api_to_mount.title}] to [{base_application.title}] at: {api_to_mount.root_path}") 36 | -------------------------------------------------------------------------------- /weather_provider_api/core/initializers/prometheus.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # SPDX-FileCopyrightText: 2019-2022 Alliander N.V. 5 | # SPDX-License-Identifier: MPL-2.0 6 | 7 | """Prometheus Middleware handler""" 8 | 9 | from fastapi import FastAPI 10 | from loguru import logger 11 | from starlette_prometheus import PrometheusMiddleware, metrics 12 | 13 | 14 | def initialize_prometheus_interface(application: FastAPI, metrics_endpoint: str = "/metrics"): 15 | """The method that attaches the Prometheus Middleware to a FastAPI application. 16 | 17 | Args: 18 | application: The FastAPI application to attach the Prometheus Middleware to. 19 | metrics_endpoint: The endpoint at which the Prometheus data should become available. 20 | 21 | Returns: 22 | Nothing. The FastAPI application itself is updated. 23 | 24 | Notes: 25 | Prometheus is a metrics system that interprets requests and results and processes those into metrics. From a 26 | Prometheus server this data can be gathered and preprocessed, allowing for more extensive data on request 27 | and the handling thereof, like information on memory usage or the time taken to process certain types of 28 | requests. 29 | Note that the Prometheus server itself also doesn't handle processing that output into views or graphs. To 30 | visualize Prometheus output, you'd need something like Elastic or Grafana. 31 | """ 32 | application.add_middleware(PrometheusMiddleware, filter_unhandled_paths=True) 33 | application.add_route(metrics_endpoint, metrics) 34 | logger.info( 35 | "Attached Prometheus Middleware to the application and opened metrics endpoint at " f"[{metrics_endpoint}]..." 36 | ) 37 | -------------------------------------------------------------------------------- /weather_provider_api/core/initializers/rate_limiter.py: -------------------------------------------------------------------------------- 1 | # !/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # SPDX-FileCopyrightText: 2019-2023 Alliander N.V. 5 | # SPDX-License-Identifier: MPL-2.0 6 | 7 | """Request Rate Limiter initializer""" 8 | 9 | from slowapi import Limiter 10 | from slowapi.util import get_remote_address 11 | 12 | API_RATE_LIMITER = Limiter(key_func=get_remote_address, default_limits=["5/minute"]) 13 | -------------------------------------------------------------------------------- /weather_provider_api/core/initializers/validation.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # SPDX-FileCopyrightText: 2019-2022 Alliander N.V. 5 | # SPDX-License-Identifier: MPL-2.0 6 | 7 | """API Validation checks""" 8 | 9 | import re 10 | from datetime import datetime 11 | 12 | from fastapi import FastAPI 13 | from loguru import logger 14 | from starlette.middleware.base import BaseHTTPMiddleware 15 | from starlette.requests import Request 16 | 17 | from weather_provider_api.config import APP_CONFIG 18 | from weather_provider_api.core.exceptions.exceptions import APIExpiredException 19 | from weather_provider_api.core.initializers.exception_handling import ( 20 | handle_http_exception, 21 | ) 22 | 23 | 24 | def initialize_api_validation(application: FastAPI): 25 | """Method for attach the API validity checker to a FastAPI application. 26 | 27 | Args: 28 | application: The FastAPI application to attach the checker to. 29 | 30 | Returns: 31 | Nothing. The application itself is updated. 32 | 33 | """ 34 | 35 | async def check_api_for_validity(request: Request, call_next): 36 | """The method that validated the API validity. 37 | 38 | Args: 39 | request: The request to evaluate 40 | call_next: The call_next object for the request (what to do next if this step doesn't raise any exceptions) 41 | 42 | Returns: 43 | The next step to execute for this request. This is either the original call_next, or an 44 | HTTP Exception trigger for an APIExpiredException. 45 | 46 | """ 47 | api_prefix = r"/api/v" 48 | api_suffix = "/weather" 49 | request_url = str(request.url) 50 | today = datetime.today().date() 51 | response = None 52 | 53 | api_version_in_url = re.search(f"{api_prefix}\\d+{api_suffix}", request_url) # Looks for mentioning of version 54 | 55 | if not api_version_in_url: 56 | return await call_next(request) # No API call, no problem 57 | 58 | # Validate the base application 59 | base_expiration_date = datetime.strptime(APP_CONFIG["base"]["expiration_date"], "%Y-%m-%d").date() 60 | if today > base_expiration_date: 61 | response = await handle_http_exception( 62 | request, 63 | APIExpiredException( 64 | f"The main project's expiry date of [{base_expiration_date}] has been reached. " 65 | "Please contact the maintainer of this project!" 66 | ), 67 | ) 68 | 69 | if not api_version_in_url: 70 | return await call_next(request) # continue as normal if no api version is involved 71 | 72 | # Determine the API interpreter version used and its expiry date 73 | start_location_of_version_number_in_url = api_version_in_url.start() + len(api_prefix) 74 | end_location_of_version_number_in_url = api_version_in_url.end() - len(api_suffix) 75 | 76 | api_version_used = request_url[start_location_of_version_number_in_url:end_location_of_version_number_in_url] 77 | 78 | # Validate the expiry date of the specific API version used 79 | version_expiration_date = datetime.strptime( 80 | APP_CONFIG[f"api_v{api_version_used}"]["expiration_date"], "%Y-%m-%d" 81 | ).date() 82 | if today > version_expiration_date: 83 | if not response: 84 | response = await handle_http_exception( 85 | request, 86 | APIExpiredException( 87 | f"The expiry date for the v{api_version_used} interface of this API has been reached: " 88 | f"[{version_expiration_date}] Please contact the maintainer of this project!" 89 | ), 90 | ) 91 | else: 92 | response = await handle_http_exception( 93 | request, 94 | APIExpiredException( 95 | f"The expiry dates for both the main project [{base_expiration_date}] and the " 96 | f"v{api_version_used} interface [{version_expiration_date}] of this API have been reached." 97 | "Please contact the maintainer of this project!" 98 | ), 99 | ) 100 | 101 | # Handle the request normally if no expiry was triggered 102 | if not response: 103 | response = await call_next(request) 104 | 105 | return response 106 | 107 | application.add_middleware(BaseHTTPMiddleware, dispatch=check_api_for_validity) 108 | logger.info("Attached API Expiration Checking to the application...") 109 | -------------------------------------------------------------------------------- /weather_provider_api/core/utils/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | -------------------------------------------------------------------------------- /weather_provider_api/core/utils/example_responses.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | 5 | def prepare_example_response(example_response, content_type: str = "application/json") -> dict: 6 | """A method to translate simple output example lists and dictionaries into FastAPI response dictionaries fit for 7 | use within the OpenAPI specifications and Swagger UI. 8 | 9 | Args: 10 | example_response (dict | list): An example response object (either a list or dict of items) 11 | content_type: A string holding the output content type. 12 | 13 | Returns: 14 | A FastAPI response dictionary fit for the OpenAPI specifications (and Swagger UI) 15 | 16 | """ 17 | return {"content": {content_type: {"example": example_response}}} 18 | -------------------------------------------------------------------------------- /weather_provider_api/routers/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # SPDX-FileCopyrightText: 2019-2022 Alliander N.V. 5 | # SPDX-License-Identifier: MPL-2.0 6 | -------------------------------------------------------------------------------- /weather_provider_api/routers/weather/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # SPDX-FileCopyrightText: 2019-2022 Alliander N.V. 5 | # SPDX-License-Identifier: MPL-2.0 6 | -------------------------------------------------------------------------------- /weather_provider_api/routers/weather/base_models/__init__.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2019-2021 Alliander N.V. 2 | # 3 | # SPDX-License-Identifier: MPL-2.0 4 | -------------------------------------------------------------------------------- /weather_provider_api/routers/weather/base_models/source.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # SPDX-FileCopyrightText: 2019-2022 Alliander N.V. 5 | # SPDX-License-Identifier: MPL-2.0 6 | 7 | 8 | class WeatherSourceBase(object): # pragma: no cover 9 | """Base class that contains the basic functionality for all sources. Any new sources should implement this as their 10 | base class! 11 | """ 12 | 13 | def setup_models(self, model_instances): 14 | self._models = {model.id: model for model in model_instances if not model.async_model} 15 | self._async_models = {model.id: model for model in model_instances if model.async_model} 16 | 17 | def get_model(self, model_id, fetch_async=False): 18 | if fetch_async: 19 | return self._async_models.get(model_id, None) 20 | return self._models.get(model_id, None) 21 | 22 | def get_models(self, fetch_async=False): 23 | if fetch_async: 24 | return list(self._async_models.values()) 25 | return list(self._models.values()) 26 | 27 | @property 28 | def models(self): 29 | return self.get_models(fetch_async=False) 30 | 31 | @property 32 | def async_models(self): 33 | return self.get_models(fetch_async=True) 34 | -------------------------------------------------------------------------------- /weather_provider_api/routers/weather/exceptions.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # SPDX-FileCopyrightText: 2019-2022 Alliander N.V. 5 | # SPDX-License-Identifier: MPL-2.0 6 | 7 | from typing import Any 8 | 9 | from fastapi import HTTPException 10 | 11 | 12 | class UnknownSourceException(HTTPException): # pragma: no cover 13 | def __init__(self, detail: Any = None): 14 | self.detail = detail or "unknown data source id" 15 | self.status_code = 404 16 | 17 | 18 | class UnknownModelException(HTTPException): # pragma: no cover 19 | def __init__(self, detail: Any = None): 20 | self.detail = detail or "unknown model id" 21 | self.status_code = 404 22 | 23 | 24 | class UnknownDataTimeException(HTTPException): 25 | pass 26 | 27 | 28 | class UnknownUnitException(HTTPException): 29 | pass 30 | -------------------------------------------------------------------------------- /weather_provider_api/routers/weather/repository/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # SPDX-FileCopyrightText: 2019-2022 Alliander N.V. 5 | # SPDX-License-Identifier: MPL-2.0 6 | -------------------------------------------------------------------------------- /weather_provider_api/routers/weather/sources/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # SPDX-FileCopyrightText: 2019-2022 Alliander N.V. 5 | # SPDX-License-Identifier: MPL-2.0 6 | -------------------------------------------------------------------------------- /weather_provider_api/routers/weather/sources/cds/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # SPDX-FileCopyrightText: 2019-2022 Alliander N.V. 5 | # SPDX-License-Identifier: MPL-2.0 6 | -------------------------------------------------------------------------------- /weather_provider_api/routers/weather/sources/cds/cds.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # SPDX-FileCopyrightText: 2019-2022 Alliander N.V. 5 | # SPDX-License-Identifier: MPL-2.0 6 | 7 | from weather_provider_api.routers.weather.base_models.source import WeatherSourceBase 8 | from weather_provider_api.routers.weather.sources.cds.models.era5sl import ERA5SLModel 9 | 10 | 11 | class CDS(WeatherSourceBase): 12 | def __init__(self): 13 | model_instances = [ 14 | ERA5SLModel(), 15 | ] 16 | 17 | self.id = "cds" 18 | self.name = "Climate Data Store" 19 | self.url = "https://cds.climate.copernicus.eu/" 20 | self._models = None 21 | self._async_models = None 22 | 23 | self.setup_models(model_instances) 24 | -------------------------------------------------------------------------------- /weather_provider_api/routers/weather/sources/cds/client/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # SPDX-FileCopyrightText: 2019-2022 Alliander N.V. 5 | # SPDX-License-Identifier: MPL-2.0 6 | -------------------------------------------------------------------------------- /weather_provider_api/routers/weather/sources/cds/client/cds_api_tools.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # SPDX-FileCopyrightText: 2019-2025 Alliander N.V. 4 | # SPDX-License-Identifier: MPL-2.0 5 | 6 | """This is the module that contains the tools for the CDS API. 7 | 8 | The cdsapi package and methods used to interact with the CDS API are defined here. 9 | These are accredited to the Copernicus Climate Data Store (CDS) and the European Centre for Medium-Range Weather 10 | Forecasts (ECMWF) and are licensed under the Apache License, Version 2.0 which can be found at 11 | https://www.apache.org/licenses/LICENSE-2.0. 12 | 13 | """ 14 | 15 | from datetime import date 16 | from enum import Enum 17 | 18 | import cdsapi 19 | from loguru import logger 20 | from pydantic import BaseModel, Field 21 | 22 | _DEFAULT_VARIABLES = [ 23 | "stl1", 24 | "stl2", 25 | "stl3", 26 | "stl4", 27 | "swvl1", 28 | "swvl2", 29 | "swvl3", 30 | "swvl4", 31 | ] 32 | 33 | 34 | def _info_callback(*args, **kwargs) -> None: # noqa: ANN002, ANN003 35 | """This is a callback function that is used to print information about the download process.""" 36 | if len(args) > 0 or len(kwargs) > 0: 37 | logger.info("Callback received:") 38 | logger.info(" - args: ", *args) 39 | logger.info(" - kwargs: ", **kwargs) 40 | 41 | 42 | class CDSDataSets(str, Enum): 43 | """Currently supported datasets for the CDS API.""" 44 | 45 | ERA5SL = "reanalysis-era5-single-levels" 46 | ERA5LAND = "reanalysis-era5-land" 47 | 48 | 49 | class CDSRequest(BaseModel): 50 | """A class that holds all necessary information for a CDS API request.""" 51 | 52 | product_type: list[str] = Field(["reanalysis"]) 53 | variables: list[str] 54 | year: list[str] = Field([date.strftime(date.today(), "%Y")]) 55 | month: list[str] = Field([date.strftime(date.today(), "%m")]) 56 | day: list[str] = Field([date.strftime(date.today(), "%d")]) 57 | time: list[str] = Field( 58 | [ 59 | "00:00", 60 | "01:00", 61 | "02:00", 62 | "03:00", 63 | "04:00", 64 | "05:00", 65 | "06:00", 66 | "07:00", 67 | "08:00", 68 | "09:00", 69 | "10:00", 70 | "11:00", 71 | "12:00", 72 | "13:00", 73 | "14:00", 74 | "15:00", 75 | "16:00", 76 | "17:00", 77 | "18:00", 78 | "19:00", 79 | "20:00", 80 | "21:00", 81 | "22:00", 82 | "23:00", 83 | ], 84 | ) 85 | data_format: str = "netcdf" 86 | download_format: str = "zip" 87 | area: tuple[float, float, float, float] = (53.7, 3.2, 50.75, 7.22) 88 | 89 | @property 90 | def request_parameters(self) -> dict[str, str | list[str] | tuple[float]]: 91 | """Returns the request parameters as a dictionary.""" 92 | return { 93 | "product_type": self.product_type, 94 | "variable": self.variables, 95 | "year": self.year, 96 | "month": self.month, 97 | "day": self.day, 98 | "time": self.time, 99 | "area": self.area, 100 | "data_format": self.data_format, 101 | "download_format": self.download_format 102 | } 103 | 104 | 105 | CDS_CLIENT = cdsapi.Client(info_callback=_info_callback(), url="https://cds.climate.copernicus.eu/api") 106 | -------------------------------------------------------------------------------- /weather_provider_api/routers/weather/sources/cds/factors.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # SPDX-FileCopyrightText: 2019-2022 Alliander N.V. 5 | # SPDX-License-Identifier: MPL-2.0 6 | 7 | # factor name mapping 8 | 9 | import json 10 | 11 | from weather_provider_api.routers.weather.utils.file_helpers import ( 12 | get_var_map_file_location, 13 | ) 14 | 15 | file_to_use_era5sl = get_var_map_file_location("era5sl_var_map.json") 16 | file_to_use_era5land = get_var_map_file_location("era5land_var_map.json") 17 | 18 | with open(file_to_use_era5sl, "r") as _f: 19 | era5sl_factors: dict = json.load(_f) 20 | # for factor in ["mwd", "mwp", "swh"]: 21 | # era5sl_factors.pop(factor) 22 | 23 | 24 | with open(file_to_use_era5land, "r") as _f: 25 | era5land_factors = json.load(_f) 26 | -------------------------------------------------------------------------------- /weather_provider_api/routers/weather/sources/cds/models/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # SPDX-FileCopyrightText: 2019-2022 Alliander N.V. 5 | # SPDX-License-Identifier: MPL-2.0 6 | -------------------------------------------------------------------------------- /weather_provider_api/routers/weather/sources/knmi/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # SPDX-FileCopyrightText: 2019-2022 Alliander N.V. 5 | # SPDX-License-Identifier: MPL-2.0 6 | -------------------------------------------------------------------------------- /weather_provider_api/routers/weather/sources/knmi/client/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # SPDX-FileCopyrightText: 2019-2022 Alliander N.V. 5 | # SPDX-License-Identifier: MPL-2.0 6 | -------------------------------------------------------------------------------- /weather_provider_api/routers/weather/sources/knmi/knmi.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # SPDX-FileCopyrightText: 2019-2022 Alliander N.V. 5 | # SPDX-License-Identifier: MPL-2.0 6 | 7 | from weather_provider_api.routers.weather.base_models.source import WeatherSourceBase 8 | from weather_provider_api.routers.weather.sources.knmi.models.actuele_waarnemingen import ( 9 | ActueleWaarnemingenModel, 10 | ) 11 | from weather_provider_api.routers.weather.sources.knmi.models.actuele_waarnemingen_register import ( 12 | ActueleWaarnemingenRegisterModel, 13 | ) 14 | from weather_provider_api.routers.weather.sources.knmi.models.daggegevens import ( 15 | DagGegevensModel, 16 | ) 17 | from weather_provider_api.routers.weather.sources.knmi.models.harmonie_arome import ( 18 | HarmonieAromeModel, 19 | ) 20 | from weather_provider_api.routers.weather.sources.knmi.models.pluim import PluimModel 21 | from weather_provider_api.routers.weather.sources.knmi.models.uurgegevens import ( 22 | UurgegevensModel, 23 | ) 24 | 25 | 26 | class KNMI(WeatherSourceBase): 27 | def __init__(self): 28 | model_instances = [ 29 | UurgegevensModel(), 30 | DagGegevensModel(), 31 | HarmonieAromeModel(), 32 | PluimModel(), 33 | ActueleWaarnemingenModel(), 34 | ActueleWaarnemingenRegisterModel(), 35 | ] 36 | 37 | self.id = "knmi" 38 | self.name = "Koninklijk Nederlands Meteorologisch Instituut (KNMI)" 39 | self.url = "https://knmi.nl/" 40 | self._models = None 41 | self._async_models = None 42 | 43 | self.setup_models(model_instances) 44 | -------------------------------------------------------------------------------- /weather_provider_api/routers/weather/sources/knmi/knmi_factors.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # SPDX-FileCopyrightText: 2019-2022 Alliander N.V. 5 | # SPDX-License-Identifier: MPL-2.0 6 | 7 | # factor name mapping 8 | 9 | import json 10 | 11 | from weather_provider_api.routers.weather.utils.file_helpers import ( 12 | get_var_map_file_location, 13 | ) 14 | 15 | file_to_use = get_var_map_file_location("arome_var_map.json") 16 | 17 | with open(file_to_use, "r") as _f: 18 | arome_factors: dict = json.load(_f) 19 | -------------------------------------------------------------------------------- /weather_provider_api/routers/weather/sources/knmi/models/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # SPDX-FileCopyrightText: 2019-2022 Alliander N.V. 5 | # SPDX-License-Identifier: MPL-2.0 6 | -------------------------------------------------------------------------------- /weather_provider_api/routers/weather/sources/knmi/models/actuele_waarnemingen.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # SPDX-FileCopyrightText: 2019-2022 Alliander N.V. 5 | # SPDX-License-Identifier: MPL-2.0 6 | 7 | """KNMI current weather data fetcher.""" 8 | 9 | import copy 10 | from typing import List, Optional 11 | 12 | import numpy as np 13 | import xarray as xr 14 | 15 | from weather_provider_api.routers.weather.base_models.model import WeatherModelBase 16 | from weather_provider_api.routers.weather.sources.knmi.stations import ( 17 | stations_actual, 18 | ) 19 | from weather_provider_api.routers.weather.sources.knmi.utils import ( 20 | download_actuele_waarnemingen_weather, 21 | find_closest_stn_list, 22 | ) 23 | from weather_provider_api.routers.weather.utils.geo_position import GeoPosition 24 | from weather_provider_api.routers.weather.utils.pandas_helpers import coords_to_pd_index 25 | 26 | 27 | class ActueleWaarnemingenModel(WeatherModelBase): 28 | """A Weather Model that incorporates the "KNMI Actuele Waarnemingen" 29 | dataset into the Weather Provider API. 30 | """ 31 | 32 | def __init__(self): 33 | super().__init__() 34 | self.id = "waarnemingen" 35 | self.name = "KNMI Actuele Waarnemingen" 36 | self.version = "" 37 | self.url = "https://www.knmi.nl/nederland-nu/weer/waarnemingen" 38 | self.predictive = False 39 | self.time_step_size_minutes = 10 40 | self.num_time_steps = 1 41 | self.description = "Current weather observations. Updated every 10 minutes." 42 | self.async_model = False 43 | 44 | self.to_si = { 45 | "weather_description": { 46 | "name": "weather_description", 47 | "convert": self.no_conversion, 48 | }, 49 | "temperature": {"convert": self.celsius_to_kelvin}, 50 | "humidity": {"convert": self.percentage_to_frac}, 51 | "wind_direction": {"convert": self.dutch_wind_direction_to_degrees}, 52 | "wind_speed": {"convert": self.no_conversion}, # m/s 53 | "visibility": {"convert": self.no_conversion}, # m 54 | "air_pressure": {"convert": lambda x: x * 100}, # hPa to Pa 55 | } 56 | self.to_human = copy.deepcopy(self.to_si) 57 | self.to_human["temperature"]["convert"] = self.no_conversion # C 58 | 59 | self.human_to_model_specific = self._create_reverse_lookup(self.to_si) 60 | 61 | def get_weather( 62 | self, 63 | coords: List[GeoPosition], 64 | begin: Optional[np.datetime64], 65 | end: Optional[np.datetime64], 66 | weather_factors: List[str] = None, 67 | ) -> xr.Dataset: 68 | """The function that gathers and processes the requested Actuele Waarnemingen weather data from the KNMI site 69 | and returns it as a Xarray Dataset. 70 | (This model interprets directly from an HTML page, but the information is also available from the data 71 | platform. Due to it being rather impractically handled, we stick to the site for now.) 72 | 73 | Args: 74 | coords: A list of GeoPositions containing the locations the data is requested for. 75 | begin: A datetime containing the start of the period to request data for. 76 | end: A datetime containing the end of the period to request data for. 77 | weather_factors: A list of weather factors to request data for (in string format) 78 | 79 | Returns: 80 | An Xarray Dataset containing the weather data for the requested period, locations and factors. 81 | 82 | Notes: 83 | As this model only return the current weather data the 'begin' and 'end' values are not actually used. 84 | """ 85 | updated_weather_factors = self._request_weather_factors(weather_factors) 86 | 87 | # Download the current weather data 88 | raw_ds = download_actuele_waarnemingen_weather() 89 | 90 | # Get a list of the relevant STNs and choose the closest STN for each coordinate 91 | coords_stn, _, _ = find_closest_stn_list(stations_actual, coords) 92 | 93 | # Select the data for the found closest STNs 94 | ds = raw_ds.sel(STN=coords_stn) 95 | 96 | data_dict = { 97 | var_name: (["time", "coord"], var.values) 98 | for var_name, var in ds.data_vars.items() 99 | if var_name in updated_weather_factors and var_name not in ["lat", "lon"] 100 | } 101 | 102 | timeline = ds.coords["time"].values 103 | 104 | ds = xr.Dataset( 105 | data_vars=data_dict, 106 | coords={"time": timeline, "coord": coords_to_pd_index(coords)}, 107 | ) 108 | ds = ds.unstack("coord") 109 | return ds 110 | 111 | def is_async(self): # pragma: no cover 112 | return self.async_model 113 | 114 | def _request_weather_factors(self, factors: Optional[List[str]]) -> List[str]: 115 | # Implementation of the Base Weather Model function that returns a list of known weather factors for the model. 116 | if factors is None: 117 | return list(self.to_si.keys()) 118 | 119 | new_factors = [] 120 | 121 | for f in factors: 122 | f_low = f.lower() 123 | if f_low in self.to_si: 124 | new_factors.append(f_low) 125 | 126 | return list(set(new_factors)) # Cleanup any duplicate values and return 127 | -------------------------------------------------------------------------------- /weather_provider_api/routers/weather/sources/knmi/models/actuele_waarnemingen_register.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # SPDX-FileCopyrightText: 2019-2023 Alliander N.V. 5 | # SPDX-License-Identifier: MPL-2.0 6 | 7 | """KNMI current weather data aggregate fetcher.""" 8 | 9 | import copy 10 | from datetime import datetime 11 | from typing import List, Optional 12 | 13 | import numpy as np 14 | import xarray as xr 15 | from dateutil.relativedelta import relativedelta 16 | 17 | from weather_provider_api.routers.weather.base_models.model import WeatherModelBase 18 | from weather_provider_api.routers.weather.sources.knmi.client.actuele_waarnemingen_register_repository import ( 19 | ActueleWaarnemingenRegisterRepository, 20 | ) 21 | from weather_provider_api.routers.weather.sources.knmi.stations import stations_actual 22 | from weather_provider_api.routers.weather.sources.knmi.utils import ( 23 | find_closest_stn_list, 24 | ) 25 | from weather_provider_api.routers.weather.utils.geo_position import GeoPosition 26 | from weather_provider_api.routers.weather.utils.pandas_helpers import coords_to_pd_index 27 | 28 | 29 | class ActueleWaarnemingenRegisterModel(WeatherModelBase): 30 | """A Weather model aimed at accessing a 24-hour register for the "KNMi Actuele Waarnemingen" dataset.""" 31 | 32 | def is_async(self): 33 | return self.async_model 34 | 35 | def __init__(self): 36 | super().__init__() 37 | self.id = "waarnemingen_register" 38 | self.name = "KNMI Actuele Waarnemingen - 48 uur register" 39 | self.version = "" 40 | self.url = "https://www.knmi.nl/nederland-nu/weer/waarnemingen" 41 | self.predictive = False 42 | self.time_step_size_minutes = 10 43 | self.num_time_steps = 12 44 | self.description = "48 Hour register for current weather observations. Updated every 10 minutes." 45 | self.async_model = False 46 | self.repository = ActueleWaarnemingenRegisterRepository() 47 | 48 | self.to_si = { 49 | "weather_description": { 50 | "name": "weather_description", 51 | "convert": self.no_conversion, 52 | }, 53 | "temperature": {"convert": self.celsius_to_kelvin}, 54 | "humidity": {"convert": self.percentage_to_frac}, 55 | "wind_direction": {"convert": self.dutch_wind_direction_to_degrees}, 56 | "wind_speed": {"convert": self.no_conversion}, # m/s 57 | "visibility": {"convert": self.no_conversion}, # m 58 | "air_pressure": {"convert": lambda x: x * 100}, # hPa to Pa 59 | } 60 | self.to_human = copy.deepcopy(self.to_si) 61 | self.to_human["temperature"]["convert"] = self.no_conversion # C 62 | 63 | self.human_to_model_specific = self._create_reverse_lookup(self.to_si) 64 | 65 | def get_weather( 66 | self, 67 | coords: List[GeoPosition], 68 | begin: Optional[np.datetime64], 69 | end: Optional[np.datetime64], 70 | weather_factors: List[str] = None, 71 | ) -> xr.Dataset: 72 | """The function that gathers and processes the requested Actuele Waarnemingen Register weather data from the 73 | 48-hour register and returns it as a Xarray Dataset. 74 | (The register for this model interprets directly from an HTML page, but the information is also available from 75 | the data platform. Due to it being rather impractically handled, we stick to the site for now.) 76 | 77 | Args: 78 | coords: A list of GeoPositions containing the locations the data is requested for. 79 | begin: A datetime containing the start of the period to request data for. 80 | end: A datetime containing the end of the period to request data for. 81 | weather_factors: A list of weather factors to request data for (in string format) 82 | 83 | Returns: 84 | An Xarray Dataset containing the weather data for the requested period, locations and factors. 85 | 86 | Notes: 87 | As this model only return the current weather data the 'begin' and 'end' values are not actually used. 88 | """ 89 | updated_weather_factors = self._request_weather_factors(weather_factors) 90 | coords_stn, _, _ = find_closest_stn_list(stations_actual, coords) 91 | 92 | now = datetime.utcnow() 93 | if now - relativedelta(days=1) > begin: 94 | raw_ds = self.repository.get_48_hour_registry_for_station(station=coords_stn) 95 | else: 96 | raw_ds = self.repository.get_24_hour_registry_for_station(station=coords_stn) 97 | 98 | data_dictionary = { 99 | var_name: (["time", "coord"], var.values) 100 | for var_name, var in raw_ds.data_vars.items() 101 | if var_name in updated_weather_factors and var_name not in ["lat", "lon"] 102 | } 103 | 104 | timeline = raw_ds.coords["time"].values 105 | 106 | output_ds = xr.Dataset( 107 | data_vars=data_dictionary, 108 | coords={"time": timeline, "coord": coords_to_pd_index(coords)}, 109 | ) 110 | output_ds = output_ds.unstack("coord") 111 | return output_ds 112 | 113 | def _request_weather_factors(self, factors: Optional[List[str]]) -> List[str]: 114 | # Implementation of the Base Weather Model function that returns a list of known weather factors for the model. 115 | if factors is None: 116 | return list(self.to_si.keys()) 117 | 118 | new_factors = [] 119 | 120 | for f in factors: 121 | f_low = f.lower() 122 | if f_low in self.to_si: 123 | new_factors.append(f_low) 124 | 125 | return list(set(new_factors)) # Cleanup any duplicate values and return 126 | -------------------------------------------------------------------------------- /weather_provider_api/routers/weather/sources/knmi/models/harmonie_arome.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # SPDX-FileCopyrightText: 2019-2022 Alliander N.V. 5 | # SPDX-License-Identifier: MPL-2.0 6 | 7 | """-- MODULE --""" 8 | 9 | import copy 10 | from datetime import datetime, time, timedelta 11 | from typing import List, Union 12 | 13 | import xarray as xr 14 | from loguru import logger 15 | 16 | from weather_provider_api.routers.weather.base_models.model import WeatherModelBase 17 | from weather_provider_api.routers.weather.sources.knmi.client.arome_repository import ( 18 | HarmonieAromeRepository, 19 | ) 20 | from weather_provider_api.routers.weather.sources.knmi.knmi_factors import arome_factors 21 | from weather_provider_api.routers.weather.utils.date_helpers import ( 22 | validate_begin_and_end, 23 | ) 24 | from weather_provider_api.routers.weather.utils.geo_position import GeoPosition 25 | 26 | 27 | class HarmonieAromeModel(WeatherModelBase): 28 | """A weather model that incorporates the 'KNMI - Harmonie Arome' weather dataset into the Weather Provider 29 | Libraries and API. 30 | 31 | 'KNMI - Harmonie Arome' is a predictive model for the upcoming 48 hours that gets generated every 6 hours. 32 | """ 33 | 34 | def __init__(self): 35 | # Pre-work 36 | super().__init__() 37 | self.id = "arome" 38 | logger.debug(f"Initializing weather model: {self.id}") 39 | 40 | # Setting the model 41 | self.repository = HarmonieAromeRepository() 42 | self.name = "harmonie_arome_cy_p1" 43 | self.version = "0.3" 44 | self.url = "ftp://data.knmi.nl/download/harmonie_arome_cy40_p1/0.2/" 45 | self.predictive = True 46 | self.time_step_size_minutes = 60 47 | self.num_time_steps = 48 48 | self.async_model = False 49 | self.dataset_name = "harmonie_arome_cy40_p1" 50 | self.dataset_version = "0.2" 51 | 52 | # Put up conversion settings 53 | si_conversion_dict = { 54 | k: {"name": k, "convert": lambda x: x} for k in arome_factors.values() 55 | } # The default output format for Arome is SI 56 | self.to_si = si_conversion_dict 57 | 58 | # Human output conversion: 59 | self.to_human = copy.deepcopy(self.to_si) 60 | self.to_human["temperature"]["convert"] = self.kelvin_to_celsius 61 | 62 | # Initialization complete 63 | logger.debug(f'The Weather model "{self.id}" was successfully initialized') 64 | 65 | def get_weather( 66 | self, 67 | coords: List[GeoPosition], 68 | begin: datetime = None, 69 | end: datetime = None, 70 | weather_factors: List[str] = None, 71 | ) -> xr.Dataset: 72 | """Implementation of the WeatherModelBase get_weather function that fetches weather data and returns it as a 73 | Xarray DataSet. 74 | 75 | """ 76 | # Handle possibly missing values for this model 77 | begin = begin or datetime.combine(datetime.today(), time.min) # Fallback: start of today 78 | end = end or datetime.combine(datetime.today() + timedelta(days=1), time.max) # Fallback: end of tomorrow 79 | 80 | # Validate the given timeframe 81 | valid_begin, valid_end = validate_begin_and_end( 82 | begin, 83 | end, 84 | self.repository.first_day_of_repo, 85 | self.repository.last_day_of_repo, 86 | ) 87 | 88 | # Gather the period from the repository 89 | weather_dataset = self.repository.gather_period(begin=valid_begin, end=valid_end, coordinates=coords) 90 | 91 | # Filter the requested factors 92 | translated_factors = [] 93 | if weather_factors is not None: 94 | for factor in weather_factors: 95 | if factor in arome_factors.keys(): 96 | new_factor = arome_factors[factor] 97 | elif factor in arome_factors.values(): 98 | new_factor = factor 99 | else: 100 | new_factor = None 101 | 102 | if new_factor is not None: 103 | translated_factors.append(new_factor) 104 | 105 | for factor in weather_dataset.keys(): 106 | if not any(item in factor for item in translated_factors): 107 | weather_dataset = weather_dataset.drop(factor) 108 | 109 | return weather_dataset 110 | 111 | def is_async(self): 112 | return self.async_model 113 | 114 | def _request_weather_factors(self, factor_list: Union[List[str], None]) -> List[str]: 115 | """Implementation of the Weather Model Base function that returns a list of the known weather factors out of 116 | a given list for this model. 117 | 118 | """ 119 | if not factor_list: 120 | return list(self.to_si.keys()) 121 | 122 | validated_factors = [] 123 | 124 | for factor in factor_list: 125 | factor_lowercase = factor.lower() 126 | 127 | if factor_lowercase in self.to_si: 128 | validated_factors.append(factor_lowercase) 129 | elif factor in self.to_si: 130 | validated_factors.append(factor) 131 | 132 | return list(set(validated_factors)) 133 | -------------------------------------------------------------------------------- /weather_provider_api/routers/weather/sources/weather_alert/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # SPDX-FileCopyrightText: 2019-2022 Alliander N.V. 5 | # SPDX-License-Identifier: MPL-2.0 6 | -------------------------------------------------------------------------------- /weather_provider_api/routers/weather/sources/weather_alert/weather_alert.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # SPDX-FileCopyrightText: 2019-2022 Alliander N.V. 5 | # SPDX-License-Identifier: MPL-2.0 6 | 7 | """Class to retrieve the current Weather Alert status according to the KNMI site""" 8 | 9 | from enum import Enum 10 | 11 | import requests 12 | from bs4 import BeautifulSoup 13 | from requests.adapters import HTTPAdapter 14 | from requests.exceptions import ProxyError, Timeout, TooManyRedirects 15 | from urllib3 import Retry 16 | 17 | 18 | class WeatherAlertCode(Enum): 19 | # Enum class with valid Weather Alert Codes 20 | green = "green" 21 | yellow = "yellow" 22 | orange = "orange" 23 | red = "red" 24 | 25 | 26 | class WeatherAlert: 27 | """A class (not a Weather Model!) that parses the Weather Alert status from the KNMI site (Weeralarm)""" 28 | 29 | def __init__(self): 30 | self.id = "weatheralert" 31 | self.name = "KNMI Weather Alert" 32 | self.version = "0.8" 33 | self.url = "https://www.knmi.nl/nederland-nu/weer/waarschuwingen/" 34 | self.predictive = True 35 | self.provinces = ( 36 | "drenthe", 37 | "friesland", 38 | "gelderland", 39 | "groningen", 40 | "flevoland", 41 | "limburg", 42 | "noord-brabant", 43 | "noord-holland", 44 | "overijssel", 45 | "utrecht", 46 | "zeeland", 47 | "zuid-holland", 48 | ) # The Dutch Provinces. Every province has its own page. 49 | 50 | def get_alarm(self): 51 | """A function that retrieves the current weather alarm stage for each of the Dutch provinces and puts those 52 | together into a formatted list of results (string-based). 53 | 54 | Returns: 55 | A list of strings holding all the provinces and their retrieved current alarm stages according to KNMI 56 | """ 57 | alarm_list = [] 58 | for province in self.provinces: 59 | # Every province is available from a different page, so we have to request all of them separately 60 | page_text = "" 61 | try: 62 | page = self._requests_retry_session().get(self.url + province) 63 | status_code = page.status_code 64 | page_text = page.text 65 | except Timeout: 66 | status_code = 408 67 | except ProxyError: 68 | status_code = 407 69 | except TooManyRedirects: 70 | status_code = 999 71 | 72 | append_string = self.process_page(page_text, status_code, province) 73 | alarm_list.append(append_string) 74 | return alarm_list 75 | 76 | @staticmethod 77 | def process_page(page_text: str, status_code, province): 78 | """A function that parses the weather alert page for a province and retrieves its current alarm stage. 79 | 80 | Args: 81 | page_text: The response content retrieved while trying to download the page 82 | status_code: The status code retrieved while trying to download the page. 83 | province: The province associated with the url, status code and alarm stage. 84 | 85 | Returns: 86 | A tuple holding the province and a result-string for that province. 87 | A result-string usually hold the alarm stage for that province, but can also hold exceptions when 88 | downloading did not succeed as intended. 89 | """ 90 | if status_code == 200 and page_text is not None: 91 | # A page was found and loaded 92 | soup = BeautifulSoup(page_text, features="lxml") 93 | 94 | classes_first_warning_block = soup.find("div", {"class": "warning-overview"}) 95 | if classes_first_warning_block is not None: 96 | classes_first_warning_block = classes_first_warning_block["class"] 97 | 98 | for class_name in classes_first_warning_block: 99 | if len(class_name) > len("warning-overview") and class_name[len("warning-overview--") :] in set( 100 | item.value for item in WeatherAlertCode 101 | ): 102 | return province, class_name[len("warning-overview--") :] 103 | 104 | # If no valid code was found return an invalid data message 105 | return province, "could not find expected data on page" 106 | elif status_code == 408: 107 | return province, "time out op loading page" 108 | elif status_code == 407: 109 | return province, "proxy error on loading page" 110 | else: 111 | return province, "page proved inaccessible" 112 | 113 | @staticmethod 114 | def _requests_retry_session( 115 | # A function for basic retrying of an url when it isn't accessible immediately. 116 | retries=8, 117 | backoff_factor=0.01, 118 | status_forcelist=(500, 502, 504), 119 | session=None, 120 | ) -> requests.Session: 121 | session = session or requests.Session() 122 | retry = Retry( 123 | total=retries, 124 | read=retries, 125 | connect=retries, 126 | backoff_factor=backoff_factor, 127 | status_forcelist=status_forcelist, 128 | ) 129 | adapter = HTTPAdapter(max_retries=retry) 130 | session.mount("https://", adapter) 131 | return session 132 | -------------------------------------------------------------------------------- /weather_provider_api/routers/weather/utils/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # SPDX-FileCopyrightText: 2019-2022 Alliander N.V. 5 | # SPDX-License-Identifier: MPL-2.0 6 | -------------------------------------------------------------------------------- /weather_provider_api/routers/weather/utils/date_helpers.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # SPDX-FileCopyrightText: 2019-2022 Alliander N.V. 5 | # SPDX-License-Identifier: MPL-2.0 6 | 7 | from datetime import datetime, timedelta 8 | from typing import Optional, Union 9 | 10 | import numpy as np 11 | import pandas as pd 12 | from fastapi import HTTPException 13 | from loguru import logger 14 | from pytz import UTC 15 | 16 | 17 | def parse_datetime( 18 | datetime_string, 19 | round_missing_time_up=False, 20 | round_to_days=False, 21 | raise_errors=False, 22 | loc=None, 23 | ) -> Optional[datetime]: 24 | if datetime_string is None: 25 | return None 26 | 27 | dt = pd.to_datetime(datetime_string, dayfirst=False, errors="coerce") 28 | 29 | if pd.isnull(dt): 30 | logger.exception("Error while parsing datetime string", input=datetime_string) 31 | if raise_errors: 32 | # Note: replace when FastAPI supports Pydantic models to define query parameters 33 | # (meaning Validators can be used) 34 | error_msg = { 35 | "loc": loc, 36 | "msg": "invalid datetime format", 37 | "type": "type_error.datetime", 38 | } 39 | raise HTTPException(status_code=422, detail=[error_msg]) 40 | 41 | dt = None 42 | 43 | if dt is not None and (round_missing_time_up or round_to_days) and time_unknown(dt, datetime_string): 44 | if round_to_days: 45 | dt = dt + timedelta(days=1) 46 | else: 47 | dt = dt.replace(hour=23, minute=59, second=59) 48 | 49 | if dt is not None: 50 | dt = np.datetime64(dt).astype(datetime) 51 | 52 | return dt 53 | 54 | 55 | def time_unknown(dt: datetime, datetime_string: str): # pragma: no cover 56 | if dt.hour == 0 and dt.minute == 0 and dt.second == 0 and ":" not in datetime_string: 57 | return True 58 | return False 59 | 60 | 61 | def validate_begin_and_end( 62 | start: datetime, 63 | end: datetime, 64 | data_start: Union[datetime, None] = None, 65 | data_end: Union[datetime, None] = None, 66 | ): 67 | """Checks the given date parameters and replaces them with default values if they aren't valid. 68 | The resulting values are then returned. 69 | """ 70 | start = start.astimezone(UTC) if start else None 71 | end = end.astimezone(UTC) if end else None 72 | data_start = data_start.astimezone(UTC) if data_start else None 73 | data_end = data_end.astimezone(UTC) if data_end else None 74 | 75 | if data_end is None: 76 | # Assuming predictions fill in this value, the most recent value for the past is before "now". 77 | data_end = datetime.now(UTC) 78 | 79 | if data_start is not None and data_start > start: 80 | # If the starting moment lies before what can be requested, put it at the moment from which it can be requested 81 | start = data_start 82 | 83 | if data_end is not None and data_end < end: 84 | end = data_end 85 | 86 | if start >= data_end: 87 | raise HTTPException( 88 | 422, 89 | f"Invalid [start] value [{start}]: value lies after last available moment for model ({data_end})", 90 | ) 91 | if data_start is not None and end <= data_start: 92 | raise HTTPException( 93 | 422, 94 | f"Invalid [end] value [{end}]: value lies before first available moment for model ({data_start})", 95 | ) 96 | 97 | if end < start: 98 | raise HTTPException( 99 | 422, 100 | f"Invalid [start] and [end] values: [end]({end}) lies before [start]({start})", 101 | ) 102 | 103 | return start, end 104 | -------------------------------------------------------------------------------- /weather_provider_api/routers/weather/utils/file_helpers.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # SPDX-FileCopyrightText: 2019-2022 Alliander N.V. 5 | # SPDX-License-Identifier: MPL-2.0 6 | 7 | import os 8 | import site 9 | from pathlib import Path 10 | 11 | from loguru import logger 12 | 13 | 14 | async def remove_file(file_path): # pragma: no cover 15 | if file_path is not None: 16 | try: 17 | file_to_rm = Path(file_path).resolve() 18 | logger.info("Removing temporary file", file_path=file_to_rm) 19 | if file_to_rm.exists() and file_to_rm.is_file(): 20 | file_to_rm.unlink() 21 | except FileNotFoundError as e: 22 | logger.exception(e) 23 | raise 24 | return True 25 | 26 | 27 | def get_var_map_file_location(filename: str) -> Path: 28 | var_map_folder = "var_maps" 29 | 30 | possible_main_folders = [ 31 | Path(os.getcwd()), # Running from main folder 32 | Path(os.getcwd()).parent, # Running from weather_provider_api folder or scripts 33 | Path(site.getsitepackages()[-1]), # Running as package 34 | ] 35 | 36 | for folder in possible_main_folders: 37 | possible_var_map_folder = folder.joinpath(var_map_folder) 38 | if possible_var_map_folder.exists(): 39 | logger.info(f'"var_maps" folder was found at: {possible_var_map_folder}') 40 | return possible_var_map_folder.joinpath(filename) 41 | 42 | logger.exception(f"File was not found: {filename}") 43 | raise FileNotFoundError 44 | -------------------------------------------------------------------------------- /weather_provider_api/routers/weather/utils/grid_helpers.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # SPDX-FileCopyrightText: 2019-2022 Alliander N.V. 5 | # SPDX-License-Identifier: MPL-2.0 6 | 7 | from typing import List, Tuple 8 | 9 | import numpy as np 10 | 11 | from weather_provider_api.routers.weather.utils.geo_position import GeoPosition 12 | 13 | 14 | def round_coordinates_to_wgs84_grid( 15 | coordinates: List[GeoPosition], 16 | grid_resolution_lat_lon: Tuple[float, float], 17 | starting_points_lat_lon: Tuple[float, float] = (0, 0), 18 | ) -> List[GeoPosition]: 19 | """A function that rounds coordinates to a WGS84 coordinate grid based on the given settings. 20 | 21 | Args: 22 | coordinates: The list of coordinates to round to the grid. 23 | grid_resolution_lat_lon: A tuple holding the grid resolution to use for rounding 24 | starting_points_lat_lon: An optional tuple holding the starting values of the grid. 25 | Useful if a grid doesn't start at a multitude of the grid resolution. 26 | 27 | Returns: 28 | A list of coordinates that have been rounded to the nearest points on the given grid. 29 | 30 | """ 31 | grid_res_lat = grid_resolution_lat_lon[0] 32 | grid_res_lon = grid_resolution_lat_lon[1] 33 | start_lat = starting_points_lat_lon[0] 34 | start_lon = starting_points_lat_lon[1] 35 | 36 | wgs84_coordinate_list = [coordinate.get_WGS84() for coordinate in coordinates] 37 | rounded_wgs84_coordinate_list = [ 38 | ( 39 | (np.round((coordinate[0] - start_lat) / grid_res_lat) * grid_res_lat) + start_lat, 40 | (np.round((coordinate[1] - start_lon) / grid_res_lon) * grid_res_lon) + start_lon, 41 | ) 42 | for coordinate in wgs84_coordinate_list 43 | ] 44 | 45 | return [GeoPosition(coordinate[0], coordinate[1]) for coordinate in rounded_wgs84_coordinate_list] 46 | -------------------------------------------------------------------------------- /weather_provider_api/routers/weather/utils/pandas_helpers.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # SPDX-FileCopyrightText: 2019-2022 Alliander N.V. 5 | # SPDX-License-Identifier: MPL-2.0 6 | 7 | from typing import List 8 | 9 | import pandas as pd 10 | 11 | from weather_provider_api.routers.weather.utils.geo_position import GeoPosition 12 | 13 | 14 | def coords_to_pd_index(coords: List[GeoPosition]) -> pd.MultiIndex: 15 | return pd.MultiIndex.from_tuples([coord.get_WGS84() for coord in coords], names=("lat", "lon")) 16 | -------------------------------------------------------------------------------- /weather_provider_api/routers/weather/utils/serializers.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # SPDX-FileCopyrightText: 2019-2022 Alliander N.V. 5 | # SPDX-License-Identifier: MPL-2.0 6 | 7 | import tempfile 8 | from typing import List, Tuple, Union 9 | 10 | import pandas as pd 11 | import xarray 12 | from starlette.responses import FileResponse 13 | 14 | from weather_provider_api.routers.weather.api_models import ( 15 | ResponseFormat, 16 | ScientificJSONResponse, 17 | WeatherContentRequestMultiLocationQuery, 18 | WeatherContentRequestQuery, 19 | ) 20 | 21 | 22 | def file_or_text_response( 23 | unserialized_data: xarray.Dataset, 24 | response_format: ResponseFormat, 25 | source_id: str, 26 | model_id: str, 27 | request: Union[WeatherContentRequestQuery, WeatherContentRequestMultiLocationQuery], 28 | coords: List[Tuple[float, float]], 29 | ) -> tuple[ScientificJSONResponse | FileResponse, str | None]: 30 | if response_format == ResponseFormat.json: 31 | return json_response(unserialized_data, coords) 32 | elif response_format == ResponseFormat.json_dataset: 33 | return json_dataset_response(unserialized_data) 34 | else: 35 | return file_response(unserialized_data, response_format, source_id, model_id, request, coords) 36 | 37 | 38 | def file_response( 39 | unserialized_data: xarray.Dataset, 40 | response_format: ResponseFormat, 41 | source_id: str, 42 | model_id: str, 43 | request: WeatherContentRequestQuery, 44 | coords: list[tuple[float, float]], 45 | ) -> tuple[FileResponse, str]: 46 | if response_format == ResponseFormat.netcdf4: 47 | file_path = to_netcdf4(unserialized_data) 48 | mime = "application/x-netcdf4" 49 | extension = ".v4.nc" 50 | elif response_format == ResponseFormat.netcdf3: 51 | file_path = to_netcdf3(unserialized_data) 52 | mime = "application/x-netcdf3" 53 | extension = ".v3.nc" 54 | elif response_format == ResponseFormat.csv: 55 | file_path = to_csv(unserialized_data, coords) 56 | mime = "text/csv" 57 | extension = ".csv" 58 | else: 59 | raise NotImplementedError(f"Cannot create file response for the {response_format.name} response format") 60 | 61 | file_name = generate_filename(source_id, model_id, request, extension) 62 | response = FileResponse(file_path, media_type=mime, filename=file_name) 63 | return response, file_path 64 | 65 | 66 | def generate_filename(source_id: str, model_id: str, request: WeatherContentRequestQuery, extension: str): 67 | file_name = f"weather_{source_id}_{model_id}_{request.begin}-{request.end}{extension}".replace(" ", "T").replace( 68 | ":", "" 69 | ) 70 | return file_name 71 | 72 | 73 | def to_netcdf4(unserialized_data: xarray.Dataset): 74 | temp_file = tempfile.NamedTemporaryFile(delete=False) 75 | temp_file_path = temp_file.name 76 | unserialized_data.to_netcdf(path=temp_file_path, mode="w", format="NETCDF4", engine="netcdf4") 77 | return temp_file_path 78 | 79 | 80 | def to_netcdf3(unserialized_data: xarray.Dataset): 81 | # SciPy handler used by Xarray for direct serialization to binary doesn't support v3. File-based one does. 82 | temp_file = tempfile.NamedTemporaryFile(delete=False) 83 | temp_file_path = temp_file.name 84 | unserialized_data.to_netcdf(path=temp_file_path, format="NETCDF3_64BIT", engine="netcdf4") 85 | return temp_file_path 86 | 87 | 88 | def to_csv(unserialized_data: xarray.Dataset, coords): 89 | csv_strings = {} 90 | cols = [] 91 | 92 | for i, c in enumerate(coords): 93 | df = get_weather_slice_for_coords(c, unserialized_data) 94 | 95 | if i == 0: 96 | cols = list(df.columns.values) 97 | cols.remove("lat") 98 | cols.remove("lon") 99 | cols = ["lat", "lon"] + cols 100 | 101 | csv_str = df.to_csv(columns=cols, header=False, float_format="%.4f") 102 | csv_strings[serialize_coords(c)] = csv_str 103 | 104 | cols = ["time"] + cols 105 | 106 | header = ",".join(cols) 107 | body = "\n".join(csv_strings.values()) 108 | payload = (header + "\n" + body).encode("utf-8") 109 | 110 | temp_file = tempfile.NamedTemporaryFile(delete=False) 111 | temp_file.write(payload) 112 | temp_file.close() 113 | 114 | return temp_file.name 115 | 116 | 117 | def get_weather_slice_for_coords(coord, unserialized_data) -> pd.DataFrame: 118 | # We use .sel to create slices from the unserialized dataset 119 | # We then switch to a Pandas dataframe 120 | if isinstance(coord, list): 121 | coord = coord[0] 122 | weather = unserialized_data.sel(lat=coord[0], lon=coord[1], method="nearest") 123 | if weather.sizes["time"] == 1: 124 | # Because a single moment in time can't be squeezed... 125 | df = weather.to_dataframe() 126 | else: 127 | # ... and multiple times need to be. 128 | df = weather.squeeze().to_dataframe() 129 | return df 130 | 131 | 132 | def json_response(unserialized_data: xarray.Dataset, coords): 133 | serialized_data = [] 134 | for coordinate in coords: 135 | df = get_weather_slice_for_coords(coordinate, unserialized_data) 136 | serialized_data.append(df.reset_index().to_dict(orient="records")) 137 | return ScientificJSONResponse(serialized_data), None 138 | 139 | 140 | def json_dataset_response(unserialized_data: xarray.Dataset): 141 | data_dict = unserialized_data.to_dict() 142 | return ScientificJSONResponse(data_dict), None 143 | 144 | 145 | def serialize_coords(coord): 146 | coord_str = ",".join([str(x) for x in coord]) 147 | return coord_str 148 | -------------------------------------------------------------------------------- /weather_provider_api/scripts/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # SPDX-FileCopyrightText: 2019-2022 Alliander N.V. 5 | # SPDX-License-Identifier: MPL-2.0 6 | -------------------------------------------------------------------------------- /weather_provider_api/scripts/erase_arome_repository.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # SPDX-FileCopyrightText: 2019-2022 Alliander N.V. 5 | # SPDX-License-Identifier: MPL-2.0 6 | 7 | from weather_provider_api.routers.weather.sources.knmi.client.arome_repository import ( 8 | HarmonieAromeRepository, 9 | ) 10 | 11 | 12 | def main(): 13 | arome_repo = HarmonieAromeRepository() 14 | arome_repo.purge_repository() 15 | 16 | 17 | if __name__ == "__main__": 18 | main() 19 | -------------------------------------------------------------------------------- /weather_provider_api/scripts/erase_era5land_repository.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # SPDX-FileCopyrightText: 2019-2022 Alliander N.V. 5 | # SPDX-License-Identifier: MPL-2.0 6 | 7 | 8 | from weather_provider_api.routers.weather.sources.cds.client.era5land_repository import ( 9 | ERA5LandRepository, 10 | ) 11 | 12 | 13 | def main(): 14 | era5land_repo = ERA5LandRepository() 15 | era5land_repo.purge_repository() 16 | 17 | 18 | if __name__ == "__main__": # pragma: no cover 19 | main() 20 | -------------------------------------------------------------------------------- /weather_provider_api/scripts/erase_era5sl_repository.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # SPDX-FileCopyrightText: 2019-2022 Alliander N.V. 5 | # SPDX-License-Identifier: MPL-2.0 6 | 7 | from weather_provider_api.routers.weather.sources.cds.client.era5sl_repository import ( 8 | ERA5SLRepository, 9 | ) 10 | 11 | 12 | def main() -> None: 13 | """Simple method wrapper for purging data.""" 14 | era5sl_repo = ERA5SLRepository() 15 | era5sl_repo.purge_repository() 16 | 17 | 18 | if __name__ == "__main__": 19 | main() 20 | -------------------------------------------------------------------------------- /weather_provider_api/scripts/erase_waarnemingen_register.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # SPDX-FileCopyrightText: 2019-2023 Alliander N.V. 5 | # SPDX-License-Identifier: MPL-2.0 6 | 7 | from weather_provider_api.routers.weather.sources.knmi.client.actuele_waarnemingen_register_repository import ( 8 | ActueleWaarnemingenRegisterRepository, 9 | ) 10 | 11 | 12 | def main(): 13 | # Simple method wrapper for purging data 14 | waarnemingen_repo = ActueleWaarnemingenRegisterRepository() 15 | waarnemingen_repo.purge_repository() 16 | 17 | 18 | if __name__ == "__main__": 19 | main() 20 | -------------------------------------------------------------------------------- /weather_provider_api/scripts/update_arome_repository.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # SPDX-FileCopyrightText: 2019-2022 Alliander N.V. 5 | # SPDX-License-Identifier: MPL-2.0 6 | 7 | from weather_provider_api.routers.weather.sources.knmi.client.arome_repository import ( 8 | HarmonieAromeRepository, 9 | ) 10 | 11 | 12 | def main(): 13 | arome_repo = HarmonieAromeRepository() 14 | arome_repo.update() 15 | 16 | 17 | if __name__ == "__main__": 18 | main() 19 | -------------------------------------------------------------------------------- /weather_provider_api/scripts/update_era5land_repository.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # SPDX-FileCopyrightText: 2019-2022 Alliander N.V. 4 | # SPDX-License-Identifier: MPL-2.0 5 | 6 | # -*- coding: utf-8 -*- 7 | from weather_provider_api.routers.weather.sources.cds.client.era5land_repository import ( 8 | ERA5LandRepository, 9 | ) 10 | 11 | 12 | def main(): 13 | era5land_repo = ERA5LandRepository() 14 | era5land_repo.update() 15 | 16 | 17 | if __name__ == "__main__": # pragma: no cover 18 | main() 19 | -------------------------------------------------------------------------------- /weather_provider_api/scripts/update_era5sl_repository.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | 4 | # SPDX-FileCopyrightText: 2019-2022 Alliander N.V. 5 | # SPDX-License-Identifier: MPL-2.0 6 | import sys 7 | 8 | from loguru import logger 9 | 10 | from weather_provider_api.routers.weather.sources.cds.client.era5sl_repository import ( 11 | ERA5SLRepository, 12 | ) 13 | 14 | 15 | def main(args) -> None: 16 | """Run the update of the ERA5SL repository.""" 17 | test_mode = False 18 | 19 | if len(args) == 2 and args[1] == "testmode": 20 | logger.warning("WARNING: Running in test mode") 21 | test_mode = True 22 | 23 | era5sl_repo = ERA5SLRepository() 24 | era5sl_repo.update(test_mode) 25 | 26 | 27 | if __name__ == "__main__": 28 | main(sys.argv) 29 | -------------------------------------------------------------------------------- /weather_provider_api/scripts/update_waarnemingen_register.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | 5 | # SPDX-FileCopyrightText: 2019-2023 Alliander N.V. 6 | # SPDX-License-Identifier: MPL-2.0 7 | from weather_provider_api.routers.weather.sources.knmi.client.actuele_waarnemingen_register_repository import ( 8 | ActueleWaarnemingenRegisterRepository, 9 | ) 10 | 11 | 12 | def main(): 13 | waarnemingen_repo = ActueleWaarnemingenRegisterRepository() 14 | waarnemingen_repo.update() 15 | 16 | 17 | if __name__ == "__main__": # pragma: no cover 18 | main() 19 | -------------------------------------------------------------------------------- /weather_provider_api/versions/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # SPDX-FileCopyrightText: 2019-2022 Alliander N.V. 5 | # SPDX-License-Identifier: MPL-2.0 6 | -------------------------------------------------------------------------------- /weather_provider_api/versions/v1.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # SPDX-FileCopyrightText: 2019-2022 Alliander N.V. 5 | # SPDX-License-Identifier: MPL-2.0 6 | 7 | from fastapi import FastAPI 8 | 9 | from weather_provider_api.config import APP_CONFIG, APP_SERVER 10 | from weather_provider_api.routers.weather.api_view_v1 import app as weather_router 11 | 12 | app = FastAPI( 13 | version=APP_CONFIG["api_v1"]["implementation"], 14 | title="Weather API (v1)", 15 | root_path="/api/v1", 16 | servers=[{"url": f"{APP_SERVER}/api/v1"}], 17 | description="The v1 endpoint interface for the Weather Provider API", 18 | contact={ 19 | "name": APP_CONFIG["maintainer"]["name"], 20 | "email": APP_CONFIG["maintainer"]["email_address"], 21 | }, 22 | ) 23 | 24 | app.openapi_version = "3.0.2" 25 | 26 | app.include_router(weather_router, prefix="/api/v1/weather") 27 | app.openapi() 28 | -------------------------------------------------------------------------------- /weather_provider_api/versions/v2.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # SPDX-FileCopyrightText: 2019-2022 Alliander N.V. 5 | # SPDX-License-Identifier: MPL-2.0 6 | 7 | from fastapi import FastAPI 8 | 9 | from weather_provider_api.config import APP_CONFIG, APP_SERVER 10 | from weather_provider_api.routers.weather.api_view_v2 import v2_router 11 | 12 | app = FastAPI( 13 | version=APP_CONFIG["api_v2"]["implementation"], 14 | title="Weather API (v2)", 15 | root_path="/api/v2", 16 | servers=[{"url": f"{APP_SERVER}/api/v2"}], 17 | description="The v2 endpoint interface for the Weather Provider API", 18 | contact={ 19 | "name": APP_CONFIG["maintainer"]["name"], 20 | "email": APP_CONFIG["maintainer"]["email_address"], 21 | }, 22 | ) 23 | 24 | app.openapi_version = "3.0.2" 25 | 26 | app.include_router(v2_router, prefix="/weather") 27 | app.openapi() 28 | --------------------------------------------------------------------------------