├── .github └── workflows │ ├── flake8.yml │ └── main.yml ├── .gitignore ├── Dockerfile ├── LICENSE.md ├── MANIFEST.in ├── Makefile ├── README.md ├── debian ├── changelog ├── compat ├── control ├── copyright ├── geomet-data-registry.cron.d ├── geomet-data-registry.install ├── postinst ├── rules └── source │ └── format ├── deploy ├── default │ ├── amqp.yml │ ├── cansips.yml │ ├── cgsl.yml │ ├── gdwps.yml │ ├── geps.yml │ ├── hrdpa.yml │ ├── model_gem_global.yml │ ├── model_gem_regional.yml │ ├── model_giops.yml │ ├── model_hrdps_continental.yml │ ├── model_raqdps-fw-ce.yml │ ├── model_raqdps-fw.yml │ ├── model_raqdps.yml │ ├── model_rdaqa-ce.yml │ ├── model_riops.yml │ ├── radar.yml │ ├── rdpa.yml │ ├── rdwps.yml │ ├── reps.yml │ ├── sarracenia │ │ ├── cansips.conf │ │ ├── cgsl.conf │ │ ├── gdwps.conf │ │ ├── geps.conf │ │ ├── hrdpa.conf │ │ ├── model_gem_global.conf │ │ ├── model_gem_regional.conf │ │ ├── model_giops.conf │ │ ├── model_hrdps_continental.conf │ │ ├── model_raqdps-fw-ce.conf │ │ ├── model_raqdps-fw.conf │ │ ├── model_raqdps.conf │ │ ├── model_rdaqa-ce.conf │ │ ├── model_riops.conf │ │ ├── radar.conf │ │ ├── rdpa.conf │ │ ├── rdwps.conf │ │ ├── rdwps_gulf.conf │ │ ├── reps.conf │ │ └── wcps.conf │ └── wcps.yml └── nightly │ └── deploy-nightly.sh ├── docker ├── Makefile ├── README.md ├── docker-compose-nightly.yml ├── docker-compose.yml └── entrypoint.sh ├── geomet-data-registry.env ├── geomet_data_registry ├── __init__.py ├── env.py ├── event │ ├── __init__.py │ ├── file_.py │ └── message.py ├── handler │ ├── __init__.py │ ├── base.py │ └── core.py ├── layer │ ├── __init__.py │ ├── base.py │ ├── cansips.py │ ├── cgsl.py │ ├── gdwps.py │ ├── geps.py │ ├── hrdpa.py │ ├── model_gem_global.py │ ├── model_gem_regional.py │ ├── model_giops.py │ ├── model_hrdps_continental.py │ ├── model_raqdps.py │ ├── model_raqdps_fw.py │ ├── model_raqdps_fw_ce.py │ ├── model_rdaqa_ce.py │ ├── model_riops.py │ ├── radar_1km.py │ ├── rdpa.py │ ├── rdwps.py │ ├── reps.py │ └── wcps.py ├── log.py ├── notifier │ ├── __init__.py │ ├── base.py │ └── celery_.py ├── plugin.py ├── store │ ├── __init__.py │ ├── base.py │ └── redis_.py ├── tileindex │ ├── __init__.py │ ├── base.py │ └── elasticsearch_.py └── util.py ├── requirements-dev.txt ├── requirements.txt ├── setup.py └── tests ├── __init__.py ├── setup_test_class.py ├── test_base.py ├── test_cansips.py ├── test_cgsl.py ├── test_gdwps.py ├── test_geps.py ├── test_hrdpa.py ├── test_model_gem_global.py ├── test_model_gem_regional.py ├── test_model_giops.py ├── test_model_hrdps_continental.py ├── test_model_raqdps.py ├── test_model_raqdps_fw.py ├── test_model_raqdps_fw_ce.py ├── test_model_rdaqa_ce.py ├── test_model_riops.py ├── test_radar_1km.py ├── test_rdpa.py ├── test_rdwps.py ├── test_reps.py └── test_wcps.py /.github/workflows/flake8.yml: -------------------------------------------------------------------------------- 1 | name: Run flake8 against codebase 🧹 2 | 3 | on: [ push, pull_request ] 4 | 5 | jobs: 6 | run_flake8: 7 | runs-on: ubuntu-latest 8 | strategy: 9 | matrix: 10 | python-version: [3.8] 11 | 12 | steps: 13 | - uses: actions/checkout@v2 14 | - uses: actions/setup-python@v2 15 | name: Set up Python ${{ matrix.python-version }} 16 | with: 17 | python-version: ${{ matrix.python-version }} 18 | - name: Install requirements 📦 19 | run: | 20 | python3 -m pip install --upgrade pip 21 | pip3 install -r requirements-dev.txt 22 | - name: run flake8 🧹 23 | run: | 24 | flake8 25 | -------------------------------------------------------------------------------- /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | name: build ⚙️ 2 | 3 | on: [ push, pull_request ] 4 | 5 | jobs: 6 | main: 7 | runs-on: ubuntu-latest 8 | strategy: 9 | matrix: 10 | python-version: [3.8] 11 | 12 | env: 13 | GDR_LOGGING_LOGLEVEL: "DEBUG" 14 | GDR_LOGGING_LOGFILE: "stdout" 15 | GDR_BASEDIR: "/opt/geomet-data-registry" 16 | GDR_DATADIR: "/data/geomet" 17 | GDR_TILEINDEX_TYPE: "Elasticsearch" 18 | GDR_TILEINDEX_BASEURL: "http://localhost:9200" 19 | GDR_TILEINDEX_NAME: "geomet-data-registry" 20 | GDR_STORE_TYPE: "Redis" 21 | GDR_STORE_URL: "redis://localhost:6379" 22 | GDR_METPX_DISCARD: "on" 23 | GDR_METPX_EVENT_FILE_PY: "/usr/lib/python3/dist-packages/geomet_data_registry/event/file_.py" 24 | GDR_METPX_EVENT_MESSAGE_PY: "/usr/lib/python3/dist-packages/geomet_data_registry/event/message.py" 25 | GDR_METPX_NOTIFY: "True" 26 | GDR_GEOMET_ONLY_USER: "username" 27 | GDR_GEOMET_ONLY_PASS: "password" 28 | GDR_GEOMET_ONLY_HOST: "example.host.com" 29 | GDR_NOTIFICATIONS: "False" 30 | GDR_NOTIFICATIONS_TYPE: "Celery" 31 | GDR_NOTIFICATIONS_URL: "redis://localhost:6379" 32 | 33 | steps: 34 | - name: Configure sysctl limits 35 | run: | 36 | sudo swapoff -a 37 | sudo sysctl -w vm.swappiness=1 38 | sudo sysctl -w fs.file-max=262144 39 | sudo sysctl -w vm.max_map_count=262144 40 | - name: Start Redis 41 | uses: supercharge/redis-github-action@1.2.0 42 | with: 43 | redis-version: 4.0.9 44 | - name: Runs Elasticsearch 45 | uses: elastic/elastic-github-actions/elasticsearch@master 46 | with: 47 | stack-version: 7.5.2 48 | - uses: actions/checkout@v2 49 | - uses: actions/setup-python@v2 50 | name: Setup Python ${{ matrix.python-version }} 51 | with: 52 | python-version: ${{ matrix.python-version }} 53 | - name: Install system dependencies 📦 54 | run: sudo apt-get install -y dh-python devscripts fakeroot debhelper python3-all python3-setuptools python3-dateutil python3-parse 55 | - name: Install requirements 📦 56 | run: | 57 | python3 -m pip install --upgrade pip 58 | pip3 install -r requirements-dev.txt 59 | - name: Install package 📦 60 | run: python3 setup.py install 61 | - name: run tests ⚙️ 62 | run: python3 setup.py test 63 | - name: build Python package 🏗️ 64 | run: python3 setup.py sdist bdist_wheel --universal 65 | - name: build Debian package 🏗️ 66 | run: sudo -E debuild --preserve-env -b -uc -us 67 | 68 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.egg-info 2 | .pybuild 3 | *.pyc 4 | build 5 | dist 6 | __pycache__ 7 | .idea 8 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # 3 | # Copyright (C) 2021 Tom Kralidis 4 | # 5 | # This program is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with this program. If not, see . 17 | # 18 | ############################################################################### 19 | 20 | FROM python:3.6-slim-buster 21 | 22 | ENV GDR_LOGGING_LOGLEVEL DEBUG 23 | ENV GDR_LOGGING_LOGFILE /tmp/geomet-data-registry-dev.log 24 | ENV GDR_CONFIG /opt/geomet-data-registry/geomet-data-registry.yml 25 | ENV GDR_BASEDIR /home/geoadm/geomet-data-registry 26 | ENV GDR_DATADIR /data/geomet/feeds 27 | ENV GDR_TILEINDEX_TYPE Elasticsearch 28 | ENV GDR_TILEINDEX_BASEURL http://localhost:9200 29 | ENV GDR_TILEINDEX_NAME geomet-data-registry-nightly 30 | ENV GDR_STORE_TYPE Redis 31 | ENV GDR_STORE_URL redis://redis:6379 32 | ENV GDR_METPX_DISCARD on 33 | ENV GDR_METPX_EVENT_FILE_PY /home/geoadm/geomet-data-registry/geomet_data_registry/event/file_.py 34 | ENV GDR_METPX_EVENT_MESSAGE_PY /home/geoadm/geomet-data-registry/geomet_data_registry/event/message.py 35 | ENV GDR_METPX_NOTIFY True 36 | ENV GDR_GEOMET_ONLY_USER username 37 | ENV GDR_GEOMET_ONLY_PASS password 38 | ENV GDR_GEOMET_ONLY_HOST feeds.example.org 39 | ENV GDR_NOTIFICATIONS False 40 | ENV GDR_NOTIFICATIONS_TYPE Celery 41 | ENV GDR_NOTIFICATIONS_URL redis://localhost:6379 42 | ENV XDG_CACHE_HOME /tmp/geomet-data-registry-sarra-logs 43 | 44 | # install commonly used dependencies 45 | RUN apt-get update \ 46 | && apt-get install -y ca-certificates curl gcc locales make redis-tools sudo \ 47 | && sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && locale-gen \ 48 | && useradd -ms /bin/bash geoadm && echo "geoadm:geoadm" | chpasswd && adduser geoadm sudo \ 49 | && echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers 50 | 51 | ENV LANG en_US.UTF-8 52 | ENV LANGUAGE en_US:en 53 | ENV LC_ALL en_US.UTF-8 54 | 55 | WORKDIR /home/geoadm 56 | 57 | # setup geomet-data-registry 58 | USER geoadm 59 | COPY . /home/geoadm/geomet-data-registry 60 | WORKDIR /home/geoadm/geomet-data-registry 61 | RUN sudo python setup.py install \ 62 | && sudo mkdir -p ${GDR_DATADIR} \ 63 | && sudo chown -R geoadm:geoadm ${GDR_DATADIR} \ 64 | && mkdir -p ${XDG_CACHE_HOME} 65 | 66 | ENTRYPOINT [ "/home/geoadm/geomet-data-registry/docker/entrypoint.sh" ] 67 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.md LICENSE.md requirements.txt 2 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # ================================================================= 2 | # 3 | # Author: Tom Kralidis 4 | # 5 | # Copyright (c) 2021 Tom Kralidis 6 | # 7 | # Permission is hereby granted, free of charge, to any person 8 | # obtaining a copy of this software and associated documentation 9 | # files (the "Software"), to deal in the Software without 10 | # restriction, including without limitation the rights to use, 11 | # copy, modify, merge, publish, distribute, sublicense, and/or sell 12 | # copies of the Software, and to permit persons to whom the 13 | # Software is furnished to do so, subject to the following 14 | # conditions: 15 | # 16 | # The above copyright notice and this permission notice shall be 17 | # included in all copies or substantial portions of the Software. 18 | # 19 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 20 | # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 21 | # OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 22 | # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 23 | # HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 24 | # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 25 | # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 26 | # OTHER DEALINGS IN THE SOFTWARE. 27 | # 28 | # ================================================================= 29 | 30 | BASEDIR= $(shell pwd) 31 | 32 | foo: 33 | @echo $(BASEDIR) 34 | 35 | clean: stop 36 | geomet-data-registry store teardown 37 | geomet-data-registry tileindex teardown 38 | 39 | clean-logs: 40 | rm -fr $(XDG_CACHE_HOME)/* 41 | 42 | flake8: 43 | find . -type f -name "*.py" | xargs flake8 44 | 45 | setup: 46 | geomet-data-registry store setup 47 | geomet-data-registry tileindex setup 48 | 49 | geomet-data-registry store set -k cansips -c deploy/default/cansips.yml 50 | geomet-data-registry store set -k cgsl -c deploy/default/cgsl.yml 51 | geomet-data-registry store set -k gdwps -c deploy/default/gdwps.yml 52 | geomet-data-registry store set -k geps -c deploy/default/geps.yml 53 | geomet-data-registry store set -k hrdpa -c deploy/default/hrdpa.yml 54 | geomet-data-registry store set -k model_gem_global -c deploy/default/model_gem_global.yml 55 | geomet-data-registry store set -k model_gem_regional -c deploy/default/model_gem_regional.yml 56 | geomet-data-registry store set -k model_giops -c deploy/default/model_giops.yml 57 | geomet-data-registry store set -k model_hrdps_continental -c deploy/default/model_hrdps_continental.yml 58 | geomet-data-registry store set -k model_raqdps-fw-ce -c deploy/default/model_raqdps-fw-ce.yml 59 | geomet-data-registry store set -k model_raqdps-fw -c deploy/default/model_raqdps-fw.yml 60 | geomet-data-registry store set -k model_raqdps -c deploy/default/model_raqdps.yml 61 | geomet-data-registry store set -k model_rdaqa-ce -c deploy/default/model_rdaqa-ce.yml 62 | geomet-data-registry store set -k model_riops -c deploy/default/model_riops.yml 63 | #geomet-data-registry store set -k radar -c deploy/default/radar.yml 64 | geomet-data-registry store set -k rdpa -c deploy/default/rdpa.yml 65 | geomet-data-registry store set -k rdwps -c deploy/default/rdwps.yml 66 | geomet-data-registry store set -k reps -c deploy/default/reps.yml 67 | geomet-data-registry store set -k wcps -c deploy/default/wcps.yml 68 | 69 | start: 70 | sr_subscribe start deploy/default/sarracenia/cansips.conf 71 | sr_subscribe start deploy/default/sarracenia/cgsl.conf 72 | sr_subscribe start deploy/default/sarracenia/gdwps.conf 73 | sr_subscribe start deploy/default/sarracenia/geps.conf 74 | sr_subscribe start deploy/default/sarr acenia/hrdpa.conf 75 | sr_subscribe start deploy/default/sarracenia/model_gem_global.conf 76 | sr_subscribe start deploy/default/sarracenia/model_gem_regional.conf 77 | sr_subscribe start deploy/default/sarracenia/model_giops.conf 78 | sr_subscribe start deploy/default/sarracenia/model_hrdps_continental.conf 79 | sr_subscribe start deploy/default/sarracenia/model_raqdps-fw-ce.conf 80 | sr_subscribe start deploy/default/sarracenia/model_raqdps-fw.conf 81 | sr_subscribe start deploy/default/sarracenia/model_raqdps.conf 82 | sr_subscribe start deploy/default/sarracenia/model_rdaqa-ce.conf 83 | sr_subscribe start deploy/default/sarracenia/model_riops.conf 84 | #sr_subscribe start deploy/default/sarracenia/radar.conf 85 | sr_subscribe start deploy/default/sarracenia/rdpa.conf 86 | sr_subscribe start deploy/default/sarracenia/rdwps.conf 87 | sr_subscribe start deploy/default/sarracenia/reps.conf 88 | sr_subscribe start deploy/default/sarracenia/wcps.conf 89 | 90 | stop: 91 | sr_subscribe stop deploy/default/sarracenia/cansips.conf 92 | sr_subscribe stop deploy/default/sarracenia/cgsl.conf 93 | sr_subscribe stop deploy/default/sarracenia/gdwps.conf 94 | sr_subscribe stop deploy/default/sarracenia/geps.conf 95 | sr_subscribe stop deploy/default/sarracenia/hrdpa.conf 96 | sr_subscribe stop deploy/default/sarracenia/model_gem_global.conf 97 | sr_subscribe stop deploy/default/sarracenia/model_gem_regional.conf 98 | sr_subscribe stop deploy/default/sarracenia/model_giops.conf 99 | sr_subscribe stop deploy/default/sarracenia/model_hrdps_continental.conf 100 | sr_subscribe stop deploy/default/sarracenia/model_raqdps-fw-ce.conf 101 | sr_subscribe stop deploy/default/sarracenia/model_raqdps-fw.conf 102 | sr_subscribe stop deploy/default/sarracenia/model_raqdps.conf 103 | sr_subscribe stop deploy/default/sarracenia/model_rdaqa-ce.conf 104 | sr_subscribe stop deploy/default/sarracenia/model_riops.conf 105 | #sr_subscribe stop deploy/default/sarracenia/radar.conf 106 | sr_subscribe stop deploy/default/sarracenia/rdpa.conf 107 | sr_subscribe stop deploy/default/sarracenia/rdwps.conf 108 | sr_subscribe stop deploy/default/sarracenia/reps.conf 109 | sr_subscribe stop deploy/default/sarracenia/wcps.conf 110 | 111 | .PHONY: clean clean-logs flake8 setup start stop 112 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # geomet-data-registry 2 | 3 | ## Overview 4 | 5 | geomet-data-registry provides a searchable real-time inventory of MSC weather, 6 | climate and water data. 7 | 8 | ## Installation 9 | 10 | ### Requirements 11 | - Python 3 12 | - [virtualenv](https://virtualenv.pypa.io/) 13 | 14 | ### Dependencies 15 | Dependencies are listed in [requirements.txt](requirements.txt). Dependencies 16 | are automatically installed during installation. 17 | 18 | ### Installing geomet-data-registry 19 | ```bash 20 | 21 | # setup virtualenv 22 | python -m venv geomet-data-registry 23 | cd geomet-data-registry 24 | . bin/activate 25 | 26 | # clone codebase and install 27 | git clone https://github.com/ECCC-MSC/geomet-data-registry.git 28 | cd geomet-data-registry 29 | pip install -r requirements.txt 30 | pip install -r requirements-dev.txt 31 | pip install -e . 32 | 33 | # configure environment 34 | cp geomet-data-registry.env dev.env 35 | vi dev.env # edit paths accordingly 36 | . dev.env 37 | ``` 38 | 39 | ## Running 40 | 41 | ```bash 42 | # help 43 | geomet-data-registry --help 44 | 45 | # get version 46 | geomet-data-registry --version 47 | 48 | # setup tileindex 49 | geomet-data-registry tileindex setup 50 | 51 | # teardown tileindex 52 | geomet-data-registry tileindex teardown 53 | 54 | # setup store 55 | geomet-data-registry store setup 56 | 57 | # list all store keys 58 | geomet-data-registry store list 59 | 60 | # list all store keys filtering on a regex 61 | geomet-data-registry store list --pattern="RADAR*" 62 | 63 | # list all store keys filtering on a fancier regex 64 | geomet-data-registry store list --pattern="RADAR*time$" 65 | 66 | # teardown store 67 | geomet-data-registry store teardown 68 | 69 | # set key/value in store 70 | geomet-data-registry store set --key=somekey --config=/path/to/file 71 | 72 | # start up 73 | sr_subscribe path/to/amqp.conf foreground 74 | 75 | # dev workflows 76 | 77 | # process a test file 78 | geomet-data-registry data add --file=/path/to/file 79 | 80 | # process a test directory of files (recursive) 81 | geomet-data-registry data add --directory=/path/to/directory 82 | ``` 83 | 84 | ## Development 85 | 86 | ### Running Tests 87 | 88 | TODO 89 | 90 | ## Releasing 91 | 92 | ```bash 93 | python setup.py sdist bdist_wheel --universal 94 | twine upload dist/* 95 | ``` 96 | 97 | ### Code Conventions 98 | 99 | * [PEP8](https://www.python.org/dev/peps/pep-0008) 100 | 101 | ### Bugs and Issues 102 | 103 | All bugs, enhancements and issues are managed on [GitHub](https://github.com/ECCC-MSC/geomet-data-registry). 104 | 105 | ## Contact 106 | 107 | * [Tom Kralidis](https://github.com/tomkralidis) 108 | -------------------------------------------------------------------------------- /debian/changelog: -------------------------------------------------------------------------------- 1 | geomet-data-registry (0.1.0) bionic; urgency=medium 2 | 3 | * initial Debian packaging 4 | 5 | -- Tom Kralidis Wed, 13 Oct 2021 11:08:50 +0000 6 | -------------------------------------------------------------------------------- /debian/compat: -------------------------------------------------------------------------------- 1 | 8 2 | -------------------------------------------------------------------------------- /debian/control: -------------------------------------------------------------------------------- 1 | Source: geomet-data-registry 2 | Section: python 3 | Priority: optional 4 | Maintainer: Tom Kralidis 5 | Build-Depends: debhelper (>= 9), python3, python3-setuptools 6 | Standards-Version: 3.9.5 7 | X-Python-Version: >= 3.6 8 | Vcs-Git: https://github.com/ECCC-MSC/geomet-data-registry.git 9 | 10 | Package: geomet-data-registry 11 | Architecture: all 12 | Depends: elasticsearch (>=7), elasticsearch (<8), python3, python3-click, python3-elasticsearch (>=7), python3-elasticsearch (<8), python3-dateutil, python3-metpx-sarracenia, python3-parse, python3-redis, python3-yaml, redis-server 13 | Homepage: https://github.com/ECCC-MSC/geomet-data-registry 14 | Description: geomet-data-registry provides a searchable real-time inventory 15 | of MSC weather, climate and water data. 16 | -------------------------------------------------------------------------------- /debian/copyright: -------------------------------------------------------------------------------- 1 | Upstream: geomet-data-registry 2 | Source: https://github.com/ECCC-MSC/geomet-data-registry 3 | Files: * 4 | Copyright: 2020 Government of Canada 5 | License: GPL-3+ 6 | geomet-data-registry provides a searchable real-time inventory of MSC weather, 7 | climate and water data. 8 | Copyright (C) 2020 Government of Canada 9 | 10 | This program is free software: you can redistribute it and/or modify 11 | it under the terms of the GNU General Public License as published by 12 | the Free Software Foundation, either version 3 of the License, or 13 | (at your option) any later version. 14 | 15 | This program is distributed in the hope that it will be useful, 16 | but WITHOUT ANY WARRANTY; without even the implied warranty of 17 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 18 | GNU General Public License for more details. 19 | 20 | You should have received a copy of the GNU General Public License 21 | along with this program. If not, see . 22 | 23 | On Debian systems, the full text of the GNU General Public 24 | License version 2 can be found in the file 25 | `/usr/share/common-licenses/GPL-3'. 26 | -------------------------------------------------------------------------------- /debian/geomet-data-registry.cron.d: -------------------------------------------------------------------------------- 1 | # ================================================================= 2 | # 3 | # Author: Etienne Pelletier 4 | # Author: Tom Kralidis 5 | # 6 | # Copyright (c) 2020 Etienne Pelletier 7 | # Copyright (c) 2020 Tom Kralidis 8 | # 9 | # Permission is hereby granted, free of charge, to any person 10 | # obtaining a copy of this software and associated documentation 11 | # files (the "Software"), to deal in the Software without 12 | # restriction, including without limitation the rights to use, 13 | # copy, modify, merge, publish, distribute, sublicense, and/or sell 14 | # copies of the Software, and to permit persons to whom the 15 | # Software is furnished to do so, subject to the following 16 | # conditions: 17 | # 18 | # The above copyright notice and this permission notice shall be 19 | # included in all copies or substantial portions of the Software. 20 | # 21 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 22 | # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 23 | # OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 24 | # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 25 | # HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 26 | # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 27 | # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 28 | # OTHER DEALINGS IN THE SOFTWARE. 29 | # 30 | # ================================================================= 31 | 32 | # IMPORTANT: When using a Unix command in a new cron job, use the full path to the command (e.g. /bin/bash, /bin/cp, etc.). 33 | 34 | ############################################################################## 35 | # data cleanups 36 | 37 | ### REPS/GEPS: every day at 00:35, clean out content older than 3 days 38 | # We never delete CanSIPS data because serving all archives is in scope for GeoMet-Weather / CCCS 39 | 35 * * * * geoadm /usr/bin/find $GDR_DATADIR/ensemble/ -type f -not -path "$GDR_DATADIR/ensemble/cansips/*" -mtime +3 -exec rm {} \; > /dev/null 2>&1 40 | 41 | ### GDPS: twice a day at 02:05 and 14:05, clean out content older than 35 hours 42 | 05 2,14 * * * geoadm /usr/bin/find $GDR_DATADIR/model_gem_global/ -type f -mmin +2100 -exec rm {} \; > /dev/null 2>&1 43 | 44 | ### HRDPS: twice a day at 02:05 and 14:05, clean out content older than 35 hours 45 | 05 2,14 * * * geoadm /usr/bin/find $GDR_DATADIR/model_hrdps/ -type f -mmin +2100 -exec rm {} \; > /dev/null 2>&1 46 | 47 | ### RDPS/CGSL: twice a day at 2:05 and 14:05, clean out content older than 30 hours 48 | 05 2,14 * * * geoadm /usr/bin/find $GDR_DATADIR/model_gem_regional/ -type f -mmin +1800 -exec rm {} \; > /dev/null 2>&1 49 | 50 | ### GIOPS: twice a day at 4:35 and 16:35, clean out netcdf content older than 4 days 51 | 35 4,16 * * * geoadm /usr/bin/find $GDR_DATADIR/model_giops/netcdf/ -type f -mtime +4 -exec rm {} \; > /dev/null 2>&1 52 | 53 | ### WCPS: twice a day at 4:35 and 16:35, clean out netcdf content older than 7 days 54 | 35 4,16 * * * geoadm /usr/bin/find $GDR_DATADIR/model_wcps/nemo/netcdf/ -type f -mtime +7 -exec rm {} \; > /dev/null 2>&1 55 | 56 | ### GDWPS/RDWPS: twice a day a 4:25 and 16:25 clean out model_wave files older than 35 hours 57 | 25 4,16 * * * geoadm /usr/bin/find $GDR_DATADIR/model_wave/ -type f -mmin +2100 -exec rm {} \; > /dev/null 2>&1 58 | 59 | # every day at 0300h, clean out empty MetPX directories 60 | 0 3 * * * geoadm /usr/bin/find $GDR_DATADIR -type d -empty -delete > /dev/null 2>&1 61 | 62 | ### RADAR: every hour, clean out content older than 240 minutes 63 | 0 * * * * geoadm /usr/bin/find $GDR_DATADIR/local/RADAR -type f -mmin +240 -delete > /dev/null 2>&1 64 | 65 | ############################################################################## 66 | -------------------------------------------------------------------------------- /debian/geomet-data-registry.install: -------------------------------------------------------------------------------- 1 | deploy/default/*.yml opt/geomet-data-registry/etc 2 | deploy/default/sarracenia/*.conf opt/geomet-data-registry/etc/sarracenia 3 | geomet_data_registry/event/file_.py opt/geomet-data-registry/event 4 | geomet_data_registry/event/message.py opt/geomet-data-registry/event 5 | -------------------------------------------------------------------------------- /debian/postinst: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # ================================================================= 3 | # 4 | # Author: Tom Kralidis 5 | # 6 | # Copyright (c) 2020 Tom Kralidis 7 | # 8 | # Permission is hereby granted, free of charge, to any person 9 | # obtaining a copy of this software and associated documentation 10 | # files (the "Software"), to deal in the Software without 11 | # restriction, including without limitation the rights to use, 12 | # copy, modify, merge, publish, distribute, sublicense, and/or sell 13 | # copies of the Software, and to permit persons to whom the 14 | # Software is furnished to do so, subject to the following 15 | # conditions: 16 | # 17 | # The above copyright notice and this permission notice shall be 18 | # included in all copies or substantial portions of the Software. 19 | # 20 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 21 | # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 22 | # OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 23 | # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 24 | # HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 25 | # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 26 | # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 27 | # OTHER DEALINGS IN THE SOFTWARE. 28 | # 29 | # ================================================================= 30 | 31 | export GDR_LOGGING_LOGLEVEL=DEBUG 32 | export GDR_LOGGING_LOGFILE=stdout 33 | export GDR_BASEDIR=/opt/geomet-data-registry 34 | export GDR_DATADIR=/data/geomet 35 | export GDR_TILEINDEX_TYPE=Elasticsearch 36 | export GDR_TILEINDEX_BASEURL=http://localhost:9200 37 | export GDR_TILEINDEX_NAME=geomet-data-registry 38 | export GDR_STORE_TYPE=Redis 39 | export GDR_STORE_URL=redis://localhost:6379 40 | export GDR_METPX_DISCARD=on 41 | export GDR_METPX_EVENT_FILE_PY=$GDR_BASEDIR/event/file_.py 42 | export GDR_METPX_EVENT_MESSAGE_PY=$GDR_BASEDIR/event/message.py 43 | 44 | export XDG_CACHE_HOME=/tmp/geomet-data-registry-sarra-logs 45 | 46 | mkdir -p $XDG_CACHE_HOME 47 | mkdir -p $GDR_DATADIR/local 48 | 49 | echo "Setting up GDR store" 50 | 51 | geomet-data-registry store setup 52 | geomet-data-registry store set -k cansips -c $GDR_BASEDIR/etc/cansips.yml 53 | geomet-data-registry store set -k cgsl -c $GDR_BASEDIR/etc/cgsl.yml 54 | geomet-data-registry store set -k gdwps -c $GDR_BASEDIR/etc/gdwps.yml 55 | geomet-data-registry store set -k geps -c $GDR_BASEDIR/etc/geps.yml 56 | geomet-data-registry store set -k hrdpa -c $GDR_BASEDIR/etc/hrdpa.yml 57 | geomet-data-registry store set -k model_gem_global -c $GDR_BASEDIR/etc/model_gem_global.yml 58 | geomet-data-registry store set -k model_gem_regional -c $GDR_BASEDIR/etc/model_gem_regional.yml 59 | geomet-data-registry store set -k model_giops -c $GDR_BASEDIR/etc/model_giops.yml 60 | geomet-data-registry store set -k model_hrdps_continental -c $GDR_BASEDIR/etc/model_hrdps_continental.yml 61 | geomet-data-registry store set -k model_raqdps-fw -c $GDR_BASEDIR/etc/model_raqdps-fw.yml 62 | geomet-data-registry store set -k model_raqdps -c $GDR_BASEDIR/etc/model_raqdps.yml 63 | geomet-data-registry store set -k rdpa -c $GDR_BASEDIR/etc/rdpa.yml 64 | geomet-data-registry store set -k rdwps -c $GDR_BASEDIR/etc/rdwps.yml 65 | geomet-data-registry store set -k reps -c $GDR_BASEDIR/etc/reps.yml 66 | geomet-data-registry store set -k wcps -c $GDR_BASEDIR/etc/wcps.yml 67 | geomet-data-registry store set -k wcps -c $GDR_BASEDIR/etc/radar.yml 68 | 69 | echo "Setting up GDR tileindex" 70 | 71 | geomet-data-registry tileindex setup 72 | 73 | if [ $? -ne 0 ];then 74 | echo "GDR tileindex exists" 75 | fi 76 | -------------------------------------------------------------------------------- /debian/rules: -------------------------------------------------------------------------------- 1 | #!/usr/bin/make -f 2 | # -*- makefile -*- 3 | # Sample debian/rules that uses debhelper. 4 | # This file was originally written by Joey Hess and Craig Small. 5 | # As a special exception, when this file is copied by dh-make into a 6 | # dh-make output file, you may use that output file without restriction. 7 | # This special exception was added by Craig Small in version 0.37 of dh-make. 8 | 9 | # Uncomment this to turn on verbose mode. 10 | #export DH_VERBOSE=1 11 | #export DH_OPTIONS=-v 12 | 13 | %: 14 | dh $@ --with python3 --buildsystem=pybuild 15 | -------------------------------------------------------------------------------- /debian/source/format: -------------------------------------------------------------------------------- 1 | 3.0 (quilt) 2 | -------------------------------------------------------------------------------- /deploy/default/amqp.yml: -------------------------------------------------------------------------------- 1 | %YAML 1.1 2 | --- 3 | # amqp configuration 4 | 5 | amqp: 6 | model_gem_global: 7 | instances: 2 8 | subtopic: model_gem_global.25km.# 9 | mirror: True 10 | report_back: False 11 | accept: .* 12 | on_file: 13 | 14 | local: 15 | radar_4km: 16 | source: 17 | on_file: 18 | radar_1km: 19 | source: 20 | on_file: 21 | -------------------------------------------------------------------------------- /deploy/default/cgsl.yml: -------------------------------------------------------------------------------- 1 | cgsl: 2 | model: CGSL 3 | filename_pattern: CMC_coupled-rdps-stlawrence-{wx_variable}_latlon0.02x0.03_{YYYYMMDD_model_run}_P{forecast_hour}.grib2 4 | model_run_retention_hours: 48 5 | model_run_interval_hours: 6 6 | variable: 7 | atmosphere: 8 | bands: 9 | 1: 10 | product: UGRD 11 | 2: 12 | product: VGRD 13 | members: null 14 | elevation: surface 15 | model_run: 16 | 00Z: 17 | files_expected: 48 18 | 06Z: 19 | files_expected: 48 20 | 12Z: 21 | files_expected: 48 22 | 18Z: 23 | files_expected: 48 24 | geomet_layers: 25 | CGSL.ETA_UU: 26 | bands: 1,2 27 | forecast_hours: 001/048/PT1H 28 | ocean: 29 | bands: 30 | 1: 31 | product: ICEC 32 | 2: 33 | product: ICETK 34 | 3: 35 | product: UICE 36 | 4: 37 | product: VICE 38 | 5: 39 | product: UOGRD 40 | 6: 41 | product: VOGRD 42 | 7: 43 | product: WTMP 44 | 8: 45 | product: ICET 46 | 9: 47 | product: ICEPRS 48 | 10: 49 | product: ICESTG 50 | members: null 51 | elevation: surface 52 | model_run: 53 | 00Z: 54 | files_expected: 48 55 | 06Z: 56 | files_expected: 48 57 | 12Z: 58 | files_expected: 48 59 | 18Z: 60 | files_expected: 48 61 | geomet_layers: 62 | CGSL.ETA_ICEC: 63 | bands: 1 64 | forecast_hours: 001/048/PT1H 65 | CGSL.ETA_ICETK: 66 | bands: 2 67 | forecast_hours: 001/048/PT1H 68 | CGSL.ETA_UICE: 69 | bands: 3,4 70 | forecast_hours: 001/048/PT1H 71 | CGSL.ETA_UOGRD: 72 | bands: 5,6 73 | forecast_hours: 001/048/PT1H 74 | CGSL.ETA_WTMP: 75 | bands: 7 76 | forecast_hours: 001/048/PT1H 77 | CGSL.ETA_ICET: 78 | bands: 8 79 | forecast_hours: 001/048/PT1H 80 | CGSL.ETA_ICEPRS: 81 | bands: 9 82 | forecast_hours: 001/048/PT1H 83 | CGSL.ETA_ICESTG: 84 | bands: 10 85 | forecast_hours: 001/048/PT1H -------------------------------------------------------------------------------- /deploy/default/hrdpa.yml: -------------------------------------------------------------------------------- 1 | hrdpa: 2 | model: HRDPA 3 | filename_pattern: CMC_HRDPA_{wx_variable}_ps2.5km_{YYYYMMDD_model_run}_{forecast_hour}.grib2 4 | variable: 5 | APCP-006-0100cutoff_SFC_0: 6 | members: null 7 | elevation: surface 8 | model_run: 9 | 00Z: 10 | files_expected: 1 11 | 06Z: 12 | files_expected: 1 13 | 12Z: 14 | files_expected: 1 15 | 18Z: 16 | files_expected: 1 17 | geomet_layers: 18 | HRDPA.6P_PR: 19 | forecast_hours: -720/000/PT6H 20 | APCP-006-0700cutoff_SFC_0: 21 | members: null 22 | elevation: surface 23 | model_run: 24 | 00Z: 25 | files_expected: 1 26 | 06Z: 27 | files_expected: 1 28 | 12Z: 29 | files_expected: 1 30 | 18Z: 31 | files_expected: 1 32 | geomet_layers: 33 | HRDPA.6F_PR: 34 | forecast_hours: -720/000/PT6H 35 | APCP-024-0100cutoff_SFC_0: 36 | members: null 37 | elevation: surface 38 | model_run: 39 | 06Z: 40 | files_expected: 1 41 | 12Z: 42 | files_expected: 1 43 | geomet_layers: 44 | HRDPA.24P_PR: 45 | forecast_hours: -720/000/PT24H 46 | APCP-024-0700cutoff_SFC_0: 47 | members: null 48 | elevation: surface 49 | model_run: 50 | 06Z: 51 | files_expected: 1 52 | 12Z: 53 | files_expected: 1 54 | geomet_layers: 55 | HRDPA.24F_PR: 56 | forecast_hours: -720/000/PT24H 57 | 58 | 59 | -------------------------------------------------------------------------------- /deploy/default/model_raqdps-fw-ce.yml: -------------------------------------------------------------------------------- 1 | model_raqdps-fw-ce: 2 | model: RAQDPS-FW Cumulative Effects 3 | filename_pattern: '{YYYYMMDD_model_run}_MSC_RAQDPS-FW_{wx_variable}_RLatLon0.09x0.09_{interval}.nc' 4 | variable: 5 | PM2.5-DIFF-MAvg-DMax_SFC: 6 | members: null 7 | elevation: surface 8 | model_run: 9 | 00Z: 10 | files_expected: 1 11 | geomet_layers: 12 | RAQDPS-FW.CE_PM2.5-DIFF-MAvg-DMax: 13 | interval: P1M 14 | PM2.5-DIFF-MAvg_SFC: 15 | members: null 16 | elevation: surface 17 | model_run: 18 | 00Z: 19 | files_expected: 1 20 | geomet_layers: 21 | RAQDPS-FW.CE_PM2.5-DIFF-MAvg: 22 | interval: P1M 23 | PM2.5-DIFF-YAvg-DMax_SFC: 24 | members: null 25 | elevation: surface 26 | model_run: 27 | 00Z: 28 | files_expected: 1 29 | geomet_layers: 30 | RAQDPS-FW.CE_PM2.5-DIFF-YAvg-DMax: 31 | interval: P1Y 32 | PM2.5-DIFF-YAvg_SFC: 33 | members: null 34 | elevation: surface 35 | model_run: 36 | 00Z: 37 | files_expected: 1 38 | geomet_layers: 39 | RAQDPS-FW.CE_PM2.5-DIFF-YAvg: 40 | interval: P1Y 41 | -------------------------------------------------------------------------------- /deploy/default/model_raqdps-fw.yml: -------------------------------------------------------------------------------- 1 | model_raqdps-fw: 2 | model: RAQDPS-FW 3 | filename_pattern: '{YYYYMMDD}T{model_run}Z_MSC_RAQDPS-FW_{wx_variable}_RLatLon0.09_PT{forecast_hour}H.grib2' 4 | dimensions: 5 | x: 729 6 | y: 599 7 | model_run_retention_hours: 48 8 | model_run_interval_hours: 12 9 | variable: 10 | PM2.5_EAtm: 11 | members: null 12 | elevation: null 13 | model_run: 14 | 00Z: 15 | files_expected: 73 16 | 12Z: 17 | files_expected: 73 18 | geomet_layers: 19 | RAQDPS-FW.EATM_PM2.5: 20 | forecast_hours: 000/072/PT1H 21 | PM2.5-Diff-RAQDPS_EAtm: 22 | members: null 23 | elevation: null 24 | model_run: 25 | 00Z: 26 | files_expected: 73 27 | 12Z: 28 | files_expected: 73 29 | geomet_layers: 30 | RAQDPS-FW.EATM_PM2.5-DIFF: 31 | forecast_hours: 000/072/PT1H 32 | PM10_EAtm: 33 | members: null 34 | elevation: null 35 | model_run: 36 | 00Z: 37 | files_expected: 73 38 | 12Z: 39 | files_expected: 73 40 | geomet_layers: 41 | RAQDPS-FW.EATM_PM10: 42 | forecast_hours: 000/072/PT1H 43 | PM10-Diff-RAQDPS_EAtm: 44 | members: null 45 | elevation: null 46 | model_run: 47 | 00Z: 48 | files_expected: 73 49 | 12Z: 50 | files_expected: 73 51 | geomet_layers: 52 | RAQDPS-FW.EATM_PM10-DIFF: 53 | forecast_hours: 000/072/PT1H 54 | PM2.5_Sfc: 55 | members: null 56 | elevation: null 57 | model_run: 58 | 00Z: 59 | files_expected: 73 60 | 12Z: 61 | files_expected: 73 62 | geomet_layers: 63 | RAQDPS-FW.SFC_PM2.5: 64 | forecast_hours: 000/072/PT1H 65 | PM2.5-Diff-RAQDPS_Sfc: 66 | members: null 67 | elevation: null 68 | model_run: 69 | 00Z: 70 | files_expected: 73 71 | 12Z: 72 | files_expected: 73 73 | geomet_layers: 74 | RAQDPS-FW.SFC_PM2.5-DIFF: 75 | forecast_hours: 000/072/PT1H 76 | PM10_Sfc: 77 | members: null 78 | elevation: null 79 | model_run: 80 | 00Z: 81 | files_expected: 73 82 | 12Z: 83 | files_expected: 73 84 | geomet_layers: 85 | RAQDPS-FW.SFC_PM10: 86 | forecast_hours: 000/072/PT1H 87 | PM10-Diff-RAQDPS_Sfc: 88 | members: null 89 | elevation: null 90 | model_run: 91 | 00Z: 92 | files_expected: 73 93 | 12Z: 94 | files_expected: 73 95 | geomet_layers: 96 | RAQDPS-FW.SFC_PM10-DIFF: 97 | forecast_hours: 000/072/PT1H 98 | -------------------------------------------------------------------------------- /deploy/default/model_raqdps.yml: -------------------------------------------------------------------------------- 1 | model_raqdps: 2 | model: RAQDPS 3 | filename_pattern: '{YYYYMMDD}T{model_run}Z_MSC_RAQDPS_{wx_variable}_RLatLon0.09_PT{forecast_hour}H.grib2' 4 | dimensions: 5 | x: 729 6 | y: 599 7 | model_run_retention_hours: 48 8 | model_run_interval_hours: 12 9 | variable: 10 | PM2.5_EAtm: 11 | members: null 12 | elevation: null 13 | model_run: 14 | 00Z: 15 | files_expected: 73 16 | 12Z: 17 | files_expected: 73 18 | geomet_layers: 19 | RAQDPS.EATM_PM2.5: 20 | forecast_hours: 000/072/PT1H 21 | PM10_EAtm: 22 | members: null 23 | elevation: null 24 | model_run: 25 | 00Z: 26 | files_expected: 73 27 | 12Z: 28 | files_expected: 73 29 | geomet_layers: 30 | RAQDPS.EATM_PM10: 31 | forecast_hours: 000/072/PT1H 32 | PM2.5_Sfc: 33 | members: null 34 | elevation: null 35 | model_run: 36 | 00Z: 37 | files_expected: 73 38 | 12Z: 39 | files_expected: 73 40 | geomet_layers: 41 | RAQDPS.SFC_PM2.5: 42 | forecast_hours: 000/072/PT1H 43 | PM10_Sfc: 44 | members: null 45 | elevation: null 46 | model_run: 47 | 00Z: 48 | files_expected: 73 49 | 12Z: 50 | files_expected: 73 51 | geomet_layers: 52 | RAQDPS.SFC_PM10: 53 | forecast_hours: 000/072/PT1H 54 | NO2_Sfc: 55 | members: null 56 | elevation: null 57 | model_run: 58 | 00Z: 59 | files_expected: 73 60 | 12Z: 61 | files_expected: 73 62 | geomet_layers: 63 | RAQDPS.SFC_NO2: 64 | forecast_hours: 000/072/PT1H 65 | NO_Sfc: 66 | members: null 67 | elevation: null 68 | model_run: 69 | 00Z: 70 | files_expected: 73 71 | 12Z: 72 | files_expected: 73 73 | geomet_layers: 74 | RAQDPS.SFC_NO: 75 | forecast_hours: 000/072/PT1H 76 | O3_Sfc: 77 | members: null 78 | elevation: null 79 | model_run: 80 | 00Z: 81 | files_expected: 73 82 | 12Z: 83 | files_expected: 73 84 | geomet_layers: 85 | RAQDPS.SFC_O3: 86 | forecast_hours: 000/072/PT1H 87 | SO2_Sfc: 88 | members: null 89 | elevation: null 90 | model_run: 91 | 00Z: 92 | files_expected: 73 93 | 12Z: 94 | files_expected: 73 95 | geomet_layers: 96 | RAQDPS.SFC_SO2: 97 | forecast_hours: 000/072/PT1H 98 | -------------------------------------------------------------------------------- /deploy/default/model_rdaqa-ce.yml: -------------------------------------------------------------------------------- 1 | model_rdaqa-ce: 2 | model: RDAQA Cumulative Effects 3 | filename_pattern: '{YYYYMMDD_model_run}_MSC_RDAQA_{wx_variable}_RLatLon0.09x0.09_{interval}.nc' 4 | variable: 5 | O3-MAvg_SFC: 6 | members: null 7 | elevation: surface 8 | model_run: 9 | 00Z: 10 | files_expected: 1 11 | geomet_layers: 12 | RDAQA.CE_O3-MAvg: 13 | interval: P1M 14 | O3-MAvg-DMax-8hRAvg_SFC: 15 | members: null 16 | elevation: surface 17 | model_run: 18 | 00Z: 19 | files_expected: 1 20 | geomet_layers: 21 | RDAQA.CE_O3-MAvg-DMax-8hRAvg: 22 | interval: P1M 23 | O3-YAvg_SFC: 24 | members: null 25 | elevation: surface 26 | model_run: 27 | 00Z: 28 | files_expected: 1 29 | geomet_layers: 30 | RDAQA.CE_O3-YAvg: 31 | interval: P1Y 32 | O3-YAvg-DMax-8hRAvg_SFC: 33 | members: null 34 | elevation: surface 35 | model_run: 36 | 00Z: 37 | files_expected: 1 38 | geomet_layers: 39 | RDAQA.CE_O3-YAvg-DMax-8hRAvg: 40 | interval: P1Y 41 | PM2.5-MAvg_SFC: 42 | members: null 43 | elevation: surface 44 | model_run: 45 | 00Z: 46 | files_expected: 1 47 | geomet_layers: 48 | RDAQA.CE_PM2.5-MAvg: 49 | interval: P1M 50 | PM2.5-MAvg-DMax_SFC: 51 | members: null 52 | elevation: surface 53 | model_run: 54 | 00Z: 55 | files_expected: 1 56 | geomet_layers: 57 | RDAQA.CE_PM2.5-MAvg-DMax: 58 | interval: P1M 59 | PM2.5-YAvg_SFC: 60 | members: null 61 | elevation: surface 62 | model_run: 63 | 00Z: 64 | files_expected: 1 65 | geomet_layers: 66 | RDAQA.CE_PM2.5-YAvg: 67 | interval: P1Y 68 | PM2.5-YAvg-DMax_SFC: 69 | members: null 70 | elevation: surface 71 | model_run: 72 | 00Z: 73 | files_expected: 1 74 | geomet_layers: 75 | RDAQA.CE_PM2.5-YAvg-DMax: 76 | interval: P1Y 77 | PM10-MAvg_SFC: 78 | members: null 79 | elevation: surface 80 | model_run: 81 | 00Z: 82 | files_expected: 1 83 | geomet_layers: 84 | RDAQA.CE_PM10-MAvg: 85 | interval: P1M 86 | PM10-MAvg-DMax_SFC: 87 | members: null 88 | elevation: surface 89 | model_run: 90 | 00Z: 91 | files_expected: 1 92 | geomet_layers: 93 | RDAQA.CE_PM10-MAvg-DMax: 94 | interval: P1M 95 | PM10-YAvg_SFC: 96 | members: null 97 | elevation: surface 98 | model_run: 99 | 00Z: 100 | files_expected: 1 101 | geomet_layers: 102 | RDAQA.CE_PM10-YAvg: 103 | interval: P1Y 104 | PM10-YAvg-DMax_SFC: 105 | members: null 106 | elevation: surface 107 | model_run: 108 | 00Z: 109 | files_expected: 1 110 | geomet_layers: 111 | RDAQA.CE_PM10-YAvg-DMax: 112 | interval: P1Y 113 | NO2-MAvg_SFC: 114 | members: null 115 | elevation: surface 116 | model_run: 117 | 00Z: 118 | files_expected: 1 119 | geomet_layers: 120 | RDAQA.CE_NO2-MAvg: 121 | interval: P1M 122 | NO2-MAvg-DMax_SFC: 123 | members: null 124 | elevation: surface 125 | model_run: 126 | 00Z: 127 | files_expected: 1 128 | geomet_layers: 129 | RDAQA.CE_NO2-MAvg-DMax: 130 | interval: P1M 131 | NO2-YAvg_SFC: 132 | members: null 133 | elevation: surface 134 | model_run: 135 | 00Z: 136 | files_expected: 1 137 | geomet_layers: 138 | RDAQA.CE_NO2-YAvg: 139 | interval: P1Y 140 | NO2-YAvg-DMax_SFC: 141 | members: null 142 | elevation: surface 143 | model_run: 144 | 00Z: 145 | files_expected: 1 146 | geomet_layers: 147 | RDAQA.CE_NO2-YAvg-DMax: 148 | interval: P1Y 149 | SO2-MAvg_SFC: 150 | members: null 151 | elevation: surface 152 | model_run: 153 | 00Z: 154 | files_expected: 1 155 | geomet_layers: 156 | RDAQA.CE_SO2-MAvg: 157 | interval: P1M 158 | SO2-MAvg-DMax_SFC: 159 | members: null 160 | elevation: surface 161 | model_run: 162 | 00Z: 163 | files_expected: 1 164 | geomet_layers: 165 | RDAQA.CE_SO2-MAvg-DMax: 166 | interval: P1M 167 | SO2-YAvg_SFC: 168 | members: null 169 | elevation: surface 170 | model_run: 171 | 00Z: 172 | files_expected: 1 173 | geomet_layers: 174 | RDAQA.CE_SO2-YAvg: 175 | interval: P1Y 176 | SO2-YAvg-DMax_SFC: 177 | members: null 178 | elevation: surface 179 | model_run: 180 | 00Z: 181 | files_expected: 1 182 | geomet_layers: 183 | RDAQA.CE_SO2-YAvg-DMax: 184 | interval: P1Y -------------------------------------------------------------------------------- /deploy/default/radar.yml: -------------------------------------------------------------------------------- 1 | radar: 2 | model: radar_1km 3 | filename_pattern: '{YYYYMMDDThhmm}Z_MSC_Radar-Composite_{precipitation_type}_1km.tif' 4 | variable: 5 | MMHR: 6 | geomet_layers: RADAR_1KM_RRAI 7 | forecast_hours: -180/000/PT10M 8 | elevation: null 9 | members: null 10 | CMHR: 11 | geomet_layers: RADAR_1KM_RSNO 12 | forecast_hours: -180/000/PT10M 13 | elevation: null 14 | members: null 15 | -------------------------------------------------------------------------------- /deploy/default/rdpa.yml: -------------------------------------------------------------------------------- 1 | rdpa: 2 | model: RDPA 3 | filename_pattern: 4 | 15km: 5 | CMC_RDPA_{wx_variable}_ps15km_{YYYYMMDD_model_run}_{forecast_hour}.grib2 6 | 10km: 7 | CMC_RDPA_{wx_variable}_ps10km_{YYYYMMDD_model_run}_{forecast_hour}.grib2 8 | variable: 9 | APCP-006-0100cutoff_SFC_0: 10 | members: null 11 | elevation: surface 12 | model_run: 13 | 00Z: 14 | files_expected: 1 15 | 06Z: 16 | files_expected: 1 17 | 12Z: 18 | files_expected: 1 19 | 18Z: 20 | files_expected: 1 21 | geomet_layers: 22 | RDPA.6P_PR: 23 | forecast_hours: -720/000/PT6H 24 | APCP-006-0700cutoff_SFC_0: 25 | members: null 26 | elevation: surface 27 | model_run: 28 | 00Z: 29 | files_expected: 1 30 | 06Z: 31 | files_expected: 1 32 | 12Z: 33 | files_expected: 1 34 | 18Z: 35 | files_expected: 1 36 | geomet_layers: 37 | RDPA.ARC_15km.6F_PR: 38 | begin: '2011-04-06T00:00:00Z' 39 | end: '2012-10-03T00:00:00Z' 40 | interval: PT6H 41 | RDPA.6F_PR: 42 | begin: '2012-10-03T06:00:00Z' 43 | interval: PT6H 44 | APCP-024-0100cutoff_SFC_0: 45 | members: null 46 | elevation: surface 47 | model_run: 48 | 06Z: 49 | files_expected: 1 50 | 12Z: 51 | files_expected: 1 52 | geomet_layers: 53 | RDPA.24P_PR: 54 | forecast_hours: -720/000/PT24H 55 | APCP-024-0700cutoff_SFC_0: 56 | members: null 57 | elevation: surface 58 | model_run: 59 | 06Z: 60 | files_expected: 1 61 | 12Z: 62 | files_expected: 1 63 | geomet_layers: 64 | RDPA.ARC_15km.24F_PR: 65 | begin: '2011-04-06T00:00:00Z' 66 | end: '2012-10-03T00:00:00Z' 67 | interval: PT24H 68 | RDPA.24F_PR: 69 | begin: '2012-10-03T06:00:00Z' 70 | interval: PT24H 71 | -------------------------------------------------------------------------------- /deploy/default/sarracenia/cansips.conf: -------------------------------------------------------------------------------- 1 | broker amqps://anonymous:anonymous@dd.weather.gc.ca/ 2 | queue_name q_${BROKER_USER}.${PROGRAM}.${CONFIG}.${HOSTNAME} 3 | directory ${GDR_DATADIR} 4 | instances 6 5 | subtopic ensemble.cansips.grib2.forecast.# 6 | mirror True 7 | report_back False 8 | accept .*latlon1.* 9 | plugin ${GDR_METPX_EVENT_FILE_PY} 10 | chmod_log 0644 11 | -------------------------------------------------------------------------------- /deploy/default/sarracenia/cgsl.conf: -------------------------------------------------------------------------------- 1 | broker amqps://anonymous:anonymous@dd.weather.gc.ca/ 2 | queue_name q_${BROKER_USER}.${PROGRAM}.${CONFIG}.${HOSTNAME} 3 | directory ${GDR_DATADIR} 4 | notify_only ${GDR_METPX_NOTIFY} 5 | instances 6 6 | subtopic model_gem_regional.coupled.gulf_st-lawrence.# 7 | mirror True 8 | report_back False 9 | accept .* 10 | discard ${GDR_METPX_DISCARD} 11 | plugin ${GDR_METPX_EVENT_MESSAGE_PY} 12 | chmod_log 0644 13 | -------------------------------------------------------------------------------- /deploy/default/sarracenia/gdwps.conf: -------------------------------------------------------------------------------- 1 | broker amqps://anonymous:anonymous@dd.weather.gc.ca/ 2 | queue_name q_${BROKER_USER}.${PROGRAM}.${CONFIG}.${HOSTNAME} 3 | directory ${GDR_DATADIR} 4 | notify_only ${GDR_METPX_NOTIFY} 5 | instances 6 6 | subtopic model_gdwps.# 7 | mirror True 8 | report_back False 9 | accept .* 10 | discard ${GDR_METPX_DISCARD} 11 | plugin ${GDR_METPX_EVENT_MESSAGE_PY} 12 | chmod_log 0644 13 | -------------------------------------------------------------------------------- /deploy/default/sarracenia/geps.conf: -------------------------------------------------------------------------------- 1 | broker amqps://anonymous:anonymous@dd.weather.gc.ca/ 2 | queue_name q_${BROKER_USER}.${PROGRAM}.${CONFIG}.${HOSTNAME} 3 | directory ${GDR_DATADIR} 4 | notify_only ${GDR_METPX_NOTIFY} 5 | instances 6 6 | subtopic ensemble.geps.grib2.products.# 7 | mirror True 8 | report_back False 9 | accept .* 10 | discard ${GDR_METPX_DISCARD} 11 | plugin ${GDR_METPX_EVENT_MESSAGE_PY} 12 | chmod_log 0644 13 | -------------------------------------------------------------------------------- /deploy/default/sarracenia/hrdpa.conf: -------------------------------------------------------------------------------- 1 | broker amqps://anonymous:anonymous@dd.weather.gc.ca/ 2 | queue_name q_${BROKER_USER}.${PROGRAM}.${CONFIG}.${HOSTNAME} 3 | directory ${GDR_DATADIR} 4 | notify_only ${GDR_METPX_NOTIFY} 5 | instances 6 6 | subtopic analysis.precip.hrdpa.# 7 | mirror True 8 | report_back False 9 | accept .* 10 | discard ${GDR_METPX_DISCARD} 11 | plugin ${GDR_METPX_EVENT_MESSAGE_PY} 12 | chmod_log 0644 13 | -------------------------------------------------------------------------------- /deploy/default/sarracenia/model_gem_global.conf: -------------------------------------------------------------------------------- 1 | broker amqps://anonymous:anonymous@dd.weather.gc.ca/ 2 | queue_name q_${BROKER_USER}.${PROGRAM}.${CONFIG}.${HOSTNAME} 3 | directory ${GDR_DATADIR} 4 | notify_only ${GDR_METPX_NOTIFY} 5 | instances 6 6 | subtopic model_gem_global.15km.# 7 | mirror True 8 | report_back False 9 | accept .* 10 | discard ${GDR_METPX_DISCARD} 11 | plugin ${GDR_METPX_EVENT_MESSAGE_PY} 12 | chmod_log 0644 13 | -------------------------------------------------------------------------------- /deploy/default/sarracenia/model_gem_regional.conf: -------------------------------------------------------------------------------- 1 | broker amqps://anonymous:anonymous@dd.weather.gc.ca/ 2 | queue_name q_${BROKER_USER}.${PROGRAM}.${CONFIG}.${HOSTNAME} 3 | directory ${GDR_DATADIR} 4 | notify_only ${GDR_METPX_NOTIFY} 5 | instances 6 6 | subtopic model_gem_regional.10km.# 7 | mirror True 8 | report_back False 9 | accept .* 10 | discard ${GDR_METPX_DISCARD} 11 | plugin ${GDR_METPX_EVENT_MESSAGE_PY} 12 | chmod_log 0644 13 | -------------------------------------------------------------------------------- /deploy/default/sarracenia/model_giops.conf: -------------------------------------------------------------------------------- 1 | broker amqps://anonymous:anonymous@dd.weather.gc.ca/ 2 | queue_name q_${BROKER_USER}.${PROGRAM}.${CONFIG}.${HOSTNAME} 3 | directory ${GDR_DATADIR} 4 | notify_only ${GDR_METPX_NOTIFY} 5 | instances 6 6 | subtopic model_giops.netcdf.lat_lon.# 7 | mirror True 8 | report_back False 9 | accept .* 10 | discard ${GDR_METPX_DISCARD} 11 | plugin ${GDR_METPX_EVENT_MESSAGE_PY} 12 | chmod_log 0644 13 | -------------------------------------------------------------------------------- /deploy/default/sarracenia/model_hrdps_continental.conf: -------------------------------------------------------------------------------- 1 | broker amqps://anonymous:anonymous@dd.weather.gc.ca/ 2 | queue_name q_${BROKER_USER}.${PROGRAM}.${CONFIG}.${HOSTNAME} 3 | directory ${GDR_DATADIR} 4 | notify_only ${GDR_METPX_NOTIFY 5 | instances 6 6 | subtopic model_hrdps.continental.# 7 | mirror True 8 | report_back False 9 | accept .* 10 | discard ${GDR_METPX_DISCARD} 11 | plugin ${GDR_METPX_EVENT_MESSAGE_PY} 12 | chmod_log 0644 13 | -------------------------------------------------------------------------------- /deploy/default/sarracenia/model_raqdps-fw-ce.conf: -------------------------------------------------------------------------------- 1 | broker amqps://${GDR_GEOMET_ONLY_USER}:${GDR_GEOMET_ONLY_PASS}@${GDR_GEOMET_ONLY_HOST}/ 2 | exchange xpublic 3 | queue_name q_${BROKER_USER}.${PROGRAM}.${CONFIG}.${HOSTNAME} 4 | directory ${GDR_DATADIR} 5 | notify_only ${GDR_METPX_NOTIFY} 6 | instances 6 7 | subtopic *.MSC-SCI-CMC-OPS.geomet_only.model_raqdps-fw.cumulative_effects.netcdf.# 8 | mirror True 9 | report_back False 10 | strip 3 11 | accept .* 12 | discard ${GDR_METPX_DISCARD} 13 | plugin ${GDR_METPX_EVENT_MESSAGE_PY} 14 | chmod_log 0644 15 | -------------------------------------------------------------------------------- /deploy/default/sarracenia/model_raqdps-fw.conf: -------------------------------------------------------------------------------- 1 | broker amqps://anonymous:anonymous@dd.weather.gc.ca/ 2 | queue_name q_${BROKER_USER}.${PROGRAM}.${CONFIG}.${HOSTNAME} 3 | directory ${GDR_DATADIR} 4 | notify_only ${GDR_METPX_NOTIFY} 5 | instances 6 6 | subtopic model_raqdps-fw.# 7 | mirror True 8 | report_back False 9 | accept .* 10 | discard ${GDR_METPX_DISCARD} 11 | plugin ${GDR_METPX_EVENT_MESSAGE_PY} 12 | chmod_log 0644 13 | -------------------------------------------------------------------------------- /deploy/default/sarracenia/model_raqdps.conf: -------------------------------------------------------------------------------- 1 | broker amqps://anonymous:anonymous@dd.weather.gc.ca/ 2 | queue_name q_${BROKER_USER}.${PROGRAM}.${CONFIG}.${HOSTNAME} 3 | directory ${GDR_DATADIR} 4 | notify_only ${GDR_METPX_NOTIFY} 5 | instances 6 6 | subtopic model_raqdps.# 7 | mirror True 8 | report_back False 9 | accept .* 10 | discard ${GDR_METPX_DISCARD} 11 | plugin ${GDR_METPX_EVENT_MESSAGE_PY} 12 | chmod_log 0644 13 | -------------------------------------------------------------------------------- /deploy/default/sarracenia/model_rdaqa-ce.conf: -------------------------------------------------------------------------------- 1 | broker amqps://${GDR_GEOMET_ONLY_USER}:${GDR_GEOMET_ONLY_PASS}@${GDR_GEOMET_ONLY_HOST}/ 2 | exchange xpublic 3 | queue_name q_${BROKER_USER}.${PROGRAM}.${CONFIG}.${HOSTNAME} 4 | directory ${GDR_DATADIR} 5 | notify_only ${GDR_METPX_NOTIFY} 6 | instances 6 7 | subtopic *.MSC-SCI-CMC-OPS.geomet_only.model_rdaqa.cumulative_effects.netcdf.# 8 | mirror True 9 | report_back False 10 | strip 3 11 | accept .* 12 | discard ${GDR_METPX_DISCARD} 13 | plugin ${GDR_METPX_EVENT_MESSAGE_PY} 14 | chmod_log 0644 15 | -------------------------------------------------------------------------------- /deploy/default/sarracenia/model_riops.conf: -------------------------------------------------------------------------------- 1 | broker amqps://anonymous:anonymous@dd.weather.gc.ca/ 2 | queue_name q_${BROKER_USER}.${PROGRAM}.${CONFIG}.${HOSTNAME} 3 | directory ${GDR_DATADIR} 4 | notify_only ${GDR_METPX_NOTIFY} 5 | instances 6 6 | subtopic model_riops.netcdf.forecast.polar_stereographic.# 7 | mirror True 8 | report_back False 9 | accept .* 10 | discard ${GDR_METPX_DISCARD} 11 | plugin ${GDR_METPX_EVENT_MESSAGE_PY} 12 | chmod_log 0644 13 | -------------------------------------------------------------------------------- /deploy/default/sarracenia/radar.conf: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # 3 | # Copyright (C) 2019 Louis-Philippe Rousseau-Lambert 4 | # 5 | # This program is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with this program. If not, see . 17 | # 18 | ############################################################################### 19 | 20 | broker amqps://anonymous:anonymous@fluxi.cmc.ec.gc.ca/ 21 | exchange xpublic 22 | queue_name q_${BROKER_USER}.${PROGRAM}.${CONFIG}.${HOSTNAME} 23 | subtopic *.MSC-RADAR.unique.GEOTIFF.COMPOSITE.# 24 | 25 | #notify_only 26 | #mirror True 27 | report_back False 28 | 29 | directory ${GDR_DATADIR}/../local/RADAR/1KM/MMHR 30 | accept .*MSC_Radar-Composite_MMHR_1km.tif.* 31 | 32 | directory ${GDR_DATADIR}/../local/RADAR/1KM/CMHR 33 | accept .*MSC_Radar-Composite_CMHR_1km.tif.* 34 | 35 | # always download radar data locally 36 | discard off 37 | 38 | plugin ${GDR_METPX_EVENT_FILE_PY} 39 | chmod_log 0644 40 | -------------------------------------------------------------------------------- /deploy/default/sarracenia/rdpa.conf: -------------------------------------------------------------------------------- 1 | broker amqps://anonymous:anonymous@dd.weather.gc.ca/ 2 | queue_name q_${BROKER_USER}.${PROGRAM}.${CONFIG}.${HOSTNAME} 3 | directory ${GDR_DATADIR} 4 | notify_only ${GDR_METPX_NOTIFY} 5 | instances 6 6 | subtopic analysis.precip.rdpa.# 7 | mirror True 8 | report_back False 9 | accept .* 10 | discard ${GDR_METPX_DISCARD} 11 | plugin ${GDR_METPX_EVENT_MESSAGE_PY} 12 | chmod_log 0644 13 | -------------------------------------------------------------------------------- /deploy/default/sarracenia/rdwps.conf: -------------------------------------------------------------------------------- 1 | broker amqps://anonymous:anonymous@dd.weather.gc.ca/ 2 | queue_name q_${BROKER_USER}.${PROGRAM}.${CONFIG}.${HOSTNAME} 3 | directory ${GDR_DATADIR} 4 | notify_only ${GDR_METPX_NOTIFY} 5 | instances 6 6 | subtopic model_rdwps.# 7 | mirror True 8 | report_back False 9 | accept .* 10 | discard ${GDR_METPX_DISCARD} 11 | plugin ${GDR_METPX_EVENT_MESSAGE_PY} 12 | chmod_log 0644 13 | -------------------------------------------------------------------------------- /deploy/default/sarracenia/rdwps_gulf.conf: -------------------------------------------------------------------------------- 1 | broker amqps://anonymous:anonymous@dd.weather.gc.ca/ 2 | queue_name q_${BROKER_USER}.${PROGRAM}.${CONFIG}.${HOSTNAME} 3 | directory ${GDR_DATADIR} 4 | notify_only ${GDR_METPX_NOTIFY} 5 | instances 6 6 | subtopic model_wave.ocean.gulf-st-lawrence.# 7 | mirror True 8 | report_back False 9 | accept .* 10 | discard ${GDR_METPX_DISCARD} 11 | plugin ${GDR_METPX_EVENT_MESSAGE_PY} 12 | chmod_log 0644 13 | -------------------------------------------------------------------------------- /deploy/default/sarracenia/reps.conf: -------------------------------------------------------------------------------- 1 | broker amqps://anonymous:anonymous@dd.weather.gc.ca/ 2 | queue_name q_${BROKER_USER}.${PROGRAM}.${CONFIG}.${HOSTNAME} 3 | directory ${GDR_DATADIR} 4 | notify_only ${GDR_METPX_NOTIFY} 5 | instances 6 6 | subtopic ensemble.reps.15km.# 7 | mirror True 8 | report_back False 9 | accept .* 10 | discard ${GDR_METPX_DISCARD} 11 | plugin ${GDR_METPX_EVENT_MESSAGE_PY} 12 | chmod_log 0644 13 | -------------------------------------------------------------------------------- /deploy/default/sarracenia/wcps.conf: -------------------------------------------------------------------------------- 1 | broker amqps://anonymous:anonymous@dd.weather.gc.ca/ 2 | queue_name q_${BROKER_USER}.${PROGRAM}.${CONFIG}.${HOSTNAME} 3 | directory ${GDR_DATADIR} 4 | notify_only ${GDR_METPX_NOTIFY} 5 | instances 6 6 | subtopic model_wcps.nemo.netcdf.# 7 | mirror True 8 | report_back False 9 | accept .* 10 | discard ${GDR_METPX_DISCARD} 11 | plugin ${GDR_METPX_EVENT_MESSAGE_PY} 12 | chmod_log 0644 13 | -------------------------------------------------------------------------------- /deploy/nightly/deploy-nightly.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ############################################################################### 3 | # 4 | # Copyright (C) 2021 Tom Kralidis 5 | # 6 | # This program is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program. If not, see . 18 | # 19 | ############################################################################### 20 | 21 | BASEDIR=/data/web/geomet-data-registry-nightly 22 | GDR_GITREPO=https://github.com/ECCC-MSC/geomet-data-registry.git 23 | DAYSTOKEEP=7 24 | 25 | # you should be okay from here 26 | 27 | DATETIME=`date +%Y%m%d` 28 | TIMESTAMP=`date +%Y%m%d.%H%M` 29 | NIGHTLYDIR=msc-geomet-data-registry-$TIMESTAMP 30 | 31 | echo "Deleting nightly builds > $DAYSTOKEEP days old" 32 | 33 | cd $BASEDIR 34 | 35 | for f in `find . -type d -name "geomet-data-registry-20*"` 36 | do 37 | DATETIME2=`echo $f | awk -F- '{print $3}' | awk -F. '{print $1}'` 38 | let DIFF=(`date +%s -d $DATETIME`-`date +%s -d $DATETIME2`)/86400 39 | if [ $DIFF -gt $DAYSTOKEEP ]; then 40 | rm -fr $f 41 | fi 42 | done 43 | 44 | rm -fr latest 45 | echo "Generating nightly build for $TIMESTAMP" 46 | mkdir $NIGHTLYDIR 47 | cd $NIGHTLYDIR 48 | git clone $GDR_GITREPO 49 | cd geomet-data-registry 50 | docker-compose -f docker/docker-compose-nightly.yml down 51 | docker-compose -f docker/docker-compose-nightly.yml build 52 | docker-compose -f docker/docker-compose-nightly.yml up -d 53 | 54 | cd ../.. 55 | 56 | ln -s $NIGHTLYDIR latest 57 | -------------------------------------------------------------------------------- /docker/Makefile: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # 3 | # Copyright (C) 2019 Tom Kralidis 4 | # 5 | # This program is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with this program. If not, see . 17 | # 18 | ############################################################################### 19 | 20 | IMAGE=geomet-data-registry-image 21 | CONTAINER=geomet-data-registry-container 22 | HOSTNAME=mandalor 23 | 24 | DATA_VOLUME=/data/geomet 25 | 26 | help: 27 | @echo 28 | @echo "make targets:" 29 | @echo " run: run container" 30 | @echo " build: build image" 31 | @echo " login: login to container" 32 | @echo " stop: stop container" 33 | @echo " clean: remove container" 34 | @echo 35 | 36 | run: 37 | docker run --name $(CONTAINER) -d -i -t -v $(DATA_VOLUME):$(DATA_VOLUME) --hostname $(HOSTNAME) -p 8001:80 -p 9200:9200 -p 6379:6379 $(IMAGE) 38 | 39 | build: 40 | docker build -t $(IMAGE) . 41 | 42 | login: 43 | docker exec -it $(CONTAINER) /bin/bash 44 | 45 | login-root: 46 | docker exec -u 0 -it $(CONTAINER) /bin/bash 47 | 48 | stop: 49 | docker stop $(CONTAINER) 50 | 51 | clean: 52 | docker rm -f $(CONTAINER) 53 | docker rmi $(IMAGE) 54 | 55 | .PHONY: help run build login login-root stop clean 56 | -------------------------------------------------------------------------------- /docker/README.md: -------------------------------------------------------------------------------- 1 | # geomet-data-registry Docker setup 2 | 3 | Development environment using Docker 4 | 5 | ## Docker 6 | 7 | To start up just geomet-data-registry without any services: 8 | 9 | ```bash 10 | # build image 11 | make build 12 | 13 | # run 14 | make run 15 | 16 | # login to a bash session 17 | make login 18 | 19 | # stop 20 | make stop 21 | ``` 22 | 23 | ## Docker Compose 24 | 25 | 26 | ```bash 27 | # run GDR with embedded services: 28 | docker network create geomet-data-registry-network 29 | docker-compose up -d 30 | 31 | # run GDR without services (i.e. ES and Redis already exist) 32 | docker-compose -f docker-compose-nightly.yml up -d 33 | ``` 34 | -------------------------------------------------------------------------------- /docker/docker-compose-nightly.yml: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # 3 | # Copyright (C) 2021 Tom Kralidis 4 | # 5 | # This program is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with this program. If not, see . 17 | # 18 | ############################################################################### 19 | 20 | version: "3" 21 | 22 | services: 23 | geomet-data-registry: 24 | container_name: geomet-data-registry 25 | environment: 26 | - GDR_LOGGING_LOGLEVEL=DEBUG 27 | - GDR_LOGGING_LOGFILE=/tmp/geomet-data-registry-nightly.log 28 | - GDR_CONFIG=/opt/geomet-data-registry/geomet-data-registry.yml 29 | - GDR_BASEDIR=/home/geoadm/geomet-data-registry 30 | - GDR_DATADIR=/data/geomet/feeds 31 | - GDR_TILEINDEX_TYPE=Elasticsearch 32 | - GDR_TILEINDEX_BASEURL=http://localhost:9200 33 | - GDR_TILEINDEX_NAME=geomet-data-registry-nightly 34 | - GDR_STORE_TYPE=Redis 35 | - GDR_STORE_URL=redis://localhost:6379?db=15 36 | - GDR_METPX_DISCARD=on 37 | - GDR_METPX_EVENT_FILE_PY=/home/geoadm/geomet-data-registry/geomet_data_registry/event/file_.py 38 | - GDR_METPX_EVENT_MESSAGE_PY=/home/geoadm/geomet-data-registry/geomet_data_registry/event/message.py 39 | - GDR_METPX_NOTIFY=True 40 | - GDR_GEOMET_ONLY_USER=username 41 | - GDR_GEOMET_ONLY_PASS=password 42 | - GDR_GEOMET_ONLY_HOST=feeds.example.org 43 | - GDR_NOTIFICATIONS=False 44 | - GDR_NOTIFICATIONS_TYPE=Celery 45 | - GDR_NOTIFICATIONS_URL=redis://localhost:6379?db=15 46 | - XDG_CACHE_HOME=/tmp/geomet-data-registry-nightly-sarra-logs 47 | 48 | volumes: 49 | - /tmp:/tmp 50 | network_mode: host 51 | 52 | build: 53 | context: .. 54 | -------------------------------------------------------------------------------- /docker/docker-compose.yml: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # 3 | # Copyright (C) 2021 Tom Kralidis 4 | # 5 | # This program is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with this program. If not, see . 17 | # 18 | ############################################################################### 19 | 20 | version: "3" 21 | 22 | volumes: 23 | esdata01: 24 | driver: local 25 | 26 | services: 27 | elasticsearch: 28 | image: docker.elastic.co/elasticsearch/elasticsearch:7.6.2 29 | container_name: elasticsearch 30 | environment: 31 | - discovery.type=single-node 32 | - node.name=elasticsearch-01 33 | - discovery.seed_hosts=elasticsearch-01 34 | - bootstrap.memory_lock=true 35 | - "ES_JAVA_OPTS=-Xms512m -Xmx512m" 36 | - http.cors.enabled=true 37 | - http.cors.allow-origin=* 38 | ulimits: 39 | nofile: 40 | soft: 524288 41 | hard: 524288 42 | memlock: 43 | soft: -1 44 | hard: -1 45 | volumes: 46 | - esdata01:/usr/share/elasticsearch/data 47 | ports: 48 | - 9200:9200 49 | 50 | redis: 51 | container_name: redis 52 | image: redis:alpine 53 | ports: 54 | - 6379:6379 55 | 56 | geomet-data-registry: 57 | container_name: geomet-data-registry 58 | depends_on: 59 | - elasticsearch 60 | - redis 61 | environment: 62 | - GDR_LOGGING_LOGLEVEL=DEBUG 63 | - GDR_LOGGING_LOGFILE=/tmp/geomet-data-registry.log 64 | - GDR_CONFIG=/opt/geomet-data-registry/geomet-data-registry.yml 65 | - GDR_BASEDIR=/home/geoadm/geomet-data-registry 66 | - GDR_DATADIR=/home/geoadm/data/feeds 67 | - GDR_TILEINDEX_TYPE=Elasticsearch 68 | - GDR_TILEINDEX_BASEURL=http://elasticsearch:9200 69 | - GDR_TILEINDEX_NAME=geomet-data-registry 70 | - GDR_STORE_TYPE=Redis 71 | - GDR_STORE_URL=redis://redis:6379 72 | - GDR_METPX_DISCARD=on 73 | - GDR_METPX_EVENT_FILE_PY=/home/geoadm/geomet-data-registry/geomet_data_registry/event/file_.py 74 | - GDR_METPX_EVENT_MESSAGE_PY=/home/geoadm/geomet-data-registry/geomet_data_registry/event/message.py 75 | - GDR_METPX_NOTIFY=True 76 | - GDR_GEOMET_ONLY_USER=username 77 | - GDR_GEOMET_ONLY_PASS=password 78 | - GDR_GEOMET_ONLY_HOST=feeds.example.org 79 | - GDR_NOTIFICATIONS=False 80 | - GDR_NOTIFICATIONS_TYPE=Celery 81 | - GDR_NOTIFICATIONS_URL=redis://redis:6379 82 | - XDG_CACHE_HOME=/tmp/geomet-data-registry-sarra-logs 83 | 84 | volumes: 85 | - /tmp:/tmp 86 | 87 | build: 88 | context: .. 89 | 90 | networks: 91 | default: 92 | external: 93 | name: geomet-data-registry-network 94 | -------------------------------------------------------------------------------- /docker/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # ================================================================= 3 | # 4 | # Author: Tom Kralidis 5 | # 6 | # Copyright (c) 2021 Tom Kralidis 7 | # 8 | # Permission is hereby granted, free of charge, to any person 9 | # obtaining a copy of this software and associated documentation 10 | # files (the "Software"), to deal in the Software without 11 | # restriction, including without limitation the rights to use, 12 | # copy, modify, merge, publish, distribute, sublicense, and/or sell 13 | # copies of the Software, and to permit persons to whom the 14 | # Software is furnished to do so, subject to the following 15 | # conditions: 16 | # 17 | # The above copyright notice and this permission notice shall be 18 | # included in all copies or substantial portions of the Software. 19 | # 20 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 21 | # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 22 | # OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 23 | # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 24 | # HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 25 | # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 26 | # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 27 | # OTHER DEALINGS IN THE SOFTWARE. 28 | # 29 | # ================================================================= 30 | 31 | echo "Verifying GDR supporting services are available" 32 | until curl --silent --output /dev/null --show-error --fail "$GDR_TILEINDEX_BASEURL" && redis-cli -u $GDR_STORE_URL ping; do 33 | >&2 echo "Elasticsearch or Redis is not unavailable - sleeping" 34 | sleep 1 35 | done 36 | 37 | >&2 echo "Elasticsearch and Redis are up - continuing" 38 | 39 | echo "Setting up GDR store and tileindex" 40 | #geomet-data-registry store setup 41 | #geomet-data-registry tileindex setup 42 | 43 | 44 | echo "Populating GDR store" 45 | geomet-data-registry store set -k cansips -c /home/geoadm/geomet-data-registry/deploy/default/cansips.yml 46 | geomet-data-registry store set -k cgsl -c /home/geoadm/geomet-data-registry/deploy/default/cgsl.yml 47 | geomet-data-registry store set -k gdwps -c /home/geoadm/geomet-data-registry/deploy/default/gdwps.yml 48 | geomet-data-registry store set -k geps -c /home/geoadm/geomet-data-registry/deploy/default/geps.yml 49 | geomet-data-registry store set -k hrdpa -c /home/geoadm/geomet-data-registry/deploy/default/hrdpa.yml 50 | geomet-data-registry store set -k model_gem_global -c /home/geoadm/geomet-data-registry/deploy/default/model_gem_global.yml 51 | geomet-data-registry store set -k model_gem_regional -c /home/geoadm/geomet-data-registry/deploy/default/model_gem_regional.yml 52 | geomet-data-registry store set -k model_giops -c /home/geoadm/geomet-data-registry/deploy/default/model_giops.yml 53 | geomet-data-registry store set -k model_hrdps_continental -c /home/geoadm/geomet-data-registry/deploy/default/model_hrdps_continental.yml 54 | geomet-data-registry store set -k model_raqdps-fw-ce -c /home/geoadm/geomet-data-registry/deploy/default/model_raqdps-fw-ce.yml 55 | geomet-data-registry store set -k model_raqdps-fw -c /home/geoadm/geomet-data-registry/deploy/default/model_raqdps-fw.yml 56 | geomet-data-registry store set -k model_raqdps -c /home/geoadm/geomet-data-registry/deploy/default/model_raqdps.yml 57 | geomet-data-registry store set -k model_rdaqa-ce -c /home/geoadm/geomet-data-registry/deploy/default/model_rdaqa-ce.yml 58 | geomet-data-registry store set -k model_riops -c /home/geoadm/geomet-data-registry/deploy/default/model_riops.yml 59 | geomet-data-registry store set -k radar -c /home/geoadm/geomet-data-registry/deploy/default/radar.yml 60 | geomet-data-registry store set -k rdpa -c /home/geoadm/geomet-data-registry/deploy/default/rdpa.yml 61 | geomet-data-registry store set -k rdwps -c /home/geoadm/geomet-data-registry/deploy/default/rdwps.yml 62 | geomet-data-registry store set -k reps -c /home/geoadm/geomet-data-registry/deploy/default/reps.yml 63 | geomet-data-registry store set -k wcps -c /home/geoadm/geomet-data-registry/deploy/default/wcps.yml 64 | 65 | echo "Starting data feeds" 66 | sr_subscribe start /home/geoadm/geomet-data-registry/deploy/default/sarracenia/cansips.conf 67 | sr_subscribe start /home/geoadm/geomet-data-registry/deploy/default/sarracenia/cgsl.conf 68 | sr_subscribe start /home/geoadm/geomet-data-registry/deploy/default/sarracenia/gdwps.conf 69 | sr_subscribe start /home/geoadm/geomet-data-registry/deploy/default/sarracenia/geps.conf 70 | sr_subscribe start /home/geoadm/geomet-data-registry/deploy/default/sarracenia/hrdpa.conf 71 | sr_subscribe start /home/geoadm/geomet-data-registry/deploy/default/sarracenia/model_gem_global.conf 72 | sr_subscribe start /home/geoadm/geomet-data-registry/deploy/default/sarracenia/model_gem_regional.conf 73 | sr_subscribe start /home/geoadm/geomet-data-registry/deploy/default/sarracenia/model_giops.conf 74 | sr_subscribe start /home/geoadm/geomet-data-registry/deploy/default/sarracenia/model_hrdps_continental.conf 75 | sr_subscribe start /home/geoadm/geomet-data-registry/deploy/default/sarracenia/model_raqdps-fw-ce.conf 76 | sr_subscribe start /home/geoadm/geomet-data-registry/deploy/default/sarracenia/model_raqdps-fw.conf 77 | sr_subscribe start /home/geoadm/geomet-data-registry/deploy/default/sarracenia/model_raqdps.conf 78 | sr_subscribe start /home/geoadm/geomet-data-registry/deploy/default/sarracenia/model_rdaqa-ce.conf 79 | sr_subscribe start /home/geoadm/geomet-data-registry/deploy/default/sarracenia/model_riops.conf 80 | sr_subscribe start /home/geoadm/geomet-data-registry/deploy/default/sarracenia/radar.conf 81 | sr_subscribe start /home/geoadm/geomet-data-registry/deploy/default/sarracenia/rdpa.conf 82 | sr_subscribe start /home/geoadm/geomet-data-registry/deploy/default/sarracenia/rdwps.conf 83 | sr_subscribe start /home/geoadm/geomet-data-registry/deploy/default/sarracenia/reps.conf 84 | sr_subscribe start /home/geoadm/geomet-data-registry/deploy/default/sarracenia/wcps.conf 85 | 86 | sleep infinity 87 | -------------------------------------------------------------------------------- /geomet-data-registry.env: -------------------------------------------------------------------------------- 1 | export GDR_LOGGING_LOGLEVEL=DEBUG 2 | export GDR_LOGGING_LOGFILE=stdout 3 | export GDR_BASEDIR=/opt/geomet-data-registry 4 | export GDR_DATADIR=/data/geomet 5 | export GDR_TILEINDEX_TYPE=Elasticsearch 6 | export GDR_TILEINDEX_BASEURL=http://localhost:9200 7 | export GDR_TILEINDEX_NAME=geomet-data-registry-dev 8 | export GDR_STORE_TYPE=Redis 9 | export GDR_STORE_URL=redis://localhost:6379 10 | export GDR_METPX_DISCARD=on 11 | export GDR_METPX_EVENT_FILE_PY=/path/to/geomet_data_registry/event/file_.py 12 | export GDR_METPX_EVENT_MESSAGE_PY=/path/to/geomet_data_registry/event/message.py 13 | export GDR_METPX_NOTIFY=True 14 | export GDR_GEOMET_ONLY_USER=username 15 | export GDR_GEOMET_ONLY_PASS=password 16 | export GDR_GEOMET_ONLY_HOST=example.host.com 17 | export GDR_NOTIFICATIONS=False 18 | export GDR_NOTIFICATIONS_TYPE=Celery 19 | export GDR_NOTIFICATIONS_URL=redis://localhost:6379 20 | -------------------------------------------------------------------------------- /geomet_data_registry/__init__.py: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # 3 | # Copyright (C) 2021 Tom Kralidis 4 | # 5 | # This program is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with this program. If not, see . 17 | # 18 | ############################################################################### 19 | 20 | import click 21 | 22 | from geomet_data_registry import env 23 | from geomet_data_registry.handler import data 24 | from geomet_data_registry.log import setup_logger 25 | from geomet_data_registry.store import store 26 | from geomet_data_registry.tileindex import tileindex 27 | 28 | __version__ = '0.1.0' 29 | 30 | setup_logger(env.LOGGING_LOGLEVEL, env.LOGGING_LOGFILE) 31 | 32 | 33 | @click.group() 34 | @click.version_option(version=__version__) 35 | def cli(): 36 | pass 37 | 38 | 39 | cli.add_command(data) 40 | cli.add_command(store) 41 | cli.add_command(tileindex) 42 | -------------------------------------------------------------------------------- /geomet_data_registry/env.py: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # 3 | # Copyright (C) 2018 Tom Kralidis 4 | # 5 | # This program is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with this program. If not, see . 17 | # 18 | ############################################################################### 19 | 20 | import logging 21 | import os 22 | 23 | from geomet_data_registry.util import str2bool 24 | 25 | LOGGER = logging.getLogger(__name__) 26 | 27 | LOGGER.info('Fetching environment variables') 28 | 29 | LOGGING_LOGLEVEL = os.getenv('GDR_LOGGING_LOGLEVEL', 'ERROR') 30 | LOGGING_LOGFILE = os.getenv('GDR_LOGGING_LOGFILE', None) 31 | BASEDIR = os.environ.get('GDR_BASEDIR', None) 32 | DATADIR = os.environ.get('GDR_DATADIR', None) 33 | TILEINDEX_TYPE = os.environ.get('GDR_TILEINDEX_TYPE', None) 34 | TILEINDEX_BASEURL = os.environ.get('GDR_TILEINDEX_BASEURL', None) 35 | TILEINDEX_NAME = os.environ.get('GDR_TILEINDEX_NAME', None) 36 | STORE_TYPE = os.environ.get('GDR_STORE_TYPE', None) 37 | STORE_URL = os.environ.get('GDR_STORE_URL', None) 38 | METPX_DISCARD = os.environ.get('GDR_METPX_DISCARD', 'on') 39 | METPX_EVENT_FILE_PY = os.environ.get('GDR_METPX_EVENT_FILE_PY', None) 40 | METPX_EVENT_MESSAGE_PY = os.environ.get('GDR_METPX_EVENT_MESSAGE_PY', None) 41 | NOTIFICATIONS = str2bool(os.environ.get('GDR_NOTIFICATIONS', False)) 42 | NOTIFICATIONS_TYPE = os.environ.get('GDR_NOTIFICATIONS_TYPE', None) 43 | NOTIFICATIONS_URL = os.environ.get('GDR_NOTIFICATIONS_URL', None) 44 | 45 | LOGGER.debug(BASEDIR) 46 | LOGGER.debug(DATADIR) 47 | LOGGER.debug(TILEINDEX_TYPE) 48 | LOGGER.debug(TILEINDEX_BASEURL) 49 | LOGGER.debug(TILEINDEX_NAME) 50 | LOGGER.debug(STORE_TYPE) 51 | LOGGER.debug(STORE_URL) 52 | LOGGER.debug(METPX_DISCARD) 53 | LOGGER.debug(NOTIFICATIONS) 54 | LOGGER.debug(NOTIFICATIONS_TYPE) 55 | LOGGER.debug(NOTIFICATIONS_URL) 56 | 57 | if None in [ 58 | BASEDIR, 59 | DATADIR, 60 | TILEINDEX_TYPE, 61 | TILEINDEX_BASEURL, 62 | TILEINDEX_NAME, 63 | STORE_TYPE, 64 | STORE_URL, 65 | METPX_EVENT_FILE_PY, 66 | METPX_EVENT_MESSAGE_PY, 67 | ]: 68 | msg = 'Environment variables not set!' 69 | LOGGER.error(msg) 70 | raise EnvironmentError(msg) 71 | 72 | STORE_PROVIDER_DEF = {'type': STORE_TYPE, 'url': STORE_URL} 73 | 74 | TILEINDEX_PROVIDER_DEF = { 75 | 'type': TILEINDEX_TYPE, 76 | 'url': TILEINDEX_BASEURL, 77 | 'name': TILEINDEX_NAME, 78 | 'group': None 79 | } 80 | 81 | NOTIFICATIONS_PROVIDER_DEF = { 82 | 'type': NOTIFICATIONS_TYPE, 83 | 'active': NOTIFICATIONS, 84 | 'url': NOTIFICATIONS_URL 85 | } 86 | -------------------------------------------------------------------------------- /geomet_data_registry/event/__init__.py: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # 3 | # Copyright (C) 2021 Tom Kralidis 4 | # 5 | # This program is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with this program. If not, see . 17 | # 18 | ############################################################################### 19 | -------------------------------------------------------------------------------- /geomet_data_registry/event/file_.py: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # 3 | # Copyright (C) 2021 Tom Kralidis 4 | # 5 | # This program is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with this program. If not, see . 17 | # 18 | ############################################################################### 19 | 20 | 21 | class FileEvent: 22 | """core event""" 23 | 24 | def __init__(self, parent): 25 | """initialize""" 26 | pass 27 | 28 | def on_file(self, parent): 29 | """ 30 | sarracenia dispatcher 31 | 32 | :param parent: `sarra.sr_subscribe.sr_subscribe` 33 | 34 | :returns: `bool` of dispatch result 35 | """ 36 | 37 | from geomet_data_registry import env 38 | from geomet_data_registry.log import setup_logger 39 | 40 | setup_logger(env.LOGGING_LOGLEVEL, env.LOGGING_LOGFILE) 41 | 42 | try: 43 | from urllib.parse import urlunparse 44 | from geomet_data_registry.handler.core import CoreHandler 45 | 46 | filepath = parent.msg.local_file 47 | parent.logger.debug('Filepath: {}'.format(filepath)) 48 | url = urlunparse(parent.msg.url) 49 | parent.logger.debug('URL: {}'.format(url)) 50 | handler = CoreHandler(filepath, url) 51 | result = handler.handle() 52 | parent.logger.debug('Result: {}'.format(result)) 53 | return True 54 | except Exception as err: 55 | parent.logger.warning(err) 56 | return False 57 | 58 | def __repr__(self): 59 | return '' 60 | 61 | 62 | self.plugin = 'FileEvent' # noqa 63 | -------------------------------------------------------------------------------- /geomet_data_registry/event/message.py: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # 3 | # Copyright (C) 2021 Tom Kralidis 4 | # 5 | # This program is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with this program. If not, see . 17 | # 18 | ############################################################################### 19 | 20 | 21 | class MessageEvent: 22 | """core event""" 23 | 24 | def __init__(self, parent): 25 | """initialize""" 26 | pass 27 | 28 | def on_message(self, parent): 29 | """ 30 | sarracenia dispatcher 31 | 32 | :param parent: `sarra.sr_subscribe.sr_subscribe` 33 | 34 | :returns: `bool` of dispatch result 35 | """ 36 | 37 | from geomet_data_registry import env 38 | from geomet_data_registry.log import setup_logger 39 | 40 | setup_logger(env.LOGGING_LOGLEVEL, env.LOGGING_LOGFILE) 41 | 42 | try: 43 | from urllib.parse import urlunparse 44 | from geomet_data_registry.handler.core import CoreHandler 45 | 46 | filepath = parent.msg.local_file 47 | parent.logger.debug('Filepath: {}'.format(filepath)) 48 | url = urlunparse(parent.msg.url) 49 | parent.logger.debug('URL: {}'.format(url)) 50 | handler = CoreHandler(filepath, url) 51 | result = handler.handle() 52 | parent.logger.debug('Result: {}'.format(result)) 53 | return True 54 | except Exception as err: 55 | parent.logger.warning(err) 56 | return False 57 | 58 | def __repr__(self): 59 | return '' 60 | 61 | 62 | self.plugin = 'MessageEvent' # noqa 63 | -------------------------------------------------------------------------------- /geomet_data_registry/handler/__init__.py: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # 3 | # Copyright (C) 2019 Tom Kralidis 4 | # 5 | # This program is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with this program. If not, see . 17 | # 18 | ############################################################################### 19 | 20 | import logging 21 | import os 22 | 23 | import click 24 | 25 | from geomet_data_registry.handler.core import CoreHandler 26 | from geomet_data_registry.util import json_pretty_print 27 | 28 | LOGGER = logging.getLogger(__name__) 29 | 30 | 31 | @click.group() 32 | def data(): 33 | """Manage geomet-data-registry data""" 34 | pass 35 | 36 | 37 | @click.command('add') 38 | @click.pass_context 39 | @click.option('--file', '-f', 'file_', 40 | type=click.Path(exists=True, resolve_path=True), 41 | help='Path to file') 42 | @click.option('--directory', '-d', 'directory', 43 | type=click.Path(exists=True, resolve_path=True, 44 | dir_okay=True, file_okay=False), 45 | help='Path to directory') 46 | @click.option('--verify', '-v', is_flag=True, help='Verify only', 47 | default=False) 48 | def add_data(ctx, file_, directory, verify=False): 49 | """add data to system""" 50 | 51 | if all([file_ is None, directory is None]): 52 | raise click.ClickException('Missing --file/-f or --dir/-d option') 53 | 54 | files_to_process = [] 55 | 56 | if file_ is not None: 57 | files_to_process = [file_] 58 | elif directory is not None: 59 | for root, dirs, files in os.walk(directory): 60 | for f in files: 61 | files_to_process.append(os.path.join(root, f)) 62 | files_to_process.sort(key=os.path.getmtime) 63 | 64 | for file_to_process in files_to_process: 65 | handler = CoreHandler(file_to_process) 66 | result = handler.handle() 67 | if result: 68 | click.echo('File properties: {}'.format( 69 | json_pretty_print(handler.layer_plugin.items))) 70 | 71 | 72 | @click.command('setup') 73 | @click.pass_context 74 | def setup_metadata(ctx): 75 | """initialize system metadata""" 76 | 77 | # connect to store 78 | # processs all YAML configurations 79 | # dump into store 80 | raise click.ClickException('Not implemented yet') 81 | 82 | 83 | data.add_command(add_data) 84 | -------------------------------------------------------------------------------- /geomet_data_registry/handler/base.py: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # 3 | # Copyright (C) 2019 Tom Kralidis 4 | # 5 | # This program is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with this program. If not, see . 17 | # 18 | ############################################################################### 19 | 20 | import logging 21 | 22 | LOGGER = logging.getLogger(__name__) 23 | 24 | 25 | class BaseHandler: 26 | """base handler""" 27 | 28 | def __init__(self, filepath, url=None): 29 | """ 30 | initializer 31 | 32 | :param filepath: path to file 33 | :param url: fully qualified URL of file 34 | """ 35 | 36 | self.filepath = filepath 37 | self.url = url 38 | LOGGER.debug('Filepath: {}'.format(self.filepath)) 39 | LOGGER.debug('URL: {}'.format(self.url)) 40 | 41 | def handle(self): 42 | """handle incoming file""" 43 | 44 | raise NotImplementedError() 45 | 46 | def __repr__(self): 47 | return ' {}'.format(self.filepath) 48 | -------------------------------------------------------------------------------- /geomet_data_registry/handler/core.py: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # 3 | # Copyright (C) 2019 Tom Kralidis 4 | # 5 | # This program is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with this program. If not, see . 17 | # 18 | ############################################################################### 19 | 20 | from fnmatch import fnmatch 21 | import logging 22 | import os 23 | 24 | from geomet_data_registry.env import NOTIFICATIONS_PROVIDER_DEF 25 | from geomet_data_registry.plugin import load_plugin, PLUGINS 26 | from geomet_data_registry.handler.base import BaseHandler 27 | from geomet_data_registry.util import get_today_and_now 28 | 29 | LOGGER = logging.getLogger(__name__) 30 | 31 | 32 | class CoreHandler(BaseHandler): 33 | """base handler""" 34 | 35 | def __init__(self, filepath, url=None): 36 | """ 37 | initializer 38 | 39 | :param filepath: path to file 40 | :param url: fully qualified URL of file 41 | 42 | :returns: `geomet_data_registry.handler.core.CoreHandler` 43 | """ 44 | 45 | self.layer_plugin = None 46 | self.notification_plugin = None 47 | 48 | super().__init__(filepath, url) 49 | 50 | def handle(self): 51 | """ 52 | handle incoming file 53 | 54 | :returns: `bool` of status result 55 | """ 56 | 57 | LOGGER.debug('Detecting filename pattern') 58 | for key, value in PLUGINS['layer'].items(): 59 | if fnmatch(os.path.basename(self.filepath), value['pattern']): 60 | plugin_def = { 61 | 'type': key 62 | } 63 | LOGGER.debug('Loading plugin {}'.format(plugin_def)) 64 | self.layer_plugin = load_plugin('layer', plugin_def) 65 | 66 | if self.layer_plugin is None: 67 | msg = 'Plugin not found' 68 | LOGGER.error(msg) 69 | raise RuntimeError(msg) 70 | 71 | LOGGER.debug('Identifying file') 72 | identify_status = self.layer_plugin.identify(self.filepath, self.url) 73 | 74 | if identify_status: 75 | self.layer_plugin.identify_datetime = get_today_and_now() 76 | LOGGER.debug('Registering file') 77 | self.layer_plugin.register() 78 | if self.layer_plugin.new_key_store: 79 | self.layer_plugin.add_time_key() 80 | 81 | for notifier in PLUGINS['notifier'].keys(): 82 | if all([notifier == NOTIFICATIONS_PROVIDER_DEF['type'], 83 | NOTIFICATIONS_PROVIDER_DEF['active']]): 84 | LOGGER.debug('Loading plugin {}'.format( 85 | NOTIFICATIONS_PROVIDER_DEF)) 86 | self.notification_plugin = load_plugin( 87 | 'notifier', NOTIFICATIONS_PROVIDER_DEF 88 | ) 89 | 90 | if self.notification_plugin is None: 91 | msg = 'Plugin not found' 92 | LOGGER.error(msg) 93 | raise RuntimeError(msg) 94 | 95 | if notifier == 'Celery': 96 | LOGGER.debug( 97 | 'Sending mapfile refresh tasks to Celery' 98 | ) 99 | self.notification_plugin.notify( 100 | self.layer_plugin.items 101 | ) 102 | 103 | return True 104 | 105 | def __repr__(self): 106 | return ' {}'.format(self.url) 107 | -------------------------------------------------------------------------------- /geomet_data_registry/layer/__init__.py: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # 3 | # Copyright (C) 2019 Tom Kralidis 4 | # 5 | # This program is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with this program. If not, see . 17 | # 18 | ############################################################################### 19 | -------------------------------------------------------------------------------- /geomet_data_registry/layer/cgsl.py: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # 3 | # Copyright (C) 2019 Tom Kralidis 4 | # 5 | # This program is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with this program. If not, see . 17 | # 18 | ############################################################################### 19 | 20 | from datetime import datetime, timedelta 21 | import json 22 | import logging 23 | import os 24 | from parse import parse 25 | import re 26 | 27 | from geomet_data_registry.layer.base import BaseLayer 28 | from geomet_data_registry.util import DATE_FORMAT 29 | 30 | LOGGER = logging.getLogger(__name__) 31 | 32 | 33 | class CgslLayer(BaseLayer): 34 | """CGSL layer""" 35 | 36 | def __init__(self, provider_def): 37 | """ 38 | Initialize object 39 | 40 | :param provider_def: provider definition dict 41 | 42 | :returns: `geomet_data_registry.layer.cgsl.CgslLayer` 43 | """ 44 | 45 | provider_def = {'name': 'cgsl'} 46 | 47 | super().__init__(provider_def) 48 | 49 | def identify(self, filepath, url=None): 50 | """ 51 | Identifies a file of the layer 52 | 53 | :param filepath: filepath from AMQP 54 | :param url: fully qualified URL of file 55 | 56 | :returns: `list` of file properties 57 | """ 58 | 59 | super().identify(filepath, url) 60 | 61 | self.model = 'cgsl' 62 | 63 | LOGGER.debug('Loading model information from store') 64 | self.file_dict = json.loads(self.store.get_key(self.model)) 65 | 66 | filename_pattern = self.file_dict[self.model]['filename_pattern'] 67 | 68 | tmp = parse(filename_pattern, os.path.basename(filepath)) 69 | 70 | file_pattern_info = { 71 | 'wx_variable': tmp.named['wx_variable'], 72 | 'time_': tmp.named['YYYYMMDD_model_run'], 73 | 'fh': tmp.named['forecast_hour'] 74 | } 75 | 76 | LOGGER.debug('Defining the different file properties') 77 | self.wx_variable = file_pattern_info['wx_variable'] 78 | 79 | if self.wx_variable not in self.file_dict[self.model]['variable']: 80 | msg = 'Variable "{}" not in ' \ 81 | 'configuration file'.format(self.wx_variable) 82 | LOGGER.warning(msg) 83 | return False 84 | 85 | runs = self.file_dict[self.model]['variable'][self.wx_variable][ 86 | 'model_run'] 87 | self.model_run_list = list(runs.keys()) 88 | 89 | time_format = '%Y%m%d%H' 90 | self.date_ = datetime.strptime(file_pattern_info['time_'], time_format) 91 | 92 | reference_datetime = self.date_ 93 | self.model_run = '{}Z'.format(self.date_.strftime('%H')) 94 | 95 | forecast_hour_datetime = self.date_ + \ 96 | timedelta(hours=int(file_pattern_info['fh'])) 97 | 98 | member = self.file_dict[self.model]['variable'][self.wx_variable][ 99 | 'members'] 100 | elevation = self.file_dict[self.model]['variable'][self.wx_variable][ 101 | 'elevation'] 102 | str_mr = re.sub('[^0-9]', 103 | '', 104 | reference_datetime.strftime(DATE_FORMAT)) 105 | str_fh = re.sub('[^0-9]', 106 | '', 107 | forecast_hour_datetime.strftime(DATE_FORMAT)) 108 | expected_count = self.file_dict[self.model]['variable'][ 109 | self.wx_variable]['model_run'][self.model_run]['files_expected'] 110 | 111 | self.geomet_layers = self.file_dict[self.model]['variable'][ 112 | self.wx_variable]['geomet_layers'] 113 | for layer_name, layer_config in self.geomet_layers.items(): 114 | identifier = '{}-{}-{}'.format(layer_name, str_mr, str_fh) 115 | 116 | vrt = 'vrt://{}?bands={}'.format(filepath, layer_config['bands']) 117 | 118 | forecast_hours = layer_config['forecast_hours'] 119 | begin, end, interval = [int(re.sub('[^0-9]', '', value)) 120 | for value in forecast_hours.split('/')] 121 | fh = int(file_pattern_info['fh']) 122 | 123 | feature_dict = { 124 | 'layer_name': layer_name, 125 | 'filepath': vrt, 126 | 'identifier': identifier, 127 | 'reference_datetime': reference_datetime.strftime(DATE_FORMAT), 128 | 'forecast_hour_datetime': forecast_hour_datetime.strftime( 129 | DATE_FORMAT), 130 | 'member': member, 131 | 'model': self.model, 132 | 'elevation': elevation, 133 | 'expected_count': expected_count, 134 | 'forecast_hours': { 135 | 'begin': begin, 136 | 'end': end, 137 | 'interval': forecast_hours.split('/')[2] 138 | }, 139 | 'layer_config': layer_config, 140 | 'register_status': True, 141 | 'refresh_config': True, 142 | } 143 | 144 | if not self.is_valid_interval(fh, begin, end, interval): 145 | feature_dict['register_status'] = False 146 | LOGGER.debug('Forecast hour {} not included in {} as ' 147 | 'defined for layer {}. File will not be ' 148 | 'added to registry for this layer' 149 | .format(fh, forecast_hours, layer_name)) 150 | 151 | self.items.append(feature_dict) 152 | 153 | return True 154 | 155 | def __repr__(self): 156 | return ' {}'.format(self.name) 157 | -------------------------------------------------------------------------------- /geomet_data_registry/layer/gdwps.py: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # 3 | # Copyright (C) 2020 Etienne Pelletier 4 | # 5 | # This program is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with this program. If not, see . 17 | # 18 | ############################################################################### 19 | 20 | from datetime import datetime, timedelta 21 | import json 22 | import logging 23 | import os 24 | import re 25 | 26 | from parse import parse 27 | from geomet_data_registry.layer.base import BaseLayer 28 | from geomet_data_registry.util import DATE_FORMAT 29 | 30 | LOGGER = logging.getLogger(__name__) 31 | 32 | 33 | class GdwpsLayer(BaseLayer): 34 | """GDWPS layer""" 35 | 36 | def __init__(self, provider_def): 37 | """ 38 | Initialize object 39 | 40 | :param provider_def: provider definition dict 41 | 42 | :returns: `geomet_data_registry.layer.gdwps.GdwpsLayer` 43 | """ 44 | 45 | provider_def = {'name': 'gdwps'} 46 | 47 | super().__init__(provider_def) 48 | 49 | def identify(self, filepath, url=None): 50 | """ 51 | Identifies a file of the layer 52 | 53 | :param filepath: filepath from AMQP 54 | :param url: fully qualified URL of file 55 | 56 | :returns: `list` of file properties 57 | """ 58 | 59 | super().identify(filepath, url) 60 | 61 | self.model = 'gdwps' 62 | 63 | LOGGER.debug('Loading model information from store') 64 | self.file_dict = json.loads(self.store.get_key(self.model)) 65 | 66 | filename_pattern = self.file_dict[self.model]['filename_pattern'] 67 | 68 | tmp = parse(filename_pattern, os.path.basename(filepath)) 69 | 70 | file_pattern_info = { 71 | 'wx_variable': tmp.named['wx_variable'], 72 | 'date': tmp.named['YYYYMMDD'], 73 | 'model_run': tmp.named['model_run'], 74 | 'fh': tmp.named['forecast_hour'] 75 | } 76 | 77 | LOGGER.debug('Defining the different file properties') 78 | self.wx_variable = file_pattern_info['wx_variable'] 79 | 80 | if self.wx_variable not in self.file_dict[self.model]['variable']: 81 | msg = 'Variable "{}" not in configuration file'.format( 82 | self.wx_variable 83 | ) 84 | LOGGER.warning(msg) 85 | return False 86 | 87 | self.dimensions = self.file_dict[self.model]['dimensions'] 88 | 89 | runs = self.file_dict[self.model]['variable'][self.wx_variable]['model_run'] # noqa 90 | self.model_run_list = list(runs.keys()) 91 | 92 | time_format = '%Y%m%d%H' 93 | self.date_ = datetime.strptime( 94 | '{}{}'.format( 95 | file_pattern_info['date'], file_pattern_info['model_run'] 96 | ), 97 | time_format 98 | ) 99 | 100 | reference_datetime = self.date_ 101 | self.model_run = '{}Z'.format(file_pattern_info['model_run']) 102 | 103 | forecast_hour_datetime = self.date_ + timedelta( 104 | hours=int(file_pattern_info['fh']) 105 | ) 106 | 107 | member = self.file_dict[self.model]['variable'][self.wx_variable]['members'] # noqa 108 | elevation = self.file_dict[self.model]['variable'][self.wx_variable]['elevation'] # noqa 109 | 110 | str_mr = re.sub('[^0-9]', '', reference_datetime.strftime(DATE_FORMAT)) 111 | str_fh = re.sub( 112 | '[^0-9]', '', forecast_hour_datetime.strftime(DATE_FORMAT) 113 | ) 114 | 115 | expected_count = self.file_dict[self.model]['variable'][self.wx_variable]['model_run'][self.model_run]['files_expected'] # noqa 116 | 117 | self.geomet_layers = self.file_dict[self.model]['variable'][self.wx_variable]['geomet_layers'] # noqa 118 | for layer_name, layer_config in self.geomet_layers.items(): 119 | identifier = '{}-{}-{}'.format(layer_name, str_mr, str_fh) 120 | 121 | forecast_hours = layer_config['forecast_hours'] 122 | begin, end, interval = [ 123 | int(re.sub('[^0-9]', '', value)) 124 | for value in forecast_hours.split('/') 125 | ] 126 | fh = int(file_pattern_info['fh']) 127 | 128 | feature_dict = { 129 | 'layer_name': layer_name, 130 | 'filepath': self.filepath, 131 | 'identifier': identifier, 132 | 'reference_datetime': reference_datetime.strftime(DATE_FORMAT), 133 | 'forecast_hour_datetime': forecast_hour_datetime.strftime( 134 | DATE_FORMAT 135 | ), 136 | 'member': member, 137 | 'model': self.model, 138 | 'elevation': elevation, 139 | 'expected_count': expected_count, 140 | 'forecast_hours': { 141 | 'begin': begin, 142 | 'end': end, 143 | 'interval': forecast_hours.split('/')[2] 144 | }, 145 | 'layer_config': layer_config, 146 | 'register_status': True, 147 | 'refresh_config': True 148 | } 149 | 150 | if 'dependencies' in layer_config: 151 | dependencies_found = self.check_layer_dependencies( 152 | layer_config['dependencies'], str_mr, str_fh 153 | ) 154 | if dependencies_found: 155 | 156 | bands_order = self.file_dict[self.model]['variable'][self.wx_variable].get('bands_order') # noqa 157 | 158 | ( 159 | feature_dict['filepath'], 160 | feature_dict['url'], 161 | feature_dict['weather_variable'], 162 | ) = self.configure_layer_with_dependencies( 163 | dependencies_found, self.dimensions, bands_order 164 | ) 165 | 166 | else: 167 | feature_dict['register_status'] = False 168 | self.items.append(feature_dict) 169 | continue 170 | 171 | if not self.is_valid_interval(fh, begin, end, interval): 172 | feature_dict['register_status'] = False 173 | LOGGER.debug( 174 | 'Forecast hour {} not included in {} as ' 175 | 'defined for layer {}. File will not be ' 176 | 'added to registry for this layer'.format( 177 | fh, forecast_hours, layer_name 178 | ) 179 | ) 180 | 181 | self.items.append(feature_dict) 182 | 183 | return True 184 | 185 | def __repr__(self): 186 | return ' {}'.format(self.name) 187 | -------------------------------------------------------------------------------- /geomet_data_registry/layer/geps.py: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # 3 | # Copyright (C) 2019 Louis-Philippe Rousseau-Lambert 4 | # Copyright (C) 2019 Etienne Pelletier 5 | # 6 | # This program is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program. If not, see . 18 | # 19 | ############################################################################### 20 | 21 | from datetime import datetime, timedelta 22 | import json 23 | import logging 24 | import os 25 | from parse import parse 26 | import re 27 | 28 | from geomet_data_registry.layer.base import BaseLayer 29 | from geomet_data_registry.util import DATE_FORMAT 30 | 31 | LOGGER = logging.getLogger(__name__) 32 | 33 | 34 | class GepsLayer(BaseLayer): 35 | """GEPS layer""" 36 | 37 | def __init__(self, provider_def): 38 | """ 39 | Initialize object 40 | 41 | :param provider_def: provider definition dict 42 | 43 | :returns: `geomet_data_registry.layer.geps.GEPSLayer` 44 | """ 45 | 46 | provider_def = {'name': 'geps'} 47 | self.type = None 48 | self.bands = None 49 | 50 | super().__init__(provider_def) 51 | 52 | def identify(self, filepath, url=None): 53 | """ 54 | Identifies a file of the layer 55 | 56 | :param filepath: filepath from AMQP 57 | :param url: fully qualified URL of file 58 | 59 | :returns: `list` of file properties 60 | """ 61 | 62 | super().identify(filepath, url) 63 | 64 | self.model = 'geps' 65 | 66 | LOGGER.debug('Loading model information from store') 67 | self.file_dict = json.loads(self.store.get_key(self.model)) 68 | 69 | if self.filepath.endswith('allmbrs.grib2'): 70 | filename_pattern = self.file_dict[self.model]['member'][ 71 | 'filename_pattern'] 72 | self.type = 'member' 73 | elif self.filepath.endswith('all-products.grib2'): 74 | filename_pattern = self.file_dict[self.model]['product'][ 75 | 'filename_pattern'] 76 | self.type = 'product' 77 | 78 | tmp = parse(filename_pattern, os.path.basename(filepath)) 79 | 80 | file_pattern_info = { 81 | 'wx_variable': tmp.named['wx_variable'], 82 | 'time_': tmp.named['YYYYMMDD_model_run'], 83 | 'fh': tmp.named['forecast_hour'] 84 | } 85 | 86 | LOGGER.debug('Defining the different file properties') 87 | self.wx_variable = file_pattern_info['wx_variable'] 88 | 89 | var_path = self.file_dict[self.model][self.type]['variable'] 90 | if self.wx_variable not in var_path: 91 | msg = 'Variable "{}" not in ' \ 92 | 'configuration file'.format(self.wx_variable) 93 | LOGGER.warning(msg) 94 | return False 95 | 96 | runs = (self.file_dict[self.model][self.type]['variable'] 97 | [self.wx_variable]['model_run']) 98 | 99 | self.model_run_list = list(runs.keys()) 100 | 101 | weather_var = self.file_dict[self.model][self.type]['variable'][ 102 | self.wx_variable] 103 | self.geomet_layers = weather_var['geomet_layers'] 104 | 105 | time_format = '%Y%m%d%H' 106 | self.date_ = datetime.strptime(file_pattern_info['time_'], time_format) 107 | reference_datetime = self.date_ 108 | self.model_run = '{}Z'.format(self.date_.strftime('%H')) 109 | forecast_hour_datetime = self.date_ + \ 110 | timedelta(hours=int(file_pattern_info['fh'])) 111 | 112 | if self.type == 'member': 113 | self.bands = self.file_dict[self.model]['member']['bands'] 114 | elif self.type == 'product': 115 | self.bands = weather_var['bands'] 116 | 117 | for band in self.bands.keys(): 118 | vrt = 'vrt://{}?bands={}'.format(self.filepath, band) 119 | 120 | elevation = weather_var['elevation'] 121 | str_mr = re.sub('[^0-9]', 122 | '', 123 | reference_datetime.strftime(DATE_FORMAT)) 124 | str_fh = re.sub('[^0-9]', 125 | '', 126 | forecast_hour_datetime.strftime(DATE_FORMAT)) 127 | 128 | expected_count = self.file_dict[self.model][self.type][ 129 | 'variable'][self.wx_variable]['model_run'][self.model_run][ 130 | 'files_expected'] 131 | 132 | for layer, layer_config in self.geomet_layers.items(): 133 | if self.type == 'member': 134 | member = self.bands[band]['member'] 135 | layer_name = layer.format(self.bands[band]['member']) 136 | 137 | elif self.type == 'product': 138 | member = None 139 | layer_name = layer.format(self.bands[band]['product']) 140 | 141 | identifier = '{}-{}-{}'.format(layer_name, str_mr, str_fh) 142 | 143 | forecast_hours = layer_config['forecast_hours'] 144 | begin, end, interval = [int(re.sub('[^0-9]', '', value)) 145 | for value in 146 | forecast_hours.split('/')] 147 | fh = int(file_pattern_info['fh']) 148 | 149 | feature_dict = { 150 | 'layer_name': layer_name, 151 | 'layer_name_unformatted': layer, 152 | 'filepath': vrt, 153 | 'identifier': identifier, 154 | 'reference_datetime': reference_datetime.strftime( 155 | DATE_FORMAT), 156 | 'forecast_hour_datetime': forecast_hour_datetime.strftime( 157 | DATE_FORMAT), 158 | 'member': member, 159 | 'model': self.model, 160 | 'elevation': elevation, 161 | 'expected_count': expected_count, 162 | 'forecast_hours': { 163 | 'begin': begin, 164 | 'end': end, 165 | 'interval': forecast_hours.split('/')[2] 166 | }, 167 | 'layer_config': layer_config, 168 | 'register_status': True, 169 | 'refresh_config': True, 170 | } 171 | 172 | if not self.is_valid_interval(fh, begin, end, interval): 173 | feature_dict['register_status'] = False 174 | LOGGER.debug('Forecast hour {} not included in {} as ' 175 | 'defined for layer {}. File will not be ' 176 | 'added to registry for this layer' 177 | .format(fh, forecast_hours, layer_name)) 178 | 179 | self.items.append(feature_dict) 180 | 181 | return True 182 | 183 | def __repr__(self): 184 | return ' {}'.format(self.name) 185 | -------------------------------------------------------------------------------- /geomet_data_registry/layer/hrdpa.py: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # 3 | # Copyright (C) 2020 Etienne Pelletier 4 | # 5 | # This program is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with this program. If not, see . 17 | # 18 | ############################################################################### 19 | 20 | from datetime import datetime, timedelta 21 | import json 22 | import logging 23 | import os 24 | import re 25 | 26 | from parse import parse 27 | from geomet_data_registry.layer.base import BaseLayer 28 | from geomet_data_registry.util import DATE_FORMAT 29 | 30 | LOGGER = logging.getLogger(__name__) 31 | 32 | 33 | class HrdpaLayer(BaseLayer): 34 | """HRDPA layer""" 35 | 36 | def __init__(self, provider_def): 37 | """ 38 | Initialize object 39 | 40 | :param provider_def: provider definition dict 41 | 42 | :returns: `geomet_data_registry.layer.HrdpaLayer` 43 | """ 44 | 45 | provider_def = {'name': 'hrdpa'} 46 | 47 | super().__init__(provider_def) 48 | 49 | def identify(self, filepath, url=None): 50 | """ 51 | Identifies a file of the layer 52 | 53 | :param filepath: filepath from AMQP 54 | :param url: fully qualified URL of file 55 | 56 | :returns: `list` of file properties 57 | """ 58 | 59 | super().identify(filepath, url) 60 | 61 | self.model = 'hrdpa' 62 | 63 | LOGGER.debug('Loading model information from store') 64 | self.file_dict = json.loads(self.store.get_key(self.model)) 65 | 66 | filename_pattern = self.file_dict[self.model]['filename_pattern'] 67 | 68 | tmp = parse(filename_pattern, os.path.basename(filepath)) 69 | 70 | file_pattern_info = { 71 | 'wx_variable': tmp.named['wx_variable'], 72 | 'time_': tmp.named['YYYYMMDD_model_run'], 73 | 'fh': tmp.named['forecast_hour'] 74 | } 75 | 76 | LOGGER.debug('Defining the different file properties') 77 | self.wx_variable = file_pattern_info['wx_variable'] 78 | 79 | if self.wx_variable not in self.file_dict[self.model]['variable']: 80 | msg = 'Variable "{}" not in ' \ 81 | 'configuration file'.format(self.wx_variable) 82 | LOGGER.warning(msg) 83 | return False 84 | 85 | runs = self.file_dict[self.model]['variable'][self.wx_variable][ 86 | 'model_run'] 87 | self.model_run_list = list(runs.keys()) 88 | 89 | time_format = '%Y%m%d%H' 90 | self.date_ = datetime.strptime(file_pattern_info['time_'], time_format) 91 | 92 | self.model_run = '{}Z'.format(self.date_.strftime('%H')) 93 | 94 | forecast_hour_datetime = self.date_ + \ 95 | timedelta(hours=int(file_pattern_info['fh'])) 96 | 97 | member = self.file_dict[self.model]['variable'][self.wx_variable][ 98 | 'members'] 99 | elevation = self.file_dict[self.model]['variable'][self.wx_variable][ 100 | 'elevation'] 101 | str_fh = re.sub('[^0-9]', 102 | '', 103 | forecast_hour_datetime.strftime(DATE_FORMAT)) 104 | expected_count = self.file_dict[self.model]['variable'][ 105 | self.wx_variable]['model_run'][self.model_run]['files_expected'] 106 | 107 | self.geomet_layers = self.file_dict[self.model]['variable'][ 108 | self.wx_variable]['geomet_layers'] 109 | for layer_name, layer_config in self.geomet_layers.items(): 110 | identifier = '{}-{}'.format(layer_name, str_fh) 111 | 112 | forecast_hours = layer_config['forecast_hours'] 113 | begin, end, interval = [int(re.sub(r'[^-\d]', '', value)) 114 | for value in forecast_hours.split('/')] 115 | 116 | feature_dict = { 117 | 'layer_name': layer_name, 118 | 'filepath': self.filepath, 119 | 'identifier': identifier, 120 | 'reference_datetime': None, 121 | 'forecast_hour_datetime': self.date_.strftime(DATE_FORMAT), 122 | 'member': member, 123 | 'model': self.model, 124 | 'elevation': elevation, 125 | 'expected_count': expected_count, 126 | 'forecast_hours': { 127 | 'begin': begin, 128 | 'end': end, 129 | 'interval': forecast_hours.split('/')[2] 130 | }, 131 | 'layer_config': layer_config, 132 | 'register_status': True, 133 | 'refresh_config': True, 134 | } 135 | self.items.append(feature_dict) 136 | 137 | return True 138 | 139 | def add_time_key(self): 140 | """ 141 | Adds default time and time extent datetime values to store for radar 142 | layers. Overrides the add_time_key method of BaseLayer class due to 143 | radar data's lack of forecast models. 144 | :return: `bool` if successfully added a new radar time key 145 | """ 146 | 147 | for item in self.items: 148 | 149 | default_time_key = '{}_default_time'.format(item['layer_name']) 150 | last_default_time_key = self.store.get_key(default_time_key) 151 | time_extent_key = '{}_time_extent'.format(item['layer_name']) 152 | 153 | start_time = self.date_ + timedelta( 154 | hours=item['forecast_hours']['begin']) 155 | start_time = start_time.strftime(DATE_FORMAT) 156 | end_time = self.date_.strftime(DATE_FORMAT) 157 | 158 | time_extent_value = '{}/{}/{}'.format(start_time, 159 | end_time, 160 | item['forecast_hours'] 161 | ['interval']) 162 | 163 | if last_default_time_key and datetime.strptime( 164 | last_default_time_key, DATE_FORMAT) > self.date_: 165 | LOGGER.debug( 166 | 'New default time value ({}) is older than the current ' 167 | 'default time in store: {}. ' 168 | 'Not updating time keys.'.format( 169 | end_time, last_default_time_key)) 170 | continue 171 | 172 | LOGGER.debug('Adding time keys in the store') 173 | self.store.set_key(default_time_key, end_time) 174 | self.store.set_key(time_extent_key, time_extent_value) 175 | 176 | return True 177 | 178 | def __repr__(self): 179 | return ' {}'.format(self.name) 180 | -------------------------------------------------------------------------------- /geomet_data_registry/layer/model_gem_global.py: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # 3 | # Copyright (C) 2019 Tom Kralidis 4 | # 5 | # This program is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with this program. If not, see . 17 | # 18 | ############################################################################### 19 | 20 | from datetime import datetime, timedelta 21 | import json 22 | import logging 23 | import os 24 | from parse import parse 25 | import re 26 | 27 | from geomet_data_registry.layer.base import BaseLayer 28 | from geomet_data_registry.util import DATE_FORMAT 29 | 30 | LOGGER = logging.getLogger(__name__) 31 | 32 | 33 | class ModelGemGlobalLayer(BaseLayer): 34 | """GDPS layer""" 35 | 36 | def __init__(self, provider_def): 37 | """ 38 | Initialize object 39 | 40 | :param provider_def: provider definition dict 41 | 42 | :returns: `geomet_data_registry.layer.model_gem_global.ModelGemGlobalLayer` # noqa 43 | """ 44 | 45 | provider_def = {'name': 'model_gem_global'} 46 | 47 | super().__init__(provider_def) 48 | 49 | def identify(self, filepath, url=None): 50 | """ 51 | Identifies a file of the layer 52 | 53 | :param filepath: filepath from AMQP 54 | :param url: fully qualified URL of file 55 | 56 | :returns: `list` of file properties 57 | """ 58 | 59 | super().identify(filepath, url) 60 | 61 | self.model = 'model_gem_global' 62 | 63 | LOGGER.debug('Loading model information from store') 64 | self.file_dict = json.loads(self.store.get_key(self.model)) 65 | 66 | filename_pattern = self.file_dict[self.model]['filename_pattern'] 67 | 68 | tmp = parse(filename_pattern, os.path.basename(filepath)) 69 | 70 | file_pattern_info = { 71 | 'wx_variable': tmp.named['wx_variable'], 72 | 'time_': tmp.named['YYYYMMDD_model_run'], 73 | 'fh': tmp.named['forecast_hour'] 74 | } 75 | 76 | LOGGER.debug('Defining the different file properties') 77 | self.wx_variable = file_pattern_info['wx_variable'] 78 | 79 | if self.wx_variable not in self.file_dict[self.model]['variable']: 80 | msg = 'Variable "{}" not in ' \ 81 | 'configuration file'.format(self.wx_variable) 82 | LOGGER.warning(msg) 83 | return False 84 | 85 | self.dimensions = self.file_dict[self.model]['dimensions'] 86 | 87 | runs = self.file_dict[self.model]['variable'][self.wx_variable][ 88 | 'model_run'] 89 | self.model_run_list = list(runs.keys()) 90 | 91 | time_format = '%Y%m%d%H' 92 | self.date_ = datetime.strptime(file_pattern_info['time_'], time_format) 93 | 94 | reference_datetime = self.date_ 95 | self.model_run = '{}Z'.format(self.date_.strftime('%H')) 96 | 97 | forecast_hour_datetime = self.date_ + \ 98 | timedelta(hours=int(file_pattern_info['fh'])) 99 | 100 | member = self.file_dict[self.model]['variable'][self.wx_variable][ 101 | 'members'] 102 | elevation = self.file_dict[self.model]['variable'][self.wx_variable][ 103 | 'elevation'] 104 | str_mr = re.sub('[^0-9]', 105 | '', 106 | reference_datetime.strftime(DATE_FORMAT)) 107 | str_fh = re.sub('[^0-9]', 108 | '', 109 | forecast_hour_datetime.strftime(DATE_FORMAT)) 110 | expected_count = self.file_dict[self.model]['variable'][ 111 | self.wx_variable]['model_run'][self.model_run]['files_expected'] 112 | 113 | self.geomet_layers = self.file_dict[self.model]['variable'][ 114 | self.wx_variable]['geomet_layers'] 115 | for layer_name, layer_config in self.geomet_layers.items(): 116 | identifier = '{}-{}-{}'.format(layer_name, str_mr, str_fh) 117 | 118 | forecast_hours = layer_config['forecast_hours'] 119 | begin, end, interval = [int(re.sub('[^0-9]', '', value)) 120 | for value in forecast_hours.split('/')] 121 | fh = int(file_pattern_info['fh']) 122 | 123 | feature_dict = { 124 | 'layer_name': layer_name, 125 | 'filepath': self.filepath, 126 | 'identifier': identifier, 127 | 'reference_datetime': reference_datetime.strftime( 128 | DATE_FORMAT), 129 | 'forecast_hour_datetime': forecast_hour_datetime.strftime( 130 | DATE_FORMAT), 131 | 'member': member, 132 | 'model': self.model, 133 | 'elevation': elevation, 134 | 'expected_count': expected_count, 135 | 'forecast_hours': { 136 | 'begin': begin, 137 | 'end': end, 138 | 'interval': forecast_hours.split('/')[2] 139 | }, 140 | 'layer_config': layer_config, 141 | 'register_status': True, 142 | 'refresh_config': True, 143 | } 144 | 145 | if 'dependencies' in layer_config: 146 | dependencies_found = self.check_layer_dependencies( 147 | layer_config['dependencies'], 148 | str_mr, 149 | str_fh) 150 | if dependencies_found: 151 | bands_order = (self.file_dict[self.model] 152 | ['variable'] 153 | [self.wx_variable].get('bands_order')) 154 | (feature_dict['filepath'], 155 | feature_dict['url'], 156 | feature_dict['weather_variable']) = ( 157 | self.configure_layer_with_dependencies( 158 | dependencies_found, 159 | self.dimensions, 160 | bands_order)) 161 | else: 162 | feature_dict['register_status'] = False 163 | self.items.append(feature_dict) 164 | continue 165 | 166 | if not self.is_valid_interval(fh, begin, end, interval): 167 | feature_dict['register_status'] = False 168 | LOGGER.debug('Forecast hour {} not included in {} as ' 169 | 'defined for layer {}. File will not be ' 170 | 'added to registry for this layer' 171 | .format(fh, forecast_hours, layer_name)) 172 | 173 | self.items.append(feature_dict) 174 | 175 | return True 176 | 177 | def __repr__(self): 178 | return ' {}'.format(self.name) 179 | -------------------------------------------------------------------------------- /geomet_data_registry/layer/model_gem_regional.py: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # 3 | # Copyright (C) 2019 Etienne Pelletier 4 | # 5 | # This program is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with this program. If not, see . 17 | # 18 | ############################################################################### 19 | 20 | from datetime import datetime, timedelta 21 | import json 22 | import logging 23 | import os 24 | from parse import parse 25 | import re 26 | 27 | from geomet_data_registry.layer.base import BaseLayer 28 | from geomet_data_registry.util import DATE_FORMAT 29 | 30 | LOGGER = logging.getLogger(__name__) 31 | 32 | 33 | class ModelGemRegionalLayer(BaseLayer): 34 | """RDPS layer""" 35 | 36 | def __init__(self, provider_def): 37 | """ 38 | Initialize object 39 | 40 | :param provider_def: provider definition dict 41 | 42 | :returns: `geomet_data_registry.layer.model_gem_global.ModelGemRegionalLayer` # noqa 43 | """ 44 | 45 | provider_def = {'name': 'model_gem_regional'} 46 | 47 | super().__init__(provider_def) 48 | 49 | def identify(self, filepath, url=None): 50 | """ 51 | Identifies a file of the layer 52 | 53 | :param filepath: filepath from AMQP 54 | :param url: fully qualified URL of file 55 | 56 | :returns: `list` of file properties 57 | """ 58 | 59 | super().identify(filepath, url) 60 | 61 | self.model = 'model_gem_regional' 62 | 63 | LOGGER.debug('Loading model information from store') 64 | self.file_dict = json.loads(self.store.get_key(self.model)) 65 | 66 | filename_pattern = self.file_dict[self.model]['filename_pattern'] 67 | 68 | tmp = parse(filename_pattern, os.path.basename(filepath)) 69 | 70 | file_pattern_info = { 71 | 'wx_variable': tmp.named['wx_variable'], 72 | 'time_': tmp.named['YYYYMMDD_model_run'], 73 | 'fh': tmp.named['forecast_hour'] 74 | } 75 | 76 | LOGGER.debug('Defining the different file properties') 77 | self.wx_variable = file_pattern_info['wx_variable'] 78 | 79 | if self.wx_variable not in self.file_dict[self.model]['variable']: 80 | msg = 'Variable "{}" not in ' \ 81 | 'configuration file'.format(self.wx_variable) 82 | LOGGER.warning(msg) 83 | return False 84 | 85 | self.dimensions = self.file_dict[self.model]['dimensions'] 86 | 87 | runs = self.file_dict[self.model]['variable'][self.wx_variable][ 88 | 'model_run'] 89 | self.model_run_list = list(runs.keys()) 90 | 91 | time_format = '%Y%m%d%H' 92 | self.date_ = datetime.strptime(file_pattern_info['time_'], time_format) 93 | 94 | reference_datetime = self.date_ 95 | self.model_run = '{}Z'.format(self.date_.strftime('%H')) 96 | 97 | forecast_hour_datetime = self.date_ + \ 98 | timedelta(hours=int(file_pattern_info['fh'])) 99 | 100 | member = self.file_dict[self.model]['variable'][self.wx_variable][ 101 | 'members'] 102 | elevation = self.file_dict[self.model]['variable'][self.wx_variable][ 103 | 'elevation'] 104 | str_mr = re.sub('[^0-9]', 105 | '', 106 | reference_datetime.strftime(DATE_FORMAT)) 107 | str_fh = re.sub('[^0-9]', 108 | '', 109 | forecast_hour_datetime.strftime(DATE_FORMAT)) 110 | expected_count = self.file_dict[self.model]['variable'][ 111 | self.wx_variable]['model_run'][self.model_run]['files_expected'] 112 | 113 | self.geomet_layers = self.file_dict[self.model]['variable'][ 114 | self.wx_variable]['geomet_layers'] 115 | for layer_name, layer_config in self.geomet_layers.items(): 116 | identifier = '{}-{}-{}'.format(layer_name, str_mr, str_fh) 117 | 118 | forecast_hours = layer_config['forecast_hours'] 119 | begin, end, interval = [int(re.sub('[^0-9]', '', value)) 120 | for value in forecast_hours.split('/')] 121 | fh = int(file_pattern_info['fh']) 122 | 123 | feature_dict = { 124 | 'layer_name': layer_name, 125 | 'filepath': self.filepath, 126 | 'identifier': identifier, 127 | 'reference_datetime': reference_datetime.strftime( 128 | DATE_FORMAT), 129 | 'forecast_hour_datetime': forecast_hour_datetime.strftime( 130 | DATE_FORMAT), 131 | 'member': member, 132 | 'model': self.model, 133 | 'elevation': elevation, 134 | 'expected_count': expected_count, 135 | 'forecast_hours': { 136 | 'begin': begin, 137 | 'end': end, 138 | 'interval': forecast_hours.split('/')[2] 139 | }, 140 | 'layer_config': layer_config, 141 | 'register_status': True, 142 | 'refresh_config': True, 143 | } 144 | 145 | if 'dependencies' in layer_config: 146 | dependencies_found = self.check_layer_dependencies( 147 | layer_config['dependencies'], 148 | str_mr, 149 | str_fh) 150 | if dependencies_found: 151 | bands_order = (self.file_dict[self.model] 152 | ['variable'] 153 | [self.wx_variable].get('bands_order')) 154 | (feature_dict['filepath'], 155 | feature_dict['url'], 156 | feature_dict['weather_variable']) = ( 157 | self.configure_layer_with_dependencies( 158 | dependencies_found, 159 | self.dimensions, 160 | bands_order)) 161 | else: 162 | feature_dict['register_status'] = False 163 | self.items.append(feature_dict) 164 | continue 165 | 166 | if not self.is_valid_interval(fh, begin, end, interval): 167 | feature_dict['register_status'] = False 168 | LOGGER.debug('Forecast hour {} not included in {} as ' 169 | 'defined for layer {}. File will not be ' 170 | 'added to registry for this layer' 171 | .format(fh, forecast_hours, layer_name)) 172 | 173 | self.items.append(feature_dict) 174 | 175 | return True 176 | 177 | def __repr__(self): 178 | return ' {}'.format(self.name) 179 | -------------------------------------------------------------------------------- /geomet_data_registry/layer/model_hrdps_continental.py: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # 3 | # Copyright (C) 2019 Etienne Pelletier 4 | # 5 | # This program is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with this program. If not, see . 17 | # 18 | ############################################################################### 19 | 20 | from datetime import datetime, timedelta 21 | import json 22 | import logging 23 | import os 24 | from parse import parse 25 | import re 26 | 27 | from geomet_data_registry.layer.base import BaseLayer 28 | from geomet_data_registry.util import DATE_FORMAT 29 | 30 | LOGGER = logging.getLogger(__name__) 31 | 32 | 33 | class ModelHrdpsContinentalLayer(BaseLayer): 34 | """HRDPS Continental layer""" 35 | 36 | def __init__(self, provider_def): 37 | """ 38 | Initialize object 39 | 40 | :param provider_def: provider definition dict 41 | 42 | :returns: `geomet_data_registry.layer.model_gem_global.ModelHrdpsContinentalLayer` # noqa 43 | """ 44 | 45 | provider_def = {'name': 'model_hrdps_continental'} 46 | 47 | super().__init__(provider_def) 48 | 49 | def identify(self, filepath, url=None): 50 | """ 51 | Identifies a file of the layer 52 | 53 | :param filepath: filepath from AMQP 54 | :param url: fully qualified URL of file 55 | 56 | :returns: `list` of file properties 57 | """ 58 | 59 | super().identify(filepath, url) 60 | 61 | self.model = 'model_hrdps_continental' 62 | 63 | LOGGER.debug('Loading model information from store') 64 | self.file_dict = json.loads(self.store.get_key(self.model)) 65 | 66 | filename_pattern = self.file_dict[self.model]['filename_pattern'] 67 | 68 | tmp = parse(filename_pattern, os.path.basename(filepath)) 69 | file_pattern_info = { 70 | 'wx_variable': tmp.named['wx_variable'], 71 | 'time_': tmp.named['YYYYMMDD_model_run'], 72 | 'fh': tmp.named['forecast_hour'] 73 | } 74 | 75 | LOGGER.debug('Defining the different file properties') 76 | self.wx_variable = file_pattern_info['wx_variable'] 77 | 78 | if self.wx_variable not in self.file_dict[self.model]['variable']: 79 | msg = 'Variable "{}" not in ' \ 80 | 'configuration file'.format(self.wx_variable) 81 | LOGGER.warning(msg) 82 | return False 83 | 84 | self.dimensions = self.file_dict[self.model]['dimensions'] 85 | 86 | runs = self.file_dict[self.model]['variable'][self.wx_variable][ 87 | 'model_run'] 88 | self.model_run_list = list(runs.keys()) 89 | 90 | time_format = '%Y%m%d%H' 91 | self.date_ = datetime.strptime(file_pattern_info['time_'], 92 | time_format) 93 | 94 | reference_datetime = self.date_ 95 | self.model_run = '{}Z'.format(self.date_.strftime('%H')) 96 | 97 | forecast_hour_datetime = self.date_ + \ 98 | timedelta(hours=int(file_pattern_info['fh'])) 99 | 100 | member = self.file_dict[self.model]['variable'][self.wx_variable][ 101 | 'members'] 102 | elevation = self.file_dict[self.model]['variable'][self.wx_variable][ 103 | 'elevation'] 104 | str_mr = re.sub('[^0-9]', 105 | '', 106 | reference_datetime.strftime(DATE_FORMAT)) 107 | str_fh = re.sub('[^0-9]', 108 | '', 109 | forecast_hour_datetime.strftime(DATE_FORMAT)) 110 | expected_count = self.file_dict[self.model]['variable'][ 111 | self.wx_variable]['model_run'][self.model_run]['files_expected'] 112 | 113 | self.geomet_layers = self.file_dict[self.model]['variable'][ 114 | self.wx_variable]['geomet_layers'] 115 | for layer_name, layer_config in self.geomet_layers.items(): 116 | identifier = '{}-{}-{}'.format(layer_name, str_mr, str_fh) 117 | 118 | forecast_hours = layer_config['forecast_hours'] 119 | begin, end, interval = [int(re.sub('[^0-9]', '', value)) 120 | for value in forecast_hours.split('/')] 121 | fh = int(file_pattern_info['fh']) 122 | 123 | feature_dict = { 124 | 'layer_name': layer_name, 125 | 'filepath': self.filepath, 126 | 'identifier': identifier, 127 | 'reference_datetime': reference_datetime.strftime( 128 | DATE_FORMAT), 129 | 'forecast_hour_datetime': forecast_hour_datetime.strftime( 130 | DATE_FORMAT), 131 | 'member': member, 132 | 'model': self.model, 133 | 'elevation': elevation, 134 | 'expected_count': expected_count, 135 | 'forecast_hours': { 136 | 'begin': begin, 137 | 'end': end, 138 | 'interval': forecast_hours.split('/')[2] 139 | }, 140 | 'layer_config': layer_config, 141 | 'register_status': True, 142 | 'refresh_config': True, 143 | } 144 | 145 | if 'dependencies' in layer_config: 146 | dependencies_found = self.check_layer_dependencies( 147 | layer_config['dependencies'], 148 | str_mr, 149 | str_fh) 150 | if dependencies_found: 151 | bands_order = (self.file_dict[self.model] 152 | ['variable'] 153 | [self.wx_variable].get('bands_order')) 154 | (feature_dict['filepath'], 155 | feature_dict['url'], 156 | feature_dict['weather_variable']) = ( 157 | self.configure_layer_with_dependencies( 158 | dependencies_found, 159 | self.dimensions, 160 | bands_order)) 161 | else: 162 | feature_dict['register_status'] = False 163 | self.items.append(feature_dict) 164 | continue 165 | 166 | if not self.is_valid_interval(fh, begin, end, interval): 167 | feature_dict['register_status'] = False 168 | LOGGER.debug('Forecast hour {} not included in {} as ' 169 | 'defined for layer {}. File will not be ' 170 | 'added to registry for this layer' 171 | .format(fh, forecast_hours, layer_name)) 172 | 173 | self.items.append(feature_dict) 174 | 175 | return True 176 | 177 | def __repr__(self): 178 | return ' {}'.format(self.name) 179 | -------------------------------------------------------------------------------- /geomet_data_registry/layer/model_raqdps.py: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # 3 | # Copyright (C) 2020 Etienne Pelletier 4 | # 5 | # This program is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with this program. If not, see . 17 | # 18 | ############################################################################### 19 | 20 | from datetime import datetime, timedelta 21 | import json 22 | import logging 23 | import os 24 | from parse import parse 25 | import re 26 | 27 | from geomet_data_registry.layer.base import BaseLayer 28 | from geomet_data_registry.util import DATE_FORMAT 29 | 30 | LOGGER = logging.getLogger(__name__) 31 | 32 | 33 | class ModelRaqdpsLayer(BaseLayer): 34 | """RAQDPS layer""" 35 | 36 | def __init__(self, provider_def): 37 | """ 38 | Initialize object 39 | 40 | :param provider_def: provider definition dict 41 | 42 | :returns: `geomet_data_registry.layer.model_raqdps.ModelRaqdpsLayer` # noqa 43 | """ 44 | 45 | provider_def = {'name': 'model_raqdps'} 46 | 47 | super().__init__(provider_def) 48 | 49 | def identify(self, filepath, url=None): 50 | """ 51 | Identifies a file of the layer 52 | 53 | :param filepath: filepath from AMQP 54 | :param url: fully qualified URL of file 55 | 56 | :returns: `list` of file properties 57 | """ 58 | 59 | super().identify(filepath, url) 60 | 61 | self.model = 'model_raqdps' 62 | 63 | LOGGER.debug('Loading model information from store') 64 | self.file_dict = json.loads(self.store.get_key(self.model)) 65 | 66 | filename_pattern = self.file_dict[self.model]['filename_pattern'] 67 | 68 | tmp = parse(filename_pattern, os.path.basename(filepath)) 69 | 70 | file_pattern_info = { 71 | 'wx_variable': tmp.named['wx_variable'], 72 | 'date': tmp.named['YYYYMMDD'], 73 | 'model_run': tmp.named['model_run'], 74 | 'fh': tmp.named['forecast_hour'], 75 | } 76 | 77 | LOGGER.debug('Defining the different file properties') 78 | self.wx_variable = file_pattern_info['wx_variable'] 79 | 80 | if self.wx_variable not in self.file_dict[self.model]['variable']: 81 | msg = 'Variable "{}" not in ' 'configuration file'.format( 82 | self.wx_variable 83 | ) 84 | LOGGER.warning(msg) 85 | return False 86 | 87 | self.dimensions = self.file_dict[self.model]['dimensions'] 88 | 89 | runs = self.file_dict[self.model]['variable'][self.wx_variable][ 90 | 'model_run' 91 | ] 92 | self.model_run_list = list(runs.keys()) 93 | 94 | time_format = '%Y%m%dT%HZ' 95 | self.date_ = datetime.strptime( 96 | '{}T{}Z'.format( 97 | file_pattern_info['date'], file_pattern_info['model_run'] 98 | ), 99 | time_format 100 | ) 101 | 102 | reference_datetime = self.date_ 103 | self.model_run = '{}Z'.format(file_pattern_info['model_run']) 104 | 105 | forecast_hour_datetime = self.date_ + timedelta( 106 | hours=int(file_pattern_info['fh']) 107 | ) 108 | 109 | member = self.file_dict[self.model]['variable'][self.wx_variable][ 110 | 'members' 111 | ] 112 | elevation = self.file_dict[self.model]['variable'][self.wx_variable][ 113 | 'elevation' 114 | ] 115 | str_mr = re.sub('[^0-9]', '', reference_datetime.strftime(DATE_FORMAT)) 116 | str_fh = re.sub( 117 | '[^0-9]', '', forecast_hour_datetime.strftime(DATE_FORMAT) 118 | ) 119 | expected_count = self.file_dict[self.model]['variable'][ 120 | self.wx_variable 121 | ]['model_run'][self.model_run]['files_expected'] 122 | 123 | self.geomet_layers = self.file_dict[self.model]['variable'][ 124 | self.wx_variable 125 | ]['geomet_layers'] 126 | for layer_name, layer_config in self.geomet_layers.items(): 127 | identifier = '{}-{}-{}'.format(layer_name, str_mr, str_fh) 128 | 129 | forecast_hours = layer_config['forecast_hours'] 130 | begin, end, interval = [ 131 | int(re.sub('[^0-9]', '', value)) 132 | for value in forecast_hours.split('/') 133 | ] 134 | fh = int(file_pattern_info['fh']) 135 | 136 | feature_dict = { 137 | 'layer_name': layer_name, 138 | 'filepath': self.filepath, 139 | 'identifier': identifier, 140 | 'reference_datetime': reference_datetime.strftime(DATE_FORMAT), 141 | 'forecast_hour_datetime': forecast_hour_datetime.strftime( 142 | DATE_FORMAT 143 | ), 144 | 'member': member, 145 | 'model': self.model, 146 | 'elevation': elevation, 147 | 'expected_count': expected_count, 148 | 'forecast_hours': { 149 | 'begin': begin, 150 | 'end': end, 151 | 'interval': forecast_hours.split('/')[2], 152 | }, 153 | 'layer_config': layer_config, 154 | 'register_status': True, 155 | 'refresh_config': True, 156 | } 157 | 158 | if not self.is_valid_interval(fh, begin, end, interval): 159 | feature_dict['register_status'] = False 160 | LOGGER.debug( 161 | 'Forecast hour {} not included in {} as ' 162 | 'defined for layer {}. File will not be ' 163 | 'added to registry for this layer'.format( 164 | fh, forecast_hours, layer_name 165 | ) 166 | ) 167 | 168 | self.items.append(feature_dict) 169 | 170 | return True 171 | 172 | def __repr__(self): 173 | return ' {}'.format(self.name) 174 | -------------------------------------------------------------------------------- /geomet_data_registry/layer/model_raqdps_fw.py: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # 3 | # Copyright (C) 2020 Etienne Pelletier 4 | # Copyright (C) 2021 Tom Kralidis 5 | # 6 | # This program is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program. If not, see . 18 | # 19 | ############################################################################### 20 | 21 | from datetime import datetime, timedelta 22 | import json 23 | import logging 24 | import os 25 | from parse import parse 26 | import re 27 | 28 | from geomet_data_registry.layer.base import BaseLayer 29 | from geomet_data_registry.util import DATE_FORMAT 30 | 31 | LOGGER = logging.getLogger(__name__) 32 | 33 | 34 | class ModelRaqdpsFwLayer(BaseLayer): 35 | """RAQDPS-FW layer""" 36 | 37 | def __init__(self, provider_def): 38 | """ 39 | Initialize object 40 | 41 | :param provider_def: provider definition dict 42 | 43 | :returns: `geomet_data_registry.layer.model_raqdps_fw.ModelRaqdpsFwLayer` # noqa 44 | """ 45 | 46 | provider_def = {'name': 'model_raqdps-fw'} 47 | 48 | super().__init__(provider_def) 49 | 50 | def identify(self, filepath, url=None): 51 | """ 52 | Identifies a file of the layer 53 | 54 | :param filepath: filepath from AMQP 55 | :param url: fully qualified URL of file 56 | 57 | :returns: `list` of file properties 58 | """ 59 | 60 | super().identify(filepath, url) 61 | 62 | self.model = 'model_raqdps-fw' 63 | 64 | LOGGER.debug('Loading model information from store') 65 | self.file_dict = json.loads(self.store.get_key(self.model)) 66 | 67 | filename_pattern = self.file_dict[self.model]['filename_pattern'] 68 | 69 | tmp = parse(filename_pattern, os.path.basename(filepath)) 70 | 71 | file_pattern_info = { 72 | 'wx_variable': tmp.named['wx_variable'], 73 | 'date': tmp.named['YYYYMMDD'], 74 | 'model_run': tmp.named['model_run'], 75 | 'fh': tmp.named['forecast_hour'], 76 | } 77 | 78 | LOGGER.debug('Defining the different file properties') 79 | self.wx_variable = file_pattern_info['wx_variable'] 80 | 81 | if self.wx_variable not in self.file_dict[self.model]['variable']: 82 | msg = 'Variable "{}" not in ' 'configuration file'.format( 83 | self.wx_variable 84 | ) 85 | LOGGER.warning(msg) 86 | return False 87 | 88 | runs = self.file_dict[self.model]['variable'][self.wx_variable][ 89 | 'model_run' 90 | ] 91 | self.model_run_list = list(runs.keys()) 92 | 93 | time_format = '%Y%m%dT%HZ' 94 | self.date_ = datetime.strptime( 95 | '{}T{}Z'.format( 96 | file_pattern_info['date'], file_pattern_info['model_run'] 97 | ), 98 | time_format 99 | ) 100 | 101 | reference_datetime = self.date_ 102 | self.model_run = '{}Z'.format(file_pattern_info['model_run']) 103 | 104 | forecast_hour_datetime = self.date_ + timedelta( 105 | hours=int(file_pattern_info['fh']) 106 | ) 107 | 108 | member = self.file_dict[self.model]['variable'][self.wx_variable][ 109 | 'members' 110 | ] 111 | elevation = self.file_dict[self.model]['variable'][self.wx_variable][ 112 | 'elevation' 113 | ] 114 | str_mr = re.sub('[^0-9]', '', reference_datetime.strftime(DATE_FORMAT)) 115 | str_fh = re.sub( 116 | '[^0-9]', '', forecast_hour_datetime.strftime(DATE_FORMAT) 117 | ) 118 | expected_count = self.file_dict[self.model]['variable'][ 119 | self.wx_variable 120 | ]['model_run'][self.model_run]['files_expected'] 121 | 122 | self.geomet_layers = self.file_dict[self.model]['variable'][ 123 | self.wx_variable 124 | ]['geomet_layers'] 125 | for layer_name, layer_config in self.geomet_layers.items(): 126 | identifier = '{}-{}-{}'.format(layer_name, str_mr, str_fh) 127 | 128 | forecast_hours = layer_config['forecast_hours'] 129 | begin, end, interval = [ 130 | int(re.sub('[^0-9]', '', value)) 131 | for value in forecast_hours.split('/') 132 | ] 133 | fh = int(file_pattern_info['fh']) 134 | 135 | feature_dict = { 136 | 'layer_name': layer_name, 137 | 'filepath': self.filepath, 138 | 'identifier': identifier, 139 | 'reference_datetime': reference_datetime.strftime(DATE_FORMAT), 140 | 'forecast_hour_datetime': forecast_hour_datetime.strftime( 141 | DATE_FORMAT 142 | ), 143 | 'member': member, 144 | 'model': self.model, 145 | 'elevation': elevation, 146 | 'expected_count': expected_count, 147 | 'forecast_hours': { 148 | 'begin': begin, 149 | 'end': end, 150 | 'interval': forecast_hours.split('/')[2], 151 | }, 152 | 'layer_config': layer_config, 153 | 'register_status': True, 154 | 'refresh_config': True, 155 | } 156 | 157 | if not self.is_valid_interval(fh, begin, end, interval): 158 | feature_dict['register_status'] = False 159 | LOGGER.debug( 160 | 'Forecast hour {} not included in {} as ' 161 | 'defined for layer {}. File will not be ' 162 | 'added to registry for this layer'.format( 163 | fh, forecast_hours, layer_name 164 | ) 165 | ) 166 | 167 | self.items.append(feature_dict) 168 | 169 | return True 170 | 171 | def __repr__(self): 172 | return ' {}'.format(self.name) 173 | -------------------------------------------------------------------------------- /geomet_data_registry/layer/radar_1km.py: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # 3 | # Copyright (C) 2019 Louis-Philippe Rousseau-Lambert 4 | # 5 | # This program is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with this program. If not, see . 17 | # 18 | ############################################################################### 19 | 20 | from datetime import datetime, timedelta 21 | import json 22 | import logging 23 | import os 24 | from parse import parse 25 | import re 26 | 27 | from geomet_data_registry.layer.base import BaseLayer 28 | from geomet_data_registry.util import DATE_FORMAT 29 | 30 | LOGGER = logging.getLogger(__name__) 31 | 32 | 33 | class Radar1kmLayer(BaseLayer): 34 | """radar layer""" 35 | 36 | def __init__(self, provider_def): 37 | """ 38 | Initialize object 39 | 40 | :param provider_def: provider definition dict 41 | 42 | :returns: `geomet_data_registry.layer.radar_1km.Radar1kmLayer` 43 | """ 44 | 45 | provider_def = {'name': 'Radar_1km'} 46 | 47 | super().__init__(provider_def) 48 | 49 | def identify(self, filepath, url=None): 50 | """ 51 | Identifies a file of the layer 52 | 53 | :param filepath: filepath from AMQP 54 | :param url: fully qualified URL of file 55 | 56 | :returns: `list` of file properties 57 | """ 58 | 59 | super().identify(filepath, url) 60 | 61 | self.model = 'radar' 62 | 63 | LOGGER.debug('Loading model information from store') 64 | self.file_dict = json.loads(self.store.get_key(self.model)) 65 | 66 | filename_pattern = self.file_dict[self.model]['filename_pattern'] 67 | 68 | tmp = parse(filename_pattern, os.path.basename(filepath)) 69 | 70 | file_pattern_info = { 71 | 'wx_variable': tmp.named['precipitation_type'], 72 | 'datetime': tmp.named['YYYYMMDDThhmm'], 73 | } 74 | 75 | LOGGER.debug('Defining the different file properties') 76 | self.wx_variable = file_pattern_info['wx_variable'] 77 | 78 | if self.wx_variable not in self.file_dict[self.model]['variable']: 79 | msg = 'Variable "{}" not in ' 'configuration file'.format( 80 | self.wx_variable 81 | ) 82 | LOGGER.warning(msg) 83 | return False 84 | 85 | time_format = '%Y%m%dT%H%M' 86 | self.date_ = datetime.strptime( 87 | file_pattern_info['datetime'], time_format 88 | ) 89 | 90 | layer_config = self.file_dict[self.model]['variable'][self.wx_variable] 91 | layer_name = layer_config['geomet_layers'] 92 | 93 | member = self.file_dict[self.model]['variable'][self.wx_variable][ 94 | 'members' 95 | ] 96 | elevation = self.file_dict[self.model]['variable'][self.wx_variable][ 97 | 'elevation' 98 | ] 99 | str_fh = re.sub('[^0-9]', '', self.date_.strftime(DATE_FORMAT)) 100 | identifier = '{}-{}'.format(layer_name, str_fh) 101 | date_format = DATE_FORMAT 102 | 103 | feature_dict = { 104 | 'layer_name': layer_name, 105 | 'layer_config': layer_config, 106 | 'filepath': self.filepath, 107 | 'identifier': identifier, 108 | 'reference_datetime': None, 109 | 'forecast_hour_datetime': self.date_.strftime(date_format), 110 | 'member': member, 111 | 'model': self.model, 112 | 'elevation': elevation, 113 | 'expected_count': None, 114 | 'layer_config': layer_config, 115 | 'register_status': True, 116 | 'refresh_config': True, 117 | } 118 | self.items.append(feature_dict) 119 | 120 | return True 121 | 122 | def add_time_key(self): 123 | """ 124 | Adds default time and time extent datetime values to store for radar 125 | layers. Overrides the add_time_key method of BaseLayer class due to 126 | radar data's lack of forecast models. 127 | :return: `bool` if successfully added a new radar time key 128 | """ 129 | 130 | layer_name = self.file_dict[self.model]['variable'][self.wx_variable][ 131 | 'geomet_layers' 132 | ] 133 | key_name = '{}_default_time'.format(layer_name) 134 | last_key = self.store.get_key(key_name) 135 | key_value = self.date_.strftime(DATE_FORMAT) 136 | extent_key = '{}_time_extent'.format(layer_name) 137 | start, end, interval = self.file_dict[self.model]['variable'][ 138 | self.wx_variable 139 | ]['forecast_hours'].split('/') 140 | start_time = self.date_ + timedelta(minutes=int(start)) 141 | start_time = start_time.strftime(DATE_FORMAT) 142 | extent_value = '{}/{}/{}'.format(start_time, key_value, interval) 143 | if last_key is None: 144 | LOGGER.warning('No previous time information in the store') 145 | else: 146 | LOGGER.debug('Adding time keys in the store') 147 | old_time = datetime.strptime(last_key, DATE_FORMAT) 148 | if old_time + timedelta(minutes=10) != self.date_: 149 | LOGGER.error( 150 | 'Missing radar between {}/{}'.format(old_time, self.date_) 151 | ) 152 | self.store.set_key(key_name, key_value) 153 | self.store.set_key(extent_key, extent_value) 154 | 155 | return True 156 | 157 | def __repr__(self): 158 | return ' {}'.format(self.name) 159 | -------------------------------------------------------------------------------- /geomet_data_registry/layer/wcps.py: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # 3 | # Copyright (C) 2020 Etienne Pelletier 4 | # 5 | # This program is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with this program. If not, see . 17 | # 18 | ############################################################################### 19 | 20 | from datetime import datetime, timedelta 21 | import json 22 | import logging 23 | import os 24 | import re 25 | 26 | from parse import parse 27 | from geomet_data_registry.layer.base import BaseLayer 28 | from geomet_data_registry.util import DATE_FORMAT 29 | 30 | LOGGER = logging.getLogger(__name__) 31 | 32 | 33 | class WcpsLayer(BaseLayer): 34 | """WCPS layer""" 35 | 36 | def __init__(self, provider_def): 37 | """ 38 | Initialize object 39 | 40 | :param provider_def: provider definition dict 41 | 42 | :returns: `geomet_data_registry.layer.wcps.WcpsLayer` 43 | """ 44 | 45 | provider_def = {'name': 'wcps'} 46 | 47 | super().__init__(provider_def) 48 | 49 | def identify(self, filepath, url=None): 50 | """ 51 | Identifies a file of the layer 52 | 53 | :param filepath: filepath from AMQP 54 | :param url: fully qualified URL of file 55 | 56 | :returns: `bool` of identification success status 57 | """ 58 | 59 | super().identify(filepath, url) 60 | 61 | self.model = 'wcps' 62 | 63 | LOGGER.debug('Loading model information from store') 64 | self.file_dict = json.loads(self.store.get_key(self.model)) 65 | 66 | filename_pattern = self.file_dict[self.model]['filename_pattern'] 67 | 68 | tmp = parse(filename_pattern, os.path.basename(filepath)) 69 | 70 | file_pattern_info = { 71 | 'wx_variable': tmp.named['wx_variable'], 72 | 'time_': tmp.named['YYYYMMDD_model_run'], 73 | 'fh': tmp.named['forecast_hour'] 74 | } 75 | 76 | LOGGER.debug('Defining the different file properties') 77 | self.wx_variable = file_pattern_info['wx_variable'] 78 | 79 | if self.wx_variable not in self.file_dict[self.model]['variable']: 80 | msg = 'Variable "{}" not in ' \ 81 | 'configuration file'.format(self.wx_variable) 82 | LOGGER.warning(msg) 83 | return False 84 | 85 | self.dimensions = self.file_dict[self.model]['dimensions'] 86 | 87 | runs = self.file_dict[self.model]['variable'][self.wx_variable][ 88 | 'model_run'] 89 | self.model_run_list = list(runs.keys()) 90 | 91 | time_format = '%Y%m%d%H' 92 | self.date_ = datetime.strptime(file_pattern_info['time_'], time_format) 93 | 94 | reference_datetime = self.date_ 95 | self.model_run = '{}Z'.format(self.date_.strftime('%H')) 96 | 97 | forecast_hour_datetime = self.date_ + \ 98 | timedelta(hours=int(file_pattern_info['fh'])) 99 | 100 | member = self.file_dict[self.model]['variable'][self.wx_variable][ 101 | 'members'] 102 | elevation = self.file_dict[self.model]['variable'][self.wx_variable][ 103 | 'elevation'] 104 | str_mr = re.sub('[^0-9]', 105 | '', 106 | reference_datetime.strftime(DATE_FORMAT)) 107 | str_fh = re.sub('[^0-9]', 108 | '', 109 | forecast_hour_datetime.strftime(DATE_FORMAT)) 110 | expected_count = self.file_dict[self.model]['variable'][ 111 | self.wx_variable]['model_run'][ 112 | self.model_run]['files_expected'] 113 | 114 | self.geomet_layers = self.file_dict[self.model]['variable'][ 115 | self.wx_variable]['geomet_layers'] 116 | for layer_name, layer_config in self.geomet_layers.items(): 117 | identifier = '{}-{}-{}'.format(layer_name, str_mr, str_fh) 118 | 119 | forecast_hours = layer_config['forecast_hours'] 120 | begin, end, interval = [int(re.sub('[^0-9]', '', value)) 121 | for value in forecast_hours.split('/')] 122 | fh = int(file_pattern_info['fh']) 123 | 124 | feature_dict = { 125 | 'layer_name': layer_name, 126 | 'filepath': self.filepath, 127 | 'identifier': identifier, 128 | 'reference_datetime': reference_datetime.strftime( 129 | DATE_FORMAT), 130 | 'forecast_hour_datetime': forecast_hour_datetime.strftime( 131 | DATE_FORMAT), 132 | 'member': member, 133 | 'model': self.model, 134 | 'elevation': elevation, 135 | 'expected_count': expected_count, 136 | 'forecast_hours': { 137 | 'begin': begin, 138 | 'end': end, 139 | 'interval': forecast_hours.split('/')[2] 140 | }, 141 | 'layer_config': layer_config, 142 | 'register_status': True, 143 | 'refresh_config': True, 144 | } 145 | 146 | if 'dependencies' in layer_config: 147 | dependencies_found = self.check_layer_dependencies( 148 | layer_config['dependencies'], 149 | str_mr, 150 | str_fh) 151 | if dependencies_found: 152 | bands_order = (self.file_dict[self.model] 153 | ['variable'] 154 | [self.wx_variable].get('bands_order')) 155 | (feature_dict['filepath'], 156 | feature_dict['url'], 157 | feature_dict['weather_variable']) = ( 158 | self.configure_layer_with_dependencies( 159 | dependencies_found, 160 | self.dimensions, 161 | bands_order)) 162 | else: 163 | feature_dict['register_status'] = False 164 | self.items.append(feature_dict) 165 | continue 166 | 167 | if not self.is_valid_interval(fh, begin, end, interval): 168 | feature_dict['register_status'] = False 169 | LOGGER.debug('Forecast hour {} not included in {} as ' 170 | 'defined for layer {}. File will not be ' 171 | 'added to registry for this layer' 172 | .format(fh, forecast_hours, layer_name)) 173 | 174 | self.items.append(feature_dict) 175 | 176 | return True 177 | 178 | def __repr__(self): 179 | return ' {}'.format(self.name) 180 | -------------------------------------------------------------------------------- /geomet_data_registry/log.py: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # 3 | # Copyright (C) 2019 Tom Kralidis 4 | # 5 | # This program is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with this program. If not, see . 17 | # 18 | ############################################################################### 19 | 20 | import logging 21 | import sys 22 | 23 | LOGGER = logging.getLogger(__name__) 24 | 25 | 26 | def setup_logger(loglevel, logfile=None): 27 | """ 28 | Setup configuration 29 | 30 | :param loglevel: logging level 31 | :param logfile: logfile location 32 | 33 | :returns: void (creates logging instance) 34 | """ 35 | 36 | log_format = \ 37 | '[%(asctime)s] %(levelname)s - %(message)s' 38 | date_format = '%Y-%m-%dT%H:%M:%SZ' 39 | 40 | loglevels = { 41 | 'CRITICAL': logging.CRITICAL, 42 | 'ERROR': logging.ERROR, 43 | 'WARNING': logging.WARNING, 44 | 'INFO': logging.INFO, 45 | 'DEBUG': logging.DEBUG, 46 | 'NOTSET': logging.NOTSET, 47 | } 48 | 49 | loglevel = loglevels[loglevel] 50 | 51 | if logfile is not None: 52 | if logfile == 'stdout': 53 | logging.basicConfig(level=loglevel, datefmt=date_format, 54 | format=log_format, stream=sys.stdout) 55 | else: 56 | logging.basicConfig(level=loglevel, datefmt=date_format, 57 | format=log_format, filename=logfile) 58 | 59 | LOGGER.debug('Logging initialized') 60 | -------------------------------------------------------------------------------- /geomet_data_registry/notifier/__init__.py: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # 3 | # Copyright (C) 2020 Etienne Pelletier 4 | # 5 | # This program is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with this program. If not, see . 17 | # 18 | ############################################################################### 19 | -------------------------------------------------------------------------------- /geomet_data_registry/notifier/base.py: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # 3 | # Copyright (C) 2020 Etienne Pelletier 4 | # 5 | # This program is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with this program. If not, see . 17 | # 18 | ############################################################################### 19 | 20 | import logging 21 | 22 | LOGGER = logging.getLogger(__name__) 23 | 24 | 25 | class BaseNotifier: 26 | """generic notifier ABC""" 27 | 28 | def __init__(self, provider_def): 29 | """ 30 | Initialize object 31 | 32 | :param provider_def: provider definition dict 33 | 34 | :returns: `geomet_data_registry.notifier.base.BaseNotifier` 35 | """ 36 | 37 | self.type = provider_def['type'] 38 | self.url = provider_def['url'] 39 | 40 | def notify(self, items=[]): 41 | """ 42 | Sends a notification 43 | 44 | :param items: `list` of items for notification 45 | 46 | :returns: `bool` of notification status 47 | """ 48 | 49 | raise NotImplementedError() 50 | 51 | def __repr__(self): 52 | return ' {}'.format(self.type) 53 | 54 | 55 | class NotifierError(Exception): 56 | """setup error""" 57 | pass 58 | 59 | 60 | class NotifierConnectionError(Exception): 61 | """setup error""" 62 | pass 63 | -------------------------------------------------------------------------------- /geomet_data_registry/notifier/celery_.py: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # 3 | # Copyright (C) 2021 Etienne Pelletier 4 | # 5 | # This program is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with this program. If not, see . 17 | # 18 | ############################################################################### 19 | 20 | import logging 21 | 22 | from celery import Celery 23 | import kombu 24 | from redis.exceptions import ConnectionError 25 | 26 | from geomet_data_registry.notifier.base import ( 27 | BaseNotifier, 28 | NotifierConnectionError, 29 | ) 30 | 31 | LOGGER = logging.getLogger(__name__) 32 | 33 | 34 | class CeleryTaskNotifier(BaseNotifier): 35 | """Celery notifier""" 36 | 37 | def __init__(self, provider_def): 38 | """ 39 | Initialize object 40 | 41 | :param provider_def: provider definition dict 42 | 43 | :returns: `geomet_data_registry.notifier.celery.CeleryTaskNotifier` 44 | """ 45 | 46 | super().__init__(provider_def) 47 | 48 | # check for valid broker connection and establish Celery app instance 49 | if self.check_broker_connection(): 50 | self.app = Celery( 51 | 'geomet-mapfile', backend=self.url, broker=self.url 52 | ) 53 | 54 | def check_broker_connection(self, timeout=5): 55 | """ 56 | Check the connection status to Celery broker. 57 | 58 | :param timeout: `float` timeout in seconds for connecting to broker. 59 | 60 | :returns: `bool` of connection status 61 | """ 62 | try: 63 | with kombu.Connection(self.url, connect_timeout=timeout) as conn: 64 | conn.connect() 65 | 66 | except (ConnectionError, ConnectionRefusedError, OSError,) as e: 67 | LOGGER.error(f'Could not connect to Celery broker ({self.url}).') 68 | raise NotifierConnectionError(e) 69 | 70 | return True 71 | 72 | def notify(self, items=[]): 73 | """ 74 | Sends a refresh_mapfile notifier task 75 | 76 | :param items: `list` of items for notification 77 | 78 | :returns: `bool` of notification status 79 | """ 80 | 81 | for item in items: 82 | published = item['layer_config'].get('published', True) 83 | if item['refresh_config'] and published: 84 | self.app.send_task( 85 | 'refresh_mapfile', args=[item['layer_name']] 86 | ) 87 | return True 88 | 89 | def __repr__(self): 90 | return ' {}'.format(self.url) 91 | -------------------------------------------------------------------------------- /geomet_data_registry/plugin.py: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # 3 | # Copyright (C) 2021 Tom Kralidis 4 | # 5 | # This program is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with this program. If not, see . 17 | # 18 | ############################################################################### 19 | 20 | import importlib 21 | import logging 22 | 23 | LOGGER = logging.getLogger(__name__) 24 | 25 | PLUGINS = { 26 | 'store': { 27 | 'Redis': { 28 | 'path': 'geomet_data_registry.store.redis_.RedisStore' 29 | } 30 | }, 31 | 'tileindex': { 32 | 'Elasticsearch': { 33 | 'path': 'geomet_data_registry.tileindex.elasticsearch_.ElasticsearchTileIndex' # noqa 34 | } 35 | }, 36 | 'notifier': { 37 | 'Celery': { 38 | 'path': 'geomet_data_registry.notifier.celery_.CeleryTaskNotifier' 39 | } 40 | }, 41 | 'layer': { 42 | 'ModelGemGlobal': { 43 | 'pattern': 'CMC_glb*', 44 | 'path': 'geomet_data_registry.layer.model_gem_global.ModelGemGlobalLayer', # noqa 45 | }, 46 | 'ModelGemRegional': { 47 | 'pattern': 'CMC_reg*', 48 | 'path': 'geomet_data_registry.layer.model_gem_regional.ModelGemRegionalLayer', # noqa 49 | }, 50 | 'ModelHrdpsContinental': { 51 | 'pattern': 'CMC_hrdps_continental*', 52 | 'path': 'geomet_data_registry.layer.model_hrdps_continental.ModelHrdpsContinentalLayer', # noqa 53 | }, 54 | 'Radar1km': { 55 | 'pattern': '*Radar-Composite*', 56 | 'path': 'geomet_data_registry.layer.radar_1km.Radar1kmLayer', 57 | }, 58 | 'CanSIPS': { 59 | 'pattern': 'cansips*', 60 | 'path': 'geomet_data_registry.layer.cansips.CansipsLayer', 61 | }, 62 | 'REPS': { 63 | 'pattern': 'CMC-reps*', 64 | 'path': 'geomet_data_registry.layer.reps.RepsLayer', 65 | }, 66 | 'GEPS': { 67 | 'pattern': 'CMC_geps*', 68 | 'path': 'geomet_data_registry.layer.geps.GepsLayer', 69 | }, 70 | 'GIOPS': { 71 | 'pattern': 'CMC_giops*', 72 | 'path': 'geomet_data_registry.layer.model_giops.GiopsLayer', 73 | }, 74 | 'RIOPS': { 75 | 'pattern': '*MSC_RIOPS*', 76 | 'path': 'geomet_data_registry.layer.model_riops.RiopsLayer', 77 | }, 78 | 'CGSL': { 79 | 'pattern': 'CMC_coupled-rdps-stlawrence*', 80 | 'path': 'geomet_data_registry.layer.cgsl.CgslLayer', 81 | }, 82 | 'RDWPS': { 83 | 'pattern': '*MSC_RDWPS*', 84 | 'path': 'geomet_data_registry.layer.rdwps.RdwpsLayer', 85 | }, 86 | 'GDWPS': { 87 | 'pattern': '*MSC_GDWPS*', 88 | 'path': 'geomet_data_registry.layer.gdwps.GdwpsLayer', 89 | }, 90 | 'WCPS': { 91 | 'pattern': 'CMC_wcps*', 92 | 'path': 'geomet_data_registry.layer.wcps.WcpsLayer', 93 | }, 94 | 'HRDPA': { 95 | 'pattern': 'CMC_HRDPA*', 96 | 'path': 'geomet_data_registry.layer.hrdpa.HrdpaLayer', 97 | }, 98 | 'RDPA': { 99 | 'pattern': 'CMC_RDPA*', 100 | 'path': 'geomet_data_registry.layer.rdpa.RdpaLayer', 101 | }, 102 | 'RAQDPS': { 103 | 'pattern': '*MSC_RAQDPS*', 104 | 'path': 'geomet_data_registry.layer.model_raqdps.ModelRaqdpsLayer', 105 | }, 106 | 'RAQDPS-FW': { 107 | 'pattern': '*MSC_RAQDPS-FW_*.grib2', 108 | 'path': 'geomet_data_registry.layer.model_raqdps_fw.ModelRaqdpsFwLayer', # noqa 109 | }, 110 | 'RAQDPS-FW-Cumulative-Effects': { 111 | 'pattern': '*MSC_RAQDPS-FW*.nc', 112 | 'path': 'geomet_data_registry.layer.model_raqdps_fw_ce.ModelRaqdpsFwCeLayer', # noqa 113 | }, 114 | 'RDAQA-Cumulative-Effects': { 115 | 'pattern': '*MSC_RDAQA*.nc', 116 | 'path': 'geomet_data_registry.layer.model_rdaqa_ce.ModelRdaqaCeLayer', # noqa 117 | }, 118 | }, 119 | } 120 | 121 | 122 | def load_plugin(plugin_type, plugin_def): 123 | """ 124 | loads plugin by type 125 | 126 | :param plugin_type: type of plugin (store, tileindex, etc.) 127 | :param plugin_def: plugin definition 128 | 129 | :returns: plugin object 130 | """ 131 | 132 | type_ = plugin_def['type'] 133 | 134 | if plugin_type not in PLUGINS.keys(): 135 | msg = 'Plugin type {} not found'.format(plugin_type) 136 | LOGGER.exception(msg) 137 | raise InvalidPluginError(msg) 138 | 139 | plugin_list = PLUGINS[plugin_type] 140 | 141 | LOGGER.debug('Plugins: {}'.format(plugin_list)) 142 | 143 | if '.' not in type_ and type_ not in plugin_list.keys(): 144 | msg = 'Plugin {} not found'.format(type_) 145 | LOGGER.exception(msg) 146 | raise InvalidPluginError(msg) 147 | 148 | if '.' in type_: # dotted path 149 | packagename, classname = type_['path'].rsplit('.', 1) 150 | else: # core formatter 151 | packagename, classname = plugin_list[type_]['path'].rsplit('.', 1) 152 | 153 | LOGGER.debug('package name: {}'.format(packagename)) 154 | LOGGER.debug('class name: {}'.format(classname)) 155 | 156 | module = importlib.import_module(packagename) 157 | class_ = getattr(module, classname) 158 | plugin = class_(plugin_def) 159 | return plugin 160 | 161 | 162 | class InvalidPluginError(Exception): 163 | """Invalid plugin""" 164 | pass 165 | -------------------------------------------------------------------------------- /geomet_data_registry/store/__init__.py: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # 3 | # Copyright (C) 2019 Tom Kralidis 4 | # 5 | # This program is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with this program. If not, see . 17 | # 18 | ############################################################################### 19 | 20 | import codecs 21 | import json 22 | import logging 23 | 24 | import click 25 | from yaml import load, Loader 26 | 27 | from geomet_data_registry.env import STORE_TYPE, STORE_URL 28 | from geomet_data_registry.plugin import load_plugin 29 | from geomet_data_registry.store.base import StoreError 30 | from geomet_data_registry.util import json_pretty_print, remove_prefix 31 | 32 | LOGGER = logging.getLogger(__name__) 33 | 34 | 35 | @click.group() 36 | def store(): 37 | """Manage the geomet-data-registry store""" 38 | pass 39 | 40 | 41 | @click.command() 42 | @click.pass_context 43 | @click.option('--group', '-g', help='group') 44 | def setup(ctx, group=None): 45 | """create store""" 46 | 47 | provider_def = { 48 | 'type': STORE_TYPE, 49 | 'url': STORE_URL, 50 | 'group': group 51 | } 52 | 53 | st = load_plugin('store', provider_def) 54 | 55 | try: 56 | click.echo('Creating store {}'.format(st.url)) 57 | st.setup() 58 | except StoreError as err: 59 | raise click.ClickException(err) 60 | click.echo('Done') 61 | 62 | 63 | @click.command() 64 | @click.pass_context 65 | @click.option('--group', '-g', help='group') 66 | def teardown(ctx, group=None): 67 | """delete store""" 68 | 69 | provider_def = { 70 | 'type': STORE_TYPE, 71 | 'url': STORE_URL, 72 | 'group': group 73 | } 74 | 75 | st = load_plugin('store', provider_def) 76 | 77 | try: 78 | click.echo('Deleting store {}'.format(st.url)) 79 | st.teardown() 80 | except StoreError as err: 81 | click.echo(err) 82 | click.echo('Done') 83 | 84 | 85 | @click.command('set') 86 | @click.pass_context 87 | @click.option('--key', '-k', help='key name for store') 88 | @click.option('--config', '-c', 'config', 89 | type=click.Path(exists=True, resolve_path=True), 90 | help='Path to config yaml file') 91 | @click.option('--raw', '-r', is_flag=True, 92 | help='set key without adding prefix') 93 | def set_key(ctx, key, config, raw): 94 | """populate store""" 95 | 96 | if all([key is None, config is None]): 97 | raise click.ClickException('Missing --key/-k or --config/-c option') 98 | 99 | provider_def = {'type': STORE_TYPE, 'url': STORE_URL} 100 | 101 | st = load_plugin('store', provider_def) 102 | 103 | try: 104 | with codecs.open(config) as ff: 105 | yml_dict = load(ff, Loader=Loader) 106 | string_ = json.dumps(yml_dict) 107 | if raw: 108 | click.echo('Setting {} key in store ({}).'.format(key, st.url)) 109 | st.set_key(key, string_, raw=True) 110 | else: 111 | click.echo( 112 | 'Setting geomet-data-registry_{} key in store ({}).' 113 | .format(key, st.url) 114 | ) 115 | st.set_key(key, string_) 116 | except StoreError as err: 117 | raise click.ClickException(err) 118 | click.echo('Done') 119 | 120 | 121 | @click.command('get') 122 | @click.pass_context 123 | @click.option('--key', '-k', help='key name to retrieve from store') 124 | @click.option('--raw', '-r', is_flag=True, 125 | help='get key without adding prefix') 126 | def get_key(ctx, key, raw): 127 | """get key from store""" 128 | 129 | if all([key is None]): 130 | raise click.ClickException('Missing --key/-k') 131 | 132 | provider_def = { 133 | 'type': STORE_TYPE, 134 | 'url': STORE_URL 135 | } 136 | 137 | st = load_plugin('store', provider_def) 138 | 139 | try: 140 | if raw: 141 | click.echo('Getting {} key from store ({}).'.format(key, st.url)) 142 | retrieved_key = st.get_key(key, raw=True) 143 | else: 144 | click.echo( 145 | 'Getting geomet-data-registry_{} key from store ({}).'.format( 146 | key, st.url) 147 | ) 148 | retrieved_key = st.get_key(key) 149 | if retrieved_key: 150 | try: 151 | click.echo('{}'.format( 152 | json_pretty_print(json.loads(retrieved_key)))) 153 | except ValueError: 154 | click.echo(retrieved_key) 155 | 156 | except StoreError as err: 157 | raise click.ClickException(err) 158 | click.echo('Done') 159 | 160 | 161 | @click.command('list') 162 | @click.option('--pattern', '-p', 163 | help='regular expression to filter keys on') 164 | @click.option('--raw', '-r', is_flag=True, 165 | help='list raw keys without removing prefix') 166 | @click.pass_context 167 | def list_keys(ctx, raw, pattern=None): 168 | """list all keys in store""" 169 | 170 | provider_def = { 171 | 'type': STORE_TYPE, 172 | 'url': STORE_URL 173 | } 174 | 175 | st = load_plugin('store', provider_def) 176 | 177 | try: 178 | pattern = 'geomet-data-registry*{}'.format(pattern if pattern else '') 179 | if raw: 180 | keys = st.list_keys(pattern) 181 | else: 182 | keys = [remove_prefix(key, 'geomet-data-registry_') for key 183 | in st.list_keys(pattern)] 184 | click.echo(json_pretty_print(keys)) 185 | except StoreError as err: 186 | raise click.ClickException(err) 187 | click.echo('Done') 188 | 189 | 190 | store.add_command(setup) 191 | store.add_command(teardown) 192 | store.add_command(set_key) 193 | store.add_command(get_key) 194 | store.add_command(list_keys) 195 | -------------------------------------------------------------------------------- /geomet_data_registry/store/base.py: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # 3 | # Copyright (C) 2019 Tom Kralidis 4 | # 5 | # This program is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with this program. If not, see . 17 | # 18 | ############################################################################### 19 | 20 | import logging 21 | 22 | LOGGER = logging.getLogger(__name__) 23 | 24 | 25 | class BaseStore: 26 | """generic key-value store ABC""" 27 | 28 | def __init__(self, provider_def): 29 | """ 30 | Initialize object 31 | 32 | :param provider_def: provider definition dict 33 | 34 | :returns: `geomet_data_registry.store.base.BaseStore` 35 | """ 36 | 37 | self.type = provider_def['type'] 38 | self.url = provider_def['url'] 39 | 40 | def setup(self): 41 | """ 42 | Create the store 43 | 44 | :returns: `bool` of process status 45 | """ 46 | 47 | raise NotImplementedError() 48 | 49 | def teardown(self): 50 | """ 51 | Delete the store 52 | 53 | :returns: `bool` of process status 54 | """ 55 | 56 | raise NotImplementedError() 57 | 58 | def get_key(self, key): 59 | """ 60 | Get key from store 61 | 62 | :param key: key to fetch 63 | 64 | :returns: string of key value from Redis store 65 | """ 66 | 67 | raise NotImplementedError() 68 | 69 | def set_key(self, key, value): 70 | """ 71 | Set key value from 72 | 73 | :param key: key to set value 74 | :param value: value to set 75 | 76 | :returns: `bool` of set success 77 | """ 78 | 79 | raise NotImplementedError() 80 | 81 | def list_keys(self, pattern=None): 82 | """ 83 | List all keys in store 84 | 85 | :param pattern: regular expression to filter keys on 86 | 87 | :returns: `list` of all store keys 88 | """ 89 | 90 | raise NotImplementedError() 91 | 92 | def __repr__(self): 93 | return ' {}'.format(self.type) 94 | 95 | 96 | class StoreError(Exception): 97 | """setup error""" 98 | pass 99 | -------------------------------------------------------------------------------- /geomet_data_registry/store/redis_.py: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # 3 | # Copyright (C) 2019 Tom Kralidis 4 | # 5 | # This program is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with this program. If not, see . 17 | # 18 | ############################################################################### 19 | 20 | import logging 21 | 22 | import redis 23 | 24 | from geomet_data_registry import __version__ 25 | from geomet_data_registry.store.base import BaseStore, StoreError 26 | 27 | LOGGER = logging.getLogger(__name__) 28 | 29 | 30 | class RedisStore(BaseStore): 31 | """Redis key-value store implementation""" 32 | 33 | def __init__(self, provider_def): 34 | """ 35 | Initialize object 36 | 37 | :param provider_def: provider definition dict 38 | 39 | :returns: `geomet_data_registry.store.redis_.RedisStore` 40 | """ 41 | 42 | super().__init__(provider_def) 43 | 44 | try: 45 | self.redis = redis.Redis.from_url(self.url, 46 | decode_responses=True) 47 | except redis.exceptions.ConnectionError as err: 48 | msg = 'Cannot connect to Redis {}: {}'.format(self.url, err) 49 | LOGGER.exception(msg) 50 | raise StoreError(msg) 51 | 52 | def setup(self): 53 | """ 54 | Create the store 55 | 56 | :returns: `bool` of process status 57 | """ 58 | 59 | return self.redis.set('geomet-data-registry-version', __version__) 60 | 61 | def teardown(self): 62 | """ 63 | Delete the store 64 | 65 | :returns: `bool` of process status 66 | """ 67 | 68 | LOGGER.debug('Deleting all Redis keys') 69 | keys = [ 70 | key for key in self.redis.scan_iter() 71 | if key.startswith('geomet-data-registry') 72 | ] 73 | for key in keys: 74 | LOGGER.debug('Deleting key {}'.format(key)) 75 | self.redis.delete(key) 76 | 77 | return True 78 | 79 | def get_key(self, key, raw=False): 80 | """ 81 | Get key from store 82 | 83 | :param key: key to fetch 84 | :param raw: `bool` indication whether to add prefix when fetching key 85 | 86 | :returns: `str` of key value from Redis store 87 | """ 88 | 89 | if raw: 90 | return self.redis.get(key) 91 | 92 | return self.redis.get('geomet-data-registry_{}'.format(key)) 93 | 94 | def set_key(self, key, value, raw=False): 95 | """ 96 | Set key value from 97 | 98 | :param key: key to set value 99 | :param value: value to set 100 | :param raw: `bool` indication whether to add prefix when setting key 101 | 102 | :returns: `bool` of set success 103 | """ 104 | 105 | if raw: 106 | return self.redis.set(key, value) 107 | 108 | return self.redis.set('geomet-data-registry_{}'.format(key), value) 109 | 110 | def list_keys(self, pattern=None): 111 | """ 112 | List all store keys 113 | 114 | :param pattern: regular expression to filter keys on 115 | 116 | :returns: `list` of all store keys 117 | """ 118 | 119 | if pattern is not None: 120 | return self.redis.keys(pattern=pattern) 121 | 122 | return self.redis.keys() 123 | 124 | def __repr__(self): 125 | return ' {}'.format(self.type) 126 | -------------------------------------------------------------------------------- /geomet_data_registry/tileindex/__init__.py: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # 3 | # Copyright (C) 2019 Tom Kralidis 4 | # 5 | # This program is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with this program. If not, see . 17 | # 18 | ############################################################################### 19 | 20 | import logging 21 | 22 | import click 23 | 24 | from geomet_data_registry.env import ( 25 | TILEINDEX_TYPE, TILEINDEX_BASEURL, TILEINDEX_NAME) 26 | from geomet_data_registry.plugin import load_plugin 27 | from geomet_data_registry.tileindex.base import TileIndexError 28 | 29 | LOGGER = logging.getLogger(__name__) 30 | 31 | 32 | @click.group() 33 | def tileindex(): 34 | """Manage the geomet-data-registry tileindex""" 35 | pass 36 | 37 | 38 | @click.command() 39 | @click.pass_context 40 | @click.option('--group', '-g', help='group') 41 | def setup(ctx, group=None): 42 | """create tileindex""" 43 | 44 | provider_def = { 45 | 'type': TILEINDEX_TYPE, 46 | 'url': TILEINDEX_BASEURL, 47 | 'name': TILEINDEX_NAME, 48 | 'group': group 49 | } 50 | 51 | ti = load_plugin('tileindex', provider_def) 52 | 53 | try: 54 | click.echo('Creating tileindex {}'.format(ti.fullpath)) 55 | ti.setup() 56 | except TileIndexError as err: 57 | raise click.ClickException(err) 58 | click.echo('Done') 59 | 60 | 61 | @click.command() 62 | @click.pass_context 63 | @click.option('--group', '-g', help='group') 64 | def teardown(ctx, group=None): 65 | """delete tileindex""" 66 | 67 | provider_def = { 68 | 'type': TILEINDEX_TYPE, 69 | 'url': TILEINDEX_BASEURL, 70 | 'name': TILEINDEX_NAME, 71 | 'group': group 72 | } 73 | 74 | ti = load_plugin('tileindex', provider_def) 75 | 76 | try: 77 | click.echo('Deleting tileindex {}'.format(ti.fullpath)) 78 | ti.teardown() 79 | except TileIndexError as err: 80 | click.echo(err) 81 | click.echo('Done') 82 | 83 | 84 | tileindex.add_command(setup) 85 | tileindex.add_command(teardown) 86 | -------------------------------------------------------------------------------- /geomet_data_registry/tileindex/base.py: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # 3 | # Copyright (C) 2019 Tom Kralidis 4 | # 5 | # This program is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with this program. If not, see . 17 | # 18 | ############################################################################### 19 | 20 | import logging 21 | import os 22 | 23 | LOGGER = logging.getLogger(__name__) 24 | 25 | 26 | class BaseTileIndex: 27 | """generic Tile Index ABC""" 28 | 29 | def __init__(self, provider_def): 30 | """ 31 | Initialize object 32 | 33 | :param provider: provider definition dict 34 | :param url: url/path of tile index 35 | :param group: provider group 36 | 37 | :returns: `geomet_data_registry.tileindex.base.BaseTileIndex` 38 | """ 39 | 40 | self.type = provider_def['type'] 41 | self.url = provider_def['url'] 42 | self.name = provider_def['name'] 43 | self.group = None 44 | 45 | LOGGER.debug('Detecting group tileindex') 46 | if 'group' in provider_def: 47 | self.group = provider_def['group'] 48 | 49 | if self.group is not None: 50 | self.name = '{}-{}'.format(self.name, self.group) 51 | 52 | self.fullpath = os.path.join(self.url, self.name) 53 | 54 | def setup(self): 55 | """ 56 | Create the tileindex 57 | 58 | :returns: `bool` of process status 59 | """ 60 | 61 | raise NotImplementedError() 62 | 63 | def teardown(self): 64 | """ 65 | Delete the tileindex 66 | 67 | :returns: `bool` of process status 68 | """ 69 | 70 | raise NotImplementedError() 71 | 72 | def query(self): 73 | """ 74 | Query the tileindex 75 | 76 | :returns: dict of 0..n GeoJSON features 77 | """ 78 | 79 | raise NotImplementedError() 80 | 81 | def get(self, identifier): 82 | """ 83 | Query the tileindex by identifier 84 | 85 | :param identifier: tileindex item identifier 86 | 87 | :returns: dict of single GeoJSON feature 88 | """ 89 | 90 | raise NotImplementedError() 91 | 92 | def add(self, identifier, data): 93 | """ 94 | Add an item to the tileindex 95 | 96 | :param identifier: tileindex item identifier 97 | :param data: GeoJSON dict 98 | 99 | :returns: `int` of status (as per HTTP status codes) 100 | """ 101 | 102 | raise NotImplementedError() 103 | 104 | def bulk_add(self, data): 105 | """ 106 | Add an many items to the tileindex 107 | 108 | :param data: GeoJSON dict 109 | 110 | :returns: list of dict {layer_id: HTTP status code} 111 | """ 112 | 113 | raise NotImplementedError() 114 | 115 | def update(self, identifier, update_dict): 116 | """ 117 | Update an item to the tileindex 118 | 119 | :param identifier: tileindex item identifier 120 | :param update_dict: `dict` of key/value updates 121 | 122 | :returns: `int` of status (as per HTTP status codes) 123 | """ 124 | 125 | raise NotImplementedError() 126 | 127 | def update_by_query(self, query_dict, update_dict): 128 | """ 129 | Add an item to the tileindex 130 | 131 | :param query_dict: `dict` of query 132 | :param update_dict: `dict` of key/value updates 133 | 134 | :returns: `int` of status (as per HTTP status codes) 135 | """ 136 | 137 | raise NotImplementedError() 138 | 139 | def remove(self, identifier): 140 | """ 141 | Remove an item from the tileindex 142 | 143 | :param identifier: tileindex item identifier 144 | 145 | :returns: `int` of status (as per HTTP status codes) 146 | """ 147 | 148 | raise NotImplementedError() 149 | 150 | def __repr__(self): 151 | return ' {}'.format(self.type) 152 | 153 | 154 | class TileIndexError(Exception): 155 | """setup error""" 156 | 157 | pass 158 | 159 | 160 | class TileNotFoundError(Exception): 161 | """BaseTileIndex.get() does not return a document""" 162 | 163 | pass 164 | -------------------------------------------------------------------------------- /requirements-dev.txt: -------------------------------------------------------------------------------- 1 | flake8 2 | pytest 3 | wheel 4 | twine 5 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | click 2 | elasticsearch 3 | metpx-sarracenia 4 | parse 5 | python-dateutil 6 | pyyaml 7 | redis 8 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # 3 | # Copyright (C) 2019 Tom Kralidis 4 | # 5 | # This program is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with this program. If not, see . 17 | # 18 | ############################################################################### 19 | 20 | import io 21 | import os 22 | import re 23 | from setuptools import Command, find_packages, setup 24 | import shutil 25 | 26 | 27 | class PyCleanBuild(Command): 28 | user_options = [] 29 | 30 | def initialize_options(self): 31 | pass 32 | 33 | def finalize_options(self): 34 | pass 35 | 36 | def run(self): 37 | remove_files = [ 38 | 'debian/files', 39 | 'debian/geomet-data-registry.debhelper.log', 40 | 'debian/geomet-data-registry.postinst.debhelper', 41 | 'debian/geomet-data-registry.prerm.debhelper', 42 | 'debian/geomet-data-registry.substvars' 43 | ] 44 | 45 | remove_dirs = [ 46 | 'debian/geomet-data-registry' 47 | ] 48 | 49 | for file_ in remove_files: 50 | try: 51 | os.remove(file_) 52 | except OSError: 53 | pass 54 | 55 | for dir_ in remove_dirs: 56 | try: 57 | shutil.rmtree(dir_) 58 | except OSError: 59 | pass 60 | 61 | for file_ in os.listdir('..'): 62 | if file_.endswith(('.deb', '.build', '.changes')): 63 | os.remove('../{}'.format(file_)) 64 | 65 | 66 | class PyTest(Command): 67 | user_options = [] 68 | 69 | def initialize_options(self): 70 | pass 71 | 72 | def finalize_options(self): 73 | pass 74 | 75 | def run(self): 76 | import subprocess 77 | errno = subprocess.call(['pytest']) 78 | raise SystemExit(errno) 79 | 80 | 81 | class PyCoverage(Command): 82 | user_options = [] 83 | 84 | def initialize_options(self): 85 | pass 86 | 87 | def finalize_options(self): 88 | pass 89 | 90 | def run(self): 91 | import subprocess 92 | 93 | errno = subprocess.call(['coverage', 'run', 94 | '--source=geomet_data_registry', 95 | '-m', 'unittest', 96 | 'geomet_data_registry.tests.run_tests']) 97 | errno = subprocess.call(['coverage', 'report', '-m']) 98 | raise SystemExit(errno) 99 | 100 | 101 | def read(filename, encoding='utf-8'): 102 | """read file contents""" 103 | full_path = os.path.join(os.path.dirname(__file__), filename) 104 | with io.open(full_path, encoding=encoding) as fh: 105 | contents = fh.read().strip() 106 | return contents 107 | 108 | 109 | def get_package_version(): 110 | """get version from top-level package init""" 111 | version_file = read('geomet_data_registry/__init__.py') 112 | version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]", 113 | version_file, re.M) 114 | if version_match: 115 | return version_match.group(1) 116 | raise RuntimeError('Unable to find version string.') 117 | 118 | 119 | LONG_DESCRIPTION = read('README.md') 120 | 121 | if os.path.exists('MANIFEST'): 122 | os.unlink('MANIFEST') 123 | 124 | setup( 125 | name='geomet-data-registry', 126 | version=get_package_version(), 127 | description='Geospatial Web Services for Canadian Weather data', 128 | long_description=LONG_DESCRIPTION, 129 | long_description_content_type='text/markdown', 130 | license='GPLv3', 131 | platforms='all', 132 | keywords=' '.join([ 133 | 'geomet', 134 | 'weather' 135 | ]), 136 | author='Meteorological Service of Canada', 137 | author_email='tom.kralidis@canada.ca', 138 | maintainer='Meteorological Service of Canada', 139 | maintainer_email='tom.kralidis@canada.ca', 140 | url='https://github.com/ECCC-MSC/geomet-data-registry', 141 | install_requires=read('requirements.txt').splitlines(), 142 | packages=find_packages(exclude=['geomet_data_registry.tests']), 143 | include_package_data=True, 144 | entry_points={ 145 | 'console_scripts': [ 146 | 'geomet-data-registry=geomet_data_registry:cli' 147 | ] 148 | }, 149 | classifiers=[ 150 | 'Development Status :: 4 - Beta', 151 | 'Environment :: Console', 152 | 'Intended Audience :: Developers', 153 | 'Intended Audience :: Science/Research', 154 | 'License :: OSI Approved :: GNU General Public License v3 (GPLv3)', 155 | 'Operating System :: OS Independent', 156 | 'Programming Language :: Python' 157 | ], 158 | cmdclass={ 159 | 'test': PyTest, 160 | 'coverage': PyCoverage, 161 | 'cleanbuild': PyCleanBuild 162 | } 163 | ) 164 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # 3 | # Copyright (C) 2021 Tom Kralidis 4 | # 5 | # This program is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with this program. If not, see . 17 | # 18 | ############################################################################### 19 | 20 | import os 21 | 22 | 23 | def set_env_vars(): 24 | os.environ['GDR_LOGGING_LOGLEVEL'] = 'ERROR' 25 | os.environ['GDR_LOGGING_LOGFILE'] = 'stdout' 26 | os.environ['GDR_BASEDIR'] = 'TODO' 27 | os.environ['GDR_DATADIR'] = 'TODO' 28 | os.environ['GDR_TILEINDEX_TYPE'] = 'TODO' 29 | os.environ['GDR_TILEINDEX_BASEURL'] = 'TODO' 30 | os.environ['GDR_TILEINDEX_NAME'] = 'TODO' 31 | os.environ['GDR_STORE_TYPE'] = 'TODO' 32 | os.environ['GDR_STORE_URL'] = 'TODO' 33 | os.environ['GDR_METPX_DISCARD'] = 'TODO' 34 | os.environ['GDR_METPX_EVENT_FILE_PY'] = 'TODO' 35 | os.environ['GDR_METPX_EVENT_MESSAGE_PY'] = 'TODO' 36 | os.environ['GDR_NOTIFICATIONS'] = 'TODO' 37 | os.environ['GDR_NOTIFICATIONS_TYPE'] = 'TODO' 38 | os.environ['GDR_NOTIFICATIONS_URL'] = 'TODO' 39 | -------------------------------------------------------------------------------- /tests/setup_test_class.py: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # 3 | # Copyright (C) 2021 Philippe Théroux 4 | # 5 | # This program is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with this program. If not, see . 17 | # 18 | ############################################################################### 19 | 20 | from datetime import datetime, timezone 21 | import importlib 22 | import json 23 | from unittest.mock import patch, DEFAULT 24 | 25 | 26 | class Setup: 27 | def __init__(self, test_file, classname, handler_name=None): 28 | # if instance.name is not identical to file name, provide it 29 | self.handler_name = handler_name or test_file 30 | 31 | module = ( 32 | importlib.import_module(f'geomet_data_registry.layer.{test_file}') 33 | ) 34 | class_ = getattr(module, classname) 35 | 36 | self.date_patcher = patch( 37 | 'geomet_data_registry.layer.base.get_today_and_now' 38 | ) 39 | self.mocked_get_date = self.date_patcher.start() 40 | 41 | self.plugin_patcher = patch( 42 | 'geomet_data_registry.layer.base.load_plugin' 43 | ) 44 | self.mocked_load_plugin = self.plugin_patcher.start() 45 | 46 | self.maxDiff = None 47 | self.today_date = ( 48 | datetime.now(timezone.utc).isoformat().replace('+00:00', 'Z') 49 | ) 50 | self.mocked_get_date.return_value = self.today_date 51 | 52 | if 'BaseLayer' in repr(class_): 53 | # if we're testing base layer, send a real provider def, otherwise 54 | # send 'whatever' since it should be overridden by the real one 55 | self.base_layer = class_({'name': self.handler_name}) 56 | else: 57 | # patch base layer init only when accessed with super().__init__() 58 | self.init_patcher = patch( 59 | f'geomet_data_registry.layer.{test_file}.BaseLayer.__init__' 60 | ) 61 | self.mocked_base_init = self.init_patcher.start() 62 | 63 | self.layer_handler = {test_file: class_({'name': 'whatever'})} 64 | self.add_init_attr(self.layer_handler[test_file]) 65 | 66 | def add_init_attr(self, instance): 67 | """Mocked BaseLayer instanciation""" 68 | 69 | instance.items = [] 70 | instance.model_run_list = [] 71 | 72 | instance.receive_datetime = self.today_date 73 | instance.identify_datetime = None 74 | instance.register_datetime = None 75 | instance.filepath = None 76 | instance.url = None 77 | instance.dimensions = None 78 | instance.model = None 79 | instance.model_run = None 80 | instance.geomet_layers = None 81 | instance.wx_variable = None 82 | instance.date_ = None 83 | instance.file_dict = None 84 | instance.new_key_store = False 85 | 86 | instance.name = self.handler_name 87 | instance.store = self.mocked_load_plugin.return_value 88 | instance.tileindex = self.mocked_load_plugin.return_value 89 | 90 | def create_item(self): 91 | """Returns a fake item.""" 92 | 93 | return { 94 | 'layer_name': 'GDPS.ETA_UGRD', 95 | 'filepath': './geomet_data_registry/tests/data/model_gem_global/15km/grib2/lat_lon/00/066/CMC_glb_UGRD_TGL_10_latlon.15x.15_2021112600_P066.grib2', # noqa 96 | 'identifier': 'GDPS.ETA_UGRD-20211126000000-20211128180000', 97 | 'reference_datetime': datetime(2021, 11, 26, 0, 0), 98 | 'forecast_hour_datetime': datetime(2021, 11, 28, 18, 0), 99 | 'member': None, 100 | 'elevation': 'surface', 101 | 'expected_count': None, 102 | 'register_status': True, 103 | 'model': 'model_gem_global', 104 | } 105 | 106 | def create_dependencies(self): 107 | """Return abitrary values to simulate dependencies.""" 108 | 109 | return [ 110 | { 111 | 'properties': { 112 | 'filepath': './geomet_data_registry/tests/data/model_gem_global/15km/grib2/lat_lon/00/066/CMC_glb_UGRD_TGL_10_latlon.15x.15_2021112600_P066.grib2', # noqa 113 | 'url': ['https://dd4.weather.gc.ca/model_gem_global/15km/grib2/lat_lon/00/066/CMC_glb_UGRD_TGL_10_latlon.15x.15_2021112600_P066.grib2'], # noqa 114 | 'weather_variable': ['UGRD_TGL_10'], 115 | } 116 | }, 117 | { 118 | 'properties': { 119 | 'filepath': './geomet_data_registry/tests/data/model_gem_global/15km/grib2/lat_lon/00/066/CMC_glb_VGRD_TGL_10_latlon.15x.15_2021112600_P066.grib2', # noqa 120 | 'url': ['https://dd4.weather.gc.ca/model_gem_global/15km/grib2/lat_lon/00/066/CMC_glb_VGRD_TGL_10_latlon.15x.15_2021112600_P066.grib2'], # noqa 121 | 'weather_variable': ['VGRD_TGL_10'], 122 | } 123 | }, 124 | ] 125 | 126 | def side_effect(self, *args, **kwargs): 127 | """Transforms dict into json inside layer handlers""" 128 | get_key = self.mocked_load_plugin.return_value.get_key.return_value 129 | if type(get_key) is dict: 130 | return json.dumps(get_key) 131 | return DEFAULT 132 | --------------------------------------------------------------------------------