├── .github ├── ISSUE_TEMPLATE │ └── issue-tracker-moved-.md └── workflows │ └── check-and-publish.yml ├── .gitignore ├── .pre-commit-config.yaml ├── LICENSE ├── MANIFEST.in ├── README.md ├── ansible ├── manually_running_ansible.sh ├── roles │ ├── aviso │ │ ├── defaults │ │ │ └── main.yml │ │ ├── files │ │ │ └── aviso_config.yaml │ │ ├── meta │ │ │ └── main.yml │ │ └── tasks │ │ │ └── aviso.yml │ ├── conda │ │ ├── defaults │ │ │ ├── .main.yml.swo │ │ │ └── main.yml │ │ └── tasks │ │ │ ├── .main.yml.swo │ │ │ └── main.yml │ ├── ecmwf-toolbox │ │ ├── defaults │ │ │ └── main.yml │ │ ├── meta │ │ │ └── main.yml │ │ └── tasks │ │ │ └── main.yml │ ├── escape2 │ │ ├── defaults │ │ │ └── main.yml │ │ ├── files │ │ │ ├── wxparaver.desktop │ │ │ └── wxparaver.png │ │ ├── meta │ │ │ └── main.yml │ │ └── tasks │ │ │ ├── escape2.yml │ │ │ └── main.yml │ ├── jupyterlab │ │ ├── defaults │ │ │ └── main.yml │ │ ├── files │ │ │ └── jupyter.conf │ │ ├── handlers │ │ │ └── main.yml │ │ ├── meta │ │ │ └── main.yml │ │ ├── tasks │ │ │ ├── jupyter.yml │ │ │ ├── main.yml │ │ │ └── proxy.yml │ │ └── templates │ │ │ └── jupyter.service.j2 │ ├── k8s-octavia │ │ ├── defaults │ │ │ └── main.yml │ │ ├── files │ │ │ ├── deployment.yaml │ │ │ └── serviceaccount.yaml │ │ ├── tasks │ │ │ └── main.yml │ │ └── templates │ │ │ └── cloudconfig.yaml.j2 │ ├── mars-client │ │ ├── defaults │ │ │ └── main.yml │ │ └── tasks │ │ │ ├── main.yml │ │ │ ├── mars-centos.yml │ │ │ └── mars-ubuntu.yml │ ├── nwp-da-training │ │ ├── defaults │ │ │ └── main.yml │ │ ├── meta │ │ │ └── main.yml │ │ └── tasks │ │ │ ├── main.yml │ │ │ └── nwp-da-setup.yml │ ├── nwp-nm-training │ │ ├── defaults │ │ │ └── main.yml │ │ ├── meta │ │ │ └── main.yml │ │ └── tasks │ │ │ ├── main.yml │ │ │ └── nwp-nm-setup.yml │ ├── nwp-pa-training │ │ ├── defaults │ │ │ └── main.yml │ │ ├── meta │ │ │ └── main.yml │ │ └── tasks │ │ │ ├── main.yml │ │ │ └── nwp-pa-setup.yml │ ├── nwp-pr-training │ │ ├── defaults │ │ │ └── main.yml │ │ ├── files │ │ │ └── scilab.desktop │ │ ├── meta │ │ │ └── main.yml │ │ └── tasks │ │ │ ├── main.yml │ │ │ └── nwp-pr-setup.yml │ ├── nwp-primer-training │ │ ├── defaults │ │ │ └── main.yml │ │ ├── meta │ │ │ └── main.yml │ │ ├── tasks │ │ │ ├── main.yml │ │ │ └── nwp-primer-setup.yml │ │ └── templates │ │ │ └── nwp-primer.sh.j2 │ ├── rdtraining │ │ ├── defaults │ │ │ └── main.yml │ │ ├── files │ │ │ ├── jupyter.desktop │ │ │ ├── jupyter.png │ │ │ ├── metview.png │ │ │ └── scilab.desktop │ │ ├── handlers │ │ │ └── main.yml │ │ ├── meta │ │ │ └── main.yml │ │ ├── tasks │ │ │ ├── acme.yml │ │ │ ├── certs.yml │ │ │ ├── conda.yml │ │ │ ├── git.yml │ │ │ ├── importcert.yml │ │ │ ├── jupyter.yml │ │ │ ├── main.yml │ │ │ ├── mountdisk.yml │ │ │ ├── proxy.yml │ │ │ ├── selfsignedcert.yml │ │ │ ├── softwaredeps.yml │ │ │ └── user.yml │ │ └── templates │ │ │ ├── README.md │ │ │ ├── ecmwf-lab-conda.j2 │ │ │ ├── ecmwf-lab-conda_concrete.yml.j2 │ │ │ ├── jupyter-revproxy.conf.j2 │ │ │ └── jupyter.service.j2 │ ├── s3clients │ │ ├── defaults │ │ │ └── main.yml │ │ ├── tasks │ │ │ └── main.yml │ │ └── templates │ │ │ ├── aws_credentials.j2 │ │ │ ├── s3cfg.j2 │ │ │ └── s3fs.j2 │ └── x2go │ │ ├── defaults │ │ └── main.yml │ │ ├── files │ │ ├── ecmwf-background.png │ │ ├── x2godesktopsharing.desktop │ │ ├── xfce-polkit.desktop │ │ └── xfce4-desktop.xml │ │ └── tasks │ │ ├── main.yml │ │ ├── x2go-centos.yml │ │ └── x2go-ubuntu.yml ├── s2s.yml └── test.yml ├── climetlab_s2s_ai_challenge ├── .gitignore ├── __init__.py ├── availability.py ├── benchmark.py ├── extra.py ├── fields.py ├── info.py ├── input.yaml ├── ncep_hindcast_only.yaml ├── observations.py ├── s2s_mergers.py ├── test_input.yaml ├── test_input_dev.yaml ├── training_input.yaml └── training_input_dev.yaml ├── data_portal.yaml ├── notebooks ├── demo_benchmark.ipynb ├── demo_grib.ipynb ├── demo_netcdf.ipynb ├── demo_observations.ipynb └── demo_zarr_experimental.ipynb ├── pyproject.toml ├── requirements-dev.txt ├── requirements.txt ├── setup.py ├── tests ├── test_availability.py ├── test_benchmarks.py ├── test_cfconventions.py ├── test_info.py ├── test_long_observations.py ├── test_merge.py ├── test_notebooks.py ├── test_observations.py ├── test_read.py └── test_read_zarr.py ├── tools ├── .gitignore ├── availability.py ├── list.py └── observations │ ├── build_dataset_observations.py │ ├── conda-packages.txt │ ├── download_from_source.sh │ └── makefile └── tox.ini /.github/ISSUE_TEMPLATE/issue-tracker-moved-.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Issue tracker moved. 3 | about: 'Link to the correct issue tracker : https://renkulab.io/gitlab/aaron.spring/s2s-ai-challenge/-/issues' 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | Please do not submit issue about the S2S ai challenge here. 11 | 12 | The issue tracker to use is located here : https://renkulab.io/gitlab/aaron.spring/s2s-ai-challenge/-/issues 13 | -------------------------------------------------------------------------------- /.github/workflows/check-and-publish.yml: -------------------------------------------------------------------------------- 1 | # This workflow will upload a Python Package using Twine when a release is created 2 | # For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries 3 | 4 | name: build 5 | 6 | on: 7 | workflow_dispatch: {} 8 | 9 | push: 10 | branches: [ main, develop ] 11 | 12 | pull_request: 13 | branches: [ main ] 14 | 15 | release: 16 | types: [created] 17 | 18 | jobs: 19 | quality: 20 | name: Code QA 21 | runs-on: ubuntu-latest 22 | steps: 23 | - uses: actions/checkout@v2 24 | - run: pip install black flake8 isort 25 | - run: isort --version 26 | - run: black --version 27 | - run: isort --check . 28 | - run: black --check . 29 | - run: flake8 . 30 | 31 | checks: 32 | strategy: 33 | fail-fast: false 34 | matrix: 35 | platform: ["ubuntu-latest"] 36 | python-version: ["3.8"] 37 | #platform: ["ubuntu-latest", "macos-latest", "windows-latest"] 38 | #python-version: ["3.6", "3.7", "3.8", "3.9"] 39 | 40 | name: Python ${{ matrix.python-version }} on ${{ matrix.platform }} 41 | runs-on: ${{ matrix.platform }} 42 | needs: quality 43 | 44 | steps: 45 | - uses: actions/checkout@v2 46 | 47 | - uses: actions/setup-python@v2 48 | with: 49 | python-version: ${{ matrix.python-version }} 50 | 51 | - name: Updating pip 52 | run: | 53 | python -m pip install --upgrade pip 54 | pip debug --verbose 55 | 56 | - run: pip install climetlab 57 | 58 | - name: Installing climetlab_s2s_ai_challenge 59 | run: pip install -e . 60 | 61 | - name: Setup test environment 62 | run: | 63 | pip install pytest 64 | pip freeze 65 | # pytest -k 'not test_notebooks' # does not work on github actions 66 | pip install zarr s3fs # dependencies for test/test_read_zarr.py 67 | 68 | - run: TEST_FAST=TRUE pytest tests/test_benchmarks.py 69 | - run: TEST_FAST=TRUE pytest tests/test_info.py 70 | # - run: TEST_FAST=TRUE pytest tests/test_notebooks.py 71 | - run: TEST_FAST=TRUE pytest tests/test_read.py 72 | - run: TEST_FAST=TRUE pytest tests/test_cfconventions.py 73 | # - run: TEST_FAST=TRUE pytest tests/test_long_observations.py # too long on github 74 | - run: TEST_FAST=TRUE pytest tests/test_merge.py 75 | - run: TEST_FAST=TRUE pytest tests/test_observations.py 76 | - run: TEST_FAST=TRUE pytest tests/test_read_zarr.py 77 | 78 | - name: Setup test environment for notebooks 79 | run: | 80 | pip install nbformat nbconvert ipykernel # dependencies for test/test_notebooks.py 81 | pip freeze 82 | - run: TEST_FAST=TRUE pytest tests/test_notebooks.py 83 | 84 | deploy: 85 | 86 | if: ${{ github.event_name == 'release' }} 87 | 88 | name: Upload to Pypi 89 | needs: checks 90 | 91 | runs-on: ubuntu-latest 92 | 93 | steps: 94 | - uses: actions/checkout@v2 95 | 96 | - name: Set up Python 97 | uses: actions/setup-python@v2 98 | with: 99 | python-version: '3.8' 100 | 101 | - name: Check version 102 | run: | 103 | release=${GITHUB_REF##*/} 104 | version=$(python setup.py --version) 105 | test "$release" == "$version" 106 | 107 | - name: Install dependencies 108 | run: | 109 | python -m pip install --upgrade pip 110 | pip install setuptools wheel twine 111 | 112 | - name: Build and publish 113 | env: 114 | TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} 115 | TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} 116 | run: | 117 | python setup.py sdist --verbose 118 | twine upload dist/* --verbose 119 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__ 2 | *.egg-info/ 3 | .ipynb_checkpoints 4 | *.swp 5 | *.swo 6 | *.swn 7 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/ambv/black 3 | rev: 21.4b0 4 | hooks: 5 | - id: black 6 | language_version: python3.8 7 | - repo: https://github.com/pycqa/isort 8 | rev: 5.8.0 9 | hooks: 10 | - id: isort 11 | name: isort (python) 12 | - id: isort 13 | name: isort (cython) 14 | types: [cython] 15 | - id: isort 16 | name: isort (pyi) 17 | types: [pyi] 18 | - repo: https://github.com/pycqa/flake8 19 | rev: 3.8.4 20 | hooks: 21 | - id: flake8 22 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include climetlab_s2s_ai_challenge/*.yaml -------------------------------------------------------------------------------- /ansible/manually_running_ansible.sh: -------------------------------------------------------------------------------- 1 | # Script to run the ansible playbok in dev mode, on a given machine 2 | # the machine should be on the .ssh/config with appropriate ssh key set up 3 | 4 | echo "To run this script you need to change roles/s3clients/defaults/main.yml and roles/rdtraining/defaults/main.yml (search for DEVMODE)" 5 | echo "and create files :" 6 | echo " pass : sudoer password of the already created machine" 7 | # not used: echo " s2sadminpassword : password you want to give to the s2sadmin user" 8 | echo " s3accesskey" # TODO get the key from the .s3cfg file 9 | echo " s3secretkey" # TODO get the key from the .s3cfg file 10 | 11 | # Install ansible with yum instead of conda. 12 | # conda deactivate 13 | 14 | ansible-playbook -i florian-ansible, s2s.yml --extra-vars "ansible_become_pass=$(cat pass)" -e "s3accesskey=$(cat s3accesskey)" -e "s3secretkey=$(cat s3secretkey)" --private-key ~/.ssh/id_rsa_s2s # -e "s2sadminpassword=$(cat s2sadminpassword)" 15 | -------------------------------------------------------------------------------- /ansible/roles/aviso/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | conda_env_name: base 3 | aviso_version: 0.7.1 4 | -------------------------------------------------------------------------------- /ansible/roles/aviso/files/aviso_config.yaml: -------------------------------------------------------------------------------- 1 | username_file: ~/.marsrc/mars.email 2 | key_file: ~/.marsrc/mars.token 3 | notification_engine: 4 | type: etcd_rest 5 | host: aviso.ecmwf.int 6 | port: 443 7 | https: true 8 | configuration_engine: 9 | type: etcd_rest 10 | host: aviso.ecmwf.int 11 | port: 443 12 | https: true 13 | schema_parser: ecmwf 14 | remote_schema: True 15 | auth_type: ecmwf 16 | -------------------------------------------------------------------------------- /ansible/roles/aviso/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | dependencies: 3 | - role: conda 4 | -------------------------------------------------------------------------------- /ansible/roles/aviso/tasks/aviso.yml: -------------------------------------------------------------------------------- 1 | - name: Install Aviso dependencies 2 | shell: | 3 | source /etc/profile.d/conda.sh 4 | conda activate {{ conda_env_name }} 5 | conda install -c conda-forge click etcd pyyaml python-json-logger requests parse pyinotify -y -q 6 | args: 7 | executable: /bin/bash 8 | 9 | - name: Install Aviso 10 | shell: | 11 | source /etc/profile.d/conda.sh 12 | conda activate {{ conda_env_name }} 13 | # pip3 install git+https://git.ecmwf.int/scm/aviso/aviso.git@{{ aviso_version}} 14 | pip3 install pyaviso 15 | args: 16 | executable: /bin/bash 17 | creates: '{{ conda_prefix }}/bin/aviso' 18 | 19 | - name: Ensure Aviso config dir exists 20 | file: 21 | path: /etc/aviso 22 | state: directory 23 | 24 | - name: Create Aviso config file 25 | copy: 26 | src: aviso_config.yaml 27 | dest: /etc/aviso/config.yaml 28 | -------------------------------------------------------------------------------- /ansible/roles/conda/defaults/.main.yml.swo: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ecmwf-lab/climetlab-s2s-ai-challenge/9a63d7ce4393145b08eed1cec89f1c8c40039895/ansible/roles/conda/defaults/.main.yml.swo -------------------------------------------------------------------------------- /ansible/roles/conda/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | conda_installer: https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh 3 | conda_prefix: /opt/anaconda3 4 | conda_update_base: no 5 | -------------------------------------------------------------------------------- /ansible/roles/conda/tasks/.main.yml.swo: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ecmwf-lab/climetlab-s2s-ai-challenge/9a63d7ce4393145b08eed1cec89f1c8c40039895/ansible/roles/conda/tasks/.main.yml.swo -------------------------------------------------------------------------------- /ansible/roles/conda/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Check if conda is installed 3 | stat: 4 | path: '{{ conda_prefix }}/bin/conda' 5 | register: conda_exe 6 | 7 | - block: 8 | - name: Download conda installer 9 | get_url: 10 | url: '{{ conda_installer}}' 11 | dest: /tmp/miniconda.sh 12 | mode: 0755 13 | 14 | - name: Install conda 15 | command: 16 | cmd: /tmp/miniconda.sh -b -p {{ conda_prefix }} 17 | creates: '{{ conda_prefix }}/bin/conda' 18 | 19 | - name: Remove conda installer 20 | file: 21 | path: /tmp/miniconda.sh 22 | state: absent 23 | when: not conda_exe.stat.exists 24 | 25 | - name: Ensure Conda is initialised in general profile 26 | file: 27 | src: '{{ conda_prefix }}/etc/profile.d/conda.sh' 28 | dest: /etc/profile.d/conda.sh 29 | state: link 30 | 31 | #- name: Initialise conda environment for user 32 | # shell: | 33 | # source /etc/profile.d/conda.sh 34 | # conda init 35 | # args: 36 | # executable: /bin/bash 37 | 38 | - name: Update conda and base environment 39 | shell: | 40 | source /etc/profile.d/conda.sh 41 | conda activate base 42 | conda update -y --all -q 43 | args: 44 | executable: /bin/bash 45 | when: conda_update_base 46 | -------------------------------------------------------------------------------- /ansible/roles/ecmwf-toolbox/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | conda_env_name: base 3 | -------------------------------------------------------------------------------- /ansible/roles/ecmwf-toolbox/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | dependencies: 3 | - role: conda 4 | -------------------------------------------------------------------------------- /ansible/roles/ecmwf-toolbox/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Ensure some system deps are present on CentOS 3 | package: 4 | name: 5 | - libXcomposite 6 | - libXcursor 7 | - libXi 8 | - libXtst 9 | - libXrandr 10 | - libXdamage 11 | - libXScrnSaver 12 | - xorg-x11-utils 13 | - mesa-libEGL 14 | - mesa-libGL 15 | - alsa-lib 16 | - curl 17 | - bzip2 18 | state: present 19 | when: ansible_distribution == 'CentOS' 20 | 21 | - name: Ensure some system deps are present on Ubuntu 22 | package: 23 | name: 24 | - libgl1-mesa-glx 25 | - libxrender1 26 | - xauth 27 | - x11-utils 28 | state: present 29 | when: ansible_distribution == 'Ubuntu' 30 | 31 | - name: Install ecmwf toolbox packages (metview, magics, eccodes, etc + deps) 32 | shell: | 33 | source /etc/profile.d/conda.sh 34 | conda activate {{ conda_env_name }} 35 | conda install -c conda-forge -y -q openblas numpy xarray pandas dask cfgrib jupyter metview magics magics-python eccodes 36 | pip3 install eccodes metview 37 | args: 38 | executable: /bin/bash 39 | creates: '{{ conda_prefix }}/bin/metview' 40 | 41 | - name: find conda qt pkgs 42 | find: 43 | paths: '{{ conda_prefix }}/pkgs/' 44 | file_type: directory 45 | patterns: 'qt-*' 46 | register: qtinstallations 47 | 48 | - name: fix conda qt permissions 49 | file: 50 | path: '{{ item.path }}/info' 51 | state: directory 52 | recurse: yes 53 | mode: 'u=rwX,g=rX,o=rX' 54 | with_items: '{{ qtinstallations.files }}' 55 | -------------------------------------------------------------------------------- /ansible/roles/escape2/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | paraver_version: 4.9.2 -------------------------------------------------------------------------------- /ansible/roles/escape2/files/wxparaver.desktop: -------------------------------------------------------------------------------- 1 | [Desktop Entry] 2 | Comment=Paraver 3 | Exec=/opt/wxparaver/bin/wxparaver 4 | GenericName=Paraver 5 | Icon=wxparaver 6 | Name=Paraver 7 | StartupNotify=false 8 | Terminal=false 9 | Type=Application 10 | Categories=Science; 11 | Keywords=Science; 12 | -------------------------------------------------------------------------------- /ansible/roles/escape2/files/wxparaver.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ecmwf-lab/climetlab-s2s-ai-challenge/9a63d7ce4393145b08eed1cec89f1c8c40039895/ansible/roles/escape2/files/wxparaver.png -------------------------------------------------------------------------------- /ansible/roles/escape2/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | dependencies: 3 | - role: rdtraining 4 | -------------------------------------------------------------------------------- /ansible/roles/escape2/tasks/escape2.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Install paraver 3 | unarchive: 4 | src: 'https://ftp.tools.bsc.es/wxparaver/wxparaver-{{ paraver_version }}-Linux_x86_64.tar.bz2' 5 | dest: /opt/ 6 | remote_src: yes 7 | owner: root 8 | group: root 9 | creates: '/opt/wxparaver-{{ paraver_version }}-Linux_x86_64/bin/wxparaver' 10 | 11 | - name: Link paraver install directory 12 | file: 13 | src: '/opt/wxparaver-{{ paraver_version }}-Linux_x86_64' 14 | dest: /opt/wxparaver 15 | state: link 16 | 17 | - name: Ensure paraver is in PATH 18 | lineinfile: 19 | path: /etc/profile.d/paraver.sh 20 | line: '[[ /opt/wxparaver/bin == *"$PATH"* ]] || export PATH=/opt/wxparaver/bin:$PATH' 21 | create: yes 22 | 23 | - name: Create paraver desktop shortcut 24 | become_user: '{{ training_user }}' 25 | copy: 26 | src: files/wxparaver.desktop 27 | dest: '~/Desktop/wxparaver.desktop' 28 | mode: '0755' 29 | 30 | - name: Ensure custom icons directory 31 | become_user: '{{ training_user }}' 32 | file: 33 | path: '~/.icons' 34 | state: directory 35 | 36 | - name: Copy jupyter icon 37 | become_user: '{{ training_user }}' 38 | copy: 39 | src: 'files/wxparaver.png' 40 | dest: '~/.icons/wxparaver.png' 41 | mode: '0755' -------------------------------------------------------------------------------- /ansible/roles/escape2/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - block: 3 | - name: Prepare ESCAPE2 requirements 4 | include_tasks: escape2.yml 5 | tags: user 6 | -------------------------------------------------------------------------------- /ansible/roles/jupyterlab/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | jupyterlab_user: jupyterlab 3 | # Use Morpheus cypher 4 | # jupyterlab_password: "{{ lookup('cypher','secret=password/jupyterlab') }}" 5 | 6 | # Use Morpheus Custom Option if run through Morpheus, otherwise fall back to default password 7 | jupyterlab_password: "{{ morpheus['customOptions']['jlpassword'] | default('test-jupyterlab123') }}" 8 | -------------------------------------------------------------------------------- /ansible/roles/jupyterlab/files/jupyter.conf: -------------------------------------------------------------------------------- 1 | # HTTP server to redirect all 80 traffic to SSL/HTTPS 2 | server { 3 | listen 80 default_server; 4 | 5 | # Tell all requests to port 80 to be 302 redirected to HTTPS 6 | return 302 https://$host$request_uri; 7 | } 8 | 9 | # HTTPS server to handle JupyterHub 10 | server { 11 | listen 443 default_server; 12 | ssl on; 13 | 14 | ssl_certificate /etc/ssl/certs/nginx-selfsigned.crt; 15 | ssl_certificate_key /etc/ssl/private/nginx-selfsigned.key; 16 | 17 | ssl_protocols TLSv1 TLSv1.1 TLSv1.2; 18 | ssl_prefer_server_ciphers on; 19 | ssl_dhparam /etc/ssl/dhparams.pem; 20 | ssl_ciphers 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:CAMELLIA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA'; 21 | ssl_session_timeout 1d; 22 | ssl_session_cache shared:SSL:50m; 23 | ssl_stapling on; 24 | ssl_stapling_verify on; 25 | add_header Strict-Transport-Security max-age=15768000; 26 | 27 | access_log /var/log/nginx/jupyter.log ; 28 | error_log /var/log/nginx/jupyter.error.log debug; 29 | 30 | location / { 31 | auth_basic "Access restricted"; 32 | auth_basic_user_file /etc/nginx/.htpasswd; 33 | proxy_set_header Host $host; 34 | proxy_set_header X-Real-IP $remote_addr; 35 | proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; 36 | proxy_set_header X-Forwarded-Proto $scheme; 37 | proxy_pass http://127.0.0.1:8888; 38 | proxy_read_timeout 90; 39 | } 40 | location ~* /(api/kernels/[^/]+/(channels|iopub|shell|stdin)|terminals/websocket)/? { 41 | proxy_pass http://127.0.0.1:8888; 42 | proxy_set_header X-Real-IP $remote_addr; 43 | proxy_set_header Host $host; 44 | proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; 45 | # WebSocket support 46 | proxy_http_version 1.1; 47 | proxy_set_header Upgrade "websocket"; 48 | proxy_set_header Connection "Upgrade"; 49 | proxy_read_timeout 86400; 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /ansible/roles/jupyterlab/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Restart nginx 3 | service: 4 | name: nginx 5 | state: restarted 6 | 7 | - name: Restart firewalld 8 | service: 9 | name: firewalld 10 | state: restarted 11 | 12 | - name: Restart jupyter 13 | service: 14 | name: jupyter 15 | state: restarted 16 | 17 | - name: Reload daemons 18 | systemd: 19 | # name: jupyterlab 20 | # state: restarted 21 | daemon_reload: yes 22 | -------------------------------------------------------------------------------- /ansible/roles/jupyterlab/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | dependencies: 3 | - role: conda 4 | -------------------------------------------------------------------------------- /ansible/roles/jupyterlab/tasks/jupyter.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Ensure ACL is available 3 | package: 4 | name: acl 5 | state: present 6 | 7 | - name: Create jupyterlab user 8 | user: 9 | name: '{{ jupyterlab_user }}' 10 | comment: Jupyter Lab User 11 | password_lock: yes 12 | state: present 13 | 14 | - name: Create conda environment 15 | become_user: '{{ jupyterlab_user }}' 16 | shell: | 17 | source /etc/profile.d/conda.sh 18 | conda init 19 | conda create --name jupyterlab -y 20 | args: 21 | executable: /bin/bash 22 | creates: /home/{{ jupyterlab_user }}/.conda/envs/jupyterlab 23 | 24 | - name: Ensure conda environment loads by default 25 | lineinfile: 26 | path: /home/{{ jupyterlab_user }}/.bashrc 27 | line: conda activate jupyterlab 28 | 29 | - name: Install jupyterlab in conda environment 30 | become_user: '{{ jupyterlab_user }}' 31 | shell: | 32 | source /etc/profile.d/conda.sh 33 | conda activate jupyterlab 34 | conda install -c conda-forge jupyterlab -y 35 | args: 36 | executable: /bin/bash 37 | creates: /home/{{ jupyterlab_user }}/.conda/envs/jupyterlab/bin/jupyter 38 | 39 | - name: Create the systemd service for jupyterlab 40 | template: 41 | src: templates/jupyter.service.j2 42 | dest: /etc/systemd/system/jupyter.service 43 | notify: 44 | - Reload daemons 45 | - Restart jupyter 46 | 47 | - name: Create jupyterlab config 48 | become_user: '{{ jupyterlab_user }}' 49 | shell: | 50 | source /etc/profile.d/conda.sh 51 | conda activate jupyterlab 52 | jupyter notebook --generate-config -y 53 | args: 54 | executable: /bin/bash 55 | creates: /home/{{ jupyterlab_user }}/.jupyter/jupyter_notebook_config.py 56 | 57 | # - name: Encrypt jupyterlab password 58 | # become_user: '{{ jupyterlab_user }}' 59 | # shell: | 60 | # source /etc/profile.d/conda.sh 61 | # conda activate jupyterlab 62 | # python -c "from notebook.auth import passwd; print(passwd('{{ jupyterlab_password }}'))" 63 | # register: encrypted 64 | # args: 65 | # executable: /bin/bash 66 | # 67 | # - name: Configure jupyterlab with password 68 | # lineinfile: 69 | # path: /home/{{ jupyterlab_user }}/.jupyter/jupyter_notebook_config.py 70 | # regexp: '^c.NotebookApp.password ' 71 | # insertafter: '^# c.NotebookApp.password ' 72 | # line: c.NotebookApp.password = u'{{ encrypted.stdout }}' 73 | 74 | - name: Configure jupyterlab with no password 75 | lineinfile: 76 | path: /home/{{ jupyterlab_user }}/.jupyter/jupyter_notebook_config.py 77 | regexp: '^c.NotebookApp.password ' 78 | insertafter: '^# c.NotebookApp.password ' 79 | line: c.NotebookApp.password = u'' 80 | notify: Restart jupyter 81 | 82 | - name: Configure jupyterlab with no token 83 | lineinfile: 84 | path: /home/{{ jupyterlab_user }}/.jupyter/jupyter_notebook_config.py 85 | regexp: '^c.NotebookApp.token ' 86 | insertafter: '^# c.NotebookApp.token ' 87 | line: c.NotebookApp.token = u'' 88 | notify: Restart jupyter 89 | 90 | - name: Ensure jupyter service is up and enabled 91 | service: 92 | name: jupyter 93 | state: started 94 | enabled: yes 95 | -------------------------------------------------------------------------------- /ansible/roles/jupyterlab/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - block: 3 | - name: Install Proxy 4 | include_tasks: proxy.yml 5 | tags: proxy 6 | 7 | - block: 8 | - name: Install Jupyter 9 | include_tasks: jupyter.yml 10 | tags: jupyter 11 | -------------------------------------------------------------------------------- /ansible/roles/jupyterlab/tasks/proxy.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Ensure EPEL repo is enabled on CentOS 3 | yum: 4 | name: epel-release 5 | state: latest 6 | when: ansible_distribution == 'CentOS' 7 | 8 | - name: Ensure nginx is installed and latest 9 | package: 10 | name: nginx 11 | state: latest 12 | 13 | - name: Firewalld exceptions 14 | block: 15 | - name: Gather the package facts 16 | package_facts: 17 | manager: auto 18 | 19 | - name: Allow HTTP/HTTPS traffic if firewalld is active 20 | command: firewall-cmd --permanent --add-service=http --add-service=https 21 | when: "'firewalld' in ansible_facts.packages" 22 | notify: Restart firewalld 23 | 24 | when: ansible_distribution == 'CentOS' 25 | 26 | - name: Ensure nginx is enabled 27 | service: 28 | name: nginx 29 | enabled: yes 30 | 31 | - name: Ensure SSL private dir on CentOS 32 | file: 33 | src: ../pki/tls/private 34 | dest: /etc/ssl/private 35 | state: link 36 | when: ansible_distribution == 'CentOS' 37 | 38 | - name: Self-signed certificate 39 | command: 40 | cmd: openssl req -x509 -nodes -days 365 -newkey rsa:2048 -subj "/C=UK/ST=England/L=Reading/O=EWCLOUD/CN=jupyterlab/emailAddress=support@europenaweather.cloud" -keyout /etc/ssl/private/nginx-selfsigned.key -out /etc/ssl/certs/nginx-selfsigned.crt 41 | creates: /etc/ssl/certs/nginx-selfsigned.crt 42 | 43 | - name: Diffie-Hellman parameters 44 | command: 45 | cmd: openssl dhparam -out /etc/ssl/dhparams.pem 2048 46 | creates: /etc/ssl/dhparams.pem 47 | 48 | # - name: Install passlib 49 | # package: 50 | # name: python-passlib 51 | # 52 | # - name: Add a user to a password file and ensure permissions are set 53 | # community.general.htpasswd: 54 | # path: /etc/nginx/.htpasswd 55 | # name: '{{ jupyterlab_user }}' 56 | # password: '{{ jupyterlab_password }}' 57 | # owner: root 58 | # group: "{{ 'www-data' if ansible_distribution == 'Ubuntu' else 'nginx' }}" 59 | # mode: 0640 60 | 61 | - name: Encode htpasswd 62 | command: 63 | cmd: openssl passwd -apr1 {{ jupyterlab_password }} 64 | register: encoded_password 65 | 66 | - name: Add a user to a password file and ensure permissions are set 67 | lineinfile: 68 | path: /etc/nginx/.htpasswd 69 | create: yes 70 | state: present 71 | line: '{{ jupyterlab_user }}:{{ encoded_password.stdout }}' 72 | regexp: '^{{ jupyterlab_user }}:' 73 | owner: root 74 | group: "{{ 'www-data' if ansible_distribution == 'Ubuntu' else 'nginx' }}" 75 | mode: 0640 76 | notify: Restart nginx 77 | 78 | # - debug: 79 | # msg: Using password {{ jupyterlab_password }} because user is {{ ansible_user }} 80 | 81 | - name: Remove default nginx site for Ubuntu 82 | file: 83 | name: /etc/nginx/sites-enabled/default 84 | state: absent 85 | when: ansible_distribution == 'Ubuntu' 86 | 87 | - name: Remove default nginx site for CentOS 88 | replace: 89 | path: /etc/nginx/nginx.conf 90 | regexp: "^([^#].* default_server.*)" 91 | replace: '#\1' 92 | when: ansible_distribution == 'CentOS' 93 | notify: Restart nginx 94 | 95 | - name: Write the site config file for Ubuntu 96 | copy: 97 | src: files/jupyter.conf 98 | dest: /etc/nginx/sites-enabled/jupyter.conf 99 | notify: Restart nginx 100 | when: ansible_distribution == 'Ubuntu' 101 | 102 | - name: Write the site config file for CentOS 103 | copy: 104 | src: files/jupyter.conf 105 | dest: /etc/nginx/conf.d/jupyter.conf 106 | notify: Restart nginx 107 | when: ansible_distribution == 'CentOS' 108 | -------------------------------------------------------------------------------- /ansible/roles/jupyterlab/templates/jupyter.service.j2: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Jupyter 3 | After=syslog.target network.target 4 | 5 | [Service] 6 | User=jupyterlab 7 | Environment="PATH=/home/{{ jupyterlab_user }}/.conda/envs/jupyterlab/bin/:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin" 8 | Environment="HOME=/home/{{ jupyterlab_user }}/" 9 | Environment="SHELL=/bin/bash" 10 | WorkingDirectory=/home/{{ jupyterlab_user }}/ 11 | ExecStart=/home/{{ jupyterlab_user }}/.conda/envs/jupyterlab/bin/jupyter lab --ip=0.0.0.0 --no-browser 12 | 13 | Restart=on-failure 14 | RestartSec=10 15 | 16 | [Install] 17 | WantedBy=multi-user.target 18 | -------------------------------------------------------------------------------- /ansible/roles/k8s-octavia/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | oic_dir: /etc/kubernetes/octavia-ingress-controller 3 | -------------------------------------------------------------------------------- /ansible/roles/k8s-octavia/files/deployment.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | kind: StatefulSet 3 | apiVersion: apps/v1 4 | metadata: 5 | name: octavia-ingress-controller 6 | namespace: kube-system 7 | labels: 8 | k8s-app: octavia-ingress-controller 9 | spec: 10 | replicas: 1 11 | selector: 12 | matchLabels: 13 | k8s-app: octavia-ingress-controller 14 | serviceName: octavia-ingress-controller 15 | template: 16 | metadata: 17 | labels: 18 | k8s-app: octavia-ingress-controller 19 | spec: 20 | serviceAccountName: octavia-ingress-controller 21 | tolerations: 22 | - effect: NoSchedule # Make sure the pod can be scheduled on master kubelet. 23 | operator: Exists 24 | - key: CriticalAddonsOnly # Mark the pod as a critical add-on for rescheduling. 25 | operator: Exists 26 | - effect: NoExecute 27 | operator: Exists 28 | containers: 29 | - name: octavia-ingress-controller 30 | image: docker.io/k8scloudprovider/octavia-ingress-controller:v1.17.0 31 | imagePullPolicy: IfNotPresent 32 | args: 33 | - /bin/octavia-ingress-controller 34 | - --config=/etc/config/octavia-ingress-controller-config.yaml 35 | volumeMounts: 36 | - mountPath: /etc/kubernetes 37 | name: kubernetes-config 38 | readOnly: true 39 | - name: ingress-config 40 | mountPath: /etc/config 41 | hostNetwork: true 42 | volumes: 43 | - name: kubernetes-config 44 | hostPath: 45 | path: /etc/kubernetes 46 | type: Directory 47 | - name: ingress-config 48 | configMap: 49 | name: octavia-ingress-controller-config 50 | items: 51 | - key: config 52 | path: octavia-ingress-controller-config.yaml 53 | -------------------------------------------------------------------------------- /ansible/roles/k8s-octavia/files/serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | kind: ServiceAccount 3 | apiVersion: v1 4 | metadata: 5 | name: octavia-ingress-controller 6 | namespace: kube-system 7 | --- 8 | kind: ClusterRoleBinding 9 | apiVersion: rbac.authorization.k8s.io/v1 10 | metadata: 11 | name: octavia-ingress-controller 12 | roleRef: 13 | apiGroup: rbac.authorization.k8s.io 14 | kind: ClusterRole 15 | name: cluster-admin 16 | subjects: 17 | - kind: ServiceAccount 18 | name: octavia-ingress-controller 19 | namespace: kube-system 20 | -------------------------------------------------------------------------------- /ansible/roles/k8s-octavia/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Ensure Octavia Ingress Controller directory exists 3 | file: 4 | path: "{{ oic_dir }}" 5 | state: directory 6 | 7 | - name: Service Account and RBAC manifest 8 | copy: 9 | src: files/serviceaccount.yaml 10 | dest: "{{ oic_dir }}/serviceaccount.yaml" 11 | 12 | - name: Cloud config manifest 13 | template: 14 | src: templates/cloudconfig.yaml.j2 15 | dest: "{{ oic_dir }}/config.yaml" 16 | mode: "600" 17 | 18 | - name: Ingress Controller manifest 19 | copy: 20 | src: files/deployment.yaml 21 | dest: "{{ oic_dir }}/deployment.yaml" 22 | 23 | - name: Prepare Ingress Controller Configuration 24 | shell: kubectl apply -f {{ oic_dir }}/config.yaml 25 | environment: 26 | KUBECONFIG: /etc/kubernetes/admin.conf 27 | 28 | - name: Create Service Account and grant permissions 29 | shell: kubectl apply -f {{ oic_dir }}/serviceaccount.yaml 30 | environment: 31 | KUBECONFIG: /etc/kubernetes/admin.conf 32 | 33 | - name: Deploy Octavia Ingress Controller 34 | shell: kubectl apply -f {{ oic_dir }}/deployment.yaml 35 | environment: 36 | KUBECONFIG: /etc/kubernetes/admin.conf 37 | -------------------------------------------------------------------------------- /ansible/roles/k8s-octavia/templates/cloudconfig.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | kind: ConfigMap 3 | apiVersion: v1 4 | metadata: 5 | name: octavia-ingress-controller-config 6 | namespace: kube-system 7 | data: 8 | config: | 9 | cluster-name: "{{ ansible_hostname | regex_replace('-master$', '') }}" 10 | openstack: 11 | auth-url: https://europeanweathercloud-pilot5.ecmwf.int:13000/v3 12 | domain-name: default 13 | username: "{{ lookup('cypher','secret=secret/k8scloudconfig:os_user') }}" 14 | password: "{{ lookup('cypher','secret=secret/k8scloudconfig:os_password') }}" 15 | project-id: "{{ lookup('cypher','secret=secret/k8scloudconfig:os_project') }}" 16 | octavia: 17 | subnet-id: "{{ lookup('cypher','secret=secret/k8scloudconfig:os_subnet') }}" 18 | floating-network-id: "{{ lookup('cypher','secret=secret/k8scloudconfig:os_public_network') }}" 19 | -------------------------------------------------------------------------------- /ansible/roles/mars-client/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | repo_base: https://nexus.ecmwf.int 3 | -------------------------------------------------------------------------------- /ansible/roles/mars-client/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - block: 3 | - name: Install Mars 4 | include_tasks: mars-{{ ansible_distribution|lower }}.yml 5 | tags: mars 6 | -------------------------------------------------------------------------------- /ansible/roles/mars-client/tasks/mars-centos.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Install yum-utils 3 | package: 4 | name: yum-utils 5 | state: latest 6 | 7 | - name: Ensure EPEL repo is enabled 8 | yum: 9 | name: epel-release 10 | state: latest 11 | 12 | - name: Add Nexus Repo 13 | yum_repository: 14 | name: nexus 15 | description: ECMWF Nexus Repository 16 | # FIXME: do not hardcode centos major/minor once the rpms are available 17 | # baseurl: '{{ repo_base }}/repository/private-centos-stable/{{ ansible_distribution_major_version }}/{{ ansible_distribution_version.split(".")[0] }}/rpms' 18 | baseurl: '{{ repo_base }}/repository/private-centos-stable/7/7/rpms' 19 | state: present 20 | 21 | - name: Install Mars Client 22 | yum: 23 | name: mars-client-cloud 24 | disable_gpg_check: yes 25 | state: latest 26 | -------------------------------------------------------------------------------- /ansible/roles/mars-client/tasks/mars-ubuntu.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Install libgomp1 3 | package: 4 | name: libgomp1 5 | state: present 6 | 7 | # Why on earth do we need this??? 8 | - name: Install libnetcdf 9 | package: 10 | name: libnetcdf13 11 | state: present 12 | 13 | - name: Install gpg-agent 14 | package: 15 | name: gpg-agent 16 | state: present 17 | 18 | - name: Add Nexus Repo key 19 | apt_key: 20 | url: https://nexus.ecmwf.int/repository/private-raw-repos-config/ubuntu/{{ ansible_distribution_release }}/stable/public.gpg.key 21 | state: present 22 | 23 | - name: Add Nexus Repo 24 | apt_repository: 25 | repo: deb https://nexus.ecmwf.int/repository/private-ubuntu-{{ ansible_distribution_release }}-stable/ {{ ansible_distribution_release }} main 26 | filename: ecmwf_nexus 27 | state: present 28 | 29 | - name: Install Mars Client 30 | apt: 31 | name: mars-client-cloud 32 | -------------------------------------------------------------------------------- /ansible/roles/nwp-da-training/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- -------------------------------------------------------------------------------- /ansible/roles/nwp-da-training/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | dependencies: 3 | - role: rdtraining 4 | -------------------------------------------------------------------------------- /ansible/roles/nwp-da-training/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - block: 3 | - name: Prepare NWP-DA requirements 4 | include_tasks: nwp-da-setup.yml 5 | tags: user 6 | -------------------------------------------------------------------------------- /ansible/roles/nwp-da-training/tasks/nwp-da-setup.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Checkout DAcourse material pack 3 | become_user: '{{ training_user }}' 4 | git: 5 | accept_hostkey: yes 6 | repo: ssh://git@git.ecmwf.int/tcd/dacourse.git 7 | dest: '~/DAcourse' 8 | 9 | -------------------------------------------------------------------------------- /ansible/roles/nwp-nm-training/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- -------------------------------------------------------------------------------- /ansible/roles/nwp-nm-training/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | dependencies: 3 | - role: rdtraining 4 | -------------------------------------------------------------------------------- /ansible/roles/nwp-nm-training/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - block: 3 | - name: Prepare NWP-NM requirements 4 | include_tasks: nwp-nm-setup.yml 5 | tags: user 6 | -------------------------------------------------------------------------------- /ansible/roles/nwp-nm-training/tasks/nwp-nm-setup.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Checkout NMcourse material pack 3 | become_user: '{{ training_user }}' 4 | git: 5 | accept_hostkey: yes 6 | repo: ssh://git@git.ecmwf.int/tcd/nmcourse.git 7 | dest: '~/NMcourse' 8 | 9 | -------------------------------------------------------------------------------- /ansible/roles/nwp-pa-training/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- -------------------------------------------------------------------------------- /ansible/roles/nwp-pa-training/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | dependencies: 3 | - role: rdtraining 4 | -------------------------------------------------------------------------------- /ansible/roles/nwp-pa-training/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - block: 3 | - name: Prepare NWP-PA requirements 4 | include_tasks: nwp-pa-setup.yml 5 | tags: user 6 | -------------------------------------------------------------------------------- /ansible/roles/nwp-pa-training/tasks/nwp-pa-setup.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: prepare ncl installation dir 3 | file: 4 | path: /opt/ncl 5 | state: directory 6 | 7 | - name: Install ncl 8 | unarchive: 9 | # src: 'https://www.earthsystemgrid.org/dataset/ncl.662.dap/file/ncl_ncarg-6.6.2-CentOS7.6_64bit_gnu485.tar.gz' 10 | # Work around website unavailability 11 | src: https://storage.ecmwf.europeanweather.cloud/ecmwf-training-labpacks/nwp-pa/ncl-6.6.2.tar.gz 12 | dest: /opt/ncl 13 | remote_src: yes 14 | owner: root 15 | group: root 16 | creates: /opt/ncl/bin/ncl 17 | 18 | - name: Ensure ncl is in PATH 19 | lineinfile: 20 | path: /etc/profile.d/ncl.sh 21 | line: '[[ /opt/ncl/bin == *"$PATH"* ]] || export PATH=/opt/ncl/bin:$PATH' 22 | create: yes 23 | 24 | - name: Ensure NCARG_ROOT is defined 25 | lineinfile: 26 | path: /etc/profile.d/ncl.sh 27 | line: 'export NCARG_ROOT=/opt/ncl' 28 | create: yes 29 | 30 | - name: Checkout PAcourse material pack 31 | become_user: '{{ training_user }}' 32 | git: 33 | accept_hostkey: yes 34 | repo: ssh://git@git.ecmwf.int/tcd/pacourse.git 35 | dest: '~/PAcourse' 36 | 37 | - name: Ensure metview dir 38 | become_user: '{{ training_user }}' 39 | file: 40 | path: '~/metview' 41 | state: directory 42 | 43 | - name: Create links for Training in Metview Dir 44 | become_user: '{{ training_user }}' 45 | file: 46 | path: '{{ item.path }}' 47 | state: link 48 | src: '{{ item.src }}' 49 | loop: 50 | - { path: '~/metview/BL-SURF', src: '~/PAcourse/BL-SURF' } 51 | - { path: '~/metview/SURF', src: '~/PAcourse/SURF' } 52 | - { path: '~/metview/scm_files', src: '~/PAcourse/scm_rclim_vtab' } 53 | 54 | #- name: Checkout PAcourse material pack 55 | # aws_s3: 56 | # s3_url: '{{ s3_url }}' 57 | # bucket: '{{ s3_bucket_name }}' 58 | # validate_certs: no 59 | # rgw: yes 60 | # mode: get 61 | # object: /pacourse.zip 62 | # dest: /tmp/pacourse.zip 63 | # overwrite: different 64 | # 65 | #- name: prepare PAcourse directory 66 | # become_user: '{{ training_user }}' 67 | # file: 68 | # path: '/home/{{ training_user }}/PAcourse' 69 | # state: directory 70 | # 71 | #- name: Unpack PAcourse material 72 | # become_user: '{{ training_user }}' 73 | # unarchive: 74 | # remote_src: yes 75 | # src: /tmp/pacourse.zip 76 | # dest: '/home/{{ training_user }}/PAcourse' 77 | 78 | -------------------------------------------------------------------------------- /ansible/roles/nwp-pr-training/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- -------------------------------------------------------------------------------- /ansible/roles/nwp-pr-training/files/scilab.desktop: -------------------------------------------------------------------------------- 1 | [Desktop Entry] 2 | Comment=Scientific software package for numerical computations 3 | Comment[fr]=Logiciel scientifique de calcul numérique 4 | Comment[de]=eine Wissenschaftssoftware für numerische Berechnungen 5 | Comment[ru]=Научная программа для численных расчётов 6 | Exec=bash -c "cd ~/lorenz1996/sci/; scilab -f init.sci" 7 | GenericName=Scientific Software Package 8 | GenericName[fr]=Logiciel de calcul numérique 9 | GenericName[de]=Wissenschaftssoftware 10 | GenericName[ru]=Научный программный комплекс 11 | Icon=scilab 12 | MimeType=application/x-scilab-sci;application/x-scilab-sce;application/x-scilab-tst;application/x-scilab-dem;application/x-scilab-sod;application/x-scilab-xcos;application/x-scilab-zcos;application/x-scilab-bin;application/x-scilab-cosf;application/x-scilab-cos; 13 | Name=Scilab PR Training 14 | StartupNotify=false 15 | Terminal=false 16 | Type=Application 17 | Categories=Science;Math; 18 | Keywords=Science;Math;Numerical;Simulation 19 | -------------------------------------------------------------------------------- /ansible/roles/nwp-pr-training/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | dependencies: 3 | - role: rdtraining 4 | -------------------------------------------------------------------------------- /ansible/roles/nwp-pr-training/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - block: 3 | - name: Prepare NWP-PR requirements 4 | include_tasks: nwp-pr-setup.yml 5 | tags: user 6 | -------------------------------------------------------------------------------- /ansible/roles/nwp-pr-training/tasks/nwp-pr-setup.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Install scilab 3 | unarchive: 4 | src: 'http://www.scilab.org/download/6.1.0/scilab-6.1.0.bin.linux-x86_64.tar.gz' 5 | dest: /opt/ 6 | remote_src: yes 7 | owner: root 8 | group: root 9 | creates: /opt/scilab/bin/scilab 10 | 11 | - name: Link scilab install directory 12 | file: 13 | src: /opt/scilab-6.1.0 14 | dest: /opt/scilab 15 | state: link 16 | 17 | - name: Ensure scilab is in PATH 18 | lineinfile: 19 | path: /etc/profile.d/scilab.sh 20 | line: '[[ /opt/scilab/bin == *"$PATH"* ]] || export PATH=/opt/scilab/bin:$PATH' 21 | create: yes 22 | 23 | - name: Add XDG_DATA_DIR for scilab 24 | lineinfile: 25 | path: /etc/profile.d/scilab.sh 26 | line: '[[ /opt/scilab/share == *"$XDG_DATA_DIRS"* ]] || export XDG_DATA_DIRS=/opt/scilab/share:$XDG_DATA_DIRS' 27 | create: yes 28 | 29 | - name: Checkout Lorenz1996 scripts pack 30 | become_user: '{{ training_user }}' 31 | git: 32 | accept_hostkey: yes 33 | repo: ssh://git@git.ecmwf.int/tcd/lorenz1996.git 34 | dest: '~/lorenz1996' 35 | 36 | - name: Checkout Lorenz1996 data pack 37 | aws_s3: 38 | s3_url: '{{ s3_url }}' 39 | bucket: '{{ s3_bucket_name }}' 40 | validate_certs: no 41 | rgw: yes 42 | mode: get 43 | object: /nwp-pr/data-lorenz1996.tar.gz 44 | dest: /tmp/data-lorenz1996.tar.gz 45 | overwrite: different 46 | 47 | - name: Unpack Lorenz1996 data 48 | become_user: '{{ training_user }}' 49 | unarchive: 50 | remote_src: yes 51 | src: /tmp/data-lorenz1996.tar.gz 52 | dest: '/home/{{ training_user }}/' 53 | 54 | - name: Create Scilab Training Desktop shortcut 55 | become_user: '{{ training_user }}' 56 | copy: 57 | src: files/scilab.desktop 58 | dest: '~/Desktop/scilab.desktop' 59 | mode: '0755' 60 | 61 | -------------------------------------------------------------------------------- /ansible/roles/nwp-primer-training/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | wipe_primer_dir: no 3 | -------------------------------------------------------------------------------- /ansible/roles/nwp-primer-training/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | dependencies: 3 | - role: rdtraining 4 | -------------------------------------------------------------------------------- /ansible/roles/nwp-primer-training/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - block: 3 | - name: Prepare NWP-Primer requirements 4 | include_tasks: nwp-primer-setup.yml 5 | tags: user 6 | -------------------------------------------------------------------------------- /ansible/roles/nwp-primer-training/tasks/nwp-primer-setup.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Fetch Primer Course material pack from S3 bucket 3 | aws_s3: 4 | s3_url: '{{ s3_url }}' 5 | bucket: '{{ s3_bucket_name }}' 6 | validate_certs: no 7 | rgw: yes 8 | mode: get 9 | object: /nwp-primer/nwp-primer.tar.gz 10 | dest: /tmp/nwp-primer.tar.gz 11 | overwrite: different 12 | 13 | - name: Wipe Primer Course material directory 14 | file: 15 | path: '/home/{{ training_user }}/nwp-primer' 16 | state: absent 17 | when: wipe_primer_dir 18 | 19 | - name: Unpack Primer Course material 20 | become_user: '{{ training_user }}' 21 | unarchive: 22 | remote_src: yes 23 | src: /tmp/nwp-primer.tar.gz 24 | dest: '/home/{{ training_user }}/' 25 | 26 | - name: Checkout OIFS notebooks 27 | become_user: '{{ training_user }}' 28 | git: 29 | accept_hostkey: yes 30 | repo: ssh://git@git.ecmwf.int/oifs/oifs-notebooks.git 31 | dest: '/home/{{ training_user }}/nwp-primer/oifs-notebooks' 32 | 33 | - name: Create the shell setup file for the course 34 | become_user: '{{ training_user }}' 35 | template: 36 | src: templates/nwp-primer.sh.j2 37 | dest: '/home/{{ training_user }}/.nwp-primer.sh' 38 | 39 | - name: Source course shell setup form bashrc 40 | become_user: '{{ training_user }}' 41 | lineinfile: 42 | path: '/home/{{ training_user }}/.bashrc' 43 | line: '[[ -f /home/{{ training_user }}/.nwp-primer.sh ]] && source /home/{{ training_user }}/.nwp-primer.sh' 44 | 45 | - name: Checkout partial PAcourse material pack 46 | become_user: '{{ training_user }}' 47 | git: 48 | accept_hostkey: yes 49 | repo: ssh://git@git.ecmwf.int/tcd/pacourse.git 50 | version: HOcourse 51 | dest: '~/PAcourse' 52 | 53 | - name: Checkout partial NMcourse material pack 54 | become_user: '{{ training_user }}' 55 | git: 56 | accept_hostkey: yes 57 | repo: ssh://git@git.ecmwf.int/tcd/nmcourse.git 58 | version: HOcourse 59 | dest: '~/NMcourse' 60 | 61 | - name: Checkout TC eccodes material pack 62 | become_user: '{{ training_user }}' 63 | git: 64 | accept_hostkey: yes 65 | repo: ssh://git@git.ecmwf.int/tcd/tc_eccodes.git 66 | dest: '~/TC_eccodes' 67 | 68 | - name: Checkout Metview hands on material pack 69 | become_user: '{{ training_user }}' 70 | git: 71 | accept_hostkey: yes 72 | repo: ssh://git@git.ecmwf.int/tcd/metview_hands_on.git 73 | dest: '~/Metview_hands_on' 74 | 75 | - name: Checkout bleeding-edge Metview Python bindings 76 | become_user: '{{ training_user }}' 77 | git: 78 | accept_hostkey: yes 79 | repo: ssh://git@git.ecmwf.int/mpy/mpy.git 80 | version: feature/MPY-291-high-level-interface 81 | dest: '~/.mpy' 82 | -------------------------------------------------------------------------------- /ansible/roles/nwp-primer-training/templates/nwp-primer.sh.j2: -------------------------------------------------------------------------------- 1 | # Changes to .bashrc for NWP Primer: 2 | # Training home directory: 3 | export THOME=/home/{{ training_user }}/nwp-primer 4 | # OpenIFS environment: 5 | export OIFS_HOME="$HOME/nwp-primer/oifs43r3" 6 | export OIFS_CYCLE="CY43R3" 7 | export PATH=$OIFS_HOME/fcm/bin:$HOME/nwp-primer/oifs43r3/bin:$PATH 8 | export OIFS_GRIB_DIR="{{ conda_prefix }}/envs/{{ conda_env_name }}" 9 | export GRIB_SAMPLES_PATH="$OIFS_GRIB_DIR/share/eccodes/ifs_samples/grib1_mlgrib2" 10 | export OIFS_GRIB_LIB="-L ${OIFS_GRIB_DIR}/lib -leccodes_f90 -leccodes" 11 | export OIFS_NETCDF_DIR="{{ conda_prefix }}/envs/{{ conda_env_name }}" 12 | export OIFS_NETCDF_INCLUDE="-I ${OIFS_NETCDF_DIR}/include" 13 | export OIFS_NETCDF_LIB="-L ${OIFS_NETCDF_DIR}/lib -lnetcdff -lnetcdf" 14 | export LD_LIBRARY_PATH="${OIFS_GRIB_DIR}/lib:$LD_LIBRARY_PATH" 15 | export OIFS_DATA_DIR="$HOME/nwp-primer/ifsdata/43r3" 16 | export OIFS_COMP="gnu" 17 | export OIFS_BUILD="opt" -------------------------------------------------------------------------------- /ansible/roles/rdtraining/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | training_host: '{{ ansible_facts.hostname }}.ecmwf-s2s-ml.ewchost.org' 3 | training_host_ip: '{{ ansible_facts.default_ipv4.address }}' 4 | #training_host_ips: '{{ ansible_facts.all_ipv4_addresses }}' 5 | #training_host: '{{ hostvars[inventory_hostname]["ansible_default_ipv4"]["address"] }}.xip.io' 6 | training_user: s2s 7 | training_user_refresh: no 8 | training_user_password_cache: '/tmp/{{ ansible_facts.hostname }}-{{ training_user }}.passwd' 9 | training_user_password: '{{ lookup("password", "{{ training_user_password_cache }} chars=ascii_letters") }}' 10 | trainer_user: s2sadmin 11 | trainer_pub_key: https://storage.ecmwf.europeanweather.cloud/s2s-ai-challenge/ewc/public/admin_rsa.pub 12 | # trainer_user_password: "{{ lookup('cypher','secret=secret/s2sadminpassword') | default('changeme') }}" # not used. DEVMODE : disable this 13 | # trainer_user_password: '{{ s2sadminpassword }}' # not used. DEVMODE : enable this 14 | conda_env_name: ecmwf-lab 15 | conda_env_concrete: yes 16 | creds_dir_name: '{{ ansible_facts.hostname }}-creds' 17 | key_name: '{{ ansible_facts.hostname }}.key' 18 | creds_dir: '/root/{{ creds_dir_name }}' 19 | 20 | # S3 Config 21 | s3_bucket_name: s2s-ai-challenge 22 | 23 | #cert_type: choose from 'import', 'selfsign', 'acme' 24 | # cert_type: import 25 | 26 | # For Imported Certificates from S3 27 | # cert_url: /creds/ssl/traininglab.crt 28 | # cert_key_url: /creds/ssl/traininglab.key 29 | 30 | # # For ACME - let's encrypt certs 31 | # enable_letsencrypt: no 32 | # acme_challenge_type: http-01 33 | # acme_directory: https://acme-v02.api.letsencrypt.org/directory 34 | # acme_version: 2 35 | # acme_email: certificate-reminders@ecmwf.europeanweather.cloud 36 | # letsencrypt_dir: /etc/letsencrypt 37 | # letsencrypt_keys_dir: /etc/letsencrypt/keys 38 | # letsencrypt_csrs_dir: /etc/letsencrypt/csrs 39 | # letsencrypt_certs_dir: /etc/letsencrypt/certs 40 | # letsencrypt_account_key: /etc/letsencrypt/account/account.key 41 | -------------------------------------------------------------------------------- /ansible/roles/rdtraining/files/jupyter.desktop: -------------------------------------------------------------------------------- 1 | [Desktop Entry] 2 | Name=Jupyter Lab 3 | Comment=Jupyter Lab 4 | Exec=chromium-browser --app=http://localhost:8888/lab 5 | Icon=jupyter 6 | Type=Application 7 | Categories=Development; -------------------------------------------------------------------------------- /ansible/roles/rdtraining/files/jupyter.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ecmwf-lab/climetlab-s2s-ai-challenge/9a63d7ce4393145b08eed1cec89f1c8c40039895/ansible/roles/rdtraining/files/jupyter.png -------------------------------------------------------------------------------- /ansible/roles/rdtraining/files/metview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ecmwf-lab/climetlab-s2s-ai-challenge/9a63d7ce4393145b08eed1cec89f1c8c40039895/ansible/roles/rdtraining/files/metview.png -------------------------------------------------------------------------------- /ansible/roles/rdtraining/files/scilab.desktop: -------------------------------------------------------------------------------- 1 | [Desktop Entry] 2 | Comment=Scientific software package for numerical computations 3 | Comment[fr]=Logiciel scientifique de calcul numérique 4 | Comment[de]=eine Wissenschaftssoftware für numerische Berechnungen 5 | Comment[ru]=Научная программа для численных расчётов 6 | Exec=bash -c "cd ~/PR-TC/sci/; scilab -f init.sci" 7 | GenericName=Scientific Software Package 8 | GenericName[fr]=Logiciel de calcul numérique 9 | GenericName[de]=Wissenschaftssoftware 10 | GenericName[ru]=Научный программный комплекс 11 | Icon=scilab 12 | MimeType=application/x-scilab-sci;application/x-scilab-sce;application/x-scilab-tst;application/x-scilab-dem;application/x-scilab-sod;application/x-scilab-xcos;application/x-scilab-zcos;application/x-scilab-bin;application/x-scilab-cosf;application/x-scilab-cos; 13 | Name=Scilab PR Training 14 | StartupNotify=false 15 | Terminal=false 16 | Type=Application 17 | Categories=Science;Math; 18 | Keywords=Science;Math;Numerical;Simulation 19 | -------------------------------------------------------------------------------- /ansible/roles/rdtraining/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Restart jupyter 3 | service: 4 | name: jupyter 5 | state: restarted 6 | 7 | - name: Restart nginx 8 | service: 9 | name: nginx 10 | state: restarted 11 | 12 | - name: Restart firewalld 13 | service: 14 | name: firewalld 15 | state: restarted 16 | 17 | - name: Reload daemons 18 | systemd: 19 | daemon_reload: yes 20 | -------------------------------------------------------------------------------- /ansible/roles/rdtraining/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | dependencies: 3 | - role: s3clients 4 | - role: conda 5 | -------------------------------------------------------------------------------- /ansible/roles/rdtraining/tasks/acme.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: "Create required directories in /etc/letsencrypt" 3 | file: 4 | path: "{{ letsencrypt_dir }}/{{ item }}" 5 | state: directory 6 | owner: root 7 | group: root 8 | mode: u=rwx,g=x,o=x 9 | with_items: 10 | - account 11 | - certs 12 | - csrs 13 | - keys 14 | 15 | - name: "Create .well-known/acme-challenge directory" 16 | file: 17 | path: /var/www/acme/.well-known/acme-challenge 18 | state: directory 19 | owner: root 20 | group: root 21 | mode: u=rwx,g=rx,o=rx 22 | 23 | - name: "Generate a Let's Encrypt account key" 24 | openssl_privatekey: 25 | path: '{{ letsencrypt_account_key }}' 26 | 27 | - name: "Generate Let's Encrypt private key" 28 | openssl_privatekey: 29 | path: '{{ letsencrypt_keys_dir }}/{{ training_host }}.key' 30 | 31 | - name: "Generate Certificate Request" 32 | openssl_csr: 33 | path: '{{ letsencrypt_csrs_dir }}/{{ training_host }}.csr' 34 | privatekey_path: '{{ letsencrypt_keys_dir }}/{{ training_host }}.key' 35 | country_name: UK 36 | organization_name: ECMWF 37 | email_address: support@europeanweather.cloud 38 | common_name: '{{ training_host }}' 39 | subject_alt_name: 'DNS:{{ training_host }}' 40 | 41 | - name: Check if Letsencrypt cert exists 42 | stat: 43 | path: /etc/ssl/certs/traininglab.crt 44 | register: certificate_present 45 | 46 | - block: 47 | - name: Link certificate 48 | file: 49 | path: /etc/ssl/certs/traininglab.crt 50 | state: link 51 | src: "{{ letsencrypt_certs_dir }}/fullchain_{{ training_host }}.crt" 52 | 53 | - name: Link certificate key 54 | file: 55 | path: /etc/ssl/private/traininglab.key 56 | state: link 57 | src: '{{ letsencrypt_keys_dir }}/{{ training_host }}.key' 58 | 59 | - name: Fallback Self-signed certificate if no Letsencrypt 60 | openssl_certificate: 61 | path: "{{ letsencrypt_certs_dir }}/fullchain_{{ training_host }}.crt" 62 | privatekey_path: '{{ letsencrypt_keys_dir }}/{{ training_host }}.key' 63 | csr_path: "{{ letsencrypt_csrs_dir }}/{{ training_host }}.csr" 64 | provider: selfsigned 65 | when: certificate_present.stat.exists == False 66 | notify: Restart nginx 67 | 68 | - name: Flush handlers 69 | meta: flush_handlers 70 | 71 | - name: "Begin Let's Encrypt challenges" 72 | acme_certificate: 73 | acme_directory: "{{ acme_directory }}" 74 | acme_version: "{{ acme_version }}" 75 | account_key_src: "{{ letsencrypt_account_key }}" 76 | account_email: "{{ acme_email }}" 77 | terms_agreed: 1 78 | challenge: "{{ acme_challenge_type }}" 79 | csr: "{{ letsencrypt_csrs_dir }}/{{ training_host }}.csr" 80 | dest: "{{ letsencrypt_certs_dir }}/{{ training_host }}.crt" 81 | fullchain_dest: "{{ letsencrypt_certs_dir }}/fullchain_{{ training_host }}.crt" 82 | remaining_days: 91 83 | register: acme_challenge 84 | 85 | - block: 86 | - name: Print challenge 87 | debug: 88 | msg: '{{ acme_challenge.challenge_data | dict2items }}' 89 | 90 | # - name: "Implement http-01 challenge files" 91 | # copy: 92 | # content: "{{ acme_challenge['challenge_data'][item]['http-01']['resource_value'] }}" 93 | # dest: "/var/www/acme/{{ acme_challenge['challenge_data'][item]['http-01']['resource'] }}" 94 | # owner: root 95 | # group: root 96 | # mode: u=rw,g=r,o=r 97 | # with_items: 98 | # - "{{ training_host }}" 99 | # - "www.{{ training_host }}" 100 | 101 | - name: "Implement http-01 challenge files" 102 | copy: 103 | dest: /var/www/acme/{{ item.value['http-01']['resource'] }} 104 | content: "{{ item.value['http-01']['resource_value'] }}" 105 | owner: root 106 | group: root 107 | mode: u=rw,g=r,o=r 108 | loop: "{{ acme_challenge.challenge_data | dict2items }}" 109 | # loop: "{{ acme_challenge.challenge_data | dictsort }}" 110 | 111 | - name: Ensure nginx is up 112 | service: 113 | name: nginx 114 | state: started 115 | 116 | - name: "Complete Let's Encrypt challenges" 117 | acme_certificate: 118 | acme_directory: "{{ acme_directory }}" 119 | acme_version: "{{ acme_version }}" 120 | account_key_src: "{{ letsencrypt_account_key }}" 121 | account_email: "{{ acme_email }}" 122 | challenge: "{{ acme_challenge_type }}" 123 | csr: "{{ letsencrypt_csrs_dir }}/{{ training_host }}.csr" 124 | dest: "{{ letsencrypt_certs_dir }}/{{ training_host }}.crt" 125 | chain_dest: "{{ letsencrypt_certs_dir }}/chain_{{ training_host }}.crt" 126 | fullchain_dest: "{{ letsencrypt_certs_dir }}/fullchain_{{ training_host }}.crt" 127 | data: "{{ acme_challenge }}" 128 | notify: Restart nginx 129 | when: acme_challenge is changed 130 | ignore_errors: yes 131 | -------------------------------------------------------------------------------- /ansible/roles/rdtraining/tasks/certs.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: "self signed certificate creation" 3 | include_tasks: selfsignedcert.yml 4 | when: cert_type == 'selfsign' 5 | 6 | - name: "Import certificate" 7 | include_tasks: importcert.yml 8 | when: cert_type == 'import' 9 | 10 | - name: "ACME certificate" 11 | include_tasks: acme.yml 12 | when: cert_type == 'acme' 13 | 14 | -------------------------------------------------------------------------------- /ansible/roles/rdtraining/tasks/conda.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Copy generic conda environment description 3 | template: 4 | src: ecmwf-lab-conda.j2 5 | dest: /tmp/environment.yml 6 | when: not conda_env_concrete 7 | register: conda_generic_env_updated 8 | 9 | - name: Copy concrete conda environment description 10 | template: 11 | src: ecmwf-lab-conda_concrete.yml.j2 12 | dest: /tmp/environment.yml 13 | when: conda_env_concrete 14 | register: conda_concrete_env_updated 15 | 16 | - name: Create conda environment for the training 17 | shell: | 18 | source /etc/profile.d/conda.sh 19 | conda env create -f /tmp/environment.yml 20 | args: 21 | executable: /bin/bash 22 | creates: '{{ conda_prefix }}/envs/{{ conda_env_name }}' 23 | register: conda_env_create 24 | 25 | - name: Update conda environment for the training 26 | shell: | 27 | source /etc/profile.d/conda.sh 28 | conda env update --prune -f /tmp/environment.yml 29 | args: 30 | executable: /bin/bash 31 | when: not conda_env_create.changed and ( conda_concrete_env_updated.changed or conda_generic_env_updated.changed ) 32 | 33 | - name: Ensure training environment loads by default 34 | become_user: '{{ training_user }}' 35 | lineinfile: 36 | path: /home/{{ training_user }}/.bashrc 37 | regexp: '^conda activate' 38 | line: 'conda activate {{ conda_env_name }}' 39 | 40 | - name: Disable PS1 conda change 41 | become_user: '{{ training_user }}' 42 | lineinfile: 43 | path: /home/{{ training_user }}/.condarc 44 | regexp: '^changeps1:' 45 | line: 'changeps1: false' 46 | create: yes 47 | 48 | - name: Ensure Desktop directory 49 | become_user: '{{ training_user }}' 50 | file: 51 | path: '~/Desktop' 52 | state: directory 53 | 54 | - name: Create Metview shortcut 55 | become_user: '{{ training_user }}' 56 | copy: 57 | src: '{{ conda_prefix }}/envs/{{ conda_env_name }}/lib/metview-bundle/share/applications/metview.desktop' 58 | dest: '~/Desktop/metview.desktop' 59 | remote_src: yes 60 | mode: '0755' 61 | 62 | - name: Ensure custom icons directory 63 | become_user: '{{ training_user }}' 64 | file: 65 | path: '~/.icons' 66 | state: directory 67 | 68 | - name: Copy metview icon 69 | become_user: '{{ training_user }}' 70 | copy: 71 | src: 'files/metview.png' 72 | dest: '~/.icons/metview.png' 73 | mode: '0755' 74 | -------------------------------------------------------------------------------- /ansible/roles/rdtraining/tasks/git.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Ensure SSL private dir on CentOS 3 | file: 4 | src: ../pki/tls/private 5 | dest: /etc/ssl/private 6 | state: link 7 | when: ansible_distribution == 'CentOS' 8 | 9 | - name: Copy morpheus key for git access - local 10 | copy: 11 | src: files/morpheus_rsa 12 | dest: '/etc/ssl/private/morpheus_rsa' 13 | mode: 0644 14 | when: morpheus is not defined 15 | 16 | - name: Fetch morpheus key for git access 17 | copy: 18 | content: '{{ lookup("cypher","secret=secret/morpheus_rsa_git owner=True") }}' 19 | dest: '/etc/ssl/private/morpheus_rsa' 20 | mode: 0644 21 | when: morpheus is defined 22 | 23 | - name: Configure git access 24 | blockinfile: 25 | path: /etc/ssh/ssh_config 26 | create: yes 27 | mode: 0644 28 | block: | 29 | Host git.ecmwf.int 30 | User morpheus 31 | IdentityFile /etc/ssl/private/morpheus_rsa 32 | 33 | - name: Trust git.ecmwf.int ssh host key 34 | known_hosts: 35 | path: /etc/ssh/ssh_known_hosts 36 | name: git.ecmwf.int 37 | key: 'git.ecmwf.int,136.156.180.229 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDov7Bi+/r0VgNGF7Nr/xJDnDZ0Y32kKYEmaxJMNswje61GBCz4OU0pMbxyp/zkVlgvWRtkhXQGVN8P/o6Et2YGAr1WvKAoFt8qYaJRMd0aGsWIYpRsFUf7zXMENoVZZW2cizHKlQwcZ6cSUYgDG1WHPcP2HYGjhdBbq+WO8u7UMi4xmwUSxRaigYT+teuZy5U3lcNtByCctUiAnhGtE7yiKpFVpcuFKOw0uFlCHFY3q+BP4O/t3vYO0p4AzUtVyvL1U/aO9iNo14Yhi3mPhR43GkrLz/cphanq1KSHs3K+Tq0IMt7iWfcumhYIOHK/nkFqQjVOehlyK1/k7zZNDa+5' 38 | 39 | -------------------------------------------------------------------------------- /ansible/roles/rdtraining/tasks/importcert.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Checkout Cert from S3 3 | aws_s3: 4 | s3_url: '{{ s3_url }}' 5 | bucket: '{{ s3_bucket_name }}' 6 | validate_certs: no 7 | rgw: yes 8 | mode: get 9 | object: '{{ cert_url }}' 10 | dest: /etc/ssl/certs/traininglab.crt 11 | overwrite: different 12 | notify: Restart nginx 13 | 14 | - name: Checkout Cert Key from S3 15 | aws_s3: 16 | s3_url: '{{ s3_url }}' 17 | bucket: '{{ s3_bucket_name }}' 18 | validate_certs: no 19 | rgw: yes 20 | mode: get 21 | object: '{{ cert_key_url }}' 22 | dest: /etc/ssl/private/traininglab.key 23 | overwrite: different 24 | notify: Restart nginx 25 | -------------------------------------------------------------------------------- /ansible/roles/rdtraining/tasks/jupyter.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Ensure ACL is available 3 | package: 4 | name: acl 5 | state: present 6 | 7 | - name: Install jupyterlab in conda environment 8 | shell: | 9 | source /etc/profile.d/conda.sh 10 | conda activate {{ conda_env_name }} 11 | conda install -c conda-forge jupyterlab -y 12 | args: 13 | executable: /bin/bash 14 | creates: '{{ conda_prefix }}/envs/{{ conda_env_name }}/bin/jupyter' 15 | 16 | - name: Install plotly jupyterlab extensions 17 | shell: | 18 | source /etc/profile.d/conda.sh 19 | conda activate {{ conda_env_name }} 20 | jupyter labextension install jupyterlab-plotly 21 | jupyter labextension install @jupyter-widgets/jupyterlab-manager plotlywidget 22 | args: 23 | executable: /bin/bash 24 | creates: '{{ conda_prefix }}/envs/{{ conda_env_name }}/share/jupyter/lab/extensions/jupyterlab-plotly-*.tgz' 25 | 26 | - name: Create jupyterlab config 27 | become_user: '{{ training_user }}' 28 | shell: | 29 | source /etc/profile.d/conda.sh 30 | conda activate {{ conda_env_name }} 31 | jupyter notebook --generate-config -y 32 | args: 33 | executable: /bin/bash 34 | creates: /home/{{ training_user }}/.jupyter/jupyter_notebook_config.py 35 | 36 | - name: Create the systemd service for jupyterlab 37 | template: 38 | src: templates/jupyter.service.j2 39 | dest: /etc/systemd/system/jupyter.service 40 | notify: 41 | - Reload daemons 42 | - Restart jupyter 43 | 44 | 45 | # - name: Encrypt jupyterlab password 46 | # become_user: '{{ training_user }}' 47 | # shell: | 48 | # source /etc/profile.d/conda.sh 49 | # conda activate jupyterlab 50 | # python -c "from notebook.auth import passwd; print(passwd('{{ jupyterlab_password }}'))" 51 | # register: encrypted 52 | # args: 53 | # executable: /bin/bash 54 | # 55 | # - name: Configure jupyterlab with password 56 | # lineinfile: 57 | # path: /home/{{ training_user }}/.jupyter/jupyter_notebook_config.py 58 | # regexp: '^c.NotebookApp.password ' 59 | # insertafter: '^# c.NotebookApp.password ' 60 | # line: c.NotebookApp.password = u'{{ encrypted.stdout }}' 61 | - 62 | - name: Configure jupyterlab with no password 63 | lineinfile: 64 | path: /home/{{ training_user }}/.jupyter/jupyter_notebook_config.py 65 | regexp: '^c.NotebookApp.password ' 66 | insertafter: '^# c.NotebookApp.password ' 67 | line: c.NotebookApp.password = u'' 68 | notify: Restart jupyter 69 | 70 | - name: Configure jupyterlab with no token 71 | lineinfile: 72 | path: /home/{{ training_user }}/.jupyter/jupyter_notebook_config.py 73 | regexp: '^c.NotebookApp.token ' 74 | insertafter: '^# c.NotebookApp.token ' 75 | line: c.NotebookApp.token = u'' 76 | notify: Restart jupyter 77 | 78 | - name: Ensure jupyter service is up and enabled 79 | service: 80 | name: jupyter 81 | state: started 82 | enabled: yes 83 | 84 | - name: Create Jupyter shortcut 85 | become_user: '{{ training_user }}' 86 | copy: 87 | src: 'files/jupyter.desktop' 88 | dest: '~/Desktop/jupyter.desktop' 89 | mode: '0755' 90 | 91 | - name: Ensure custom icons directory 92 | become_user: '{{ training_user }}' 93 | file: 94 | path: '~/.icons' 95 | state: directory 96 | 97 | - name: Copy jupyter icon 98 | become_user: '{{ training_user }}' 99 | copy: 100 | src: 'files/jupyter.png' 101 | dest: '~/.icons/jupyter.png' 102 | mode: '0755' 103 | -------------------------------------------------------------------------------- /ansible/roles/rdtraining/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - block: 3 | - name: Prepare Software layer 4 | include_tasks: softwaredeps.yml 5 | tags: softwaredeps 6 | 7 | - block: 8 | - name: Mount /data disk 9 | include_tasks: mountdisk.yml 10 | 11 | #- block: 12 | # - name: Prepare Git settings 13 | # include_tasks: git.yml 14 | # tags: git 15 | 16 | - block: 17 | - name: Prepare Training user 18 | include_tasks: user.yml 19 | tags: user 20 | 21 | #- block: 22 | # - name: Prepare Conda environment 23 | # include_tasks: conda.yml 24 | # tags: conda 25 | # 26 | #- block: 27 | # - name: Prepare Jupyter service 28 | # include_tasks: jupyter.yml 29 | # tags: jupyter 30 | # 31 | #- block: 32 | # - name: Enable HTTP(s) reverse proxy 33 | # include_tasks: proxy.yml 34 | # tags: proxy 35 | 36 | -------------------------------------------------------------------------------- /ansible/roles/rdtraining/tasks/mountdisk.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: "create a ext4 filesystem on vdb disk" 3 | filesystem: 4 | fstype: ext4 5 | dev: /dev/vdb 6 | opts: -L data 7 | - name: Creates /data directory 8 | file: 9 | path: /data 10 | state: directory 11 | - name: "Mount /data disk" 12 | shell: | 13 | echo "LABEL=data /data ext4 defaults 0 0" | sudo tee -a /etc/fstab > /dev/null 14 | mount -av 15 | -------------------------------------------------------------------------------- /ansible/roles/rdtraining/tasks/proxy.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Ensure EPEL repo is enabled on CentOS 3 | yum: 4 | name: epel-release 5 | state: latest 6 | when: ansible_distribution == 'CentOS' 7 | 8 | - name: Ensure nginx is installed and latest 9 | package: 10 | name: nginx 11 | state: latest 12 | 13 | - name: Firewalld exceptions 14 | block: 15 | - name: Gather the package facts 16 | package_facts: 17 | manager: auto 18 | # - name: Allow HTTP/HTTPS traffic if firewalld is active 19 | # command: firewall-cmd --permanent --add-service=http --add-service=https 20 | # when: "'firewalld' in ansible_facts.packages" 21 | # notify: Restart firewalld 22 | - name: Allow HTTP/HTTPS traffic if firewalld is active 23 | # ansible.posix.firewalld: 24 | firewalld: 25 | service: '{{ item }}' 26 | permanent: yes 27 | state: enabled 28 | loop: 29 | - http 30 | - https 31 | when: "'firewalld' in ansible_facts.packages" 32 | notify: Restart firewalld 33 | when: ansible_distribution == 'CentOS' 34 | 35 | - name: Ensure nginx is enabled 36 | service: 37 | name: nginx 38 | enabled: yes 39 | 40 | - name: Ensure SSL private dir on CentOS 41 | file: 42 | src: ../pki/tls/private 43 | dest: /etc/ssl/private 44 | state: link 45 | when: ansible_distribution == 'CentOS' 46 | 47 | - name: Diffie-Hellman parameters 48 | openssl_dhparam: 49 | path: /etc/ssl/dhparams.pem 50 | size: 2048 51 | # command: 52 | # cmd: openssl dhparam -out /etc/ssl/dhparams.pem 2048 53 | # creates: /etc/ssl/dhparams.pem 54 | 55 | - name: Install passlib 56 | package: 57 | name: python-passlib 58 | 59 | - name: Set up HTTP authentication 60 | htpasswd: 61 | path: /etc/nginx/.htpasswd 62 | name: '{{ training_user }}' 63 | password: '{{ training_user_password }}' 64 | owner: root 65 | group: "{{ 'www-data' if ansible_distribution == 'Ubuntu' else 'nginx' }}" 66 | mode: 0640 67 | notify: Restart nginx 68 | 69 | #- name: Encode htpasswd 70 | # command: 71 | # cmd: 'openssl passwd -apr1 {{ lookup("password", "/root/{{training_user}}.passwd chars=ascii_letters") }}' 72 | # register: encoded_password 73 | # 74 | #- name: Add a user to a password file and ensure permissions are set 75 | # lineinfile: 76 | # path: /etc/nginx/.htpasswd 77 | # create: yes 78 | # state: present 79 | # line: '{{ training_user }}:' 80 | # regexp: '^{{ training_user }}:' 81 | # owner: root 82 | # group: "{{ 'www-data' if ansible_distribution == 'Ubuntu' else 'nginx' }}" 83 | # mode: 0640 84 | # notify: Restart nginx 85 | 86 | # - debug: 87 | # msg: Using password {{ jupyterlab_password }} because user is {{ ansible_user }} 88 | 89 | - name: Remove default nginx site for Ubuntu 90 | file: 91 | name: /etc/nginx/sites-enabled/default 92 | state: absent 93 | when: ansible_distribution == 'Ubuntu' 94 | 95 | - name: Remove default nginx site for CentOS 96 | replace: 97 | path: /etc/nginx/nginx.conf 98 | regexp: "^([^#].* default_server.*)" 99 | replace: '#\1' 100 | when: ansible_distribution == 'CentOS' 101 | notify: Restart nginx 102 | 103 | - name: Write the site config file for Ubuntu 104 | template: 105 | src: templates/jupyter-revproxy.conf.j2 106 | dest: /etc/nginx/sites-enabled/jupyter-revproxy.conf 107 | notify: Restart nginx 108 | when: ansible_distribution == 'Ubuntu' 109 | 110 | - name: Write the site config file for CentOS 111 | template: 112 | src: templates/jupyter-revproxy.conf.j2 113 | dest: /etc/nginx/conf.d/jupyter-revproxy.conf 114 | notify: Restart nginx 115 | when: ansible_distribution == 'CentOS' 116 | 117 | - name: Prepare Certificates 118 | include_tasks: certs.yml 119 | -------------------------------------------------------------------------------- /ansible/roles/rdtraining/tasks/selfsignedcert.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: "Generate Cert private key" 3 | openssl_privatekey: 4 | path: '/etc/ssl/private/traininglab-self.key' 5 | 6 | - name: "Generate Certificate Request" 7 | openssl_csr: 8 | path: '/etc/ssl/private/traininglab-self.csr' 9 | privatekey_path: '/etc/ssl/private/traininglab-self.key' 10 | country_name: UK 11 | organization_name: ECMWF 12 | email_address: support@europeanweather.cloud 13 | common_name: '{{ training_host }}' 14 | subject_alt_name: 'DNS:{{ training_host }}' 15 | 16 | - name: Self-signed certificate 17 | openssl_certificate: 18 | path: "/etc/ssl/certs/traininglab-self.crt" 19 | privatekey_path: '/etc/ssl/private/traininglab-self.key' 20 | csr_path: "/etc/ssl/private/traininglab-self.csr" 21 | provider: selfsigned 22 | 23 | - name: Link certificate 24 | file: 25 | path: /etc/ssl/certs/traininglab.crt 26 | state: link 27 | src: /etc/ssl/certs/traininglab-self.crt 28 | notify: Restart nginx 29 | 30 | - name: Link certificate key 31 | file: 32 | path: /etc/ssl/private/traininglab.key 33 | state: link 34 | src: /etc/ssl/private/traininglab-self.key 35 | notify: Restart nginx 36 | -------------------------------------------------------------------------------- /ansible/roles/rdtraining/tasks/softwaredeps.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Ensure some system deps are present on CentOS 3 | package: 4 | name: 5 | - libXcomposite 6 | - libXcursor 7 | - libXi 8 | - libXtst 9 | - libXrandr 10 | - libXdamage 11 | - libXScrnSaver 12 | - xorg-x11-utils 13 | - mesa-libEGL 14 | - mesa-libGL 15 | - alsa-lib 16 | - curl 17 | - bzip2 18 | - cmake3 19 | - boost 20 | - cpp 21 | - vim 22 | - tree 23 | - htop 24 | - tmux 25 | - s3fs-fuse 26 | - git 27 | - xterm 28 | - "@Development tools" 29 | - pandoc 30 | - texlive 31 | - texlive-latex 32 | - python-boto 33 | - python-boto3 34 | - python2-botocore 35 | - emacs 36 | - nano 37 | state: present 38 | when: ansible_distribution == 'CentOS' 39 | 40 | - name: Link cmake3 binaries to generic equivalents 41 | file: 42 | src: '/usr/bin/{{ item }}3' 43 | dest: '/usr/bin/{{ item }}' 44 | state: link 45 | with_items: 46 | - cmake 47 | - ccmake 48 | - ctest 49 | - cpack 50 | when: ansible_distribution == 'CentOS' 51 | 52 | - name: Ensure some system deps are present on Ubuntu 53 | package: 54 | name: '{{ item }}' 55 | state: present 56 | with_items: 57 | - libgl1-mesa-glx 58 | - libxrender1 59 | - xauth 60 | - x11-utils 61 | - cmake 62 | - boost 63 | - cpp 64 | - emacs 65 | - nano 66 | when: ansible_distribution == 'Ubuntu' 67 | -------------------------------------------------------------------------------- /ansible/roles/rdtraining/tasks/user.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - block: 3 | - name: Delete cached password 4 | become: no 5 | local_action: 6 | module: file 7 | path: '{{ training_user_password_cache }}' 8 | state: absent 9 | 10 | - name: Kill all training user processes 11 | shell: 'killall -u {{ training_user }} || true' 12 | 13 | - name: Wipe /tmp files for training user 14 | command: 'find /tmp -user {{ training_user }} -delete' 15 | 16 | - name: Wipe training user 17 | user: 18 | name: '{{ training_user }}' 19 | state: absent 20 | remove: yes 21 | notify: Restart jupyter 22 | when: training_user_refresh 23 | 24 | - name: Create training user 25 | user: 26 | name: '{{ training_user }}' 27 | groups: 'wheel' 28 | comment: Training user 29 | generate_ssh_key: yes 30 | ssh_key_bits: 2048 31 | ssh_key_file: .ssh/id_rsa 32 | ssh_key_comment: '{{ training_user }}@{{ ansible_host }}' 33 | password: '{{ training_user_password | password_hash("sha512", 65534 | random(seed=inventory_hostname) | string) }}' 34 | state: present 35 | 36 | - name: Open up training user home to group 37 | file: 38 | path: '/home/{{ training_user }}' 39 | state: directory 40 | mode: '0750' 41 | 42 | - name: Read SSH Public key 43 | slurp: 44 | src: '/home/{{ training_user }}/.ssh/id_rsa.pub' 45 | register: ssh_pub_key_b64 46 | 47 | - name: Set authorised SSH key 48 | authorized_key: 49 | user: '{{ training_user }}' 50 | state: present 51 | key: "{{ ssh_pub_key_b64.content | b64decode }}" 52 | 53 | - name: Prepare credentials directory 54 | file: 55 | path: '{{ creds_dir }}' 56 | state: directory 57 | 58 | - name: Copy README 59 | template: 60 | src: README.md 61 | dest: '{{ creds_dir }}' 62 | register: readmemd 63 | 64 | - name: Generate PDF 65 | shell: 'pandoc {{ creds_dir }}/README.md -o {{ creds_dir }}/README.pdf' 66 | args: 67 | executable: /bin/bash 68 | when: readmemd.changed 69 | 70 | - name: Prepare credentials 71 | copy: 72 | src: '/home/{{ training_user }}/.ssh/id_rsa' 73 | dest: '{{ creds_dir }}/{{ key_name }}' 74 | mode: preserve 75 | remote_src: yes 76 | 77 | - name: Prepare credentials zip 78 | archive: 79 | path: '{{ creds_dir }}' 80 | dest: '{{ creds_dir }}.zip' 81 | exclude_path: 82 | - '{{ creds_dir }}/README.md' 83 | format: zip 84 | 85 | - name: Retrieve credentials zip 86 | fetch: 87 | src: '{{ creds_dir }}.zip' 88 | dest: '{{ creds_dir_name }}.zip' 89 | flat: yes 90 | when: morpheus is not defined 91 | 92 | - name: Create S3 bucket 93 | s3_bucket: 94 | s3_url: '{{ s3_url }}' 95 | validate_certs: no 96 | ceph: yes 97 | name: '{{ s3_bucket_name }}' 98 | 99 | - name: Push credentials pack to S3 100 | aws_s3: 101 | s3_url: '{{ s3_url }}' 102 | bucket: '{{ s3_bucket_name }}' 103 | validate_certs: no 104 | rgw: yes 105 | mode: put 106 | object: '/ewc/creds/{{ creds_dir | basename }}.zip' 107 | src: '{{ creds_dir }}.zip' 108 | overwrite: different 109 | 110 | - name: Write IP and hostname 111 | shell: 'echo {{ training_host }},{{ training_host_ip }} > {{ creds_dir }}/{{ training_host }}.ip' 112 | args: 113 | executable: /bin/bash 114 | - name: Push IP to S3 115 | aws_s3: 116 | s3_url: '{{ s3_url }}' 117 | bucket: '{{ s3_bucket_name }}' 118 | validate_certs: no 119 | rgw: yes 120 | mode: put 121 | object: '/ewc/creds/{{ creds_dir | basename }}.ip' 122 | src: '{{ creds_dir }}/{{ training_host}}.ip' 123 | overwrite: different 124 | 125 | 126 | - name: Create trainer user 127 | user: 128 | name: '{{ trainer_user }}' 129 | comment: Trainer 130 | groups: '{{ training_user }},wheel' 131 | # password: '{{ trainer_user_password | string }}' 132 | state: present 133 | 134 | 135 | - name: Set authorised SSH key for trainer 136 | authorized_key: 137 | user: '{{ trainer_user }}' 138 | state: present 139 | key: "{{ trainer_pub_key }}" 140 | when: trainer_pub_key is defined 141 | 142 | - name: sudo without password for wheel group 143 | copy: 144 | content: '%wheel ALL=(ALL:ALL) NOPASSWD:ALL' 145 | # sudo with password : 146 | # content: '%wheel ALL=(ALL:ALL) ALL' 147 | dest: /etc/sudoers.d/wheel_sudo_no_password 148 | mode: 0440 149 | 150 | -------------------------------------------------------------------------------- /ansible/roles/rdtraining/templates/README.md: -------------------------------------------------------------------------------- 1 | # Welcome to your ECMWF S2S instance 2 | 3 | Here are the details for your session: 4 | 5 | * Host: {{ training_host }} 6 | * User: {{ training_user }} 7 | * Password: {{ training_user_password }} 8 | * SSH key file: {{ key_name }} 9 | 10 | ## How to connect via command line 11 | 12 | Alternatively, you can also connect to your lab on the command line, with no graphical desktop, using SSH. 13 | On Linux or Mac, or if running on Windows using WSL, you can do: 14 | 15 | ssh -i {{ key_name }} -l {{ training_user }} {{ training_host }} 16 | 17 | On Windows, you may also use any other SSH client such as PuTTY or MobaXterm. 18 | 19 | Note that you don't need to type the password, authentication is done through the key file provided. 20 | 21 | ## Environment 22 | 23 | The /home disk is small and will be easily filled up. 24 | There is a larger disk on /data: you should use /data/{{ training_user }} to store your data. 25 | 26 | Conda and CliMetLab are pre-installed, as root, with some default packages. 27 | You can create your own conda environment as the "{{ training_user }}" user. 28 | CliMetLab is preinstalled for the user "{{ training_user }}", its cache directory has been set to /data/{{ training_user }}/tmp-climetlab/. 29 | 30 | This setup is experimental, it is expected to fit your needs, but has not been extensively tested. Do not hesitate to adapt it. 31 | For instance, you may want to setup a Jupyter lab instance and use SSH tunnel to access it. 32 | 33 | This setup is intented for a single user "{{ training_user }}". Feel free to create more users if needed. Please do not remove the already existing users: mafp and s2sadmin. 34 | 35 | ## Support 36 | 37 | This virtual machine is provided on a best effort basis. 38 | When using these resources, feedback is very welcome (on https://renkulab.io/gitlab/aaron.spring/s2s-ai-challenge/-/issues) but no support is provided to maintain the system. 39 | That is the reason why you have sudo permissions on the machine. 40 | 41 | ## Terms and conditions 42 | 43 | The credentials have been provided to you for the sole purpose of participating to the "S2S challenge" as described in https://s2s-ai-challenge.github.io/ to improve subseasonal-to-seasonal precipitation and temperature forecasts with Machine Learning/Artificial Intelligence. 44 | To use the EWC, you must agree with the terms and conditions (https://confluence.ecmwf.int/display/EWCLOUDKB/Terms+and+Conditions). If you do not agree with them, do not use these credentials. 45 | Unless otherwise notified, the credentials will be disabled and the virtual machines deleted (including the data) upon termination of the "S2S challenge". 46 | -------------------------------------------------------------------------------- /ansible/roles/rdtraining/templates/ecmwf-lab-conda.j2: -------------------------------------------------------------------------------- 1 | name: {{ conda_env_name }} 2 | channels: 3 | - conda-forge 4 | - defaults 5 | dependencies: 6 | - compilers 7 | - python 8 | - openmpi 9 | - libnetcdf 10 | - netcdf-fortran 11 | - jupyterlab 12 | - jupyterlab-git 13 | - jupyterlab-favorites 14 | - ipywidgets 15 | - metview 16 | - metview-python 17 | - nodejs 18 | - pandas 19 | - xarray 20 | - seaborn 21 | - pyspharm 22 | - pyfftw 23 | - joblib 24 | - tqdm 25 | - gmsh 26 | - perl 27 | - perl-xml-parser 28 | - git 29 | - boost 30 | - plotly 31 | - yaml 32 | - pyyaml 33 | - pip 34 | - pip: 35 | - eccodes 36 | -------------------------------------------------------------------------------- /ansible/roles/rdtraining/templates/ecmwf-lab-conda_concrete.yml.j2: -------------------------------------------------------------------------------- 1 | name: {{ conda_env_name }} 2 | channels: 3 | - conda-forge 4 | - defaults 5 | dependencies: 6 | - _libgcc_mutex=0.1=conda_forge 7 | - _openmp_mutex=4.5=1_gnu 8 | - alsa-lib=1.2.3=h516909a_0 9 | - anyio=3.1.0=py39hf3d152e_0 10 | - argon2-cffi=20.1.0=py39h3811e60_2 11 | - async_generator=1.10=py_0 12 | - attrs=21.2.0=pyhd8ed1ab_0 13 | - babel=2.9.1=pyh44b312d_0 14 | - backcall=0.2.0=pyh9f0ad1d_0 15 | - backports=1.0=py_2 16 | - backports.functools_lru_cache=1.6.4=pyhd8ed1ab_0 17 | - binutils=2.35.1=hdd6e379_2 18 | - binutils_impl_linux-64=2.35.1=h193b22a_2 19 | - binutils_linux-64=2.35=h67ddf6f_30 20 | - bison=3.4=h58526e2_1 21 | - bleach=3.3.0=pyh44b312d_0 22 | - boost=1.76.0=py39h5472131_0 23 | - boost-cpp=1.76.0=hc6e9bd1_0 24 | - brotlipy=0.7.0=py39h3811e60_1001 25 | - bzip2=1.0.8=h7f98852_4 26 | - c-ares=1.17.1=h7f98852_1 27 | - c-compiler=1.1.3=h7f98852_0 28 | - ca-certificates=2020.12.5=ha878542_0 29 | - cairo=1.16.0=h6cf1ce9_1008 30 | - certifi=2020.12.5=py39hf3d152e_1 31 | - cffi=1.14.5=py39he32792d_0 32 | - cfgrib=0.9.9.0=pyhd8ed1ab_1 33 | - chardet=4.0.0=py39hf3d152e_1 34 | - click=8.0.1=py39hf3d152e_0 35 | - colorama=0.4.4=pyh9f0ad1d_0 36 | - compilers=1.1.3=ha770c72_0 37 | - cryptography=3.4.7=py39hbca0aa6_0 38 | - curl=7.76.1=hea6ffbf_2 39 | - cxx-compiler=1.1.3=h4bd325d_0 40 | - cycler=0.10.0=py_2 41 | - dbus=1.13.6=h48d8840_2 42 | - decorator=5.0.9=pyhd8ed1ab_0 43 | - defusedxml=0.7.1=pyhd8ed1ab_0 44 | - eccodes=2.22.0=he2bb022_0 45 | - entrypoints=0.3=pyhd8ed1ab_1003 46 | - expat=2.3.0=h9c3ff4c_0 47 | - fftw=3.3.9=nompi_h74d3f13_101 48 | - flex=2.6.4=h58526e2_1004 49 | - font-ttf-dejavu-sans-mono=2.37=hab24e00_0 50 | - font-ttf-inconsolata=3.000=h77eed37_0 51 | - font-ttf-source-code-pro=2.038=h77eed37_0 52 | - font-ttf-ubuntu=0.83=hab24e00_0 53 | - fontconfig=2.13.1=hba837de_1005 54 | - fonts-conda-ecosystem=1=0 55 | - fonts-conda-forge=1=0 56 | - fortran-compiler=1.1.3=h1990efc_0 57 | - freetype=2.10.4=h0708190_1 58 | - fribidi=1.0.10=h36c2ea0_0 59 | - gcc_impl_linux-64=9.3.0=h70c0ae5_19 60 | - gcc_linux-64=9.3.0=hf25ea35_30 61 | - gdbm=1.18=h0a1914f_2 62 | - gettext=0.19.8.1=h0b5b191_1005 63 | - gfortran_impl_linux-64=9.3.0=hc4a2995_19 64 | - gfortran_linux-64=9.3.0=hdc58fab_30 65 | - git=2.30.2=pl5320h24fefe6_1 66 | - gitdb=4.0.7=pyhd8ed1ab_0 67 | - gitpython=3.1.17=pyhd8ed1ab_0 68 | - glib=2.68.2=h9c3ff4c_0 69 | - glib-tools=2.68.2=h9c3ff4c_0 70 | - gmp=6.2.1=h58526e2_0 71 | - gmsh=4.6.0=hd134328_0 72 | - graphite2=1.3.13=h58526e2_1001 73 | - gst-plugins-base=1.18.4=hf529b03_2 74 | - gstreamer=1.18.4=h76c114f_2 75 | - gxx_impl_linux-64=9.3.0=hd87eabc_19 76 | - gxx_linux-64=9.3.0=h3fbe746_30 77 | - harfbuzz=2.8.1=h83ec7ef_0 78 | - hdf4=4.2.15=h10796ff_3 79 | - hdf5=1.10.6=nompi_h6a2412b_1114 80 | - icu=68.1=h58526e2_0 81 | - idna=2.10=pyh9f0ad1d_0 82 | - importlib-metadata=4.0.1=py39hf3d152e_0 83 | - ipykernel=5.5.5=py39hef51801_0 84 | - ipython=7.23.1=py39hef51801_0 85 | - ipython_genutils=0.2.0=py_1 86 | - ipywidgets=7.6.3=pyhd3deb0d_0 87 | - jasper=1.900.1=h07fcdf6_1006 88 | - jedi=0.18.0=py39hf3d152e_2 89 | - jinja2=3.0.1=pyhd8ed1ab_0 90 | - joblib=1.0.1=pyhd8ed1ab_0 91 | - jpeg=9d=h36c2ea0_0 92 | - json5=0.9.5=pyh9f0ad1d_0 93 | - jsonschema=3.2.0=pyhd8ed1ab_3 94 | - jupyter_client=6.1.12=pyhd8ed1ab_0 95 | - jupyter_core=4.7.1=py39hf3d152e_0 96 | - jupyter_server=1.7.0=py39hf3d152e_1 97 | - jupyter-server-mathjax=0.2.2=pyhd8ed1ab_0 98 | - jupyterlab=3.0.16=pyhd8ed1ab_0 99 | - jupyterlab-favorites=3.0.0=pyhd8ed1ab_0 100 | - jupyterlab-git=0.30.1=pyhd8ed1ab_0 101 | - jupyterlab_pygments=0.1.2=pyh9f0ad1d_0 102 | - jupyterlab_server=2.5.2=pyhd8ed1ab_0 103 | - jupyterlab_widgets=1.0.0=pyhd8ed1ab_1 104 | - kernel-headers_linux-64=2.6.32=h77966d4_13 105 | - kiwisolver=1.3.1=py39h1a9c180_1 106 | - krb5=1.19.1=hcc1bbae_0 107 | - lcms2=2.12=hddcbb42_0 108 | - ld_impl_linux-64=2.35.1=hea4e1c9_2 109 | - libaec=1.0.4=h9c3ff4c_1 110 | - libblas=3.9.0=9_openblas 111 | - libcblas=3.9.0=9_openblas 112 | - libclang=11.1.0=default_ha53f305_1 113 | - libcurl=7.76.1=h2574ce0_2 114 | - libedit=3.1.20191231=he28a2e2_2 115 | - libev=4.33=h516909a_1 116 | - libevent=2.1.10=hcdb4288_3 117 | - libffi=3.3=h58526e2_2 118 | - libgcc-devel_linux-64=9.3.0=h7864c58_19 119 | - libgcc-ng=9.3.0=h2828fa1_19 120 | - libgfortran-ng=9.3.0=hff62375_19 121 | - libgfortran5=9.3.0=hff62375_19 122 | - libglib=2.68.2=h3e27bee_0 123 | - libglu=9.0.0=he1b5a44_1001 124 | - libgomp=9.3.0=h2828fa1_19 125 | - libiconv=1.16=h516909a_0 126 | - liblapack=3.9.0=9_openblas 127 | - libllvm11=11.1.0=hf817b99_2 128 | - libnetcdf=4.8.0=nompi_hcd642e3_103 129 | - libnghttp2=1.43.0=h812cca2_0 130 | - libogg=1.3.4=h7f98852_1 131 | - libopenblas=0.3.15=pthreads_h8fe5266_1 132 | - libopus=1.3.1=h7f98852_1 133 | - libpng=1.6.37=h21135ba_2 134 | - libpq=13.3=hd57d9b9_0 135 | - libsodium=1.0.18=h36c2ea0_1 136 | - libssh2=1.9.0=ha56f1ee_6 137 | - libstdcxx-devel_linux-64=9.3.0=hb016644_19 138 | - libstdcxx-ng=9.3.0=h6de172a_19 139 | - libtiff=4.2.0=hbd63e13_2 140 | - libuuid=2.32.1=h7f98852_1000 141 | - libuv=1.41.0=h7f98852_0 142 | - libvorbis=1.3.7=h9c3ff4c_0 143 | - libwebp-base=1.2.0=h7f98852_2 144 | - libxcb=1.13=h7f98852_1003 145 | - libxkbcommon=1.0.3=he3ba5ed_0 146 | - libxml2=2.9.12=h72842e0_0 147 | - libzip=1.7.3=h4de3113_0 148 | - lz4-c=1.9.3=h9c3ff4c_0 149 | - m4=1.4.18=h516909a_1001 150 | - magics-metview=4.8.0=he8d2e36_0 151 | - markupsafe=2.0.1=py39h3811e60_0 152 | - matplotlib-base=3.4.2=py39h2fa2bec_0 153 | - matplotlib-inline=0.1.2=pyhd8ed1ab_2 154 | - metview=5.12.0=hd9a45fa_0 155 | - metview-python=1.7.2=pyhd8ed1ab_0 156 | - mistune=0.8.4=py39h3811e60_1003 157 | - mpi=1.0=openmpi 158 | - mysql-common=8.0.23=ha770c72_2 159 | - mysql-libs=8.0.23=h935591d_2 160 | - nbclassic=0.2.8=pyhd8ed1ab_0 161 | - nbclient=0.5.3=pyhd8ed1ab_0 162 | - nbconvert=6.0.7=py39hf3d152e_3 163 | - nbdime=3.1.0=pyhd8ed1ab_0 164 | - nbformat=5.1.3=pyhd8ed1ab_0 165 | - ncurses=6.2=h58526e2_4 166 | - nest-asyncio=1.5.1=pyhd8ed1ab_0 167 | - netcdf-fortran=4.5.3=nompi_hf3f1587_104 168 | - nodejs=15.14.0=h92b4a50_0 169 | - notebook=6.4.0=pyha770c72_0 170 | - nspr=4.30=h9c3ff4c_0 171 | - nss=3.65=hb5efdd6_0 172 | - numpy=1.20.2=py39hdbf815f_0 173 | - occt=7.4.0=h9121d39_3 174 | - olefile=0.46=pyh9f0ad1d_1 175 | - openblas=0.3.15=pthreads_h4748800_1 176 | - openjpeg=2.4.0=hb52868f_1 177 | - openmpi=4.1.1=hbfc84c5_0 178 | - openssl=1.1.1k=h7f98852_0 179 | - packaging=20.9=pyh44b312d_0 180 | - pandas=1.2.4=py39hde0f152_0 181 | - pandoc=2.13=h7f98852_0 182 | - pandocfilters=1.4.2=py_1 183 | - pango=1.48.5=hb8ff022_0 184 | - parso=0.8.2=pyhd8ed1ab_0 185 | - patsy=0.5.1=py_0 186 | - pcre=8.44=he1b5a44_0 187 | - perl=5.26.2=h36c2ea0_1008 188 | - perl-xml-parser=2.44_01=pl5262hc3e0081_1002 189 | - pexpect=4.8.0=pyh9f0ad1d_2 190 | - pickleshare=0.7.5=py_1003 191 | - pillow=8.2.0=py39hf95b381_1 192 | - pip=21.1.1=pyhd8ed1ab_0 193 | - pixman=0.40.0=h36c2ea0_0 194 | - plotly=4.14.3=pyh44b312d_0 195 | - proj=8.0.0=h277dcde_0 196 | - prometheus_client=0.10.1=pyhd8ed1ab_0 197 | - prompt-toolkit=3.0.18=pyha770c72_0 198 | - pthread-stubs=0.4=h36c2ea0_1001 199 | - ptyprocess=0.7.0=pyhd3deb0d_0 200 | - pycparser=2.20=pyh9f0ad1d_2 201 | - pyfftw=0.12.0=py39h51d1ae8_2 202 | - pygments=2.9.0=pyhd8ed1ab_0 203 | - pyopenssl=20.0.1=pyhd8ed1ab_0 204 | - pyparsing=2.4.7=pyh9f0ad1d_0 205 | - pyrsistent=0.17.3=py39h3811e60_2 206 | - pysocks=1.7.1=py39hf3d152e_3 207 | - pyspharm=1.0.9=py39h984b4d8_1007 208 | - python=3.9.4=hffdb5ce_0_cpython 209 | - python-dateutil=2.8.1=py_0 210 | - python_abi=3.9=1_cp39 211 | - pytz=2021.1=pyhd8ed1ab_0 212 | - pyyaml=5.4.1=py39h3811e60_0 213 | - pyzmq=22.0.3=py39h37b5a0c_1 214 | - qt=5.12.9=hda022c4_4 215 | - readline=8.1=h46c0cb4_0 216 | - requests=2.25.1=pyhd3deb0d_0 217 | - retrying=1.3.3=py_2 218 | - scipy=1.6.3=py39hee8e79c_0 219 | - seaborn=0.11.1=hd8ed1ab_1 220 | - seaborn-base=0.11.1=pyhd8ed1ab_1 221 | - send2trash=1.5.0=py_0 222 | - setuptools=49.6.0=py39hf3d152e_3 223 | - simplejson=3.17.2=py39h3811e60_2 224 | - six=1.16.0=pyh6c4a22f_0 225 | - smmap=3.0.5=pyh44b312d_0 226 | - sniffio=1.2.0=py39hf3d152e_1 227 | - sqlite=3.35.5=h74cdb3f_0 228 | - statsmodels=0.12.2=py39hce5d2b2_0 229 | - sysroot_linux-64=2.12=h77966d4_13 230 | - tbb=2020.2=h4bd325d_4 231 | - terminado=0.9.4=py39hf3d152e_0 232 | - testpath=0.5.0=pyhd8ed1ab_0 233 | - tk=8.6.10=h21135ba_1 234 | - tornado=6.1=py39h3811e60_1 235 | - tqdm=4.60.0=pyhd8ed1ab_0 236 | - traitlets=5.0.5=py_0 237 | - typing_extensions=3.7.4.3=py_0 238 | - tzdata=2021a=he74cb21_0 239 | - urllib3=1.26.4=pyhd8ed1ab_0 240 | - wcwidth=0.2.5=pyh9f0ad1d_2 241 | - webencodings=0.5.1=py_1 242 | - websocket-client=0.57.0=py39hf3d152e_4 243 | - wheel=0.36.2=pyhd3deb0d_0 244 | - widgetsnbextension=3.5.1=py39hf3d152e_4 245 | - xarray=0.18.2=pyhd8ed1ab_0 246 | - xorg-fixesproto=5.0=h7f98852_1002 247 | - xorg-kbproto=1.0.7=h7f98852_1002 248 | - xorg-libice=1.0.10=h7f98852_0 249 | - xorg-libsm=1.2.3=hd9c2040_1000 250 | - xorg-libx11=1.7.1=h7f98852_0 251 | - xorg-libxau=1.0.9=h7f98852_0 252 | - xorg-libxdmcp=1.1.3=h7f98852_0 253 | - xorg-libxext=1.3.4=h7f98852_1 254 | - xorg-libxfixes=5.0.3=h7f98852_1004 255 | - xorg-libxmu=1.1.3=h7f98852_0 256 | - xorg-libxrender=0.9.10=h7f98852_1003 257 | - xorg-libxt=1.2.1=h7f98852_2 258 | - xorg-renderproto=0.11.1=h7f98852_1002 259 | - xorg-xextproto=7.3.0=h7f98852_1002 260 | - xorg-xproto=7.0.31=h7f98852_1007 261 | - xz=5.2.5=h516909a_1 262 | - yaml=0.2.5=h516909a_0 263 | - zeromq=4.3.4=h9c3ff4c_0 264 | - zipp=3.4.1=pyhd8ed1ab_0 265 | - zlib=1.2.11=h516909a_1010 266 | - zstd=1.4.9=ha95c52a_0 267 | - pip: 268 | - eccodes==1.3.2 -------------------------------------------------------------------------------- /ansible/roles/rdtraining/templates/jupyter-revproxy.conf.j2: -------------------------------------------------------------------------------- 1 | # HTTP server to redirect all 80 traffic to SSL/HTTPS 2 | server { 3 | listen 80 default_server; 4 | 5 | # Tell all requests to port 80 to be 302 redirected to HTTPS 6 | return 302 https://$host$request_uri; 7 | } 8 | 9 | # HTTPS server to handle JupyterHub 10 | server { 11 | listen 443 default_server ssl; 12 | 13 | # ssl_certificate /etc/ssl/certs/nginx-selfsigned.crt; 14 | # ssl_certificate_key /etc/ssl/private/nginx-selfsigned.key; 15 | ssl_certificate /etc/ssl/certs/traininglab.crt; 16 | ssl_certificate_key /etc/ssl/private/traininglab.key; 17 | # ssl_certificate {{ letsencrypt_certs_dir }}/fullchain_{{ training_host }}.crt; 18 | # ssl_certificate_key {{ letsencrypt_keys_dir }}/{{ training_host }}.key; 19 | 20 | ssl_protocols TLSv1 TLSv1.1 TLSv1.2; 21 | ssl_prefer_server_ciphers on; 22 | ssl_dhparam /etc/ssl/dhparams.pem; 23 | ssl_ciphers 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:CAMELLIA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA'; 24 | ssl_session_timeout 1d; 25 | ssl_session_cache shared:SSL:50m; 26 | ssl_stapling on; 27 | ssl_stapling_verify on; 28 | add_header Strict-Transport-Security max-age=15768000; 29 | 30 | # Needed for Jupyterlab to save notebooks. 31 | client_max_body_size 0; 32 | 33 | access_log /var/log/nginx/jupyter.log ; 34 | error_log /var/log/nginx/jupyter.error.log debug; 35 | 36 | location '/.well-known/acme-challenge' { 37 | default_type "text/plain"; 38 | root /var/www/acme; 39 | } 40 | 41 | location / { 42 | auth_basic "Access restricted"; 43 | auth_basic_user_file /etc/nginx/.htpasswd; 44 | proxy_set_header Host $host; 45 | proxy_set_header X-Real-IP $remote_addr; 46 | proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; 47 | proxy_set_header X-Forwarded-Proto $scheme; 48 | proxy_pass http://127.0.0.1:8888; 49 | proxy_read_timeout 90; 50 | } 51 | location ~* /(api/kernels/[^/]+/(channels|iopub|shell|stdin)|terminals/websocket)/? { 52 | proxy_pass http://127.0.0.1:8888; 53 | proxy_set_header X-Real-IP $remote_addr; 54 | proxy_set_header Host $host; 55 | proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; 56 | # WebSocket support 57 | proxy_http_version 1.1; 58 | proxy_set_header Upgrade "websocket"; 59 | proxy_set_header Connection "Upgrade"; 60 | proxy_read_timeout 86400; 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /ansible/roles/rdtraining/templates/jupyter.service.j2: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Jupyter 3 | After=syslog.target network.target 4 | 5 | [Service] 6 | User={{ training_user }} 7 | Environment="PATH={{ conda_prefix }}/envs/{{ conda_env_name }}/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin" 8 | Environment="HOME=/home/{{ training_user }}" 9 | Environment="SHELL=/bin/bash" 10 | WorkingDirectory=/home/{{ training_user }} 11 | ExecStart={{ conda_prefix }}/envs/{{ conda_env_name }}/bin/jupyter lab --ip=0.0.0.0 --no-browser 12 | StandardOutput=syslog 13 | StandardError=syslog 14 | SyslogIdentifier=jupyter 15 | 16 | Restart=on-failure 17 | RestartSec=10 18 | 19 | [Install] 20 | WantedBy=multi-user.target 21 | -------------------------------------------------------------------------------- /ansible/roles/s3clients/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | s3_location: ecmwf 3 | s3_url: "https://storage.{{ s3_location }}.europeanweather.cloud" 4 | s3_access_key: "{{ lookup('cypher','secret=secret/s3accesskey') | default('changeme') }}" # DEVMODE : disable this 5 | s3_secret_key: "{{ lookup('cypher','secret=secret/s3secretkey') | default('changeme') }}" # DEVMODE : disable this 6 | # s3_access_key: "{{ s3accesskey }}" # DEVMODE : enable this 7 | # s3_secret_key: "{{ s3secretkey }}" # DEVMODE : enable this 8 | -------------------------------------------------------------------------------- /ansible/roles/s3clients/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Get passwd info 3 | getent: 4 | database: passwd 5 | 6 | - name: Get Standard Users 7 | set_fact: 8 | # host_users: "{{ getent_passwd| 9 | # dict2items| 10 | # json_query('[? to_number(value[1]) >= `1000` && value[5] != `/sbin/nologin`].key') }}" 11 | host_users: "{{ getent_passwd| 12 | dict2items| 13 | json_query('[? (to_number(value[1]) == `0` || to_number(value[1]) == `1000`) && value[5] != `/sbin/nologin`].key') }}" 14 | 15 | - name: Install s3fs client 16 | package: 17 | name: '{{ "s3fs-fuse" if ansible_distribution == "CentOS" else "s3fs" }}' 18 | 19 | - name: Configure S3FS 20 | become_user: '{{ item }}' 21 | template: 22 | src: s3fs.j2 23 | dest: '~/.passwd-s3fs' 24 | mode: 0600 25 | loop: '{{ host_users }}' 26 | ignore_errors: yes 27 | 28 | - name: Install s3cmd client 29 | package: 30 | name: s3cmd 31 | state: present 32 | 33 | - name: Configure S3cmd 34 | become_user: '{{ item }}' 35 | template: 36 | src: s3cfg.j2 37 | dest: '~/.s3cfg' 38 | mode: 0600 39 | loop: '{{ host_users }}' 40 | ignore_errors: yes 41 | 42 | - name: Create ~/.aws directory 43 | become_user: '{{ item }}' 44 | file: 45 | path: '~/.aws' 46 | state: directory 47 | loop: '{{ host_users }}' 48 | ignore_errors: yes 49 | 50 | - name: Configure AWS credentials (for boto3 based apps) 51 | become_user: '{{ item }}' 52 | template: 53 | src: aws_credentials.j2 54 | dest: '~/.aws/credentials' 55 | mode: 0600 56 | loop: '{{ host_users }}' 57 | ignore_errors: yes 58 | -------------------------------------------------------------------------------- /ansible/roles/s3clients/templates/aws_credentials.j2: -------------------------------------------------------------------------------- 1 | [default] 2 | aws_access_key_id={{ s3_access_key }} 3 | aws_secret_access_key={{ s3_secret_key }} 4 | -------------------------------------------------------------------------------- /ansible/roles/s3clients/templates/s3cfg.j2: -------------------------------------------------------------------------------- 1 | host_base = {{ s3_url }} 2 | host_bucket = 3 | access_key = {{ s3_access_key }} 4 | secret_key = {{ s3_secret_key }} 5 | use_https = True 6 | -------------------------------------------------------------------------------- /ansible/roles/s3clients/templates/s3fs.j2: -------------------------------------------------------------------------------- 1 | {{ s3_access_key }}:{{ s3_secret_key }} 2 | -------------------------------------------------------------------------------- /ansible/roles/x2go/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | xfce_ecmwf_background: yes 3 | xfce_kiosk: yes 4 | -------------------------------------------------------------------------------- /ansible/roles/x2go/files/ecmwf-background.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ecmwf-lab/climetlab-s2s-ai-challenge/9a63d7ce4393145b08eed1cec89f1c8c40039895/ansible/roles/x2go/files/ecmwf-background.png -------------------------------------------------------------------------------- /ansible/roles/x2go/files/x2godesktopsharing.desktop: -------------------------------------------------------------------------------- 1 | [Desktop Entry] 2 | Version=1.0 3 | Type=Application 4 | Name=X2Go Desktop Sharing 5 | GenericName=Share your Desktop with X2Go 6 | Exec=bash -c "sleep 20 && /usr/bin/x2godesktopsharing --activate-desktop-sharing" 7 | Icon=x2godesktopsharing 8 | StartupWMClass=x2godesktopsharing 9 | X-Window-Icon=x2godesktopsharing 10 | Terminal=false 11 | Categories=Qt;KDE;Network;RemoteAccess; 12 | StartupNotify=false 13 | Keywords=X2Go;desktop;sharing;shadow;session;view;control;access;full; 14 | X-Desktop-File-Install-Version=0.23 15 | Comment= -------------------------------------------------------------------------------- /ansible/roles/x2go/files/xfce-polkit.desktop: -------------------------------------------------------------------------------- 1 | [Desktop Entry] 2 | Hidden=true -------------------------------------------------------------------------------- /ansible/roles/x2go/files/xfce4-desktop.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | -------------------------------------------------------------------------------- /ansible/roles/x2go/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - block: 3 | - name: Install X2Go 4 | include_tasks: x2go-{{ ansible_distribution|lower }}.yml 5 | 6 | - name: Create background directory 7 | file: 8 | path: /usr/share/backgrounds/images 9 | state: directory 10 | 11 | - name: Copy ECMWF background 12 | copy: 13 | src: files/ecmwf-background.png 14 | dest: /usr/share/backgrounds/images/default.png 15 | when: xfce_ecmwf_background 16 | 17 | - block: 18 | - name: Ensure kiosk directory exists 19 | file: 20 | name: /etc/xdg/xfce4/kiosk/ 21 | state: directory 22 | 23 | - name: Disable Shutdown capabilities 24 | ini_file: 25 | path: /etc/xdg/xfce4/kiosk/kioskrc 26 | section: xfce4-session 27 | option: Shutdown 28 | value: root 29 | backup: yes 30 | when: xfce_kiosk 31 | 32 | - name: Create default XFCE Panel directory for new users 33 | file: 34 | path: /etc/skel/.config/xfce4/xfconf/xfce-perchannel-xml 35 | state: directory 36 | 37 | - name: Set default XFCE Panel directory for new users 38 | copy: 39 | src: /etc/xdg/xfce4/panel/default.xml 40 | dest: /etc/skel/.config/xfce4/xfconf/xfce-perchannel-xml/xfce4-panel.xml 41 | remote_src: yes 42 | 43 | - name: Set default XFCE Desktop config for new users 44 | copy: 45 | src: files/xfce4-desktop.xml 46 | dest: /etc/skel/.config/xfce4/xfconf/xfce-perchannel-xml/xfce4-desktop.xml 47 | 48 | - name: Create default autostart directory for new users 49 | file: 50 | path: /etc/skel/.config/autostart 51 | state: directory 52 | 53 | - name: Set Autostart x2godesktopsharing 54 | copy: 55 | src: files/x2godesktopsharing.desktop 56 | dest: /etc/skel/.config/autostart/x2godesktopsharing.desktop 57 | 58 | - name: Disable Autostart Polkit 59 | copy: 60 | src: files/xfce-polkit.desktop 61 | dest: /etc/skel/.config/autostart/xfce-polkit.desktop 62 | -------------------------------------------------------------------------------- /ansible/roles/x2go/tasks/x2go-centos.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Ensure EPEL repo is enabled 3 | yum: 4 | name: epel-release 5 | state: latest 6 | 7 | - name: Install x2go packages 8 | package: 9 | name: 10 | - x2goserver-xsession 11 | - x2godesktopsharing 12 | - "@Xfce" 13 | - gvfs 14 | - xdg-utils 15 | - chromium 16 | - firefox 17 | - mousepad 18 | - atril 19 | - mate-calc 20 | - ristretto 21 | - gv 22 | state: present 23 | 24 | - name: Fix tab autocompletion bug 25 | lineinfile: 26 | path: /etc/xdg/xfce4/xfconf/xfce-perchannel-xml/xfce4-keyboard-shortcuts.xml 27 | state: absent 28 | regexp: "switch_window_key" -------------------------------------------------------------------------------- /ansible/roles/x2go/tasks/x2go-ubuntu.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Add x2go repo 3 | apt_repository: 4 | repo: ppa:x2go/stable 5 | state: present 6 | 7 | - name: Install x2go packages 8 | package: 9 | name: "{{ item }}" 10 | state: present 11 | loop: 12 | - x2goserver 13 | - x2goserver-xsession 14 | - desktopsharing 15 | - xfce4 16 | - gvfs 17 | - xfce4-terminal 18 | - firefox 19 | - mousepad 20 | - atril 21 | - mate-calc 22 | - ristretto 23 | - gv 24 | -------------------------------------------------------------------------------- /ansible/s2s.yml: -------------------------------------------------------------------------------- 1 | - name: Setting S2S challenge machine 2 | hosts: all 3 | become: yes 4 | pre_tasks: 5 | - name: Update system 6 | package: 7 | name: '*' 8 | state: latest 9 | roles: 10 | - conda 11 | - rdtraining 12 | tasks: 13 | - name: Print all available facts 14 | ansible.builtin.debug: 15 | var: ansible_facts 16 | - name: Install some conda packages 17 | shell: | 18 | source /etc/profile.d/conda.sh 19 | conda activate base 20 | conda install xarray pandas -y || echo 'already installed' 21 | conda install s3fs zarr -y || echo 'already installed' 22 | args: 23 | executable: /bin/bash 24 | - name: Install climetlab and s2s plugin 25 | become_user: "{{ training_user }}" 26 | shell: | 27 | source /etc/profile.d/conda.sh 28 | # source /home/{{ training_user }}/.bashrc 29 | conda activate base 30 | pip install -U climetlab 31 | pip install -U climetlab_s2s_ai_challenge 32 | args: 33 | executable: /bin/bash 34 | - name: "Creates /data/{{ training_user }} directory" 35 | file: 36 | path: /data/{{ training_user }} 37 | state: directory 38 | owner: "{{ training_user }}" 39 | group: "{{ training_user }}" 40 | - name: Set climelab cache to /data/s2s/tmp-climetlab 41 | become_user: "{{ training_user }}" 42 | shell: | 43 | source /etc/profile.d/conda.sh 44 | # source /home/{{ training_user }}/.bashrc 45 | conda activate base 46 | mkdir -p "/data/s2s/tmp-climetlab" 47 | python -c 'import climetlab; climetlab.settings.set("cache-directory", "/data/s2s/tmp-climetlab")' || echo 48 | args: 49 | executable: /bin/bash 50 | - name: Enable conda at startup 51 | become_user: "{{ training_user }}" 52 | shell: | 53 | echo 'source /etc/profile.d/conda.sh' >> /home/{{ training_user }}/.bashrc 54 | echo 'conda activate base' >> /home/{{ training_user }}/.bashrc 55 | args: 56 | executable: /bin/bash 57 | - name: Remove root credentials 58 | shell: | 59 | cd /root && rm .aws/* .passwd-s3fs .s3cfg 60 | args: 61 | executable: /bin/bash 62 | 63 | 64 | -------------------------------------------------------------------------------- /ansible/test.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - name: Network Getting Started First Playbook 4 | hosts: all 5 | become: yes 6 | tasks: 7 | - name: Hello 8 | shell: | 9 | echo 'toto' > /tt 10 | - name: morinfo 11 | debug: 12 | msg: "lkj {{ var }}" 13 | -------------------------------------------------------------------------------- /climetlab_s2s_ai_challenge/.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__ 2 | -------------------------------------------------------------------------------- /climetlab_s2s_ai_challenge/__init__.py: -------------------------------------------------------------------------------- 1 | # (C) Copyright 2020 ECMWF. # 2 | # This software is licensed under the terms of the Apache Licence Version 2.0 3 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 4 | # In applying this licence, ECMWF does not waive the privileges and immunities 5 | # granted to it by virtue of its status as an intergovernmental organisation 6 | # nor does it submit to any jurisdiction. 7 | # 8 | from __future__ import annotations 9 | 10 | from climetlab import Dataset 11 | 12 | from . import extra # noqa F401 13 | 14 | # note : this version number is the plugin version. It has nothing to do with the version number of the dataset 15 | __version__ = "0.9.0" 16 | DATA_VERSION = "0.3.0" 17 | OBSERVATIONS_DATA_VERSION = "0.3.1" 18 | 19 | URL = "https://object-store.os-api.cci1.ecmwf.int" 20 | DATA = "s2s-ai-challenge/data" 21 | 22 | # fmt:off 23 | PATTERN_GRIB = "{url}/{data}/{dataset}/{version}/grib/{origin}-{fctype}-{parameter}-{date}.grib" 24 | PATTERN_NCDF = "{url}/{data}/{dataset}/{version}/netcdf/{origin}-{fctype}-{parameter}-{date}.nc" 25 | PATTERN_ZARR = "{url}/{data}/{dataset}/{version}/zarr/{origin}-{fctype}-{parameter}.zarr" 26 | # fmt:on 27 | 28 | PARAMETER_LIST = [ 29 | "t2m", 30 | "siconc", 31 | "gh", 32 | "lsm", 33 | "msl", 34 | "q", 35 | "rsn", 36 | "sm100", 37 | "sm20", 38 | "sp", 39 | "sst", 40 | "st100", 41 | "st20", 42 | "t", 43 | "tcc", 44 | "tcw", 45 | "tp", 46 | "ttr", 47 | "u", 48 | "v", 49 | ] 50 | 51 | 52 | class S2sDataset(Dataset): 53 | name = None 54 | home_page = "-" 55 | licence = "https://apps.ecmwf.int/datasets/data/s2s/licence/" 56 | documentation = "-" 57 | citation = "-" 58 | 59 | terms_of_use = ( 60 | "By downloading data from this dataset, you agree to the terms and conditions defined at " 61 | "https://apps.ecmwf.int/datasets/data/s2s/licence/. " 62 | "If you do not agree with such terms, do not download the data. " 63 | ) 64 | 65 | 66 | ALIAS_ORIGIN = { 67 | "ecmwf": "ecmwf", 68 | "ecmf": "ecmwf", 69 | "cwao": "eccc", 70 | "eccc": "eccc", 71 | "kwbc": "ncep", 72 | "ncep": "ncep", 73 | } 74 | 75 | ALIAS_MARSORIGIN = { 76 | "ecmwf": "ecmf", 77 | "ecmf": "ecmf", 78 | "cwao": "cwao", 79 | "eccc": "cwao", 80 | "kwbc": "kwbc", 81 | "ncep": "kwbc", 82 | } 83 | 84 | ALIAS_FCTYPE = { 85 | "hindcast": "hindcast", 86 | "reforecast": "hindcast", 87 | "forecast": "forecast", 88 | "realtime": "forecast", 89 | "hc": "hindcast", 90 | "rt": "forecast", 91 | "fc": "forecast", 92 | } 93 | 94 | ALIAS_DATASETNAMES = { 95 | "hindcast-input": "training-input", 96 | "forecast-input": "test-input", 97 | "hindcast-input-dev": "training-input-dev", 98 | "forecast-input-dev": "test-input-dev", 99 | "hindcast-like-observations": "training-output-reference", 100 | "forecast-like-observations": "test-output-reference", 101 | "forecast-benchmark": "test-output-benchmark", 102 | } 103 | for v in list(ALIAS_DATASETNAMES.values()): 104 | ALIAS_DATASETNAMES[v] = v 105 | ALIAS_DATASETNAMES["ncep-hindcast-only"] = "ncep-hindcast-only" 106 | 107 | CF_CELL_METHODS = { 108 | "t2p": None, 109 | "tpp": None, 110 | "t2m": "average", 111 | "sst": "average", 112 | "siconc": "average", 113 | "rsn": "average", 114 | "tcc": "average", 115 | "tcw": "average", 116 | "sm20": "average", 117 | "sm100": "average", 118 | "st20": "average", 119 | "st100": "average", 120 | "tp": "sum", # accumulated 121 | "ttr": "sum", # accumulated 122 | "sp": "point", 123 | "msl": "point", 124 | "lsm": "point", 125 | "u": "point", 126 | "v": "point", 127 | "gh": "point", 128 | "t": "point", 129 | "q": "point", 130 | } 131 | # 'q': '3d', 'u':'3d','v':'3d','gh':'3d','t':'3d', 132 | -------------------------------------------------------------------------------- /climetlab_s2s_ai_challenge/availability.py: -------------------------------------------------------------------------------- 1 | # (C) Copyright 2020 ECMWF. 2 | # 3 | # This software is licensed under the terms of the Apache Licence Version 2.0 4 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 5 | # In applying this licence, ECMWF does not waive the privileges and immunities 6 | # granted to it by virtue of its status as an intergovernmental organisation 7 | # nor does it submit to any jurisdiction. 8 | # 9 | 10 | 11 | def s2s_availability_parser(v): 12 | # do not use date in availability because type='date-list' and availability are not yet implemented in climetlab 13 | # if "alldates" in v: 14 | # dates = list(pd.date_range(**v["alldates"])) 15 | # # v["date"] = dates 16 | # # v["date"] = [d.strftime("%Y%m%d") for d in dates] 17 | 18 | # # v["alldates"] = v["alldates"]["start"] + "/" + v["alldates"]["end"] 19 | 20 | if "number" in v: 21 | s, _, e = v["number"].split("/") 22 | v["number"] = [x for x in range(int(s), int(e) + 1)] 23 | 24 | if "param" in v: 25 | aliases = {"2t": "t2m", "ci": "siconc"} 26 | v["parameter"] = [aliases.get(p, p) for p in v["param"]] + ["ALL"] 27 | 28 | for remove in [ 29 | "grid", 30 | "stream", 31 | "step", 32 | "stepintervals", 33 | "level", 34 | "levelbis", 35 | "param", 36 | "alldates", 37 | ]: 38 | v.pop(remove, None) 39 | 40 | return v 41 | -------------------------------------------------------------------------------- /climetlab_s2s_ai_challenge/benchmark.py: -------------------------------------------------------------------------------- 1 | # (C) Copyright 2020 ECMWF. 2 | # 3 | # This software is licensed under the terms of the Apache Licence Version 2.0 4 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 5 | # In applying this licence, ECMWF does not waive the privileges and immunities 6 | # granted to it by virtue of its status as an intergovernmental organisation 7 | # nor does it submit to any jurisdiction. 8 | # 9 | from __future__ import annotations 10 | 11 | import climetlab as cml 12 | from climetlab import Dataset 13 | 14 | from . import DATA, URL 15 | from .extra import cf_conventions 16 | 17 | PATTERN = "{url}/{data}/{dataset}/{parameter}.nc" 18 | 19 | 20 | def benchmark_builder(datasetname): 21 | class Benchmark(Dataset): 22 | terms_of_use = ( 23 | "By downloading data from this dataset, you agree to the terms and conditions defined at " 24 | "https://apps.ecmwf.int/datasets/data/s2s/licence/. " 25 | "If you do not agree with such terms, do not download the data. " 26 | ) 27 | 28 | # valid_parameters = ["t2m", "tp"] 29 | 30 | def __init__(self, parameter): 31 | parameter = cf_conventions(parameter) 32 | self.dataset = datasetname 33 | request = dict(url=URL, data=DATA, parameter=parameter, dataset=self.dataset) 34 | self.source = cml.load_source( 35 | "url-pattern", 36 | PATTERN, 37 | request, 38 | merger="merge()", 39 | ) 40 | 41 | return Benchmark 42 | 43 | 44 | TestOutputBenchmark = benchmark_builder("test-output-benchmark") 45 | TrainingOutputBenchmark = benchmark_builder("training-output-benchmark") 46 | 47 | ForecastBenchmark = TestOutputBenchmark 48 | HindcastBenchmark = TrainingOutputBenchmark 49 | -------------------------------------------------------------------------------- /climetlab_s2s_ai_challenge/extra.py: -------------------------------------------------------------------------------- 1 | # (C) Copyright 2020 ECMWF. 2 | # 3 | # This software is licensed under the terms of the Apache Licence Version 2.0 4 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 5 | # In applying this licence, ECMWF does not waive the privileges and immunities 6 | # granted to it by virtue of its status as an intergovernmental organisation 7 | # nor does it submit to any jurisdiction. 8 | # 9 | import warnings 10 | 11 | import pandas as pd 12 | import xarray as xr 13 | 14 | 15 | def cf_conventions(parameter): 16 | # this code should be removed once climetlab code handle aliases nicely 17 | # with "normalize" 18 | fix = {"2t": "t2m", "ci": "siconc"} 19 | if isinstance(parameter, str): 20 | return fix.get(parameter, parameter) 21 | 22 | if isinstance(parameter, (tuple, list)): 23 | return [fix.get(p, p) for p in parameter] 24 | 25 | return parameter 26 | 27 | 28 | def create_valid_time_from_forecast_time_and_lead_time(inits, leads): 29 | """Take forecast_time and add lead_time into the future creating two-dimensional 30 | valid_time.""" 31 | inits = xr.DataArray( 32 | inits, 33 | dims="forecast_time", 34 | coords={"forecast_time": inits}, 35 | ) 36 | valid_times = xr.concat( 37 | [ 38 | xr.DataArray( 39 | inits + x, 40 | dims="forecast_time", 41 | coords={"forecast_time": inits}, 42 | ) 43 | for x in leads 44 | ], 45 | "lead_time", 46 | ) 47 | valid_times = valid_times.assign_coords(lead_time=leads) 48 | return valid_times.rename("valid_time") 49 | 50 | 51 | def create_lead_time_and_forecast_time_from_time(forecast, obs_time): 52 | """Create observation with dimensions forecast_time and lead_time and valid_time 53 | coordinate from observations with time dimension""" 54 | if "valid_time" not in forecast.coords or "valid_time" in forecast.dims: 55 | raise ValueError("expect valid_time coords and not as dim") 56 | if "time" not in obs_time.dims: 57 | raise ValueError("expect time as dim in obs_time") 58 | # cannot be lazy dask 59 | forecast["valid_time"] = forecast["valid_time"].compute() 60 | obs_lead_init = obs_time.rename({"time": "valid_time"}).sel(valid_time=forecast.valid_time) 61 | return obs_lead_init 62 | 63 | 64 | def forecast_like_observations(forecast, obs_time): 65 | """Create observation with dimensions `forecast_time` and `lead_time` and 66 | `valid_time` coordinate from observations with `time` dimension 67 | while accumulating precipitation_flux `pr` to precipitation_amount `tp`. 68 | 69 | Note that the newly created output follows the ECMWF S2S convention: 70 | - `tp`: 71 | * accumulated for each day from the beginning of the forecast 72 | * `lead_time = 1 days` accumulates precipitation_flux `pr` from hourly 73 | steps 0-24 at `forecast_time`, 0 days = 0 (no `tp`) by definition, 74 | i.e. `lead_time` defines the end of the end of the aggregation period. 75 | * week 3-4: day 28 minus day 14 76 | * week 5-6: day 42 minus day 28 77 | * https://confluence.ecmwf.int/display/S2S/S2S+Total+Precipitation 78 | - `t2m`: 79 | * averaged each day 80 | * `lead_time = 0 days` averages daily from hourly steps 0-24, 81 | i.e. averaging conditions over the day of `forecast_time` 82 | * week 3-4: average [day 14, day 27] 83 | * week 5-6: average [day 28, day 41] 84 | * https://confluence.ecmwf.int/display/S2S/S2S+Surface+Air+Temperature 85 | 86 | 87 | Args: 88 | forecast (xr.DataArray, xr.Dataset): initialized forecast with `lead_time` 89 | (continuous with daily strides for `tp`) and `forecast_time` dimension and 90 | `valid_time` coordinate 91 | obs_time (xr.Dataset): observations with `time` dimension 92 | 93 | Return: 94 | xr.Dataset: observations with `lead_time` and `forecast_time` dimension and 95 | `valid_time` coordinate for data_vars from obs_time. Converts `pr` to `tp`. 96 | All other variables are not aggregated. 97 | 98 | Example: 99 | >>> import climetlab as cml 100 | >>> forecast = cml.load_dataset('s2s-ai-challenge-training-input', 101 | ... date=20100107, origin='ncep', parameter='tp', 102 | ... format='netcdf').to_xarray() 103 | >>> obs_lead_forecast_time = cml.load_dataset('s2s-ai-challenge-observations', 104 | ... parameter=['pr', 't2m']).to_xarray(like=forecast) 105 | >>> obs_lead_forecast_time 106 | 107 | Dimensions: (forecast_time: 12, latitude: 121, lead_time: 44, 108 | longitude: 240, realization: 4) 109 | Coordinates: 110 | * realization (realization) int64 0 1 2 3 111 | * forecast_time (forecast_time) datetime64[ns] 1999-01-07 ... 2010-01-07 112 | * lead_time (lead_time) timedelta64[ns] 1 days 2 days ... 43 days 44 days 113 | * latitude (latitude) float64 90.0 88.5 87.0 85.5 ... -87.0 -88.5 -90.0 114 | * longitude (longitude) float64 0.0 1.5 3.0 4.5 ... 355.5 357.0 358.5 115 | valid_time (forecast_time, lead_time) datetime64[ns] 116 | Data variables: 117 | tp (realization, forecast_time, lead_time, latitude, longitude) 118 | t2m (realization, forecast_time, lead_time, latitude, longitude) 119 | 120 | Or explicitly with the function `forecast_like_observations` 121 | 122 | >>> from climetlab_s2s_ai_challenge.extra import forecast_like_observations 123 | >>> forecast = cml.load_dataset('s2s-ai-challenge-training-input', 124 | ... date=20100107, origin='ncep', parameter='tp', 125 | ... format='netcdf').to_xarray() 126 | >>> obs_time = cml.load_dataset('s2s-ai-challenge-observations', 127 | ... parameter=['pr', 't2m']).to_xarray() 128 | >>> obs_lead_time_forecast_time = forecast_like_observations(forecast, obs_time) 129 | >>> obs_lead_time_forecast_time 130 | 131 | Dimensions: (forecast_time: 12, latitude: 121, lead_time: 44, 132 | longitude: 240, realization: 4) 133 | Coordinates: 134 | * realization (realization) int64 0 1 2 3 135 | * forecast_time (forecast_time) datetime64[ns] 1999-01-07 ... 2010-01-07 136 | * lead_time (lead_time) timedelta64[ns] 1 days 2 days ... 43 days 44 days 137 | * latitude (latitude) float64 90.0 88.5 87.0 85.5 ... -87.0 -88.5 -90.0 138 | * longitude (longitude) float64 0.0 1.5 3.0 4.5 ... 355.5 357.0 358.5 139 | valid_time (forecast_time, lead_time) datetime64[ns] 140 | Data variables: 141 | tp (realization, forecast_time, lead_time, latitude, longitude) 142 | t2m (realization, forecast_time, lead_time, latitude, longitude) 143 | """ 144 | assert isinstance(obs_time, xr.Dataset) 145 | 146 | obs_lead_init = create_lead_time_and_forecast_time_from_time(forecast, obs_time) 147 | # cumsum pr into tp 148 | if "pr" in obs_time.data_vars: 149 | forecast_lead_strides = forecast.lead_time.diff("lead_time").to_index() 150 | if forecast_lead_strides.mean() != pd.Timedelta("1 days") or forecast_lead_strides.std() != pd.Timedelta( 151 | "0 days" 152 | ): 153 | warnings.warn( 154 | "function `forecast_like_observations(forecast, obs_time)` expects " 155 | "equal daily stides in `forecast.lead_time`, " 156 | f"found strides {forecast_lead_strides} in {forecast.lead_time}" 157 | ) 158 | obs_lead_init_tp = (obs_lead_init[["pr"]].cumsum("lead_time", keep_attrs=True, skipna=True)).rename( 159 | {"pr": "tp"} 160 | ) 161 | # mask all NaNs - cannot do cumsum(skipna=False) because then all grid cells get NaNs (related to leap days?) 162 | obs_lead_init_tp["tp"] = obs_lead_init_tp["tp"].where(~obs_time["pr"].isnull().all("time")) 163 | # shift valid_time and lead_time one unit forward as 164 | # pr describes observed precipitation_flux at given date, e.g. Jan 01 165 | # tp describes observed precipitation_amount, e.g. Jan 01 00:00 to Jan 01 23:59 166 | # therefore labeled by the end of the period Jan 02 167 | shift = forecast_lead_strides.mean() 168 | obs_lead_init_tp = obs_lead_init_tp.assign_coords(valid_time=forecast.valid_time + shift).assign_coords( 169 | lead_time=forecast.lead_time + shift 170 | ) 171 | del obs_lead_init["pr"] 172 | # lead_time 0 days tp stays all NaNs 173 | obs_lead_init["tp"] = obs_lead_init_tp["tp"] 174 | # add attrs 175 | obs_lead_init["tp"].attrs.update( 176 | { 177 | "units": "kg m-2", 178 | "standard_name": "precipitation_amount", 179 | "long_name": "total precipitation", 180 | "aggregation": "precipitation_flux `pr` is accumulated daily from " 181 | + "`forecast_time` up to the date of `valid_time` " 182 | + "(but not including the `valid_time` date) over `lead_time`", 183 | } 184 | ) 185 | # add Dataset metadata 186 | obs_lead_init.attrs.update({"function": "climetlab_s2s_ai_challenge.extra.forecast_like_observations"}) 187 | return obs_lead_init 188 | -------------------------------------------------------------------------------- /climetlab_s2s_ai_challenge/fields.py: -------------------------------------------------------------------------------- 1 | # (C) Copyright 2020 ECMWF. 2 | # 3 | # This software is licensed under the terms of the Apache Licence Version 2.0 4 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 5 | # In applying this licence, ECMWF does not waive the privileges and immunities 6 | # granted to it by virtue of its status as an intergovernmental organisation 7 | # nor does it submit to any jurisdiction. 8 | # 9 | import climetlab as cml 10 | from climetlab.decorators import availability, normalize 11 | 12 | from . import ( # ALIAS_MARSORIGIN, 13 | ALIAS_FCTYPE, 14 | DATA, 15 | DATA_VERSION, 16 | PATTERN_GRIB, 17 | PATTERN_NCDF, 18 | PATTERN_ZARR, 19 | URL, 20 | S2sDataset, 21 | ) 22 | from .availability import s2s_availability_parser 23 | from .info import Info 24 | from .s2s_mergers import ensure_naming_conventions 25 | 26 | PARAMS = [ 27 | "t2m", 28 | "siconc", 29 | "gh", 30 | "lsm", 31 | "msl", 32 | "q", 33 | "rsn", 34 | "sm100", 35 | "sm20", 36 | "sp", 37 | "sst", 38 | "st100", 39 | "st20", 40 | "t", 41 | "tcc", 42 | "tcw", 43 | "tp", 44 | "ttr", 45 | "u", 46 | "v", 47 | ] 48 | 49 | 50 | class FieldS2sDataset(S2sDataset): 51 | dataset = None 52 | 53 | @availability("input.yaml", parser=s2s_availability_parser) 54 | @normalize("origin", ["ecmwf", "eccc", "ncep"], aliases={"ecmf": "ecmwf", "cwao": "eccc", "kwbc": "ncep"}) 55 | @normalize("fctype", ["forecast", "hindcast"], aliases=ALIAS_FCTYPE) 56 | @normalize("parameter", ["ALL"] + PARAMS, multiple=True, aliases={"2t": "t2m", "ci": "siconc"}) 57 | # @normalize("date", multiple=True) 58 | @normalize("date", "date-list(%Y%m%d)") 59 | def __init__(self, origin, fctype, format, dev, parameter="ALL", version=DATA_VERSION, date=None): 60 | self._development_dataset = dev 61 | self.origin = origin 62 | self.fctype = fctype 63 | self.version = version 64 | self.format = { 65 | "grib": Grib(), 66 | "netcdf": Netcdf(), 67 | "zarr": Zarr(), 68 | }[format] 69 | if date is None: 70 | date = [d.strftime("%Y%m%d") for d in self.get_all_reference_dates()] 71 | self.date = date 72 | if parameter == ["ALL"]: 73 | parameter = self._info().get_param_list( 74 | origin=origin, 75 | fctype=fctype, 76 | ) 77 | 78 | sources = [] 79 | for p in parameter: 80 | request = self._make_request(p) 81 | sources.append(self.format._load(request)) 82 | self.source = cml.load_source("multi", sources, merger="merge()") 83 | 84 | def to_xarray(self, *args, **kwargs): 85 | ds = self.source.to_xarray(*args, **kwargs) 86 | ds = ensure_naming_conventions(ds) 87 | return ds 88 | 89 | @classmethod 90 | def cls_get_all_reference_dates(cls, origin, fctype): 91 | return cls._info()._get_config("alldates", origin=origin, fctype=fctype) 92 | 93 | def get_all_reference_dates(self): 94 | return self._info()._get_config("alldates", origin=self.origin, fctype=self.fctype) 95 | 96 | @classmethod 97 | def _info(cls): 98 | return Info(cls.dataset) 99 | 100 | def _make_request(self, p): 101 | dataset = self.dataset 102 | if self._development_dataset: 103 | dataset = dataset + "-dev" 104 | request = dict( 105 | url=URL, 106 | data=DATA, 107 | dataset=dataset, 108 | origin=self.origin, 109 | version=self.version, 110 | parameter=p, 111 | fctype=self.fctype, 112 | date=self.date, 113 | ) 114 | return request 115 | 116 | 117 | class Grib: 118 | def _load(self, request): 119 | return cml.load_source( 120 | "url-pattern", 121 | PATTERN_GRIB, 122 | request, 123 | ) 124 | 125 | 126 | class Netcdf: 127 | def _load(self, request): 128 | return cml.load_source( 129 | "url-pattern", 130 | PATTERN_NCDF, 131 | request, 132 | merger="concat(concat_dim=forecast_time)", 133 | # maybe need to add combine="nested" in xarray merge 134 | ) 135 | 136 | 137 | class Zarr: 138 | def _load(self, request, *args, **kwargs): 139 | from climetlab.utils.patterns import Pattern 140 | 141 | request.pop("date") 142 | 143 | urls = Pattern(PATTERN_ZARR).substitute(request) 144 | 145 | return cml.load_source("zarr-s3", urls) 146 | 147 | 148 | class TrainingInput(FieldS2sDataset): 149 | dataset = "training-input" 150 | 151 | def __init__(self, origin="ecmwf", format="netcdf", fctype="hindcast", dev=False, **kwargs): 152 | super().__init__(format=format, origin=origin, fctype=fctype, dev=dev, **kwargs) 153 | 154 | 155 | class TestInput(FieldS2sDataset): 156 | dataset = "test-input" 157 | 158 | def __init__(self, origin="ecmwf", format="netcdf", fctype="forecast", dev=False, **kwargs): 159 | super().__init__(format=format, origin=origin, fctype=fctype, dev=dev, **kwargs) 160 | -------------------------------------------------------------------------------- /climetlab_s2s_ai_challenge/info.py: -------------------------------------------------------------------------------- 1 | # (C) Copyright 2020 ECMWF. 2 | # 3 | # This software is licensed under the terms of the Apache Licence Version 2.0 4 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 5 | # In applying this licence, ECMWF does not waive the privileges and immunities 6 | # granted to it by virtue of its status as an intergovernmental organisation 7 | # nor does it submit to any jurisdiction. 8 | # 9 | import os 10 | 11 | import climetlab as cml 12 | import climetlab.utils 13 | import climetlab.utils.conventions 14 | import pandas 15 | import yaml 16 | 17 | from . import ( 18 | ALIAS_DATASETNAMES, 19 | ALIAS_FCTYPE, 20 | ALIAS_ORIGIN, 21 | DATA_VERSION, 22 | PATTERN_GRIB, 23 | PATTERN_NCDF, 24 | ) 25 | 26 | 27 | class Info: 28 | def __init__(self, dataset): 29 | if "_" in dataset and dataset not in ALIAS_DATASETNAMES.keys(): 30 | raise ValueError(f'Cannot find {dataset}. Did you mean {dataset.replace("_", "-")} maybe ?') 31 | dataset = ALIAS_DATASETNAMES[dataset] 32 | self.dataset = dataset 33 | 34 | filename = self.dataset.replace("-", "_") + ".yaml" 35 | path = os.path.join(os.path.dirname(os.path.abspath(__file__)), filename) 36 | with open(path) as f: 37 | self.config = yaml.load(f.read(), Loader=yaml.SafeLoader) 38 | for k, v in self.config.items(): 39 | if "alldates" in v: 40 | v["alldates"] = pandas.date_range(**v["alldates"]) 41 | 42 | self.fctype = self._guess_fctype() 43 | 44 | def _guess_fctype(self): 45 | keys = self.config.keys() 46 | fctypes = [k.split("-")[-1] for k in keys] 47 | fctypes = list(set(fctypes)) # make unique 48 | assert len(fctypes) == 1 49 | return fctypes[0] 50 | 51 | def _get_cf_name(self, param): 52 | return cml.utils.conventions.normalise_string(param, convention="cf") 53 | 54 | # TODO add _ 55 | def get_category_param(self, param): 56 | if param in "t2m/siconc/2t/sst/sm20/sm100/st20/st100/ci/rsn/tcc/tcw".split("/"): 57 | return "daily_average" 58 | if param in "sp/msl/ttr/tp".split("/"): 59 | return "instantaneous" 60 | if param in "lsm".split("/"): 61 | return "instantaneous_only_control" 62 | if param in "u/v/gh/t".split("/"): 63 | return "3d" 64 | if param in "q".split("/"): 65 | return "3dbis" 66 | raise NotImplementedError(param) 67 | 68 | def _get_config_keys(self): 69 | return self.config.keys() 70 | 71 | def _get_s3path_grib(self, origin, fctype, parameter, date, url="s3://", version=DATA_VERSION): 72 | origin = ALIAS_ORIGIN[origin] 73 | fctype = ALIAS_FCTYPE[fctype] 74 | return PATTERN_GRIB.format( 75 | url=url, 76 | data="s2s-ai-challenge/data", 77 | dataset=self.dataset, 78 | fctype=fctype, 79 | origin=origin, 80 | version=version, 81 | parameter=parameter, 82 | date=date, 83 | ) 84 | 85 | def _get_s3path_netcdf(self, origin, fctype, parameter, date, url="s3://", version=DATA_VERSION): 86 | origin = ALIAS_ORIGIN[origin] 87 | fctype = ALIAS_FCTYPE[fctype] 88 | return PATTERN_NCDF.format( 89 | url=url, 90 | data="s2s-ai-challenge/data", 91 | dataset=self.dataset, 92 | fctype=fctype, 93 | origin=origin, 94 | version=version, 95 | parameter=parameter, 96 | date=date, 97 | ) 98 | 99 | def _get_config(self, key, origin, fctype=None, date=None, param=None): 100 | origin = ALIAS_ORIGIN[origin] 101 | 102 | if fctype is None: 103 | fctype = self.fctype 104 | fctype = ALIAS_FCTYPE[fctype] 105 | 106 | origin_fctype = f"{origin}-{fctype}" 107 | 108 | import pandas as pd 109 | 110 | if key == "hdate": 111 | if origin == "ncep" and fctype == "hindcast": 112 | return pd.date_range(end=date, periods=12, freq=pd.DateOffset(years=1)) 113 | 114 | if key == "marsdate": 115 | if origin == "ncep" and fctype == "hindcast": 116 | only_one_date = "2011-03-01" 117 | return pd.to_datetime(only_one_date) 118 | else: 119 | return date 120 | 121 | if param is None: 122 | return self.config[origin_fctype][key] 123 | return self.config[origin_fctype][param][key] 124 | 125 | def get_param_list(self, origin, fctype=None): 126 | lst = self._get_config("param", origin, fctype) 127 | lst = [self._get_cf_name(p) for p in lst] 128 | return lst 129 | -------------------------------------------------------------------------------- /climetlab_s2s_ai_challenge/input.yaml: -------------------------------------------------------------------------------- 1 | - origin: ecmwf 2 | fctype: hindcast 3 | param: ['2t', 'ci', 'gh', 'lsm', 'msl', 'q', 'rsn', 'sm100', 'sm20', 'sp', 'sst', 'st100', 'st20', 't', 'tcc', 'tcw', 'tp', 'ttr', 'u', 'v'] 4 | # here, it should not be 1/to/50 5 | number: 1/to/50 6 | stream: enfh 7 | step: 0/to/1104/by/24 8 | stepintervals: 0-24/24-48/48-72/72-96/96-120/120-144/144-168/168-192/192-216/216-240/240-264/264-288/288-312/312-336/336-360/360-384/384-408/408-432/432-456/456-480/480-504/504-528/528-552/552-576/576-600/600-624/624-648/648-672/672-696/696-720/720-744/744-768/768-792/792-816/816-840/840-864/864-888/888-912/912-936/936-960/960-984/984-1008/1008-1032/1032-1056/1056-1080/1080-1104 9 | levels: 1000/925/850/700/500/300/200/100/50/10 10 | levelsbis: 1000/925/850/700/300/500/200 11 | grid: null 12 | hdate: ALL 13 | alldates: {start: '2020-01-02', end: '2020-12-31', freq: 'w-thu'} 14 | 15 | - origin: ecmwf 16 | fctype: forecast 17 | param: ['2t', 'ci', 'gh', 'lsm', 'msl', 'q', 'rsn', 'sm100', 'sm20', 'sp', 'sst', 'st100', 'st20', 't', 'tcc', 'tcw', 'tp', 'ttr', 'u', 'v'] 18 | number: 1/to/50 19 | stream: enfo 20 | step: 0/to/1104/by/24 21 | stepintervals: 0-24/24-48/48-72/72-96/96-120/120-144/144-168/168-192/192-216/216-240/240-264/264-288/288-312/312-336/336-360/360-384/384-408/408-432/432-456/456-480/480-504/504-528/528-552/552-576/576-600/600-624/624-648/648-672/672-696/696-720/720-744/744-768/768-792/792-816/816-840/840-864/864-888/888-912/912-936/936-960/960-984/984-1008/1008-1032/1032-1056/1056-1080/1080-1104 22 | levels: 1000/925/850/700/500/300/200/100/50/10 23 | levelsbis: 1000/925/850/700/300/500/200 24 | grid: null 25 | hdate: None 26 | alldates: {start: '2020-01-02', end: '2020-12-31', freq: 'w-thu'} 27 | 28 | - origin: eccc 29 | fctype: hindcast 30 | param: ['2t', 'ci', 'gh', 'lsm', 'msl', 'q', 'rsn', 'sp', 'sst', 't', 'tcc', 'tcw', 'tp', 'ttr', 'u', 'v'] 31 | number: 1/to/50 32 | stream: enfh 33 | step: 24/to/768/by/24 34 | stepintervals: 0-24/24-48/48-72/72-96/96-120/120-144/144-168/168-192/192-216/216-240/240-264/264-288/288-312/312-336/336-360/360-384/384-408/408-432/432-456/456-480/480-504/504-528/528-552/552-576/576-600/600-624/624-648/648-672/672-696/696-720/720-744/744-768 35 | levels: 1000/925/850/700/500/300/200/100/50/10 36 | levelsbis: 1000/925/850/700/300/500/200 37 | grid: null 38 | hdate: ALL 39 | alldates: {start: '2020-01-02', end: '2020-12-31', freq: 'w-thu'} 40 | 41 | - origin: eccc 42 | fctype: forecast 43 | param: ['2t', 'ci', 'gh', 'lsm', 'msl', 'q', 'rsn', 'sp', 'sst', 't', 'tcc', 'tcw', 'tp', 'ttr', 'u', 'v'] 44 | number: 1/to/20 45 | stream: enfo 46 | step: 24/to/768/by/24 47 | stepintervals: 0-24/24-48/48-72/72-96/96-120/120-144/144-168/168-192/192-216/216-240/240-264/264-288/288-312/312-336/336-360/360-384/384-408/408-432/432-456/456-480/480-504/504-528/528-552/552-576/576-600/600-624/624-648/648-672/672-696/696-720/720-744/744-768 48 | levels: 1000/925/850/700/500/300/200/100/50/10 49 | levelsbis: 1000/925/850/700/300/500/200 50 | grid: null 51 | hdate: None 52 | alldates: {start: '2020-01-02', end: '2020-12-31', freq: 'w-thu'} 53 | 54 | # ncep hindcast has run only once, with date = 2011-03-01 55 | - origin: ncep 56 | fctype: hindcast 57 | param: ['2t', 'ci', 'gh', 'lsm', 'msl', 'q', 'sm100', 'sm20', 'sp', 'sst', 'st100', 'st20', 't', 'tcc', 'tcw', 'tp', 'ttr', 'u', 'v'] 58 | number: 1/to/3 59 | stream: enfh 60 | step: 24/to/1056/by/24 61 | stepintervals: 24-48/48-72/72-96/96-120/120-144/144-168/168-192/192-216/216-240/240-264/264-288/288-312/312-336/336-360/360-384/384-408/408-432/432-456/456-480/480-504/504-528/528-552/552-576/576-600/600-624/624-648/648-672/672-696/696-720/720-744/744-768/768-792/792-816/816-840/840-864/864-888/888-912/912-936/936-960/960-984/984-1008/1008-1032/1032-1056 62 | levels: 1000/925/850/700/500/300/200/100/50/10 63 | levelsbis: 1000/925/850/700/300/500/200 64 | grid: null 65 | # note that this is 2010, that is why the date for ncep is not starting on 2010-01-02 (which is not a thursday btw) 66 | alldates: {start: '2010-01-07', end: '2010-12-29', freq: 'w-thu'} 67 | 68 | - origin: ncep 69 | fctype: forecast 70 | param: ['2t', 'ci', 'gh', 'lsm', 'msl', 'q', 'sm100', 'sm20', 'sp', 'sst', 'st100', 'st20', 't', 'tcc', 'tcw', 'tp', 'ttr', 'u', 'v'] 71 | number: 1/to/15 72 | stream: enfo 73 | step: 24/to/1056/by/24 74 | stepintervals: 24-48/48-72/72-96/96-120/120-144/144-168/168-192/192-216/216-240/240-264/264-288/288-312/312-336/336-360/360-384/384-408/408-432/432-456/456-480/480-504/504-528/528-552/552-576/576-600/600-624/624-648/648-672/672-696/696-720/720-744/744-768/768-792/792-816/816-840/840-864/864-888/888-912/912-936/936-960/960-984/984-1008/1008-1032/1032-1056 75 | levels: 1000/925/850/700/500/300/200/100/50/10 76 | levelsbis: 1000/925/850/700/300/500/200 77 | grid: null 78 | hdate: None 79 | alldates: {start: '2020-01-02', end: '2020-12-31', freq: 'w-thu'} -------------------------------------------------------------------------------- /climetlab_s2s_ai_challenge/ncep_hindcast_only.yaml: -------------------------------------------------------------------------------- 1 | ncep-hindcast: 2 | param: ['2t', 'ci', 'gh', 'lsm', 'msl', 'q', 'sm100', 'sm20', 'sp', 'sst', 'st100', 'st20', 't', 'tcc', 'tcw', 'tp', 'ttr', 'u', 'v'] 3 | number: 1/to/3 4 | stream: enfh 5 | step: 24/to/1056/by/24 6 | stepintervals: 24-48/48-72/72-96/96-120/120-144/144-168/168-192/192-216/216-240/240-264/264-288/288-312/312-336/336-360/360-384/384-408/408-432/432-456/456-480/480-504/504-528/528-552/552-576/576-600/600-624/624-648/648-672/672-696/696-720/720-744/744-768/768-792/792-816/816-840/840-864/864-888/888-912/912-936/936-960/960-984/984-1008/1008-1032/1032-1056 -------------------------------------------------------------------------------- /climetlab_s2s_ai_challenge/observations.py: -------------------------------------------------------------------------------- 1 | # (C) Copyright 2020 ECMWF. 2 | # 3 | # This software is licensed under the terms of the Apache Licence Version 2.0 4 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 5 | # In applying this licence, ECMWF does not waive the privileges and immunities 6 | # granted to it by virtue of its status as an intergovernmental organisation 7 | # nor does it submit to any jurisdiction. 8 | # 9 | from __future__ import annotations 10 | 11 | import climetlab as cml 12 | import xarray as xr 13 | from climetlab.decorators import normalize 14 | from climetlab.normalize import normalize_args 15 | 16 | from . import DATA, OBSERVATIONS_DATA_VERSION, URL, S2sDataset 17 | from .s2s_mergers import ensure_naming_conventions 18 | 19 | PATTERN_OBS = "{url}/{data}/{dataset}/{version}/{parameter}-{date}.nc" 20 | PATTERN_RAWOBS = "{url}/{data}/{dataset}/{version}/{parameter}{grid_string}.nc" 21 | 22 | GRID_STRING = { 23 | "240x121": "", 24 | "121x240": "", 25 | "720x360": "_720x360", 26 | "360x720": "_720x360", 27 | } 28 | 29 | 30 | class Observations(S2sDataset): 31 | terms_of_use = ( 32 | S2sDataset.terms_of_use 33 | + "\n" 34 | + ( 35 | " This dataset has been dowloaded from IRIDL. By downloading this data you also agree to " 36 | "the terms and conditions defined at https://iridl.ldeo.columbia.edu." 37 | ) 38 | ) 39 | 40 | 41 | class RawObservations(Observations): 42 | dataset = "observations" 43 | 44 | @normalize("parameter", [None, "t2m", "pr"], multiple=True) 45 | def __init__(self, parameter=None, grid="240x121", version=OBSERVATIONS_DATA_VERSION): 46 | if parameter == [None] or parameter is None: 47 | parameter = ["t2m", "pr"] 48 | self.version = version 49 | self.grid_string = GRID_STRING[grid] 50 | 51 | request = dict( 52 | url=URL, 53 | data=DATA, 54 | parameter=parameter, 55 | dataset=self.dataset, 56 | version=self.version, 57 | grid_string=self.grid_string, 58 | ) 59 | 60 | self.source = cml.load_source("url-pattern", PATTERN_RAWOBS, request, merger="merge()") 61 | 62 | def to_xarray(self, like=None): 63 | ds = self.source.to_xarray() 64 | if isinstance(like, xr.Dataset): 65 | from .extra import forecast_like_observations 66 | 67 | ds = forecast_like_observations(like, ds) 68 | return ds 69 | 70 | 71 | class PreprocessedObservations(Observations): 72 | dataset = None 73 | 74 | @normalize_args(date="date-list(%Y%m%d)") 75 | @normalize("parameter", [None, "t2m", "tp"], multiple=True) 76 | def __init__(self, date, parameter=None, version=OBSERVATIONS_DATA_VERSION): 77 | if parameter == [None] or parameter is None: 78 | parameter = ["t2m", "pr"] 79 | self.version = version 80 | self.date = date 81 | self.parameter = parameter 82 | 83 | sources = [] 84 | for p in parameter: 85 | request = self._make_request(p) 86 | sources.append( 87 | cml.load_source("url-pattern", PATTERN_OBS, request, merger="concat(concat_dim=time_forecast)") 88 | ) 89 | self.source = cml.load_source("multi", sources, merger="merge()") 90 | 91 | def to_xarray(self, *args, **kwargs): 92 | ds = self.source.to_xarray() 93 | ds = ensure_naming_conventions(ds) 94 | return ds 95 | 96 | def _make_request(self, parameter): 97 | request = dict( 98 | url=URL, 99 | data=DATA, 100 | parameter=parameter, 101 | dataset=self.dataset, 102 | date=self.date, 103 | version=self.version, 104 | ) 105 | return request 106 | 107 | 108 | class TrainingOutputReference(PreprocessedObservations): 109 | dataset = "training-output-reference" 110 | 111 | 112 | class TestOutputReference(PreprocessedObservations): 113 | dataset = "test-output-reference" 114 | 115 | 116 | HindcastLikeObservations = TrainingOutputReference 117 | ForecastLikeObservations = TestOutputReference 118 | -------------------------------------------------------------------------------- /climetlab_s2s_ai_challenge/s2s_mergers.py: -------------------------------------------------------------------------------- 1 | # (C) Copyright 2020 ECMWF. # 2 | # This software is licensed under the terms of the Apache Licence Version 2.0 3 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 4 | # In applying this licence, ECMWF does not waive the privileges and immunities 5 | # granted to it by virtue of its status as an intergovernmental organisation 6 | # nor does it submit to any jurisdiction. 7 | # 8 | import logging 9 | 10 | import xarray as xr 11 | from climetlab.utils.conventions import normalise_string 12 | 13 | from . import CF_CELL_METHODS 14 | 15 | 16 | def remove_unused_coord(ds, name): 17 | if name not in list(ds.coords): 18 | return ds 19 | for var in ds.variables: 20 | if name in ds[var].dims: 21 | return ds 22 | return ds.drop(name) 23 | 24 | 25 | def rename_without_overwrite(ds, before, after): 26 | if before in list(ds.variables) and after not in list(ds.coords): 27 | # print("renaming", before, after) 28 | ds = ds.rename({before: after}) 29 | return ds 30 | 31 | 32 | def _roundtrip(ds, strict_check=True, copy_filename=None, verbose=False): 33 | if not copy_filename: 34 | # import uuid 35 | # uniq = uuid.uuid4() 36 | # copy_filename = f"test_{uniq}.nc" 37 | 38 | import os 39 | import tempfile 40 | 41 | fd, copy_filename = tempfile.mkstemp() 42 | os.close(fd) 43 | coords = " ".join(sorted(list(ds.coords))) 44 | ds.to_netcdf(copy_filename) 45 | copy = xr.open_dataset(copy_filename) 46 | coords2 = " ".join(sorted(list(copy.coords))) 47 | if verbose: 48 | print(f"{copy_filename} :\n in = {coords} \n out= {coords2}") 49 | if strict_check and coords != coords2: 50 | raise (Exception(f"Round trip failed seed {copy_filename}")) 51 | return copy 52 | 53 | 54 | def ensure_naming_conventions(ds, round_trip_hack=False): # noqa C901 55 | # we may want also to add this too : 56 | # import cf2cdm # this is from the package cfgrib 57 | # ds = cf2cdm.translate_coords(ds, cf2cdm.CDS) 58 | # or 59 | # ds = cf2cdm.translate_coords(ds, cf2cdm.ECMWF) 60 | assert isinstance(ds, xr.Dataset), ds 61 | 62 | ds = rename_without_overwrite(ds, "number", "realization") 63 | ds = rename_without_overwrite(ds, "depthBelowLandLayer", "depth_below_and_layer") 64 | ds = rename_without_overwrite(ds, "entireAtmospheretime", "entire_atmosphere") 65 | ds = rename_without_overwrite(ds, "nominalTop", "nominal_top") 66 | ds = rename_without_overwrite(ds, "time", "forecast_time") 67 | ds = rename_without_overwrite(ds, "time", "valid_time") # must be after previous line 68 | ds = rename_without_overwrite(ds, "step", "lead_time") 69 | ds = rename_without_overwrite(ds, "isobaricInhPa", "plev") 70 | ds = rename_without_overwrite(ds, "heightAboveGround", "height_above_ground") 71 | 72 | if "t2p" in list(ds.variables): # special case for benchmark dataset 73 | ds = rename_without_overwrite(ds, "realization", "category") 74 | 75 | ds = remove_unused_coord(ds, "plev") 76 | ds = remove_unused_coord(ds, "surface") 77 | ds = remove_unused_coord(ds, "height_above_ground") 78 | 79 | if round_trip_hack: # see https://github.com/pydata/xarray/issues/5170 80 | ds = _roundtrip(ds, strict_check=False, copy_filename=round_trip_hack) 81 | 82 | if "valid_time" in list(ds.variables) and "valid_time" not in list(ds.coords): 83 | ds = ds.set_coords("valid_time") 84 | 85 | for name in list(ds.variables): 86 | if name not in list(ds.coords): 87 | ds = ds.rename({name: normalise_string(name, convention="cf")}) 88 | 89 | lead_time = "lead_time" 90 | for name, da in ds.data_vars.items(): 91 | method = CF_CELL_METHODS[name] 92 | if method is not None: 93 | da.attrs["cell_method"] = f"{lead_time}: {method}" 94 | else: 95 | logging.warn(f"no cell method known for {name}") 96 | 97 | return ds 98 | 99 | 100 | class S2sMerger: 101 | def __init__(self, engine, concat_dim="forecast_time", options=None): 102 | self.engine = engine 103 | self.concat_dim = concat_dim 104 | self.options = options if options is not None else {} 105 | 106 | def to_xarray(self, paths, **kwargs): 107 | return xr.open_mfdataset( 108 | paths, 109 | engine=self.engine, 110 | # preprocess=ensure_naming_conventions, 111 | concat_dim=self.concat_dim, 112 | **self.options, 113 | ) 114 | -------------------------------------------------------------------------------- /climetlab_s2s_ai_challenge/test_input.yaml: -------------------------------------------------------------------------------- 1 | ecmwf-forecast: 2 | param: ['2t', 'ci', 'gh', 'lsm', 'msl', 'q', 'rsn', 'sm100', 'sm20', 'sp', 'sst', 'st100', 'st20', 't', 'tcc', 'tcw', 'tp', 'ttr', 'u', 'v'] 3 | number: 1/to/50 4 | stream: enfo 5 | step: 0/to/1104/by/24 6 | stepintervals: 0-24/24-48/48-72/72-96/96-120/120-144/144-168/168-192/192-216/216-240/240-264/264-288/288-312/312-336/336-360/360-384/384-408/408-432/432-456/456-480/480-504/504-528/528-552/552-576/576-600/600-624/624-648/648-672/672-696/696-720/720-744/744-768/768-792/792-816/816-840/840-864/864-888/888-912/912-936/936-960/960-984/984-1008/1008-1032/1032-1056/1056-1080/1080-1104 7 | levels: 1000/925/850/700/500/300/200/100/50/10 8 | levelsbis: 1000/925/850/700/300/500/200 9 | grid: null 10 | hdate: None 11 | alldates: {start: '2020-01-02', end: '2020-12-31', freq: 'w-thu'} 12 | eccc-forecast: 13 | param: ['2t', 'ci', 'gh', 'lsm', 'msl', 'q', 'rsn', 'sp', 'sst', 't', 'tcc', 'tcw', 'tp', 'ttr', 'u', 'v'] 14 | number: 1/to/20 15 | stream: enfo 16 | step: 24/to/768/by/24 17 | stepintervals: 0-24/24-48/48-72/72-96/96-120/120-144/144-168/168-192/192-216/216-240/240-264/264-288/288-312/312-336/336-360/360-384/384-408/408-432/432-456/456-480/480-504/504-528/528-552/552-576/576-600/600-624/624-648/648-672/672-696/696-720/720-744/744-768 18 | levels: 1000/925/850/700/500/300/200/100/50/10 19 | levelsbis: 1000/925/850/700/300/500/200 20 | grid: null 21 | hdate: None 22 | alldates: {start: '2020-01-02', end: '2020-12-31', freq: 'w-thu'} 23 | ncep-forecast: 24 | param: ['2t', 'ci', 'gh', 'lsm', 'msl', 'q', 'sm100', 'sm20', 'sp', 'sst', 'st100', 'st20', 't', 'tcc', 'tcw', 'tp', 'ttr', 'u', 'v'] 25 | number: 1/to/15 26 | stream: enfo 27 | step: 24/to/1056/by/24 28 | stepintervals: 24-48/48-72/72-96/96-120/120-144/144-168/168-192/192-216/216-240/240-264/264-288/288-312/312-336/336-360/360-384/384-408/408-432/432-456/456-480/480-504/504-528/528-552/552-576/576-600/600-624/624-648/648-672/672-696/696-720/720-744/744-768/768-792/792-816/816-840/840-864/864-888/888-912/912-936/936-960/960-984/984-1008/1008-1032/1032-1056 29 | levels: 1000/925/850/700/500/300/200/100/50/10 30 | levelsbis: 1000/925/850/700/300/500/200 31 | grid: null 32 | hdate: None 33 | alldates: {start: '2020-01-02', end: '2020-12-31', freq: 'w-thu'} 34 | -------------------------------------------------------------------------------- /climetlab_s2s_ai_challenge/test_input_dev.yaml: -------------------------------------------------------------------------------- 1 | ecmwf-forecast: 2 | param: ['2t', 'tp', 'lsm', 'u', 'q'] 3 | number: 1/to/2 4 | stream: enfo 5 | step: 0/to/72/by/24 6 | stepintervals: 0-24/24-48/48-72 7 | levels: 1000/200/10 8 | levelsbis: 1000/200 9 | grid: 10/10 10 | hdate: None 11 | alldates: {start: '2020-01-02', end: '2020-02-15', freq: 'w-thu'} 12 | eccc-forecast: 13 | param: ['2t', 'tp', 'lsm', 'u', 'q'] 14 | number: 1/to/2 15 | stream: enfo 16 | step: 24/to/72/by/24 17 | stepintervals: 0-24/24-48/48-72 18 | levels: 1000/200/10 19 | levelsbis: 1000/200 20 | grid: 10/10 21 | hdate: None 22 | alldates: {start: '2020-01-02', end: '2020-02-15', freq: 'w-thu'} 23 | ncep-forecast: 24 | param: ['2t', 'tp', 'lsm', 'u', 'q'] 25 | number: 1/to/2 26 | stream: enfo 27 | step: 24/to/72/by/24 28 | stepintervals: 24-48/48-72 29 | levels: 1000/200/10 30 | levelsbis: 1000/200 31 | grid: 10/10 32 | hdate: None 33 | alldates: {start: '2020-01-02', end: '2020-02-15', freq: 'w-thu'} 34 | -------------------------------------------------------------------------------- /climetlab_s2s_ai_challenge/training_input.yaml: -------------------------------------------------------------------------------- 1 | ecmwf-hindcast: 2 | param: ['2t', 'ci', 'gh', 'lsm', 'msl', 'q', 'rsn', 'sm100', 'sm20', 'sp', 'sst', 'st100', 'st20', 't', 'tcc', 'tcw', 'tp', 'ttr', 'u', 'v'] 3 | number: 1/to/50 4 | stream: enfh 5 | step: 0/to/1104/by/24 6 | stepintervals: 0-24/24-48/48-72/72-96/96-120/120-144/144-168/168-192/192-216/216-240/240-264/264-288/288-312/312-336/336-360/360-384/384-408/408-432/432-456/456-480/480-504/504-528/528-552/552-576/576-600/600-624/624-648/648-672/672-696/696-720/720-744/744-768/768-792/792-816/816-840/840-864/864-888/888-912/912-936/936-960/960-984/984-1008/1008-1032/1032-1056/1056-1080/1080-1104 7 | levels: 1000/925/850/700/500/300/200/100/50/10 8 | levelsbis: 1000/925/850/700/300/500/200 9 | grid: null 10 | hdate: ALL 11 | alldates: {start: '2020-01-02', end: '2020-12-31', freq: 'w-thu'} 12 | eccc-hindcast: 13 | param: ['2t', 'ci', 'gh', 'lsm', 'msl', 'q', 'rsn', 'sp', 'sst', 't', 'tcc', 'tcw', 'tp', 'ttr', 'u', 'v'] 14 | number: 1/to/50 15 | stream: enfh 16 | step: 24/to/768/by/24 17 | stepintervals: 0-24/24-48/48-72/72-96/96-120/120-144/144-168/168-192/192-216/216-240/240-264/264-288/288-312/312-336/336-360/360-384/384-408/408-432/432-456/456-480/480-504/504-528/528-552/552-576/576-600/600-624/624-648/648-672/672-696/696-720/720-744/744-768 18 | levels: 1000/925/850/700/500/300/200/100/50/10 19 | levelsbis: 1000/925/850/700/300/500/200 20 | grid: null 21 | hdate: ALL 22 | alldates: {start: '2020-01-02', end: '2020-12-31', freq: 'w-thu'} 23 | ncep-hindcast: # ncep hindcast has run only once, with date = 2011-03-01 24 | param: ['2t', 'ci', 'gh', 'lsm', 'msl', 'q', 'sm100', 'sm20', 'sp', 'sst', 'st100', 'st20', 't', 'tcc', 'tcw', 'tp', 'ttr', 'u', 'v'] 25 | number: 1/to/3 26 | stream: enfh 27 | step: 24/to/1056/by/24 28 | stepintervals: 24-48/48-72/72-96/96-120/120-144/144-168/168-192/192-216/216-240/240-264/264-288/288-312/312-336/336-360/360-384/384-408/408-432/432-456/456-480/480-504/504-528/528-552/552-576/576-600/600-624/624-648/648-672/672-696/696-720/720-744/744-768/768-792/792-816/816-840/840-864/864-888/888-912/912-936/936-960/960-984/984-1008/1008-1032/1032-1056 29 | levels: 1000/925/850/700/500/300/200/100/50/10 30 | levelsbis: 1000/925/850/700/300/500/200 31 | grid: null 32 | # note that this is 2010, that is why the date for ncep is not starting on 2010-01-02 (which is not a thursday btw) 33 | alldates: {start: '2010-01-07', end: '2010-12-29', freq: 'w-thu'} 34 | -------------------------------------------------------------------------------- /climetlab_s2s_ai_challenge/training_input_dev.yaml: -------------------------------------------------------------------------------- 1 | ecmwf-hindcast: 2 | param: ['2t', 'tp', 'lsm', 'u', 'q'] 3 | number: 1/to/2 4 | stream: enfh 5 | step: 0/to/72/by/24 6 | stepintervals: 0-24/24-48/48-72 7 | levels: 1000/200/10 8 | levelsbis: 1000/200 9 | grid: 10/10 10 | hdate: ALL 11 | alldates: {start: '2020-01-02', end: '2020-02-15', freq: 'w-thu'} 12 | eccc-hindcast: 13 | param: ['2t', 'tp', 'lsm', 'u', 'q'] 14 | number: 1/to/2 15 | stream: enfh 16 | step: 24/to/72/by/24 17 | stepintervals: 0-24/24-48/48-72 18 | levels: 1000/200/10 19 | levelsbis: 1000/200 20 | grid: 10/10 21 | hdate: ALL 22 | alldates: {start: '2020-01-02', end: '2020-02-15', freq: 'w-thu'} 23 | ncep-hindcast: # ncep hindcast has run only once, with date = 2011-03-01 24 | param: ['2t', 'tp', 'lsm', 'u', 'q'] 25 | number: 1/to/2 26 | stream: enfh 27 | step: 24/to/72/by/24 28 | stepintervals: 24-48/48-72 29 | levels: 1000/200/10 30 | levelsbis: 1000/200 31 | grid: 10/10 32 | # note that this is 2010, that is why the date for ncep is not starting on 2010-01-02 (which is not a thursday btw) 33 | alldates: {start: '2010-01-07', end: '2010-02-15', freq: 'w-thu'} 34 | -------------------------------------------------------------------------------- /data_portal.yaml: -------------------------------------------------------------------------------- 1 | { 2 | "description": "Sub-seasonal to Seasonal (S2S) Artificial Intelligence Challenge 2021.", 3 | "start_date": "1999-01-07", 4 | "end_date": "2010-12-30", 5 | "encoding": [ 6 | "grib", 7 | "netcdf", 8 | "zarr" 9 | ], 10 | "variables": [ 11 | { 12 | "shortname": "t2m", 13 | "description": "Temperature at 2m. See here.", 14 | "name": "Temperature at 2m", 15 | "id": 0 16 | }, 17 | { 18 | "shortname": "tp", 19 | "description": "Total precipitation. See here.", 20 | "name": "Total precipitation", 21 | "id": 1 22 | }, 23 | ], 24 | "delivery": [ 25 | "climetlab", 26 | "http", 27 | ], 28 | "provided_by": "ECMWF", 29 | "machine_learning": ["true"], 30 | "title": "S2S AI challenge 2021", 31 | "reanalysis": [ 32 | "false" 33 | ], 34 | "created_by": "ECMWF-IRIDL-NCEP-ECCC", 35 | "access": [ 36 | "public" 37 | ], 38 | "type": "Observation and model output", 39 | "nature": [ 40 | "unknown" 41 | ] 42 | } 43 | -------------------------------------------------------------------------------- /notebooks/demo_observations.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": { 7 | "ExecuteTime": { 8 | "end_time": "2021-03-08T12:51:50.883656Z", 9 | "start_time": "2021-03-08T12:51:50.881555Z" 10 | } 11 | }, 12 | "outputs": [], 13 | "source": [ 14 | "#! pip install -U climetlab --quiet\n", 15 | "#! pip install -U climetlab_s2s_ai_challenge --quiet" 16 | ] 17 | }, 18 | { 19 | "cell_type": "code", 20 | "execution_count": 2, 21 | "metadata": { 22 | "ExecuteTime": { 23 | "end_time": "2021-03-08T12:51:51.541510Z", 24 | "start_time": "2021-03-08T12:51:51.536658Z" 25 | }, 26 | "scrolled": true 27 | }, 28 | "outputs": [ 29 | { 30 | "name": "stdout", 31 | "output_type": "stream", 32 | "text": [ 33 | "Climetlab version : 0.9.1\n", 34 | "Climetlab-s2s-ai-challenge plugin version : 0.8.1\n" 35 | ] 36 | } 37 | ], 38 | "source": [ 39 | "import climetlab as cml\n", 40 | "\n", 41 | "import climetlab_s2s_ai_challenge\n", 42 | "\n", 43 | "print(f\"Climetlab version : {cml.__version__}\")\n", 44 | "print(f\"Climetlab-s2s-ai-challenge plugin version : {climetlab_s2s_ai_challenge.__version__}\")" 45 | ] 46 | }, 47 | { 48 | "cell_type": "markdown", 49 | "metadata": {}, 50 | "source": [ 51 | "# Observations data from training" 52 | ] 53 | }, 54 | { 55 | "cell_type": "markdown", 56 | "metadata": {}, 57 | "source": [ 58 | "Climetlab provides the observation datasets. They can be used as a xarray.Dataset :" 59 | ] 60 | }, 61 | { 62 | "cell_type": "code", 63 | "execution_count": 3, 64 | "metadata": {}, 65 | "outputs": [ 66 | { 67 | "name": "stdout", 68 | "output_type": "stream", 69 | "text": [ 70 | "By downloading data from this dataset, you agree to the terms and conditions defined at https://apps.ecmwf.int/datasets/data/s2s/licence/. If you do not agree with such terms, do not download the data. \n", 71 | " This dataset has been dowloaded from IRIDL. By downloading this data you also agree to the terms and conditions defined at https://iridl.ldeo.columbia.edu.\n" 72 | ] 73 | }, 74 | { 75 | "data": { 76 | "text/plain": [ 77 | "Coordinates:\n", 78 | " valid_time (lead_time, forecast_time) datetime64[ns] dask.array\n", 79 | " * longitude (longitude) float64 0.0 1.5 3.0 4.5 ... 355.5 357.0 358.5\n", 80 | " * latitude (latitude) float64 90.0 88.5 87.0 85.5 ... -87.0 -88.5 -90.0\n", 81 | " * forecast_time (forecast_time) datetime64[ns] 2000-01-02 ... 2019-01-02\n", 82 | " * lead_time (lead_time) timedelta64[ns] 0 days 1 days ... 45 days 46 days" 83 | ] 84 | }, 85 | "execution_count": 3, 86 | "metadata": {}, 87 | "output_type": "execute_result" 88 | } 89 | ], 90 | "source": [ 91 | "cmlds = cml.load_dataset(\"s2s-ai-challenge-training-output-reference\", date=20200102, parameter=\"t2m\")\n", 92 | "cmlds.to_xarray().coords" 93 | ] 94 | }, 95 | { 96 | "cell_type": "markdown", 97 | "metadata": {}, 98 | "source": [ 99 | "# Observations data like forecast data\n", 100 | "\n", 101 | "The hindcast `training-input` for `origin='ncep'` is only available from `forecast_time` 1999 - 2010." 102 | ] 103 | }, 104 | { 105 | "cell_type": "code", 106 | "execution_count": 4, 107 | "metadata": {}, 108 | "outputs": [ 109 | { 110 | "name": "stdout", 111 | "output_type": "stream", 112 | "text": [ 113 | "By downloading data from this dataset, you agree to the terms and conditions defined at https://apps.ecmwf.int/datasets/data/s2s/licence/. If you do not agree with such terms, do not download the data. \n" 114 | ] 115 | }, 116 | { 117 | "data": { 118 | "text/plain": [ 119 | "Coordinates:\n", 120 | " * realization (realization) int64 0 1 2 3\n", 121 | " * forecast_time (forecast_time) datetime64[ns] 1999-01-07 ... 2010-01-07\n", 122 | " * lead_time (lead_time) timedelta64[ns] 1 days 2 days ... 43 days 44 days\n", 123 | " * latitude (latitude) float64 90.0 88.5 87.0 85.5 ... -87.0 -88.5 -90.0\n", 124 | " * longitude (longitude) float64 0.0 1.5 3.0 4.5 ... 355.5 357.0 358.5\n", 125 | " valid_time (forecast_time, lead_time) datetime64[ns] dask.array" 126 | ] 127 | }, 128 | "execution_count": 4, 129 | "metadata": {}, 130 | "output_type": "execute_result" 131 | } 132 | ], 133 | "source": [ 134 | "forecast = cml.load_dataset(\n", 135 | " \"s2s-ai-challenge-training-input\", date=[20100107], origin=\"ncep\", parameter=\"tp\", format=\"netcdf\"\n", 136 | ").to_xarray()\n", 137 | "forecast.coords" 138 | ] 139 | }, 140 | { 141 | "cell_type": "markdown", 142 | "metadata": {}, 143 | "source": [ 144 | "Download `observations` for precipitation flux `pr` (also works for 2m-temperature `t2m`) with a `time` dimension.\n", 145 | "Use `climetlab_s2s_ai_challenge.extra.forecast_like_observations` to convert like a forecast, which converts `pr` to total precipitation `tp`." 146 | ] 147 | }, 148 | { 149 | "cell_type": "code", 150 | "execution_count": 5, 151 | "metadata": {}, 152 | "outputs": [ 153 | { 154 | "name": "stdout", 155 | "output_type": "stream", 156 | "text": [ 157 | "By downloading data from this dataset, you agree to the terms and conditions defined at https://apps.ecmwf.int/datasets/data/s2s/licence/. If you do not agree with such terms, do not download the data. \n", 158 | " This dataset has been dowloaded from IRIDL. By downloading this data you also agree to the terms and conditions defined at https://iridl.ldeo.columbia.edu.\n" 159 | ] 160 | }, 161 | { 162 | "data": { 163 | "text/plain": [ 164 | "Coordinates:\n", 165 | " * lead_time (lead_time) timedelta64[ns] 1 days 2 days ... 43 days 44 days\n", 166 | " valid_time (forecast_time, lead_time) datetime64[ns] 1999-01-08 ... 2...\n", 167 | " * longitude (longitude) float64 0.0 1.5 3.0 4.5 ... 355.5 357.0 358.5\n", 168 | " * latitude (latitude) float64 90.0 88.5 87.0 85.5 ... -87.0 -88.5 -90.0\n", 169 | " * forecast_time (forecast_time) datetime64[ns] 1999-01-07 ... 2010-01-07" 170 | ] 171 | }, 172 | "execution_count": 5, 173 | "metadata": {}, 174 | "output_type": "execute_result" 175 | } 176 | ], 177 | "source": [ 178 | "obs_ds = cml.load_dataset(\"s2s-ai-challenge-observations\", parameter=[\"pr\"]).to_xarray()\n", 179 | "from climetlab_s2s_ai_challenge.extra import forecast_like_observations\n", 180 | "\n", 181 | "obs_lead_time_forecast_time = forecast_like_observations(forecast, obs_ds)\n", 182 | "obs_lead_time_forecast_time.coords" 183 | ] 184 | }, 185 | { 186 | "cell_type": "markdown", 187 | "metadata": {}, 188 | "source": [ 189 | "This is equivalent to `.to_xarray(like=forecast)`." 190 | ] 191 | }, 192 | { 193 | "cell_type": "code", 194 | "execution_count": 6, 195 | "metadata": {}, 196 | "outputs": [], 197 | "source": [ 198 | "obs_like = cml.load_dataset(\"s2s-ai-challenge-observations\", parameter=[\"pr\"]).to_xarray(like=forecast)" 199 | ] 200 | }, 201 | { 202 | "cell_type": "code", 203 | "execution_count": 7, 204 | "metadata": {}, 205 | "outputs": [], 206 | "source": [ 207 | "import xarray\n", 208 | "\n", 209 | "xarray.testing.assert_equal(obs_like, obs_lead_time_forecast_time)" 210 | ] 211 | }, 212 | { 213 | "cell_type": "markdown", 214 | "metadata": {}, 215 | "source": [ 216 | "> Note that you can use this with any initialized forecast `xr.Dataset` with coordinate `valid_time(forecast_time, lead_time)`,\n", 217 | "> i.e. any initialized NMME, SubX or S2S output" 218 | ] 219 | }, 220 | { 221 | "cell_type": "code", 222 | "execution_count": null, 223 | "metadata": {}, 224 | "outputs": [], 225 | "source": [] 226 | } 227 | ], 228 | "metadata": { 229 | "kernelspec": { 230 | "display_name": "Python 3 (ipykernel)", 231 | "language": "python", 232 | "name": "python3" 233 | }, 234 | "language_info": { 235 | "codemirror_mode": { 236 | "name": "ipython", 237 | "version": 3 238 | }, 239 | "file_extension": ".py", 240 | "mimetype": "text/x-python", 241 | "name": "python", 242 | "nbconvert_exporter": "python", 243 | "pygments_lexer": "ipython3", 244 | "version": "3.8.10" 245 | }, 246 | "toc": { 247 | "base_numbering": 1, 248 | "nav_menu": {}, 249 | "number_sections": true, 250 | "sideBar": true, 251 | "skip_h1_title": false, 252 | "title_cell": "Table of Contents", 253 | "title_sidebar": "Contents", 254 | "toc_cell": false, 255 | "toc_position": { 256 | "height": "calc(100% - 180px)", 257 | "left": "10px", 258 | "top": "150px", 259 | "width": "378.4px" 260 | }, 261 | "toc_section_display": true, 262 | "toc_window_display": true 263 | } 264 | }, 265 | "nbformat": 4, 266 | "nbformat_minor": 4 267 | } 268 | -------------------------------------------------------------------------------- /notebooks/demo_zarr_experimental.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": { 7 | "ExecuteTime": { 8 | "end_time": "2021-03-08T13:24:37.469229Z", 9 | "start_time": "2021-03-08T13:24:37.466666Z" 10 | }, 11 | "scrolled": true 12 | }, 13 | "outputs": [], 14 | "source": [ 15 | "#! pip install climetlab_s2s_ai_challenge --quiet" 16 | ] 17 | }, 18 | { 19 | "cell_type": "code", 20 | "execution_count": 2, 21 | "metadata": { 22 | "ExecuteTime": { 23 | "end_time": "2021-03-08T13:24:38.159473Z", 24 | "start_time": "2021-03-08T13:24:38.088436Z" 25 | }, 26 | "scrolled": true 27 | }, 28 | "outputs": [ 29 | { 30 | "name": "stdout", 31 | "output_type": "stream", 32 | "text": [ 33 | "Climetlab version : 0.9.1\n", 34 | "Climetlab-s2s-ai-challenge plugin version : 0.8.1\n" 35 | ] 36 | } 37 | ], 38 | "source": [ 39 | "import climetlab as cml\n", 40 | "import xarray as xr\n", 41 | "\n", 42 | "xr.set_options(display_style=\"text\")\n", 43 | "\n", 44 | "import climetlab_s2s_ai_challenge\n", 45 | "\n", 46 | "print(f\"Climetlab version : {cml.__version__}\")\n", 47 | "print(f\"Climetlab-s2s-ai-challenge plugin version : {climetlab_s2s_ai_challenge.__version__}\")" 48 | ] 49 | }, 50 | { 51 | "cell_type": "markdown", 52 | "metadata": {}, 53 | "source": [ 54 | "# Using Zarr data" 55 | ] 56 | }, 57 | { 58 | "cell_type": "markdown", 59 | "metadata": {}, 60 | "source": [ 61 | "Let us get the zarr pointer to the cloud data." 62 | ] 63 | }, 64 | { 65 | "cell_type": "markdown", 66 | "metadata": {}, 67 | "source": [ 68 | "# hindcast" 69 | ] 70 | }, 71 | { 72 | "cell_type": "code", 73 | "execution_count": 3, 74 | "metadata": { 75 | "ExecuteTime": { 76 | "end_time": "2021-03-08T13:24:56.865551Z", 77 | "start_time": "2021-03-08T13:24:38.223140Z" 78 | } 79 | }, 80 | "outputs": [ 81 | { 82 | "name": "stdout", 83 | "output_type": "stream", 84 | "text": [ 85 | "By downloading data from this dataset, you agree to the terms and conditions defined at https://apps.ecmwf.int/datasets/data/s2s/licence/. If you do not agree with such terms, do not download the data. \n" 86 | ] 87 | }, 88 | { 89 | "name": "stderr", 90 | "output_type": "stream", 91 | "text": [ 92 | "/Users/aaron.spring/anaconda3/envs/climetlab/lib/python3.7/site-packages/xarray/core/dataset.py:413: UserWarning: Specified Dask chunks (5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 3) would separate on disks chunk shape 10 for dimension forecast_time. This could degrade performance. Consider rechunking after loading instead.\n", 93 | " _check_chunks_compatibility(var, output_chunks, preferred_chunks)\n", 94 | "/Users/aaron.spring/anaconda3/envs/climetlab/lib/python3.7/site-packages/xarray/core/dataset.py:413: UserWarning: Specified Dask chunks (120, 120) would separate on disks chunk shape 240 for dimension longitude. This could degrade performance. Consider rechunking after loading instead.\n", 95 | " _check_chunks_compatibility(var, output_chunks, preferred_chunks)\n" 96 | ] 97 | }, 98 | { 99 | "data": { 100 | "text/plain": [ 101 | "Coordinates:\n", 102 | " * forecast_time (forecast_time) datetime64[ns] 2000-01-02 ... 2012-01-16\n", 103 | " * latitude (latitude) float64 90.0 88.5 87.0 85.5 ... -87.0 -88.5 -90.0\n", 104 | " * lead_time (lead_time) timedelta64[ns] 0 days 1 days ... 45 days 46 days\n", 105 | " * longitude (longitude) float64 0.0 1.5 3.0 4.5 ... 355.5 357.0 358.5\n", 106 | " * realization (realization) int64 0 1 2 3 4 5 6 7 8 9 10\n", 107 | " valid_time (forecast_time, lead_time) datetime64[ns] dask.array" 108 | ] 109 | }, 110 | "execution_count": 3, 111 | "metadata": {}, 112 | "output_type": "execute_result" 113 | } 114 | ], 115 | "source": [ 116 | "hindcast = cml.load_dataset(\n", 117 | " \"s2s-ai-challenge-training-input\", origin=\"ecmwf\", parameter=\"tp\", format=\"zarr\"\n", 118 | ").to_xarray()\n", 119 | "\n", 120 | "hindcast.coords" 121 | ] 122 | }, 123 | { 124 | "cell_type": "markdown", 125 | "metadata": {}, 126 | "source": [ 127 | "# forecast" 128 | ] 129 | }, 130 | { 131 | "cell_type": "code", 132 | "execution_count": 4, 133 | "metadata": { 134 | "ExecuteTime": { 135 | "start_time": "2021-03-08T13:24:36.784Z" 136 | }, 137 | "scrolled": true 138 | }, 139 | "outputs": [ 140 | { 141 | "name": "stdout", 142 | "output_type": "stream", 143 | "text": [ 144 | "By downloading data from this dataset, you agree to the terms and conditions defined at https://apps.ecmwf.int/datasets/data/s2s/licence/. If you do not agree with such terms, do not download the data. \n" 145 | ] 146 | }, 147 | { 148 | "data": { 149 | "text/html": [ 150 | "
<xarray.Dataset>\n",
151 |        "Dimensions:        (forecast_time: 53, latitude: 121, lead_time: 47, longitude: 240, realization: 51)\n",
152 |        "Coordinates:\n",
153 |        "  * forecast_time  (forecast_time) datetime64[ns] 2020-01-02 ... 2020-12-31\n",
154 |        "  * latitude       (latitude) float64 90.0 88.5 87.0 85.5 ... -87.0 -88.5 -90.0\n",
155 |        "  * lead_time      (lead_time) timedelta64[ns] 0 days 1 days ... 45 days 46 days\n",
156 |        "  * longitude      (longitude) float64 0.0 1.5 3.0 4.5 ... 355.5 357.0 358.5\n",
157 |        "  * realization    (realization) int64 0 1 2 3 4 5 6 7 ... 44 45 46 47 48 49 50\n",
158 |        "    valid_time     (forecast_time, lead_time) datetime64[ns] dask.array<chunksize=(53, 47), meta=np.ndarray>\n",
159 |        "Data variables:\n",
160 |        "    tp             (realization, forecast_time, lead_time, latitude, longitude) float32 dask.array<chunksize=(6, 2, 47, 121, 240), meta=np.ndarray>\n",
161 |        "Attributes:\n",
162 |        "    Conventions:             CF-1.7\n",
163 |        "    GRIB_centre:             ecmf\n",
164 |        "    GRIB_centreDescription:  European Centre for Medium-Range Weather Forecasts\n",
165 |        "    GRIB_edition:            2\n",
166 |        "    GRIB_subCentre:          0\n",
167 |        "    history:                 2021-05-10T15:46:13 GRIB to CDM+CF via cfgrib-0....\n",
168 |        "    institution:             European Centre for Medium-Range Weather Forecasts
" 169 | ], 170 | "text/plain": [ 171 | "\n", 172 | "Dimensions: (forecast_time: 53, latitude: 121, lead_time: 47, longitude: 240, realization: 51)\n", 173 | "Coordinates:\n", 174 | " * forecast_time (forecast_time) datetime64[ns] 2020-01-02 ... 2020-12-31\n", 175 | " * latitude (latitude) float64 90.0 88.5 87.0 85.5 ... -87.0 -88.5 -90.0\n", 176 | " * lead_time (lead_time) timedelta64[ns] 0 days 1 days ... 45 days 46 days\n", 177 | " * longitude (longitude) float64 0.0 1.5 3.0 4.5 ... 355.5 357.0 358.5\n", 178 | " * realization (realization) int64 0 1 2 3 4 5 6 7 ... 44 45 46 47 48 49 50\n", 179 | " valid_time (forecast_time, lead_time) datetime64[ns] dask.array\n", 180 | "Data variables:\n", 181 | " tp (realization, forecast_time, lead_time, latitude, longitude) float32 dask.array\n", 182 | "Attributes:\n", 183 | " Conventions: CF-1.7\n", 184 | " GRIB_centre: ecmf\n", 185 | " GRIB_centreDescription: European Centre for Medium-Range Weather Forecasts\n", 186 | " GRIB_edition: 2\n", 187 | " GRIB_subCentre: 0\n", 188 | " history: 2021-05-10T15:46:13 GRIB to CDM+CF via cfgrib-0....\n", 189 | " institution: European Centre for Medium-Range Weather Forecasts" 190 | ] 191 | }, 192 | "execution_count": 4, 193 | "metadata": {}, 194 | "output_type": "execute_result" 195 | } 196 | ], 197 | "source": [ 198 | "forecast = cml.load_dataset(\"s2s-ai-challenge-test-input\", origin=\"ecmwf\", parameter=[\"tp\"], format=\"zarr\").to_xarray()\n", 199 | "\n", 200 | "forecast" 201 | ] 202 | }, 203 | { 204 | "cell_type": "code", 205 | "execution_count": null, 206 | "metadata": {}, 207 | "outputs": [], 208 | "source": [] 209 | } 210 | ], 211 | "metadata": { 212 | "kernelspec": { 213 | "display_name": "Python 3", 214 | "language": "python", 215 | "name": "python3" 216 | }, 217 | "language_info": { 218 | "codemirror_mode": { 219 | "name": "ipython", 220 | "version": 3 221 | }, 222 | "file_extension": ".py", 223 | "mimetype": "text/x-python", 224 | "name": "python", 225 | "nbconvert_exporter": "python", 226 | "pygments_lexer": "ipython3", 227 | "version": "3.7.10" 228 | }, 229 | "toc": { 230 | "base_numbering": 1, 231 | "nav_menu": {}, 232 | "number_sections": true, 233 | "sideBar": true, 234 | "skip_h1_title": false, 235 | "title_cell": "Table of Contents", 236 | "title_sidebar": "Contents", 237 | "toc_cell": false, 238 | "toc_position": { 239 | "height": "calc(100% - 180px)", 240 | "left": "10px", 241 | "top": "150px", 242 | "width": "378.4px" 243 | }, 244 | "toc_section_display": true, 245 | "toc_window_display": true 246 | } 247 | }, 248 | "nbformat": 4, 249 | "nbformat_minor": 4 250 | } 251 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.black] 2 | line-length = 120 3 | -------------------------------------------------------------------------------- /requirements-dev.txt: -------------------------------------------------------------------------------- 1 | scipy 2 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | climetlab 2 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # (C) Copyright 2020 ECMWF. 3 | # 4 | # This software is licensed under the terms of the Apache Licence Version 2.0 5 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 6 | # In applying this licence, ECMWF does not waive the privileges and immunities 7 | # granted to it by virtue of its status as an intergovernmental organisation 8 | # nor does it submit to any jurisdiction. 9 | # 10 | 11 | 12 | import io 13 | import os 14 | 15 | import setuptools 16 | 17 | 18 | def read(fname): 19 | file_path = os.path.join(os.path.dirname(__file__), fname) 20 | return io.open(file_path, encoding="utf-8").read() 21 | 22 | 23 | package_name = "climetlab-s2s-ai-challenge" 24 | 25 | version = None 26 | init_py = os.path.join(package_name.replace("-", "_"), "__init__.py") 27 | for line in read(init_py).split("\n"): 28 | if line.startswith("__version__"): 29 | version = line.split("=")[-1].strip()[1:-1] 30 | assert version 31 | 32 | 33 | extras_require = {"zarr": ["zarr", "s3fs"]} 34 | 35 | setuptools.setup( 36 | name=package_name, 37 | version=version, 38 | description="Climetlab external dataset plugin for the S2S AI competition organised by ECMWF", 39 | long_description=read("README.md"), 40 | long_description_content_type="text/markdown", 41 | author="European Centre for Medium-Range Weather Forecasts (ECMWF)", 42 | author_email="software.support@ecmwf.int", 43 | license="Apache License Version 2.0", 44 | url="https://github.com/ecmwf-lab/climetlab-s2s-ai-challenge", 45 | packages=setuptools.find_packages(), 46 | include_package_data=True, 47 | install_requires=["climetlab>=0.9.3"], 48 | extras_require=extras_require, 49 | zip_safe=True, 50 | entry_points={ 51 | "climetlab.datasets": [ 52 | "s2s-ai-challenge-observations = climetlab_s2s_ai_challenge.observations:RawObservations", 53 | # Domain style 54 | "s2s-ai-challenge-hindcast-input = climetlab_s2s_ai_challenge.fields:TrainingInput", 55 | "s2s-ai-challenge-forecast-input = climetlab_s2s_ai_challenge.fields:TestInput", 56 | "s2s-ai-challenge-hindcast-like-observations = climetlab_s2s_ai_challenge.observations:HindcastLikeObservations", 57 | "s2s-ai-challenge-forecast-like-observations = climetlab_s2s_ai_challenge.observations:ForecastLikeObservations", 58 | "s2s-ai-challenge-hindcast-benchmark = climetlab_s2s_ai_challenge.benchmark:HindcastBenchmark", 59 | "s2s-ai-challenge-forecast-benchmark = climetlab_s2s_ai_challenge.benchmark:ForecastBenchmark", 60 | # ML style 61 | "s2s-ai-challenge-training-input = climetlab_s2s_ai_challenge.fields:TrainingInput", 62 | "s2s-ai-challenge-test-input = climetlab_s2s_ai_challenge.fields:TestInput", 63 | "s2s-ai-challenge-training-output-reference = climetlab_s2s_ai_challenge.observations:TrainingOutputReference", # noqa: E501 64 | "s2s-ai-challenge-training-output-benchmark = climetlab_s2s_ai_challenge.benchmark:TrainingOutputBenchmark", 65 | "s2s-ai-challenge-test-output-reference = climetlab_s2s_ai_challenge.observations:TestOutputReference", 66 | "s2s-ai-challenge-test-output-benchmark = climetlab_s2s_ai_challenge.benchmark:TestOutputBenchmark", 67 | ] 68 | }, 69 | keywords="meteorology", 70 | classifiers=[ 71 | "Development Status :: 3 - Alpha", 72 | "Intended Audience :: Developers", 73 | "License :: OSI Approved :: Apache Software License", 74 | "Programming Language :: Python :: 3", 75 | "Programming Language :: Python :: 3.6", 76 | "Programming Language :: Python :: 3.7", 77 | "Programming Language :: Python :: 3.8", 78 | "Programming Language :: Python :: Implementation :: CPython", 79 | "Programming Language :: Python :: Implementation :: PyPy", 80 | "Operating System :: OS Independent", 81 | ], 82 | ) 83 | -------------------------------------------------------------------------------- /tests/test_availability.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # (C) Copyright 2020 ECMWF. 4 | # 5 | # This software is licensed under the terms of the Apache Licence Version 2.0 6 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 7 | # In applying this licence, ECMWF does not waive the privileges and immunities 8 | # granted to it by virtue of its status as an intergovernmental organisation 9 | # nor does it submit to any jurisdiction. 10 | # 11 | 12 | import os 13 | 14 | import climetlab as cml 15 | import pytest 16 | 17 | is_test = os.environ.get("TEST_FAST", False) 18 | 19 | 20 | def get_dataset(origin, param): 21 | return cml.load_dataset( 22 | "s2s-ai-challenge-test-input", 23 | dev=is_test, 24 | origin=origin, 25 | date="20200102", 26 | parameter=param, 27 | format="netcdf", 28 | ) 29 | 30 | 31 | @pytest.mark.parametrize( 32 | "args", 33 | [ 34 | ["ecmwf", "t2m"], 35 | ["eccc", "t2m"], 36 | ], 37 | ) 38 | def test_availabilty_1(args): 39 | print(get_dataset(origin=args[0], param=args[1]).to_xarray()) 40 | 41 | 42 | @pytest.mark.parametrize( 43 | "args", 44 | [ 45 | ["eccc", "st100"], 46 | ["ncep", "rsn"], 47 | ], 48 | ) 49 | def test_availability_2(args): 50 | with pytest.raises(ValueError): 51 | print(get_dataset(origin=args[0], param=args[1]).to_xarray()) 52 | 53 | 54 | def test_availability_3(): 55 | cml.load_dataset( 56 | "s2s-ai-challenge-training-input", date=[20100107], origin="ncep", parameter="tp", format="netcdf" 57 | ).to_xarray() 58 | 59 | 60 | if __name__ == "__main__": 61 | from climetlab.testing import main 62 | 63 | main(__file__) 64 | -------------------------------------------------------------------------------- /tests/test_benchmarks.py: -------------------------------------------------------------------------------- 1 | import climetlab as cml 2 | 3 | PARAMS = ["t2m", "tp"] 4 | 5 | 6 | def test_benchmark_1(): 7 | ds = cml.load_dataset("s2s-ai-challenge-test-output-benchmark", parameter=PARAMS) 8 | print(ds.to_xarray()) 9 | 10 | 11 | def test_benchmark_2(): 12 | for p in PARAMS: 13 | ds = cml.load_dataset( 14 | "s2s-ai-challenge-test-output-benchmark", 15 | parameter=p, 16 | ) 17 | print(ds.to_xarray()) 18 | -------------------------------------------------------------------------------- /tests/test_cfconventions.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # (C) Copyright 2020 ECMWF. 4 | # 5 | # This software is licensed under the terms of the Apache Licence Version 2.0 6 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 7 | # In applying this licence, ECMWF does not waive the privileges and immunities 8 | # granted to it by virtue of its status as an intergovernmental organisation 9 | # nor does it submit to any jurisdiction. 10 | # 11 | 12 | import os 13 | 14 | import climetlab as cml 15 | import pytest 16 | 17 | is_test = os.environ.get("TEST_FAST", False) 18 | 19 | 20 | def get_dataset(format, param): 21 | return cml.load_dataset( 22 | "s2s-ai-challenge-test-input", 23 | dev=is_test, 24 | origin="ecmwf", 25 | date="20200102", 26 | parameter=param, 27 | format=format, 28 | ) 29 | 30 | 31 | @pytest.mark.skipif(not os.environ.get("TEST_FAST", None) is None, reason="siconc/ci not in dev dataset") 32 | @pytest.mark.parametrize("param", ["2t", "ci", "t2m", ["t2m", "ci"]]) 33 | def test_read_grib_to_xarray(param): 34 | dsgrib = get_dataset("grib", param) 35 | dsgrib = dsgrib.to_xarray() 36 | dsnetcdf = get_dataset("netcdf", param).to_xarray() 37 | print(dsgrib) 38 | print(dsnetcdf) 39 | assert dsgrib.attrs == dsgrib.attrs 40 | 41 | 42 | @pytest.mark.parametrize("param", ["2t", "t2m"]) 43 | def test_read_grib_to_xarray_2(param): 44 | dsgrib = get_dataset("grib", param) 45 | dsgrib = dsgrib.to_xarray() 46 | dsnetcdf = get_dataset("netcdf", param).to_xarray() 47 | print(dsgrib) 48 | print(dsnetcdf) 49 | assert dsgrib.attrs == dsgrib.attrs 50 | 51 | 52 | if __name__ == "__main__": 53 | from climetlab.testing import main 54 | 55 | main(__file__) 56 | -------------------------------------------------------------------------------- /tests/test_info.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # (C) Copyright 2020 ECMWF. 4 | # 5 | # This software is licensed under the terms of the Apache Licence Version 2.0 6 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 7 | # In applying this licence, ECMWF does not waive the privileges and immunities 8 | # granted to it by virtue of its status as an intergovernmental organisation 9 | # nor does it submit to any jurisdiction. 10 | # 11 | import pandas as pd 12 | 13 | from climetlab_s2s_ai_challenge.info import Info 14 | 15 | 16 | def test_info(): 17 | for n in ( 18 | "ncep-hindcast-only", 19 | "test-input-dev", 20 | "training-input", 21 | "test-input", 22 | "training-input-dev", 23 | ): 24 | info = Info(n) 25 | print(info) 26 | 27 | 28 | def test_get_param_list(): 29 | lst = Info("training-input").get_param_list(origin="ncep", fctype="hindcast") 30 | assert len(lst) == 19 31 | assert lst[0] == "t2m" 32 | assert lst[1] == "siconc" 33 | assert lst[-1] == "v" 34 | 35 | lst = Info("training-input").get_param_list(origin="ecmwf") 36 | assert len(lst) == 20 37 | 38 | lst = Info("training-input-dev").get_param_list(origin="ecmwf") 39 | assert len(lst) == 5 40 | 41 | 42 | def test_get_all_dates(): 43 | lst = Info("training-input")._get_config("alldates", origin="ncep") 44 | assert len(lst) == 51 45 | assert lst[0] == pd.Timestamp("2010-01-07 00:00:00") 46 | assert lst[1] == pd.Timestamp("2010-01-14 00:00:00") 47 | assert lst[-1] == pd.Timestamp("2010-12-23 00:00:00") 48 | 49 | lst = Info("training-input-dev")._get_config("alldates", origin="ncep") 50 | assert len(lst) == 6 51 | 52 | 53 | if __name__ == "__main__": 54 | # test_read_2t_ecmwf_grib_cf_convention() 55 | test_info() 56 | -------------------------------------------------------------------------------- /tests/test_long_observations.py: -------------------------------------------------------------------------------- 1 | import climetlab as cml 2 | 3 | # import pytest 4 | 5 | 6 | def test_observations_merged(): 7 | cmlds = cml.load_dataset( 8 | "s2s-ai-challenge-observations", 9 | parameter=["pr", "t2m"], 10 | ) 11 | ds = cmlds.to_xarray() 12 | print(ds) 13 | 14 | 15 | def test_observations(): 16 | for p in ["pr", "t2m"]: 17 | cmlds = cml.load_dataset( 18 | "s2s-ai-challenge-observations", 19 | parameter=p, 20 | ) 21 | ds = cmlds.to_xarray() 22 | print(ds) 23 | 24 | 25 | def test_observations_720x360(): 26 | cmlds = cml.load_dataset("s2s-ai-challenge-observations", parameter="pr", grid="720x360") 27 | ds = cmlds.to_xarray() 28 | cmlds = cml.load_dataset("s2s-ai-challenge-observations", parameter="t2m", grid="720x360") 29 | ds = cmlds.to_xarray() 30 | print(ds) 31 | 32 | 33 | # @pytest.mark.skipif(True, reason="Disabled because it needs a lot of memory") 34 | def test_observations_720x360_merged_1(): 35 | cmlds = cml.load_dataset("s2s-ai-challenge-observations", parameter=["pr", "t2m"], grid="720x360") 36 | ds = cmlds.to_xarray() 37 | print(ds) 38 | 39 | 40 | # @pytest.mark.skipif(True, reason="Disabled because it needs a lot of memory") 41 | def test_observations_720x360_merged_2(): 42 | cmlds = cml.load_dataset("s2s-ai-challenge-observations", grid="720x360") 43 | ds = cmlds.to_xarray() 44 | print(ds) 45 | -------------------------------------------------------------------------------- /tests/test_merge.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import climetlab as cml 4 | import pytest 5 | import xarray as xr 6 | 7 | is_test = os.environ.get("TEST_FAST", False) 8 | 9 | 10 | def short_print(ds): 11 | print(dict(ds.dims), list(ds.keys())) 12 | 13 | 14 | @pytest.mark.parametrize("format1", ["grib", "netcdf"]) 15 | @pytest.mark.parametrize("format2", ["grib", "netcdf"]) 16 | def test_merge_2020_01_02_and_2020_01_09(format1, format2): 17 | merge_multiple_dates(["20200102", "20200109"], format1, format2) 18 | 19 | 20 | def test_merge_2020_01_02(): 21 | merge("20200102") 22 | 23 | 24 | # not uploaded yet 25 | # def test_merge_2020_12_31(): 26 | # merge("20201231") 27 | 28 | 29 | def merge(date): 30 | dslist = [] 31 | ds = cml.load_dataset( 32 | "s2s-ai-challenge-forecast-input", 33 | dev=is_test, 34 | origin="cwao", 35 | date=date, 36 | parameter="2t", 37 | format="grib", 38 | ) 39 | dslist.append(ds.to_xarray()) 40 | ds = cml.load_dataset( 41 | "s2s-ai-challenge-forecast-input", 42 | dev=is_test, 43 | origin="cwao", 44 | date=date, 45 | parameter="tp", 46 | format="grib", 47 | ) 48 | dslist.append(ds.to_xarray()) 49 | 50 | for ds in dslist: 51 | short_print(ds) 52 | 53 | ds = xr.merge(dslist) 54 | print("-- Merged into --") 55 | short_print(ds) 56 | 57 | # failing on test data. 58 | # assert dslist[0].lead_time.values[0] == dslist[1].lead_time.values[0] 59 | # assert dslist[0].lead_time.values[-1] == dslist[1].lead_time.values[-1] 60 | 61 | 62 | def merge_multiple_dates(dates, format1, format2): 63 | dslist = [] 64 | for date in dates: 65 | ds = cml.load_dataset( 66 | "s2s-ai-challenge-forecast-input", 67 | dev=is_test, 68 | origin="cwao", 69 | date=date, 70 | parameter="2t", 71 | format=format1, 72 | ) 73 | dslist.append(ds.to_xarray()) 74 | for ds in dslist: 75 | short_print(ds) 76 | print(ds) 77 | 78 | ds = xr.merge(dslist) 79 | print("-- Merged into --") 80 | short_print(ds) 81 | 82 | ds2 = cml.load_dataset( 83 | "s2s-ai-challenge-forecast-input", 84 | dev=is_test, 85 | origin="cwao", 86 | date=dates, 87 | parameter="2t", 88 | format=format2, 89 | ) 90 | ds2 = ds2.to_xarray() 91 | print("-- direct merge --") 92 | short_print(ds2) 93 | print(ds2) 94 | 95 | 96 | def test_get_obs_merge_concat(): 97 | cmlds = cml.load_dataset( 98 | "s2s-ai-challenge-test-output-reference", 99 | date=20200312, 100 | parameter=["t2m", "tp"], 101 | ) 102 | ds = cmlds.to_xarray() 103 | print(ds) 104 | 105 | 106 | if __name__ == "__main__": 107 | merge_multiple_dates(["20200102", "20200109"]) 108 | merge("20200102") 109 | merge("20201231") 110 | -------------------------------------------------------------------------------- /tests/test_notebooks.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # (C) Copyright 2020 ECMWF. 4 | # 5 | # This software is licensed under the terms of the Apache Licence Version 2.0 6 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 7 | # In applying this licence, ECMWF does not waive the privileges and immunities 8 | # granted to it by virtue of its status as an intergovernmental organisation 9 | # nor does it submit to any jurisdiction. 10 | # 11 | 12 | import os 13 | import re 14 | 15 | import nbformat 16 | import pytest 17 | from nbconvert.preprocessors import ExecutePreprocessor 18 | 19 | # See https://www.blog.pythonlibrary.org/2018/10/16/testing-jupyter-notebooks/ 20 | 21 | 22 | EXAMPLES = os.path.join(os.path.dirname(os.path.dirname(__file__)), "notebooks") 23 | 24 | SKIP = ( 25 | "demo_zarr_experimental.ipynb", 26 | "demo_zarr.ipynb", 27 | "demo_forecast_benchmark.ipynb", 28 | ) 29 | 30 | 31 | def notebooks_list(): 32 | notebooks = [] 33 | for path in os.listdir(EXAMPLES): 34 | if not path.startswith("demo_"): # test only demo notebooks 35 | continue 36 | if not re.match(r"[^_].*\.ipynb$", path): # ignore notebooks starting with '_' 37 | continue 38 | if "Copy" in path: # ignore notebooks including 'Copy' 39 | continue 40 | if path.startswith("Untitled"): # ignore untitled notebooks 41 | continue 42 | notebooks.append(path) 43 | 44 | return sorted(notebooks) 45 | 46 | 47 | @pytest.mark.parametrize("path", notebooks_list()) 48 | def test_notebook(path): 49 | print(path) 50 | 51 | if path in SKIP: 52 | pytest.skip("Notebook marked as 'skip'") 53 | 54 | with open(os.path.join(EXAMPLES, path)) as f: 55 | nb = nbformat.read(f, as_version=4) 56 | 57 | proc = ExecutePreprocessor(timeout=60 * 60, kernel_name="python3") 58 | proc.preprocess(nb, {"metadata": {"path": EXAMPLES}}) 59 | 60 | 61 | if __name__ == "__main__": 62 | for k, f in sorted(globals().items()): 63 | if k.startswith("test_") and callable(f): 64 | print(k) 65 | f() 66 | -------------------------------------------------------------------------------- /tests/test_observations.py: -------------------------------------------------------------------------------- 1 | import climetlab as cml 2 | import numpy as np 3 | import xarray as xr 4 | 5 | 6 | def test_test_get_rain_obs(): 7 | cmlds = cml.load_dataset( 8 | "s2s-ai-challenge-test-output-reference", 9 | date=20200312, 10 | parameter="tp", 11 | ) 12 | ds = cmlds.to_xarray() 13 | print(ds) 14 | 15 | 16 | def test_test_get_rain_obs_2(): 17 | cmlds = cml.load_dataset( 18 | "s2s-ai-challenge-training-output-reference", 19 | date=[20200102, 20200312], 20 | parameter="tp", 21 | ) 22 | ds = cmlds.to_xarray() 23 | print(ds) 24 | 25 | 26 | def test_train_get_rain_obs(): 27 | cmlds = cml.load_dataset( 28 | "s2s-ai-challenge-training-output-reference", 29 | date=20200312, 30 | parameter="tp", 31 | ) 32 | ds = cmlds.to_xarray() 33 | print(ds) 34 | 35 | 36 | def test_test_get_t2m_obs(): 37 | cmlds = cml.load_dataset( 38 | "s2s-ai-challenge-test-output-reference", 39 | date=20200312, 40 | parameter="t2m", 41 | ) 42 | ds = cmlds.to_xarray() 43 | print(ds) 44 | 45 | 46 | def test_test_get_t2m_obs_2(): 47 | cmlds = cml.load_dataset( 48 | "s2s-ai-challenge-test-output-reference", 49 | date=20200312, 50 | parameter="t2m", 51 | ) 52 | ds = cmlds.to_xarray() 53 | print(ds) 54 | 55 | 56 | def test_test_get_t2m_obs_3(): 57 | cmlds = cml.load_dataset( 58 | "s2s-ai-challenge-test-output-reference", 59 | date="2020-03-12", 60 | parameter="t2m", 61 | ) 62 | ds = cmlds.to_xarray() 63 | print(ds) 64 | 65 | 66 | def test_train_get_t2m_obs(): 67 | cmlds = cml.load_dataset( 68 | "s2s-ai-challenge-training-output-reference", 69 | date=20200312, 70 | parameter="t2m", 71 | ) 72 | ds = cmlds.to_xarray() 73 | print(ds) 74 | 75 | 76 | def test_get_obs(): 77 | cmlds = cml.load_dataset( 78 | "s2s-ai-challenge-test-output-reference", 79 | date=20200312, 80 | parameter="t2m", 81 | ) 82 | ds = cmlds.to_xarray() 83 | print(ds) 84 | 85 | 86 | def test_forecast_like_observations_script(): 87 | """Create synthetic observations object with time dim 88 | and forecast object with lead_time and forecast_time, 89 | to create observation with forecast_time and lead_time 90 | while accumulating pr to tp. 91 | """ 92 | import pandas as pd 93 | 94 | from climetlab_s2s_ai_challenge.extra import ( 95 | create_lead_time_and_forecast_time_from_time, 96 | create_valid_time_from_forecast_time_and_lead_time, 97 | forecast_like_observations, 98 | ) 99 | 100 | # create obs with time dimension 101 | n_time = 100 102 | time = np.arange(n_time) 103 | time_coord = pd.date_range(start="2000", freq="1D", periods=n_time) 104 | ds_time = xr.DataArray(time, dims="time", coords={"time": time_coord}) 105 | 106 | # create valid_time 107 | i_time = 10 108 | init_coord = pd.date_range(start="2000", freq="W-THU", periods=i_time) 109 | inits = xr.DataArray(np.arange(i_time), dims="forecast_time", coords={"forecast_time": init_coord}) 110 | leads = [pd.Timedelta(f"{d} d") for d in range(10)] 111 | valid_times = create_valid_time_from_forecast_time_and_lead_time(inits.forecast_time, leads) 112 | assert "lead_time" in valid_times.dims 113 | assert "forecast_time" in valid_times.dims 114 | 115 | # create a forecast with 10 forecast_time and 10 lead_time and add valid_time 116 | forecast = xr.DataArray( 117 | ds_time.values.reshape(10, 10), 118 | dims=["forecast_time", "lead_time"], 119 | coords={"forecast_time": valid_times.forecast_time, "lead_time": valid_times.lead_time}, 120 | ) 121 | forecast = forecast.assign_coords(valid_time=valid_times) 122 | 123 | # add dimensions lead_time and forecast_time from dim time 124 | ds_lead_init = create_lead_time_and_forecast_time_from_time(forecast, ds_time) 125 | 126 | for d in ["lead_time", "forecast_time"]: 127 | assert d in ds_lead_init.dims 128 | 129 | # promote to dataset 130 | forecast = forecast.to_dataset(name="pr") 131 | forecast["t2m"] = forecast["pr"] 132 | ds_time = ds_time.to_dataset(name="pr") 133 | ds_time["t2m"] = ds_time["pr"] 134 | 135 | # testing forecast_like_observations 136 | obs_lead_init = forecast_like_observations(forecast, ds_time) 137 | assert "tp" in obs_lead_init.data_vars 138 | assert "pr" not in obs_lead_init.data_vars 139 | assert not obs_lead_init["tp"].identical(obs_lead_init["t2m"]) 140 | assert obs_lead_init["tp"].attrs["standard_name"] == "precipitation_amount" 141 | -------------------------------------------------------------------------------- /tests/test_read.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # (C) Copyright 2020 ECMWF. 4 | # 5 | # This software is licensed under the terms of the Apache Licence Version 2.0 6 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 7 | # In applying this licence, ECMWF does not waive the privileges and immunities 8 | # granted to it by virtue of its status as an intergovernmental organisation 9 | # nor does it submit to any jurisdiction. 10 | # 11 | 12 | import os 13 | 14 | import climetlab as cml 15 | import pytest 16 | 17 | is_test = os.environ.get("TEST_FAST", False) 18 | 19 | 20 | def _generic_test_read( 21 | parameter, 22 | origin, 23 | format, 24 | date="20200102", 25 | fctype="forecast", 26 | datasetname="s2s-ai-challenge-forecast-input", 27 | dev=is_test, 28 | ): 29 | ds = cml.load_dataset( 30 | datasetname, 31 | origin=origin, 32 | date=date, 33 | parameter=parameter, 34 | format=format, 35 | fctype=fctype, 36 | ) 37 | xds = ds.to_xarray() 38 | print(xds) 39 | 40 | 41 | def test_read_tp_ecmwf_grib__(): 42 | _generic_test_read(parameter="tp", origin="ecmwf", format="grib") 43 | 44 | 45 | def test_read_domain_name(): 46 | _generic_test_read( 47 | parameter="tp", origin="ecmwf", format="grib", datasetname="s2s-ai-challenge-forecast-input", dev=is_test 48 | ), 49 | 50 | 51 | def test_read_ml_name(): 52 | _generic_test_read( 53 | parameter="tp", origin="ecmwf", format="grib", datasetname="s2s-ai-challenge-test-input", dev=is_test 54 | ), 55 | 56 | 57 | def test_read_tp_ecmwf_netcdf(): 58 | _generic_test_read(parameter="tp", origin="ecmwf", format="netcdf") 59 | 60 | 61 | def test_read_tp_cwao_grib__(): 62 | _generic_test_read(parameter="tp", origin="cwao", format="grib") 63 | 64 | 65 | def test_read_tp_cwao_netcdf(): 66 | _generic_test_read(parameter="tp", origin="cwao", format="netcdf") 67 | 68 | 69 | def test_read_tp_kwbc_grib__(): 70 | _generic_test_read(parameter="tp", origin="kwbc", format="grib") 71 | 72 | 73 | def test_read_tp_kwbc_netcdf(): 74 | _generic_test_read(parameter="tp", origin="kwbc", format="netcdf") 75 | 76 | 77 | def test_read_2t_ecmwf_grib_mars_convention(): 78 | _generic_test_read(parameter="2t", origin="ecmwf", format="grib") 79 | 80 | 81 | def test_read_2t_ecmwf_grib_cf_convention(): 82 | _generic_test_read(parameter="t2m", origin="ecmwf", format="grib") 83 | 84 | 85 | def test_read_2dates_cwao(): 86 | _generic_test_read(parameter="t2m", origin="cwao", format="grib", date=["20200102", "20200109"]) 87 | 88 | 89 | def test_read_2dates_kwbc(): 90 | _generic_test_read(parameter="t2m", origin="kwbc", format="grib", date=["20200102", "20200109"]) 91 | 92 | 93 | def test_read_hindcast_grib(): 94 | _generic_test_read(parameter="t2m", origin="ecmwf", format="grib") 95 | 96 | 97 | def test_read_hindcast_netcdf(): 98 | _generic_test_read(parameter="t2m", origin="ecmwf", format="netcdf") 99 | 100 | 101 | @pytest.mark.skipif(not os.environ.get("TEST_FAST", None) is None, reason="TEST_FAST is set") 102 | def test_read_hindcast_netcdf_2(): 103 | _generic_test_read(parameter="rsn", origin="ecmwf", format="netcdf") 104 | 105 | 106 | @pytest.mark.skipif(not os.environ.get("TEST_FAST", None) is None, reason="TEST_FAST is set") 107 | def test_read_2dates_cwao_2(): 108 | _generic_test_read(parameter="t2m", origin="cwao", format="grib", date=["20200102", "20201231"]) 109 | 110 | 111 | @pytest.mark.skipif(not os.environ.get("TEST_FAST", None) is None, reason="TEST_FAST is set") 112 | def test_read_2dates_kwbc_2(): 113 | _generic_test_read(parameter="t2m", origin="kwbc", format="grib", date=["20200102", "20201231"]) 114 | -------------------------------------------------------------------------------- /tests/test_read_zarr.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # (C) Copyright 2020 ECMWF. 4 | # 5 | # This software is licensed under the terms of the Apache Licence Version 2.0 6 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 7 | # In applying this licence, ECMWF does not waive the privileges and immunities 8 | # granted to it by virtue of its status as an intergovernmental organisation 9 | # nor does it submit to any jurisdiction. 10 | # 11 | 12 | import climetlab as cml 13 | 14 | 15 | def test_read_zarr(): 16 | return # TODO re-enable test when data is uploaded 17 | for parameter in ["2t"] + ["t2m"]: 18 | for fctype in ["forecast"]: # ["forecast", "hindcast"]: 19 | for origin in ["ecmwf"]: # ["cwao", "ecmwf", "kwbc"]: 20 | ds = cml.load_dataset( 21 | "s2s-ai-challenge-test-input", 22 | origin=origin, 23 | fctype=fctype, 24 | format="zarr", 25 | parameter=parameter, 26 | ) 27 | xds = ds.to_xarray() 28 | print(xds) 29 | 30 | 31 | if __name__ == "__main__": 32 | test_read_zarr() 33 | -------------------------------------------------------------------------------- /tools/.gitignore: -------------------------------------------------------------------------------- 1 | *.json 2 | -------------------------------------------------------------------------------- /tools/availability.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from climetlab.utils.availability import Availability 4 | 5 | a = Availability("availability.json") 6 | 7 | for p in a.iterate(): 8 | print(p) 9 | 10 | 11 | print() 12 | print(a.tree()) 13 | -------------------------------------------------------------------------------- /tools/list.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import json 4 | import time 5 | from itertools import product 6 | 7 | import pandas as pd 8 | import requests 9 | 10 | VERSION = "0.1.43" 11 | 12 | URL = "https://object-store.os-api.cci1.ecmwf.int" 13 | DATA = "s2s-ai-challenge/data" 14 | 15 | PATTERN = "{URL}/{DATA}/{dataset}-{fctype}-{origin}/{VERSION}/grib/{parameter}-{date}.grib" 16 | 17 | DATASET = ( 18 | "training-set", 19 | "reference-set", 20 | ) 21 | ORIGIN = ( 22 | "ecmf", 23 | "kwbc", 24 | "cwao", 25 | ) 26 | 27 | FCTYPE = ( 28 | "forecast", 29 | "hindcast", 30 | ) 31 | 32 | PARAMETER = ("2t", "tp") 33 | 34 | DATES = [d.strftime("%Y%m%d") for d in pd.date_range(start="2020-01-01", end="2020-12-31")] 35 | 36 | avail = [] 37 | 38 | for origin, dataset, fctype, parameter, date in product(ORIGIN, DATASET, FCTYPE, PARAMETER, DATES): 39 | url = PATTERN.format(**locals()) 40 | print(url) 41 | while True: 42 | try: 43 | r = requests.head(url) 44 | break 45 | except Exception as e: 46 | print(e) 47 | time.sleep(10) 48 | if r.status_code == 200: 49 | avail.append( 50 | dict( 51 | origin=origin, 52 | dataset=dataset, 53 | fctype=fctype, 54 | parameter=parameter, 55 | date=date, 56 | ) 57 | ) 58 | with open("availability.json", "wt") as f: 59 | print(json.dumps(avail, indent=4, sort_keys=True), file=f) 60 | -------------------------------------------------------------------------------- /tools/observations/conda-packages.txt: -------------------------------------------------------------------------------- 1 | cf_xarray>=0.6.0 2 | esmf>=8.1.0 3 | 4 | -------------------------------------------------------------------------------- /tools/observations/download_from_source.sh: -------------------------------------------------------------------------------- 1 | #!/bin/env bash 2 | 3 | SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" 4 | 5 | OUTDIR=${1:-/s2s-obs/} 6 | 7 | cd $OUTDIR 8 | 9 | for t in tmin; do 10 | mkdir -p $t; 11 | for i in {1979..2021}; do 12 | wget http://iridl.ldeo.columbia.edu/SOURCES/.NOAA/.NCEP/.CPC/.temperature/.daily/.${t}/T/%281%20Jan%20${i}%29%2831%20Dec%20${i}%29RANGEEDGES/data.nc -O $t/data.$i.nc; 13 | done; 14 | done 15 | 16 | for t in tmax; do 17 | mkdir -p $t; 18 | for i in {1979..2021}; do 19 | wget http://iridl.ldeo.columbia.edu/SOURCES/.NOAA/.NCEP/.CPC/.temperature/.daily/.${t}/T/%281%20Jan%20${i}%29%2831%20Dec%20${i}%29RANGEEDGES/data.nc -O $t/data.$i.nc; 20 | done; 21 | done 22 | 23 | for t in rain; do 24 | mkdir -p $t; 25 | for i in {1979..2021}; do 26 | wget http://iridl.ldeo.columbia.edu/SOURCES/.NOAA/.NCEP/.CPC/.UNIFIED_PRCP/.GAUGE_BASED/.GLOBAL/.v1p0/.extREALTIME/.rain/T/%280000%201%20Jan%20${i}%29%280000%2031%20Dec%20${i}%29RANGEEDGES/data.nc -O $t/data.$i.nc; 27 | done; 28 | done 29 | 30 | echo "data downloaded" 31 | -------------------------------------------------------------------------------- /tools/observations/makefile: -------------------------------------------------------------------------------- 1 | OUTDIR=/s2s-obs/tmp/ 2 | 3 | 4 | # TODO : turn this into a ecflow suite. If needed. 5 | # To run this do : 6 | # make build 7 | # make publish 8 | 9 | download: ${OUTDIR}/tmax ${OUTDIR}/tmin ${OUTDIR}/rain 10 | 11 | ${OUTDIR}/tmax: 12 | mkdir -p ${OUTDIR} && ./download_from_source.sh ${OUTDIR} 13 | ${OUTDIR}/tmin: 14 | mkdir -p ${OUTDIR} && ./download_from_source.sh ${OUTDIR} 15 | ${OUTDIR}/rain: 16 | mkdir -p ${OUTDIR} && ./download_from_source.sh ${OUTDIR} 17 | 18 | build: ${OUTDIR}/tmax ${OUTDIR}/tmin ${OUTDIR}/rain 19 | ./build_dataset_observations.py --outdir ${OUTDIR} --input ${OUTDIR} --temperature && \ 20 | ./build_dataset_observations.py --outdir ${OUTDIR} --input ${OUTDIR} --rain 21 | 22 | upload: 23 | cd ${OUTDIR} && \ 24 | for i in training-output-reference/*/*; do echo $$i; s3cmd put $$i s3://s2s-ai-challenge/data/$$i; done && \ 25 | for i in test-output-reference/*/*; do echo $$i; s3cmd put $$i s3://s2s-ai-challenge/data/$$i; done && \ 26 | for i in observations/*/*; do echo $$i; s3cmd put $$i s3://s2s-ai-challenge/data/$$i; done 27 | 28 | publish: upload 29 | s3cmd setacl s3://s2s-ai-challenge/data/test-output-reference --recursive --acl-public && \ 30 | s3cmd setacl s3://s2s-ai-challenge/data/test-output-reference/ --recursive --acl-public && \ 31 | s3cmd setacl s3://s2s-ai-challenge/data/training-output-reference --recursive --acl-public && \ 32 | s3cmd setacl s3://s2s-ai-challenge/data/training-output-reference/ --recursive --acl-public && \ 33 | s3cmd setacl s3://s2s-ai-challenge/data/observations --recursive --acl-public && \ 34 | s3cmd setacl s3://s2s-ai-challenge/data/observations/ --recursive --acl-public 35 | 36 | nuke: 37 | s3cmd rm s3://s2s-ai-challenge/data/training-output-reference --recursive 38 | s3cmd rm s3://s2s-ai-challenge/data/test-output-reference --recursive 39 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [isort] 2 | profile=black 3 | 4 | [flake8] 5 | max-line-length = 125 6 | max-complexity = 10 7 | --------------------------------------------------------------------------------