├── .github ├── CODEOWNERS └── workflows │ ├── coverage.yml │ ├── qa.yml │ └── test-and-release.yml ├── .gitignore ├── .pre-commit-config.yaml ├── .readthedocs.yaml ├── .vscode ├── launch.json ├── settings.json └── tasks.json ├── CHANGELOG.md ├── HOWTO.md ├── LICENSE ├── MANIFEST.in ├── README.md ├── climetlab-and-ecmwflibs-install.md ├── climetlab-and-plugins.code-workspace ├── climetlab.code-workspace ├── dev ├── HOWTOS.md ├── TODO.md ├── constants.py ├── dev.ipynb ├── in-progress.ipynb ├── tf.ipynb ├── tf.py ├── timeseries.csv └── timeseries.ipynb ├── docs ├── Makefile ├── _static │ ├── Cli-Met-Lab-Horiz-text-right.eps │ ├── Cli-Met-Lab-Vert-text-left.eps │ ├── climetlab.png │ ├── climetlab.svg │ ├── dataset-example.svg │ ├── datasource-example.svg │ ├── example-plots │ │ ├── test-grib-plot.svg │ │ └── test-netcdf-plot.svg │ ├── gallery │ │ ├── layers │ │ │ ├── default-background.svg │ │ │ ├── default-foreground.svg │ │ │ └── land-sea.svg │ │ ├── projections │ │ │ ├── africa.svg │ │ │ ├── asia.svg │ │ │ ├── bonne.svg │ │ │ ├── collignon.svg │ │ │ ├── euro-atlantic.svg │ │ │ ├── europe-cylindrical.svg │ │ │ ├── europe.svg │ │ │ ├── global.svg │ │ │ ├── goode.svg │ │ │ ├── mercator.svg │ │ │ ├── mollweide.svg │ │ │ ├── north-america.svg │ │ │ ├── north-america1.svg │ │ │ ├── north-atlantic.svg │ │ │ ├── north-hemisphere.svg │ │ │ ├── polar-north.svg │ │ │ ├── robinson.svg │ │ │ ├── south-america.svg │ │ │ ├── south-atlantic.svg │ │ │ ├── south-hemisphere.svg │ │ │ ├── south-pacific.svg │ │ │ ├── tropics-east.svg │ │ │ ├── tropics-west.svg │ │ │ └── web-mercator.svg │ │ └── styles │ │ │ ├── ct_sst_blk_fM2t36i2_i4.svg │ │ │ ├── ct_sst_blue_fM2t36i2_i4.svg │ │ │ ├── ct_sst_multicolour.svg │ │ │ ├── cyclone-track.svg │ │ │ ├── default-style-fields.svg │ │ │ ├── default-style-observations.svg │ │ │ ├── land-sea-mask.svg │ │ │ ├── no-style.svg │ │ │ ├── orography.svg │ │ │ ├── rainbow-markers.svg │ │ │ ├── sh_sst_fM2t36i2.svg │ │ │ └── sh_sst_fM5t45.svg │ ├── guide │ │ └── plotting-attributes.svg │ ├── irma.svg │ ├── logo.png │ ├── logo1.svg │ ├── logo2.svg │ ├── map.svg │ ├── style.css │ ├── uma.svg │ └── x.drawio.svg ├── _templates │ ├── .gitignore │ └── dont-use-layout.html ├── apply-fmt.sh ├── check-index.sh ├── conf.py ├── contributing │ ├── alias-argument-1.py │ ├── alias-argument-2.py │ ├── alias_argument.rst │ ├── availability.rst │ ├── datasets.rst │ ├── datasets_yaml.rst │ ├── grib.rst │ ├── normalize-example-alias-2.py │ ├── normalize-example-alias.py │ ├── normalize-example-date.py │ ├── normalize-example-enum.py │ ├── normalize-example-int.py │ ├── normalize.rst │ ├── overview.rst │ ├── plotting.rst │ └── sources.rst ├── dataset-example.py ├── datasource-example.py ├── developer │ ├── Makefile │ ├── api.rst │ ├── architecture.rst │ ├── datasets.rst │ ├── delegation.dot │ ├── gallery.rst │ ├── legend.dot │ ├── overview.rst │ ├── plotting.rst │ ├── plugins.rst │ ├── sources.rst │ ├── todolist.rst │ └── xml2rst.py ├── examples.rst ├── examples │ ├── .gitignore │ ├── 01-source-file.ipynb │ ├── 02-source-url.ipynb │ ├── 03-source-cds.ipynb │ ├── 04-source-mars.ipynb │ ├── 05-source-open-data.ipynb │ ├── 06-era5-temperature.ipynb │ ├── 07-high-lows.ipynb │ ├── 08-bufr-data.ipynb │ ├── 09-mars-odb.ipynb │ ├── 10-meteonet.ipynb │ ├── 11-weatherbench.ipynb │ ├── 12-hurricane-database.ipynb │ ├── 13-icoads.ipynb │ ├── 14-external-plugins.ipynb │ ├── 15-more-plotting.ipynb │ ├── 16-gruan.ipynb │ ├── 17-interactive-maps.ipynb │ ├── 50-working-with-gribs.ipynb │ ├── 51-saving-to-grib.ipynb │ ├── 99-availability.ipynb │ ├── Makefile │ ├── renumber.sh │ ├── test.db │ ├── test.grib │ ├── test.nc │ └── test4.grib ├── firststeps.rst ├── generate-examples-maps.py ├── guide │ ├── caching.rst │ ├── cmdline.rst │ ├── dask.rst │ ├── data_handling.rst │ ├── datasets.rst │ ├── howtos.rst │ ├── mltools.rst │ ├── plotting-attributes.py │ ├── plotting-options.py │ ├── plotting.rst │ ├── pluginlist.rst │ ├── settings-1-get.py │ ├── settings-2-set.py │ ├── settings-3-reset.py │ ├── settings.rst │ └── sources.rst ├── index.rst ├── installing.rst ├── overview.rst ├── print-magics-objects.py └── requirements.txt ├── experiments ├── README ├── animate.ipynb ├── bias.py ├── dask-client.py ├── dask │ ├── .gitignore │ ├── Vagrantfile │ ├── config.yaml │ └── provision.sh ├── dask_tests.py ├── era5-vs-oper.py ├── fdb.py ├── gaussian.ipynb ├── gaussian.py ├── grib-caching.py ├── grib-coding.py ├── io │ ├── io.ipynb │ └── io.py ├── load-zarr-fourcastnet.yaml ├── load-zarr-from-db-pl-and-sfc.yaml ├── load-zarr.yaml ├── order.py ├── scaling.py ├── start_dask.py ├── t_draft.py ├── test-cubes.py ├── test-cubes2.py ├── timeseries.py ├── transposition.py ├── virtual1.py └── virtual2.py ├── pyproject.toml ├── pytest.ini ├── requirements.txt ├── schemas ├── definitions.json ├── magics.json ├── mcoast.json ├── mcont.json ├── mmap.json ├── msymb.json └── mtable.json ├── src └── climetlab │ ├── __init__.py │ ├── __main__.py │ ├── aaa.py │ ├── arguments │ ├── __init__.py │ ├── args_kwargs.py │ ├── argument.py │ ├── climetlab_types.py │ ├── guess.py │ ├── input_manager.py │ └── transformers.py │ ├── config │ └── units.yaml │ ├── core │ ├── __init__.py │ ├── caching.py │ ├── constants.py │ ├── data.py │ ├── index.py │ ├── initialise.py │ ├── ipython.py │ ├── metadata.py │ ├── order.py │ ├── plugins.py │ ├── select.py │ ├── settings.py │ ├── statistics.py │ ├── temporary.py │ └── thread.py │ ├── dask │ └── __init__.py │ ├── data │ ├── css │ │ └── table.css │ ├── dask │ │ ├── local.yaml │ │ ├── slurm.yaml │ │ └── ssh.yaml │ ├── domains │ │ └── verification.yaml │ ├── layers │ │ ├── default-background.yaml │ │ ├── default-foreground.yaml │ │ ├── example-foreground.yaml │ │ └── land-sea.yaml │ ├── projections │ │ ├── africa.yaml │ │ ├── asia.yaml │ │ ├── bonne.yaml │ │ ├── collignon.yaml │ │ ├── euro-atlantic.yaml │ │ ├── europe-cylindrical.yaml │ │ ├── europe.yaml │ │ ├── global.yaml │ │ ├── goode.yaml │ │ ├── mercator.yaml │ │ ├── mollweide.yaml │ │ ├── north-america.yaml │ │ ├── north-america1.yaml │ │ ├── north-atlantic.yaml │ │ ├── north-hemisphere.yaml │ │ ├── polar-north.yaml │ │ ├── robinson.yaml │ │ ├── south-america.yaml │ │ ├── south-atlantic.yaml │ │ ├── south-hemisphere.yaml │ │ ├── south-pacific.yaml │ │ ├── tapestry.yaml │ │ ├── tropics-east.yaml │ │ ├── tropics-west.yaml │ │ └── web-mercator.yaml │ └── styles │ │ ├── cyclone-track.yaml │ │ ├── default-style-fields.yaml │ │ ├── default-style-observations.yaml │ │ ├── land-sea-mask.yaml │ │ ├── no-style.yaml │ │ ├── orography.yaml │ │ ├── rainbow-markers.yaml │ │ └── tapestry.yaml │ ├── datasets │ ├── __init__.py │ ├── era5_precipitations.py │ ├── era5_single_levels.py │ ├── era5_temperature.py │ ├── example-dataset.yaml │ ├── high_low.py │ ├── hurricane_database.py │ ├── meteonet_samples │ │ ├── __init__.py │ │ ├── dataset.yaml │ │ ├── ground_stations.py │ │ ├── masks.py │ │ ├── radar.py │ │ ├── styles │ │ │ └── meteonet-radar-rainfall.yaml │ │ └── weather_models.py │ ├── sample-bufr-data.yaml │ ├── sample-grib-data.yaml │ └── weather_bench.py │ ├── debug.py │ ├── decorators.py │ ├── distributed │ ├── __init__.py │ └── dask.py │ ├── exceptions.py │ ├── grids │ └── __init__.py │ ├── indexing │ ├── __init__.py │ ├── cube.py │ ├── database │ │ ├── __init__.py │ │ ├── json.py │ │ └── sql.py │ └── fieldset.py │ ├── mergers │ ├── __init__.py │ ├── pandas.py │ ├── tfdataset.py │ └── xarray.py │ ├── metview │ └── __init__.py │ ├── mirrors │ ├── __init__.py │ └── directory_mirror.py │ ├── ml │ ├── __init__.py │ ├── data_io.py │ ├── filters.py │ ├── tf.py │ ├── torch.py │ └── utils.py │ ├── mockup.py │ ├── normalize.py │ ├── notebook │ ├── __init__.py │ └── table.py │ ├── plotting │ ├── __init__.py │ ├── backends │ │ ├── __init__.py │ │ └── magics │ │ │ ├── __init__.py │ │ │ ├── actions.py │ │ │ ├── apply.py │ │ │ ├── backend.py │ │ │ ├── colour.py │ │ │ ├── convertions.py │ │ │ └── magics.yaml │ ├── options.py │ └── wms │ │ ├── __init__.py │ │ ├── _folium.py │ │ ├── _ipyleafet.py │ │ ├── wms.j2 │ │ └── wms.js │ ├── profiling.py │ ├── prompt.py │ ├── readers │ ├── __init__.py │ ├── archive.py │ ├── bufr.py │ ├── bzip.py │ ├── csv.py │ ├── directory.py │ ├── fwf.py │ ├── grib │ │ ├── __init__.py │ │ ├── codes.py │ │ ├── fieldset.py │ │ ├── index │ │ │ ├── __init__.py │ │ │ ├── db.py │ │ │ ├── file.py │ │ │ ├── json.py │ │ │ └── sql.py │ │ ├── output.py │ │ ├── pandas.py │ │ ├── parsing.py │ │ ├── pytorch.py │ │ ├── reader.py │ │ ├── tensorflow.py │ │ └── xarray.py │ ├── matlab.py │ ├── netcdf │ │ ├── __init__.py │ │ ├── coords.py │ │ ├── dataset.py │ │ ├── field.py │ │ └── fieldset.py │ ├── numpy.py │ ├── odb.py │ ├── tar.py │ ├── text.py │ ├── tfrecord.py │ ├── unknown.py │ └── zip.py │ ├── scripts │ ├── __init__.py │ ├── availability.py │ ├── benchmark.py │ ├── benchmarks │ │ ├── __init__.py │ │ └── indexed_url.py │ ├── cache.py │ ├── check.py │ ├── completion.py │ ├── grib.py │ ├── grib_info.py │ ├── main.py │ ├── settings.py │ ├── test_data.py │ └── tools.py │ ├── sources │ ├── __init__.py │ ├── ads.py │ ├── cds.py │ ├── climetlab_testing.py │ ├── constants.py │ ├── dummy.grib │ ├── ecmwf_api.py │ ├── ecmwf_data_server.py │ ├── ecmwf_data_server_base.py │ ├── ecmwf_open_data.py │ ├── ecmwf_research_experiment.py │ ├── empty.py │ ├── era5_accumulations.py │ ├── fdb.py │ ├── file.py │ ├── file_pattern.py │ ├── indexed.py │ ├── indexed_directory.py │ ├── indexed_url.py │ ├── indexed_url_with_json_index.py │ ├── indexed_urls.py │ ├── loader.py │ ├── mars.py │ ├── metview.py │ ├── multi.py │ ├── multi_url.py │ ├── opendap.py │ ├── oper_accumulations.py │ ├── prompt.py │ ├── sentinel_hub.py │ ├── url.py │ ├── url_pattern.py │ ├── virtual.py │ ├── zarr.py │ ├── zarr_s3.py │ └── zenodo.py │ ├── sphinxext │ ├── __init__.py │ ├── command_output.py │ ├── docs │ │ └── _static │ │ │ └── gallery │ │ │ ├── layers │ │ │ ├── default-background.svg │ │ │ ├── default-foreground.svg │ │ │ └── land-sea.svg │ │ │ ├── projections │ │ │ ├── africa.svg │ │ │ ├── asia.svg │ │ │ ├── bonne.svg │ │ │ ├── collignon.svg │ │ │ ├── euro-atlantic.svg │ │ │ ├── europe-cylindrical.svg │ │ │ ├── europe.svg │ │ │ ├── global.svg │ │ │ ├── goode.svg │ │ │ ├── mercator.svg │ │ │ ├── mollweide.svg │ │ │ ├── north-america.svg │ │ │ ├── north-america1.svg │ │ │ └── north-atlantic.svg │ │ │ └── styles │ │ │ ├── cyclone-track.svg │ │ │ ├── default-style-fields.svg │ │ │ ├── default-style-observations.svg │ │ │ ├── land-sea-mask.svg │ │ │ └── rainbow-markers.svg │ ├── file_content.py │ ├── generate_cmdline_help.py │ ├── generate_gallery_rst.py │ ├── generate_settings_rst.py │ └── module_output.py │ ├── testing.py │ ├── utils │ ├── __init__.py │ ├── availability.py │ ├── bbox.py │ ├── config.py │ ├── conventions.py │ ├── dates.py │ ├── domains.py │ ├── factorise.py │ ├── html.py │ ├── humanize.py │ ├── kwargs.py │ ├── lazy.py │ ├── parts.py │ ├── patterns.py │ └── serialise.py │ ├── vocabularies │ ├── __init__.py │ ├── aliases.py │ ├── cf.py │ ├── grib-paramid.csv │ └── grib.py │ └── wrappers │ ├── __init__.py │ ├── date.py │ ├── integer.py │ ├── ndarray.py │ ├── none.py │ ├── pandas.py │ ├── string.py │ ├── tensor.py │ └── xarray.py ├── tests ├── conftest.py ├── converters │ ├── test_metview.py │ ├── test_numpy.py │ └── test_tfdataset.py ├── core │ ├── test_cache.py │ ├── test_plugins.py │ ├── test_settings.py │ └── test_version.py ├── documentation │ ├── test_examples.py │ ├── test_generate_doc.py │ └── test_notebooks.py ├── example.json ├── example.ya_ml ├── example.yaml ├── example.yml ├── indexing │ ├── test_order_kwargs.py │ └── test_selection_kwargs.py ├── normalize │ ├── aliases.json │ ├── availability.json │ ├── test_normalize_aliases.py │ ├── test_normalize_aliases_grib.py │ ├── test_normalize_args.py │ ├── test_normalize_availability.py │ ├── test_normalize_bbox.py │ ├── test_normalize_date.py │ ├── test_normalize_enum.py │ ├── test_normalize_errors.py │ ├── test_normalize_int_list.py │ ├── test_normalize_kwargs.py │ ├── test_normalize_parameter.py │ └── test_transformers.py ├── readers │ ├── little_endian.mat │ ├── test_csv_reader.py │ ├── test_fwf_reader.py │ ├── test_grib_reader.py │ ├── test_matlab_reader.py │ ├── test_netcdf_reader.py │ ├── test_tar_reader.py │ ├── test_tfdataset_reader.py │ ├── test_unknown_reader.py │ ├── test_zip_reader.py │ ├── unknown_file.unknown_ext │ └── unknown_text_file.unknown_ext ├── sources │ ├── test_cds.py │ ├── test_constants.py │ ├── test_file.py │ ├── test_indexed_url.py │ ├── test_indexed_urls.py │ ├── test_mars.py │ ├── test_merge.py │ ├── test_mirror.py │ ├── test_multi.py │ ├── test_open_data.py │ ├── test_prompt.py │ ├── test_url.py │ ├── test_url_pattern.py │ ├── test_zarr.py │ ├── test_zenodo.py │ └── zedono-dataset-1.yaml ├── test_annotations.py ├── test_bbox.py ├── test_cli.py ├── test_consume.py ├── test_datasets.py ├── test_dates.py ├── test_domains.py ├── test_download.py ├── test_factorise.py ├── test_imports.py ├── test_kwargs.py ├── test_magics.py ├── test_meteonet.py ├── test_output.py ├── test_patterns.py ├── test_plotting.py ├── test_thread.py ├── test_unpack.py ├── test_utils.py ├── test_yaml.py └── wrappers │ └── test_xarray_wrapper.py └── tools ├── import-eccharts.py ├── imports.py └── test-data └── .gitignore /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | /climetlab/version @floriankrb @b8raoult 2 | -------------------------------------------------------------------------------- /.github/workflows/coverage.yml: -------------------------------------------------------------------------------- 1 | # (C) Copyright 2021 ECMWF. 2 | # 3 | # This software is licensed under the terms of the Apache Licence Version 2.0 4 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 5 | # In applying this licence, ECMWF does not waive the privileges and immunities 6 | # granted to it by virtue of its status as an intergovernmental organisation 7 | # nor does it submit to any jurisdiction. 8 | # 9 | name: Coverage 10 | 11 | on: 12 | # schedule: 13 | # # Every day at 3:07am 14 | # - cron: '07 3 * * *' 15 | 16 | workflow_dispatch: {} 17 | 18 | jobs: 19 | coverage: 20 | 21 | runs-on: "ubuntu-latest" 22 | steps: 23 | - uses: actions/checkout@v3 24 | - uses: actions/setup-python@v4 25 | with: 26 | python-version: "3.10" 27 | 28 | - name: Tokens 29 | shell: python 30 | env: 31 | ECMWFAPIRC: ${{ secrets.ECMWFAPIRC }} 32 | CDSAPIRC: ${{ secrets.CDSAPIRC }} 33 | run: | 34 | import os 35 | for n in ('ECMWFAPIRC', 'CDSAPIRC'): 36 | m = os.path.expanduser("~/." + n.lower()) 37 | if os.environ[n]: 38 | with open(m, "w") as f: 39 | print(os.environ[n], file=f) 40 | 41 | - name: Install climetlab 42 | run: pip install -e .[interactive,zarr,tensorflow] 43 | 44 | - name: Install test tools 45 | run: pip install pytest pytest-cov nbformat nbconvert ipykernel 46 | 47 | - name: Check imports 48 | env: 49 | SKIP_TEST_IMPORTS: 0 50 | run: pytest tests/test_imports.py 51 | 52 | - name: Tests with code coverage 53 | run: pytest --cov=climetlab --cov-report=html --cov-report=xml -E release 54 | 55 | - name: Codecov Upload 56 | uses: codecov/codecov-action@v2 57 | -------------------------------------------------------------------------------- /.github/workflows/qa.yml: -------------------------------------------------------------------------------- 1 | # (C) Copyright 2021 ECMWF. 2 | # 3 | # This software is licensed under the terms of the Apache Licence Version 2.0 4 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 5 | # In applying this licence, ECMWF does not waive the privileges and immunities 6 | # granted to it by virtue of its status as an intergovernmental organisation 7 | # nor does it submit to any jurisdiction. 8 | # 9 | 10 | name: QA and short tests 11 | 12 | on: 13 | workflow_dispatch: {} 14 | 15 | # schedule: 16 | # # Every day at 3:07am 17 | # # on the "default" branch set in github (currently is develop) 18 | # - cron: '07 3 * * *' 19 | 20 | pull_request: 21 | 22 | jobs: 23 | quality: 24 | name: Code QA 25 | runs-on: ubuntu-latest 26 | steps: 27 | - run: sudo apt-get install -y pandoc # Needed by sphinx for notebooks 28 | - uses: actions/checkout@v4 29 | - uses: actions/setup-python@v5 30 | with: 31 | python-version: 3.x 32 | - uses: pre-commit/action@v3.0.1 33 | 34 | short-tests: 35 | name: Short tests 36 | runs-on: "ubuntu-latest" 37 | steps: 38 | - uses: actions/checkout@v3 39 | - uses: actions/setup-python@v4 40 | with: 41 | python-version: "3.10" 42 | - name: Install climetlab 43 | run: pip install -e . 44 | - run: pip install pytest 45 | - run: pytest -vv -E short 46 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | 3 | 4 | - repo: https://github.com/pre-commit/pre-commit-hooks 5 | rev: v4.4.0 6 | hooks: 7 | - id: check-yaml # Check YAML files for syntax errors only 8 | args: [--unsafe, --allow-multiple-documents] 9 | - id: debug-statements # Check for debugger imports and py37+ breakpoint() 10 | - id: end-of-file-fixer # Ensure files end in a newline 11 | - id: trailing-whitespace # Trailing whitespace checker 12 | - id: no-commit-to-branch # Prevent committing to main / master 13 | - id: check-added-large-files # Check for large files added to git 14 | - id: check-merge-conflict # Check for files that contain merge conflict 15 | 16 | - repo: https://github.com/psf/black-pre-commit-mirror 17 | rev: 24.1.1 18 | hooks: 19 | - id: black 20 | args: [--line-length=120] 21 | 22 | - repo: https://github.com/pycqa/isort 23 | rev: 5.13.2 24 | hooks: 25 | - id: isort 26 | args: 27 | - -l 120 28 | - --force-single-line-imports 29 | - --profile black 30 | 31 | 32 | - repo: https://github.com/astral-sh/ruff-pre-commit 33 | rev: v0.3.0 34 | hooks: 35 | - id: ruff 36 | args: 37 | - --line-length=120 38 | - --fix 39 | - --exit-non-zero-on-fix 40 | - --preview 41 | -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | # (C) Copyright 2023 ECMWF. 2 | # 3 | # This software is licensed under the terms of the Apache Licence Version 2.0 4 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 5 | # In applying this licence, ECMWF does not waive the privileges and immunities 6 | # granted to it by virtue of its status as an intergovernmental organisation 7 | # nor does it submit to any jurisdiction. 8 | # 9 | # See https://docs.readthedocs.io/en/stable/config-file/v2.html 10 | 11 | version: 2 12 | 13 | build: 14 | os: ubuntu-22.04 15 | tools: 16 | python: "3.10" 17 | 18 | python: 19 | install: 20 | - requirements: requirements.txt 21 | - requirements: docs/requirements.txt 22 | - method: pip 23 | path: . 24 | 25 | # Build PDF & ePub 26 | formats: 27 | - epub 28 | - pdf 29 | -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | // Use IntelliSense to learn about possible attributes. 3 | // Hover to view descriptions of existing attributes. 4 | // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 5 | "version": "0.2.0", 6 | "configurations": [ 7 | { 8 | "name": "Python: Current File", 9 | "type": "python", 10 | "request": "launch", 11 | "program": "${file}", 12 | "console": "integratedTerminal", 13 | "justMyCode": false 14 | } 15 | ] 16 | } 17 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "python.testing.unittestEnabled": false, 3 | "python.testing.pytestEnabled": true 4 | } 5 | -------------------------------------------------------------------------------- /.vscode/tasks.json: -------------------------------------------------------------------------------- 1 | { 2 | // See https://go.microsoft.com/fwlink/?LinkId=733558 3 | // for the documentation about the tasks.json format 4 | "version": "2.0.0", 5 | "tasks": [ 6 | { 7 | "label": "pytest", 8 | "type": "shell", 9 | "command": "pytest", 10 | "problemMatcher": [], 11 | "group": "build" 12 | }, 13 | { 14 | "label": "black", 15 | "type": "shell", 16 | "command": "black .", 17 | "problemMatcher": [], 18 | "group": { 19 | "kind": "build", 20 | "isDefault": true 21 | } 22 | }, 23 | { 24 | "label": "isort", 25 | "type": "shell", 26 | "command": "isort -y", 27 | "problemMatcher": [], 28 | "group": { 29 | "kind": "build", 30 | "isDefault": true 31 | } 32 | }, 33 | { 34 | "label": "flake8", 35 | "type": "shell", 36 | "command": " flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics", 37 | "problemMatcher": [], 38 | "group": { 39 | "kind": "build", 40 | "isDefault": true 41 | } 42 | } 43 | ] 44 | } 45 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | All notable changes to CliMetLab will be documented in this file. 4 | 5 | Notice that CliMetLab is still alpha. Things will break, public API will change. 6 | 7 | ## [0.14.30] 8 | 9 | ## [0.11.1] - 2022-04-11 10 | 11 | - Added support to python 3.10. 12 | - Dropped support for python 3.6 (not supported by numpy anymore). 13 | 14 | ## [0.10.1] - 2022-02-17 15 | 16 | ### Changed 17 | - When using GRIB files. ``to_xarray()`` does not remove empty coordinates anymore. 18 | Use ``to_xarray(xarray_open_dataset_kwargs=dict(squeeze=True))`` to revert to 19 | previous behaviour. 20 | 21 | ### Removed 22 | - ``DateListNormaliser`` has been removed. The same functionality is provided 23 | by the decorator ``cml.decorator.normalize``. 24 | 25 | 26 | 27 | ## [0.0.0] - YYYY-MM-DD 28 | 29 | *Added* for new features. 30 | *Changed* for changes in existing functionality. 31 | *Deprecated* for once-stable features removed in upcoming releases. 32 | *Removed* for deprecated features removed in this release. 33 | *Fixed* for any bug fixes. 34 | *Security* to invite users to upgrade in case of vulnerabilities. 35 | -------------------------------------------------------------------------------- /HOWTO.md: -------------------------------------------------------------------------------- 1 | # Instructions for developers 2 | 3 | (This is a place holder for developer's documentation). 4 | 5 | ## Generate documentation 6 | 7 | TODO: Add documentation 8 | 9 | ## Upload sample data 10 | 11 | To push sample data to ECMWF's object storage (authorised persons only): 12 | 13 | First install `s3cmd` if needed: 14 | ```bash 15 | % pip install s3cmd 16 | ``` 17 | 18 | Then set `~/.s3cfg` to: 19 | ``` 20 | host_base = object-store.os-api.cci1.ecmwf.int 21 | host_bucket = 22 | access_key = xxxxxxxxxxxx 23 | secret_key = yyyyyyyyyyyy 24 | use_https = True 25 | ``` 26 | 27 | First time (already done): 28 | ```bash 29 | % s3cmd mb --no-preserve s3://climetlab 30 | ``` 31 | 32 | ```bash 33 | % s3cmd put --acl-public --no-preserve example.file s3://climetlab/directory/example.file 34 | ``` 35 | 36 | The file is then available at: https://object-store.os-api.cci1.ecmwf.int/climetlab/samples/example.file. 37 | 38 | Please note that only `https` is supported, even if `s3cmd` reports a url starting with `http`. 39 | 40 | 41 | If you use several S3 account, use: 42 | ```bash 43 | % s3cmd -c ~/.s3cfg.climetlab ... 44 | ``` 45 | 46 | Make sure that you can re-create, re-upload all data. 47 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include climetlab/*/*.yaml 2 | include climetlab/*/*/*.yaml 3 | include climetlab/*/*/*/*.yaml 4 | include climetlab/*/*/*/*/*.yaml 5 | include climetlab/*/*/*/*/*/*.yaml 6 | include climetlab/data/css/*.css 7 | include climetlab/sources/dummy.grib 8 | include climetlab/version 9 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # CliMetLab 2 | 3 | [![PyPI version fury.io](https://badge.fury.io/py/climetlab.svg)](https://pypi.python.org/pypi/climetlab/) 4 | [![PyPI pyversions](https://img.shields.io/pypi/pyversions/climetlab.svg)](https://pypi.python.org/pypi/climetlab/) 5 | [![Build](https://github.com/ecmwf/climetlab/actions/workflows/long-test.yml/badge.svg)](https://github.com/ecmwf/climetlab/actions/workflows/long-test.yml) 6 | 7 | **CliMetLab** is a Python package aiming at simplifying access to climate and meteorological datasets, allowing users to focus on science instead of 8 | technical issues such as data access and data formats. It is mostly intended to be used in Jupyter notebooks, and be interoperable with all popular 9 | data analytic packages, such as Numpy, Pandas, Xarray, SciPy, Matplotlib, etc. as well as machine learning frameworks, such as Tensorflow, Keras or PyTorch. 10 | 11 | The documentation can be found at . 12 | 13 | ### Plugins 14 | 15 | See https://climetlab.readthedocs.io/en/latest/guide/pluginlist.html. 16 | 17 | ### License 18 | [Apache License 2.0](LICENSE) In applying this licence, ECMWF does not waive the privileges and immunities 19 | granted to it by virtue of its status as an intergovernmental organisation nor does it submit to any jurisdiction. 20 | -------------------------------------------------------------------------------- /climetlab-and-plugins.code-workspace: -------------------------------------------------------------------------------- 1 | { 2 | "folders": [ 3 | { 4 | "path": "." 5 | }, 6 | { 7 | "path": "../climetlab-meteonet" 8 | }, 9 | { 10 | "path": "../climetlab-demo-dataset" 11 | }, 12 | { 13 | "path": "../climetlab-demo-source" 14 | }, 15 | { 16 | "path": "../climetlab-s2s-ai-challenge" 17 | }, 18 | { 19 | "path": "../climetlab-maelstrom-radiation" 20 | }, 21 | { 22 | "path": "../climetlab-eumetsat" 23 | }, 24 | { 25 | "path": "../climetlab-plugin-tools" 26 | } 27 | ], 28 | "settings": { 29 | 30 | "cSpell.language": "en-GB", 31 | 32 | "yaml.schemas": { 33 | "./schemas/magics.json": ["*.yaml"] 34 | } 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /climetlab.code-workspace: -------------------------------------------------------------------------------- 1 | { 2 | "folders": [ 3 | { 4 | "path": "." 5 | } 6 | ], 7 | "settings": {"cSpell.language": "en-GB", 8 | "yaml.schemas": { 9 | "./schemas/magics.json": [ 10 | "*.yaml" 11 | ] 12 | }, 13 | "esbonio.sphinx.confDir": "" 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /dev/HOWTOS.md: -------------------------------------------------------------------------------- 1 | # Documentation 2 | 3 | Currently, the documentation is hosted at https://climetlab.readthedocs.io/ and is updated each time the `main` branch is updated. 4 | 5 | This may be useful to develop the documentation further: ``sphinx-autobuild docs docs/_build/html`` 6 | 7 | ## Example notebooks 8 | 9 | TODO 10 | 11 | Always run the notebooks twice before committing changes so that they do not show the data downloads. 12 | 13 | ## Python examples 14 | 15 | TODO 16 | 17 | ## Inline example plots 18 | 19 | To regenerate the plots from the examples, run `docs/generate-examples-maps.py`. 20 | 21 | ## Checking links 22 | 23 | TODO 24 | 25 | ## 26 | -------------------------------------------------------------------------------- /dev/constants.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | 3 | import climetlab as cml 4 | 5 | sample = cml.load_source( 6 | "mars", 7 | levtype="sfc", 8 | param="2t", 9 | grid="O96", 10 | time=1200, 11 | date=20010101, 12 | ) 13 | 14 | 15 | start = sample[0].datetime() 16 | first_step = 6 17 | last_step = 240 18 | step_increment = 6 19 | dates = [] 20 | for step in range(first_step, last_step + step_increment, step_increment): 21 | dates.append(start + datetime.timedelta(hours=step)) 22 | 23 | 24 | params = [ 25 | "cos_latitude", 26 | "cos_longitude", 27 | "sin_latitude", 28 | "sin_longitude", 29 | "cos_julian_day", 30 | "cos_local_time", 31 | "sin_julian_day", 32 | "sin_local_time", 33 | "insolation", 34 | ] 35 | 36 | ds = cml.load_source( 37 | "constants", 38 | sample, 39 | date=dates, 40 | param=params, 41 | ) 42 | 43 | 44 | for f in ds: 45 | print(f) 46 | 47 | assert len(ds) == len(params) * len(dates) 48 | -------------------------------------------------------------------------------- /dev/timeseries.csv: -------------------------------------------------------------------------------- 1 | date,value,value2 2 | 2000-01-01,1,43 3 | 2000-01-03,3,-6 4 | 2000-01-04,1,8 5 | 2000-01-05,5,12 6 | 2000-01-06,2,4 7 | 2000-01-07,7,5 8 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env make -f 2 | 3 | # Minimal makefile for Sphinx documentation 4 | # 5 | 6 | # You can set these variables from the command line, and also 7 | # from the environment for the first two. 8 | SPHINXOPTS ?= 9 | SPHINXBUILD ?= sphinx-build 10 | SOURCEDIR = . 11 | BUILDDIR = _build 12 | 13 | # Put it first so that "make" without argument is like "make help". 14 | help: 15 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 16 | 17 | .PHONY: help Makefile 18 | 19 | # Catch-all target: route all unknown targets to Sphinx using the new 20 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 21 | %: Makefile 22 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 23 | -------------------------------------------------------------------------------- /docs/_static/Cli-Met-Lab-Horiz-text-right.eps: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ecmwf/climetlab/a55f9c8590c2ede64a2b0fdc1b163501c81a34f9/docs/_static/Cli-Met-Lab-Horiz-text-right.eps -------------------------------------------------------------------------------- /docs/_static/Cli-Met-Lab-Vert-text-left.eps: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ecmwf/climetlab/a55f9c8590c2ede64a2b0fdc1b163501c81a34f9/docs/_static/Cli-Met-Lab-Vert-text-left.eps -------------------------------------------------------------------------------- /docs/_static/climetlab.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ecmwf/climetlab/a55f9c8590c2ede64a2b0fdc1b163501c81a34f9/docs/_static/climetlab.png -------------------------------------------------------------------------------- /docs/_static/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ecmwf/climetlab/a55f9c8590c2ede64a2b0fdc1b163501c81a34f9/docs/_static/logo.png -------------------------------------------------------------------------------- /docs/_static/style.css: -------------------------------------------------------------------------------- 1 | .wy-side-nav-search { 2 | background-color: #5d89bb; 3 | } 4 | 5 | /*There is a clash between xarray notebook styles and readthedoc*/ 6 | 7 | .rst-content dl.xr-attrs dt { 8 | all: revert; 9 | font-size: 95%; 10 | white-space: nowrap; 11 | } 12 | 13 | .rst-content dl.xr-attrs dd { 14 | font-size: 95%; 15 | } 16 | 17 | .xr-wrap { 18 | font-size: 85%; 19 | } 20 | 21 | img.cmlbadge { 22 | height: 20px; 23 | } 24 | 25 | 26 | .wy-table-responsive table td, 27 | .wy-table-responsive table th { 28 | white-space: normal !important; 29 | vertical-align: top !important; 30 | } 31 | 32 | .wy-table-responsive { 33 | margin-bottom: 24px; 34 | max-width: 100%; 35 | overflow: visible; 36 | } 37 | 38 | /* Hide notebooks warnings */ 39 | .nboutput .stderr { 40 | display: none; 41 | } 42 | -------------------------------------------------------------------------------- /docs/_templates/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ecmwf/climetlab/a55f9c8590c2ede64a2b0fdc1b163501c81a34f9/docs/_templates/.gitignore -------------------------------------------------------------------------------- /docs/_templates/dont-use-layout.html: -------------------------------------------------------------------------------- 1 | {% extends "!layout.html" %} 2 | 3 | -------------------------------------------------------------------------------- /docs/apply-fmt.sh: -------------------------------------------------------------------------------- 1 | : 2 | for n in $(find . -name '*.rst') 3 | do 4 | rstfmt $n 5 | done 6 | -------------------------------------------------------------------------------- /docs/check-index.sh: -------------------------------------------------------------------------------- 1 | : 2 | # See https://github.com/vscode-restructuredtext/vscode-restructuredtext/issues/280 3 | for n in $(find . -name '*.rst') 4 | do 5 | m=$(echo $n | sed 's/\.rst//' | sed 's,^\./,,') 6 | egrep ":doc:.$m" index.rst > /dev/null || echo $m 7 | done 8 | -------------------------------------------------------------------------------- /docs/contributing/alias-argument-1.py: -------------------------------------------------------------------------------- 1 | from climetlab.decorators import alias_argument 2 | 3 | 4 | @alias_argument(param="parameter") 5 | def func(param, other): 6 | return "param=" + param 7 | 8 | 9 | func(param="tp", other=1) 10 | # -> param=tp 11 | 12 | func(parameter="tp", other=1) 13 | # -> param=tp 14 | -------------------------------------------------------------------------------- /docs/contributing/alias-argument-2.py: -------------------------------------------------------------------------------- 1 | from climetlab.decorators import alias_argument 2 | 3 | 4 | @alias_argument(param=["parameter", "variable"]) 5 | def func(param, other=1): 6 | return "param=" + param 7 | 8 | 9 | func(param="tp") 10 | # -> param=tp 11 | 12 | func(parameter="tp") 13 | # -> param=tp 14 | 15 | func(variable="tp") 16 | # -> param=tp 17 | -------------------------------------------------------------------------------- /docs/contributing/alias_argument.rst: -------------------------------------------------------------------------------- 1 | .. _alias_argument: 2 | 3 | Alias argument decorator 4 | ======================== 5 | 6 | This section discusses the purpose of the `@alias_argument` decorator, 7 | shows how to use it and provides reference documentation. 8 | 9 | Purpose 10 | ------- 11 | 12 | This decorator allows a function to accept a range of parameters names, 13 | to make the interface easier to use for the end-user. 14 | 15 | 16 | .. literalinclude:: alias-argument-1.py 17 | 18 | The decorator also accept a list of alias values. 19 | 20 | .. literalinclude:: alias-argument-2.py 21 | -------------------------------------------------------------------------------- /docs/contributing/availability.rst: -------------------------------------------------------------------------------- 1 | .. _availability: 2 | 3 | Availability decorator 4 | ====================== 5 | 6 | .. todo:: 7 | 8 | Not implemented yet 9 | 10 | 11 | Purpose 12 | ------- 13 | 14 | When sharing code, a large amount of code is usually dedicated to 15 | processing the arguments of the functions or methods where users 16 | are requesting data. This boilerplate code needs to check their 17 | values and ensure that data is actually available. In case of 18 | failure, it would be helpful to generate an appropriate error message 19 | to the user, in order to help them understand why the call to the 20 | function was not successful. 21 | 22 | CliMetLab offers predefined shortcuts to implement this. The short 23 | API aims to address the most common use cases using the ``@availability`` 24 | decorator. 25 | 26 | 27 | Example: 28 | 29 | .. code-block:: python 30 | 31 | @availability("availability.json") 32 | def func(date, option): 33 | do_stuff(date, option) 34 | 35 | 36 | 37 | 38 | Usage 39 | ----- 40 | 41 | 42 | Reference 43 | --------- 44 | -------------------------------------------------------------------------------- /docs/contributing/datasets_yaml.rst: -------------------------------------------------------------------------------- 1 | .. _dataset-yaml: 2 | 3 | How to create a dataset plugin (YAML) 4 | ------------------------------------- 5 | 6 | YAML file definitions can be used for simple datasets which rely on 7 | existing built-in :ref:`data source `, and cannot be 8 | as flexible to end-users. The following example shows how to use a 9 | source when the data consists of a single file downloadable from a URL. 10 | 11 | .. code-block:: yaml 12 | 13 | --- 14 | dataset: 15 | source: url 16 | args: 17 | url: http://get.ecmwf.int/test-data/metview/gallery/temp.bufr 18 | 19 | metadata: 20 | documentation: Sample BUFR file containing TEMP messages 21 | 22 | 23 | .. todo:: 24 | Document the YAML file way to create a dataset. 25 | Choose a good way to implement the workflow. 26 | 27 | - Create a dataset YAML file. 28 | - Distribute it. 29 | -------------------------------------------------------------------------------- /docs/contributing/normalize-example-alias-2.py: -------------------------------------------------------------------------------- 1 | from climetlab.decorators import normalize 2 | 3 | DATES = dict( 4 | april=["20210401", "20210402", "20210403"], 5 | june=["20210610", "20210611"], 6 | ) 7 | 8 | 9 | @normalize("x", "date-list(%Y%m%d)", aliases=DATES) 10 | def f(x): 11 | return x 12 | 13 | 14 | assert f("2021-06-10") == ["20210610"] 15 | assert f("june") == ["20210610", "20210611"] 16 | assert f("1999-01-01") == ["19990101"] 17 | -------------------------------------------------------------------------------- /docs/contributing/normalize-example-alias.py: -------------------------------------------------------------------------------- 1 | from climetlab.decorators import normalize 2 | 3 | 4 | @normalize("param", ["tp", "gh"]) 5 | def f(param): 6 | return param 7 | 8 | 9 | assert f(param="tp") == "tp" 10 | # f(param="t2m") # fails 11 | -------------------------------------------------------------------------------- /docs/contributing/normalize-example-date.py: -------------------------------------------------------------------------------- 1 | from climetlab.decorators import normalize 2 | 3 | 4 | @normalize("option", "date(%Y-%m-%d)") 5 | def f(option): 6 | return option 7 | 8 | 9 | assert f(option="2022-12-31") == "2022-12-31" 10 | assert f(option="20221231") == "2022-12-31" 11 | assert f(option=20221231) == "2022-12-31" 12 | -------------------------------------------------------------------------------- /docs/contributing/normalize-example-enum.py: -------------------------------------------------------------------------------- 1 | from climetlab.decorators import normalize 2 | 3 | 4 | @normalize("param", ["tp", "gh"]) 5 | def f(self, param): 6 | print(param) 7 | -------------------------------------------------------------------------------- /docs/contributing/normalize-example-int.py: -------------------------------------------------------------------------------- 1 | from climetlab.decorators import normalize 2 | 3 | 4 | @normalize("option", "int", multiple=True) 5 | def f(option): 6 | return option 7 | 8 | 9 | # Alternative shorter version 10 | @normalize("option", "int-list") 11 | def g(option): 12 | return option 13 | 14 | 15 | assert f(option="2022") == [2022] 16 | assert g(option="2022") == [2022] 17 | assert f(option=[48, 72.0, "96"]) == [48, 72, 96] 18 | assert g(option=[48, 72.0, "96"]) == [48, 72, 96] 19 | -------------------------------------------------------------------------------- /docs/contributing/plotting.rst: -------------------------------------------------------------------------------- 1 | .. warning:: 2 | 3 | This part of CliMetLab is still a work in progress. Documentation and code behaviour will change. 4 | 5 | Plotting 6 | ======== 7 | 8 | 9 | .. todo:: 10 | 11 | Explain how to contribute layers, projections and styles 12 | -------------------------------------------------------------------------------- /docs/dataset-example.py: -------------------------------------------------------------------------------- 1 | # import climetlab as cml 2 | 3 | # data = cml.load_dataset("hurricane-database", bassin="atlantic") 4 | # print(data.home_page) 5 | 6 | # irma = data.to_pandas(name="irma", year=2017) 7 | # cml.plot_map(irma) 8 | -------------------------------------------------------------------------------- /docs/datasource-example.py: -------------------------------------------------------------------------------- 1 | import climetlab as cml 2 | 3 | data = cml.load_source( 4 | "url", 5 | ( 6 | "https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs" 7 | "/v04r00/access/csv/ibtracs.SP.list.v04r00.csv" 8 | ), 9 | ) 10 | 11 | pd = data.to_pandas() 12 | uma = pd[pd.NAME == "UMA:VELI"] 13 | cml.plot_map(uma, style="cyclone-track") 14 | -------------------------------------------------------------------------------- /docs/developer/Makefile: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env make -f 2 | 3 | all: plotting.rst actions.py ../../climetlab/plotting/backends/magics/magics.yaml ../../../climetlab/schemas/definitions.json 4 | 5 | plotting.rst: ~/git/magics/src/params/*.xml xml2rst.py 6 | ./xml2rst.py --types ~/git/magics/src/params/types.yaml --rst ~/git/magics/src/params/*.xml > plotting.tmp 7 | mv plotting.tmp plotting.rst 8 | 9 | actions.py: ~/git/magics/src/params/*.xml xml2rst.py 10 | ./xml2rst.py --types ~/git/magics/src/params/types.yaml --python ~/git/magics/src/params/*.xml > actions.tmp 11 | mv actions.tmp actions.py 12 | 13 | ../../climetlab/plotting/backends/magics/magics.yaml: ~/git/magics/src/params/*.xml xml2rst.py 14 | ./xml2rst.py --types ~/git/magics/src/params/types.yaml --yaml ~/git/magics/src/params/*.xml > magics.tmp 15 | mv magics.tmp ../../climetlab/plotting/backends/magics/magics.yaml 16 | 17 | ../../../climetlab/schemas/definitions.json: ~/git/magics/src/params/*.xml xml2rst.py 18 | ./xml2rst.py --schemas ../../../climetlab/schemas/ --types ~/git/magics/src/params/types.yaml ~/git/magics/src/params/*.xml 19 | -------------------------------------------------------------------------------- /docs/developer/api.rst: -------------------------------------------------------------------------------- 1 | API reference 2 | ============= 3 | 4 | .. todo:: 5 | 6 | Explain how extend the code (write data sources and datasets). 7 | 8 | Public 9 | ------ 10 | 11 | .. autosummary:: 12 | 13 | climetlab.load_dataset 14 | climetlab.load_source 15 | climetlab.plot_map 16 | climetlab.new_plot 17 | 18 | Module 19 | ------ 20 | 21 | 22 | .. automodule:: climetlab.core 23 | :members: 24 | 25 | Function 26 | -------- 27 | 28 | .. autofunction:: climetlab.plot_map 29 | 30 | 31 | Plotting 32 | -------- 33 | 34 | .. .. autoclass:: climetlab.plotting.Backend 35 | .. :members: 36 | 37 | 38 | Other 39 | ----- 40 | 41 | .. autosummary:: 42 | 43 | .. climetlab.Dataset 44 | .. climetlab.DataSource 45 | -------------------------------------------------------------------------------- /docs/developer/architecture.rst: -------------------------------------------------------------------------------- 1 | Architecture 2 | ============ 3 | 4 | Message dispatch 5 | 6 | .. graphviz:: delegation.dot 7 | 8 | 9 | With: 10 | 11 | .. graphviz:: legend.dot 12 | -------------------------------------------------------------------------------- /docs/developer/datasets.rst: -------------------------------------------------------------------------------- 1 | Datasets 2 | ======== 3 | 4 | .. todo:: 5 | 6 | Describe what is a class inheriting from :py:class:`cml.Dataset`. 7 | -------------------------------------------------------------------------------- /docs/developer/delegation.dot: -------------------------------------------------------------------------------- 1 | digraph delegation { 2 | graph [fontname = "helvetica";]; 3 | node [fontname = "helvetica";]; 4 | edge [fontname = "helvetica";]; 5 | 6 | node [shape=box] 7 | 8 | 9 | Dataset; 10 | FileSource; 11 | SomeDataset 12 | SomeReader; 13 | SomeSource; 14 | Reader; 15 | Source; 16 | 17 | // SomeDataset -> SomeSource ; 18 | 19 | 20 | // Inheritance 21 | SomeDataset -> Dataset; 22 | SomeSource -> FileSource; 23 | FileSource -> Source; 24 | SomeReader -> Reader; 25 | 26 | // Delegation 27 | // "SomeDataset" -> "SomeSource"; 28 | Dataset -> SomeSource [label=< source >; style=dashed;]; 29 | FileSource -> SomeReader [label=< reader >; style=dashed;]; 30 | 31 | {rank=same; FileSource; SomeReader;} 32 | {rank=same; Dataset; SomeSource;} 33 | 34 | } 35 | -------------------------------------------------------------------------------- /docs/developer/gallery.rst: -------------------------------------------------------------------------------- 1 | Gallery 2 | ======= 3 | 4 | .. module-output:: generate_gallery_rst 5 | -------------------------------------------------------------------------------- /docs/developer/legend.dot: -------------------------------------------------------------------------------- 1 | digraph legend { 2 | rankdir=LR; 3 | graph [fontname = "helvetica"]; 4 | node [fontname = "helvetica"]; 5 | edge [fontname = "helvetica";]; 6 | 7 | node [shape=point; height=0] 8 | 9 | A -> B [label=< inherits from >;]; 10 | 11 | B -> C [style=invis]; 12 | C -> D [label=< delegates to >; style=dashed;]; 13 | } 14 | -------------------------------------------------------------------------------- /docs/developer/overview.rst: -------------------------------------------------------------------------------- 1 | Overview 2 | ======== 3 | 4 | The is the **CliMetLab** developer guide part of the CliMetLab documentation. 5 | 6 | The CliMetLab documentation is split as follow: 7 | 8 | - *Getting started*: General introduction with the main idea described there. 9 | - *User guide*: This is the part you should read if you are using CliMetLab 10 | and plugins developed by others. 11 | - *Plugin Developer guide*: describes how to create plugins (or YAML files) to add 12 | data and functionalities to CliMetLab, to make it available to the users 13 | above. In order to avoid rewriting the same code over and over, consider 14 | distributing it, the design of CliMetLab allows to do this with plugins. 15 | - **CliMetLab Developers guide**: Please refer to this part either if you are willing to 16 | develop further CliMetLab or if you want to achieve something that is not 17 | possible with the current plugin framework. 18 | -------------------------------------------------------------------------------- /docs/developer/sources.rst: -------------------------------------------------------------------------------- 1 | Data sources 2 | ============ 3 | 4 | .. todo:: 5 | 6 | Describe what is a class inheriting from :py:class:`cml.Source`. 7 | -------------------------------------------------------------------------------- /docs/developer/todolist.rst: -------------------------------------------------------------------------------- 1 | Help wanted 2 | =========== 3 | 4 | CliMetLab is still experimental and its development status is still Alpha. 5 | Here are the parts that would need work and/or benefit from a community 6 | contribution. 7 | 8 | Backward compatibility is not ensured as long a version 1.0 is not released. 9 | Nevertheless the functionalities are expected to be usable and stable, 10 | please submit a issue if needed (pull request welcome). 11 | 12 | .. _todolist: 13 | 14 | How can I to contribute ? 15 | ------------------------- 16 | 17 | - Submit bug reports, propose enhancements, on github. 18 | - You can also contribute to the core code by forking and 19 | submitting a pull request. 20 | - TODO: fill this todo list. 21 | -------------------------------------------------------------------------------- /docs/examples.rst: -------------------------------------------------------------------------------- 1 | .. _Examples: 2 | 3 | 4 | Examples 5 | ======== 6 | 7 | Here is a list of example notebooks to illustrate how to access data, 8 | create plots, and do machine learning using CliMetLab. 9 | 10 | .. toctree:: 11 | :maxdepth: 2 12 | :glob: 13 | 14 | examples/* 15 | -------------------------------------------------------------------------------- /docs/examples/.gitignore: -------------------------------------------------------------------------------- 1 | *.py 2 | *.test 3 | -------------------------------------------------------------------------------- /docs/examples/Makefile: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env make -f 2 | 3 | tests := $(patsubst %.ipynb,%.test,$(wildcard ??-*.ipynb)) 4 | 5 | all : $(tests) 6 | 7 | 8 | .ipynb.py: 9 | jupyter nbconvert --to script $< 10 | 11 | .py.test: 12 | python3 -mIPython $< 13 | touch $@ 14 | 15 | clean: 16 | rm -f *.test *.py 17 | 18 | .SUFFIXES: .py .ipynb .test 19 | -------------------------------------------------------------------------------- /docs/examples/renumber.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | for n in $(seq $1 89) 4 | do 5 | m=$((n+1)) 6 | f1=$(printf '%02d-' $n) 7 | f2=$(printf '%02d-' $m) 8 | p=$(ls $f1*.ipynb 2>/dev/null) 9 | if [[ -f $p ]] 10 | then 11 | q=$(echo $p | sed s/$f1/$f2/) 12 | echo $p $q 13 | git mv $p $q 14 | fi 15 | done 16 | -------------------------------------------------------------------------------- /docs/examples/test.db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ecmwf/climetlab/a55f9c8590c2ede64a2b0fdc1b163501c81a34f9/docs/examples/test.db -------------------------------------------------------------------------------- /docs/examples/test.grib: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ecmwf/climetlab/a55f9c8590c2ede64a2b0fdc1b163501c81a34f9/docs/examples/test.grib -------------------------------------------------------------------------------- /docs/examples/test.nc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ecmwf/climetlab/a55f9c8590c2ede64a2b0fdc1b163501c81a34f9/docs/examples/test.nc -------------------------------------------------------------------------------- /docs/examples/test4.grib: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ecmwf/climetlab/a55f9c8590c2ede64a2b0fdc1b163501c81a34f9/docs/examples/test4.grib -------------------------------------------------------------------------------- /docs/generate-examples-maps.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # (C) Copyright 2020 ECMWF. 3 | # 4 | # This software is licensed under the terms of the Apache Licence Version 2.0 5 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 6 | # In applying this licence, ECMWF does not waive the privileges and immunities 7 | # granted to it by virtue of its status as an intergovernmental organisation 8 | # nor does it submit to any jurisdiction. 9 | # 10 | 11 | import os 12 | 13 | me = os.path.basename(__file__) 14 | here = os.path.dirname(__file__) 15 | 16 | CODE = """ 17 | import climetlab as cml 18 | cml.plotting_options(path="{}") 19 | {} 20 | """ 21 | 22 | for root, _, files in os.walk(here): 23 | for file in files: 24 | if file.endswith(".py") and file != me: 25 | full = os.path.join(root, file) 26 | with open(full) as f: 27 | code = f.read() 28 | if "plot_map" in code: 29 | n = len(here) + 1 30 | path = os.path.join(here, "_static", full[n:]) 31 | path = path.replace(".py", ".svg") 32 | if not os.path.exists(os.path.dirname(path)): 33 | os.makedirs(os.path.dirname(path)) 34 | print("PATH", path) 35 | code = CODE.format(path, code) 36 | exec(code, dict(__file__=full), {}) 37 | -------------------------------------------------------------------------------- /docs/guide/cmdline.rst: -------------------------------------------------------------------------------- 1 | Command line tool 2 | ================= 3 | 4 | Installing climetlab also make available the ``climetlab`` utility 5 | command line interface. 6 | 7 | Usage 8 | ------ 9 | 10 | .. code:: bash 11 | 12 | $ climetlab [options] 13 | 14 | .. code:: bash 15 | 16 | $ climetlab 17 | (climetlab) [options] 18 | 19 | Interactive prompt 20 | ------------------ 21 | :code:`climetlab` 22 | 23 | Running the ``climetlab`` command with no argument starts the 24 | interactive prompt. Autocompletion is enabled on the interactive 25 | prompt. To exit the interactive prompt use Control+D. 26 | 27 | .. code:: bash 28 | 29 | $ climetlab 30 | (climetlab) [options] 31 | (climetlab) 32 | 33 | 34 | .. module-output:: generate_cmdline_help 35 | -------------------------------------------------------------------------------- /docs/guide/mltools.rst: -------------------------------------------------------------------------------- 1 | Machine learning tools 2 | ====================== 3 | 4 | .. warning:: 5 | 6 | This part of CliMetLab is still a work in progress. Documentation and code behaviour will change. 7 | 8 | .. todo:: 9 | 10 | TODO: Develop and document machine learning related tools 11 | 12 | 13 | to_tfdataset() 14 | -------------- 15 | 16 | To use a CliMetLab dataset with tensorflow, 17 | use the ``_to_tfdataset()`` method. 18 | 19 | .. code-block:: python 20 | 21 | >>> import climetlab as cml 22 | >>> ds = cml.load_dataset(x) 23 | >>> x = ds.to_tfdataset(options) 24 | >>> model.fit(x, ....) 25 | 26 | The discussion is still open to decide whether ``to_dataset()`` returns: 27 | 28 | - ``tf.keras.utils.experimental.DatasetCreator`` 29 | - ``tf.data.Dataset`` 30 | - ``tf.keras.utils.Sequence`` 31 | - A custom CliMetLab class 32 | 33 | PyTorch support 34 | --------------- 35 | 36 | .. todo:: 37 | 38 | A long-term goal of CliMetLab is to provide a easy way to use a dataset with Pytorch. 39 | A merge request within this respect would be welcome. 40 | The CliMetLab API (exposed to the end-user) should be mostly identical for Tensorflow or Pytorch. 41 | -------------------------------------------------------------------------------- /docs/guide/plotting-attributes.py: -------------------------------------------------------------------------------- 1 | import climetlab as cml 2 | 3 | # cml.settings.set("plotting-options", {'dump_yaml': True}) 4 | 5 | dataset = cml.load_dataset("example-dataset") 6 | data = dataset[0] 7 | 8 | cml.plot_map(data, foreground=False) 9 | 10 | cml.plot_map(data, foreground=True) 11 | 12 | 13 | cml.plot_map(data, foreground="example-foreground") 14 | 15 | cml.plot_map( 16 | data, 17 | foreground=dict( 18 | map_grid=False, 19 | map_label=False, 20 | map_grid_frame=True, 21 | map_grid_frame_thickness=5, 22 | map_boundaries=True, 23 | ), 24 | ) 25 | 26 | # Partial update of the current `foreground` 27 | # How to do is still to be decided 28 | 29 | # Option 1 30 | cml.plot_map( 31 | data, 32 | foreground={ 33 | "+map_rivers": True, 34 | "+map_cities": True, 35 | "+map_label": True, 36 | "-map_boundaries": None, 37 | }, 38 | ) 39 | 40 | # Option 2 41 | cml.plot_map( 42 | data, 43 | foreground={ 44 | "set": {"map_rivers": True, "map_cities": True, "map_label": True}, 45 | "clear": ["map_boundaries"], 46 | }, 47 | ) 48 | # Option 3 49 | cml.plot_map( 50 | data, 51 | foreground={ 52 | "+": {"map_rivers": True, "map_cities": True, "map_label": True}, 53 | "-": ["map_boundaries"], 54 | }, 55 | ) 56 | # Option 4 57 | cml.plot_map( 58 | data, 59 | update_foreground={ 60 | "map_rivers": True, 61 | "map_cities": True, 62 | "map_label": True, 63 | "map_boundaries": None, 64 | }, 65 | ) 66 | 67 | # Option 5 68 | cml.plot_map( 69 | data, 70 | update={ 71 | "foreground": { 72 | "map_rivers": True, 73 | "map_cities": True, 74 | "map_label": True, 75 | "map_boundaries": None, 76 | }, 77 | }, 78 | ) 79 | -------------------------------------------------------------------------------- /docs/guide/plotting-options.py: -------------------------------------------------------------------------------- 1 | import climetlab as cml 2 | 3 | cml.plotting_options(width=400, foreground=False) 4 | -------------------------------------------------------------------------------- /docs/guide/plotting.rst: -------------------------------------------------------------------------------- 1 | .. _plotting: 2 | 3 | 4 | Plotting 5 | ======== 6 | 7 | See also : https://confluence.ecmwf.int/display/MAGP/Reference+guide 8 | 9 | .. todo:: 10 | 11 | Introduce advance plotting options 12 | 13 | 14 | .. literalinclude:: plotting-attributes.py 15 | 16 | 17 | .. literalinclude:: plotting-options.py 18 | 19 | 20 | .. _projections: 21 | 22 | Projections 23 | ----------- 24 | 25 | .. module-output:: file_content data/projections/global.YAML 26 | 27 | See `mmap `_. 28 | 29 | .. _styles: 30 | 31 | Styles 32 | ------ 33 | 34 | TODO 35 | 36 | See `mcont `_ and `msymb `_. 37 | 38 | 39 | .. _layers: 40 | 41 | Layers 42 | ------ 43 | 44 | See `mcoast `_. 45 | 46 | 47 | TODO 48 | -------------------------------------------------------------------------------- /docs/guide/settings-1-get.py: -------------------------------------------------------------------------------- 1 | import climetlab as cml 2 | 3 | # Access one of the settings 4 | cache_path = cml.settings.get("cache-directory") 5 | print(cache_path) 6 | 7 | # If this is the last line of a Notebook cell, this 8 | # will display a table with all the current settings 9 | cml.settings 10 | -------------------------------------------------------------------------------- /docs/guide/settings-2-set.py: -------------------------------------------------------------------------------- 1 | import climetlab as cml 2 | 3 | # Change the location of the cache: 4 | cml.settings.set("cache-directory", "/big-disk/climetlab-cache") 5 | 6 | # Set some default plotting options (e.g. all maps will 7 | # be 400 pixels wide by default): 8 | cml.settings.set("plotting-options", width=400) 9 | -------------------------------------------------------------------------------- /docs/guide/settings-3-reset.py: -------------------------------------------------------------------------------- 1 | import climetlab as cml 2 | 3 | # Reset a named setting to its default value 4 | cml.settings.reset("cache-directory") 5 | 6 | # Reset all settings to their default values 7 | cml.settings.reset() 8 | -------------------------------------------------------------------------------- /docs/print-magics-objects.py: -------------------------------------------------------------------------------- 1 | import climetlab.plotting 2 | 3 | # List of possible projections 4 | for p in climetlab.plotting.projections(): 5 | print(p) 6 | 7 | # List of possible styles 8 | for p in climetlab.plotting.styles(): 9 | print(p) 10 | 11 | # List of possible backgrounds/foregrounds 12 | for p in climetlab.plotting.layers(): 13 | print(p) 14 | -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | # These are the requirements for readthedoc 2 | eccodes 3 | ecmwflibs 4 | entrypoints 5 | filelock 6 | ipykernel 7 | magics 8 | markdown 9 | markupsafe==2.0.1 10 | multiurl 11 | nbsphinx 12 | pandas 13 | pyyaml 14 | Pygments>=2.6.1 15 | sphinx-panels 16 | sphinx_rtd_theme 17 | termcolor 18 | tqdm 19 | -------------------------------------------------------------------------------- /experiments/README: -------------------------------------------------------------------------------- 1 | Just a few experiments 2 | -------------------------------------------------------------------------------- /experiments/dask-client.py: -------------------------------------------------------------------------------- 1 | # flake8: noqa 2 | import logging 3 | import os 4 | import time 5 | 6 | import dask 7 | 8 | import climetlab as cml 9 | 10 | # https://github.com/dask/dask-jobqueue/issues/548 11 | # http://localhost:8787/status 12 | 13 | 14 | hosts = [f"node{i}" for i in range(0, 4)] 15 | 16 | workers = list(i + 1 for i in range(len(hosts) - 1)) 17 | 18 | client = cml.start_dask( 19 | "ssh", 20 | cluster_kwargs=dict( 21 | hosts=hosts, 22 | connect_options=dict( 23 | config=os.path.expanduser("~/.ssh/vagrant_ssh_config"), 24 | known_hosts=None, 25 | ), 26 | remote_python="/usr/bin/python3", 27 | scheduler_options=dict( 28 | host="node0", 29 | port=9000, 30 | dashboard_address=":8787", 31 | ), 32 | # worker_options=dict(worker_port=[9000+i for i in workers]) 33 | ), 34 | ).client 35 | 36 | 37 | def inc(x): 38 | from distributed.worker import logger 39 | 40 | w = dask.distributed.get_worker() # .log('hello') 41 | logger.info("xxxx", x) 42 | # assert False, type(w) 43 | return x + 1 44 | 45 | 46 | x = client.submit(inc, 10) 47 | 48 | print(client.get_events()) 49 | # print(client.scheduler.story()) 50 | 51 | time.sleep(10) 52 | 53 | print(x.result()) 54 | -------------------------------------------------------------------------------- /experiments/dask/.gitignore: -------------------------------------------------------------------------------- 1 | .vagrant/ 2 | -------------------------------------------------------------------------------- /experiments/dask/Vagrantfile: -------------------------------------------------------------------------------- 1 | # brew install qemu vagrant 2 | # On MAC M1: 3 | # vagrant plugin install vagrant-qemu 4 | # vagrant up --provider=qemu 5 | # Elsewhere: 6 | # vagrant up --provider=virtualbox 7 | # vagrant ssh-config > ~/.ssh/vagrant_ssh_config 8 | 9 | Vagrant.configure(2) do |config| 10 | config.vm.box = "generic/ubuntu2204" 11 | # config.vm.box_version = "4.1.4" 12 | config.vm.synced_folder ".", "/vagrant", disabled: true 13 | 14 | config.vm.provision "file", source: "~/.cdsapirc", destination: ".cdsapirc" 15 | config.vm.provision "file", source: "~/.ecmwfapirc", destination: ".ecmwfapirc" 16 | 17 | config.vm.provision "file", source: "config.yaml", destination: "~/.config/dask/config.yaml" 18 | 19 | N = 3 20 | 21 | (0..N).each do |i| 22 | config.vm.define "node#{i}" do |node| 23 | 24 | node.vm.network "private_network", ip: "192.168.56.#{i+10}" 25 | 26 | 27 | node.vm.provision "shell" do |s| 28 | s.path = "provision.sh" 29 | s.args = [i, N] 30 | end 31 | 32 | node.vm.provider "qemu" do |qe| 33 | qe.memory = "2G" 34 | qe.arch = "x86_64" 35 | qe.machine = "q35" 36 | qe.cpu = "max" 37 | qe.net_device = "virtio-net-pci" 38 | qe.ssh_port = 5022 + i 39 | end 40 | 41 | if (i == 0) 42 | node.vm.network :forwarded_port, guest: 9000, host: 9000 # scheduler 43 | node.vm.network :forwarded_port, guest: 8787, host: 8787 # dashboard 44 | else 45 | node.vm.network :forwarded_port, guest: 9000+i, host: 9000+i # worker 46 | end 47 | end 48 | end 49 | end 50 | -------------------------------------------------------------------------------- /experiments/dask/config.yaml: -------------------------------------------------------------------------------- 1 | logging: 2 | version: 1 3 | format: "%(asctime)s %(levelname)s %(message)s" 4 | handlers: 5 | file: 6 | class: logging.handlers.RotatingFileHandler 7 | filename: output.log 8 | level: DEBUG 9 | console: 10 | class: logging.StreamHandler 11 | level: DEBUG 12 | loggers: 13 | distributed.worker: 14 | level: DEBUG 15 | handlers: 16 | - file 17 | - console 18 | distributed.scheduler: 19 | level: DEBUG 20 | handlers: 21 | - file 22 | - console 23 | -------------------------------------------------------------------------------- /experiments/dask/provision.sh: -------------------------------------------------------------------------------- 1 | 2 | #!/bin/bash 3 | set -x 4 | node=$1 5 | total=$2 6 | 7 | if [[ ! -f /usr/bin/pip3 ]]; then 8 | apt-get update 9 | apt-get install -y python3-pip 10 | fi 11 | 12 | if [[ ! -f /usr/bin/simpleproxy ]]; then 13 | apt-get install -y simpleproxy 14 | fi 15 | 16 | pip3 freeze | grep climetlab 17 | if [[ $? -ne 0 ]]; then 18 | pip3 install bokeh 19 | pip3 install "dask[distributed]" 20 | pip3 install git+https://github.com/ecmwf/climetlab.git@develop 21 | fi 22 | 23 | for n in $(seq 0 $total) 24 | do 25 | grep node$n /etc/hosts 26 | if [[ $? -ne 0 ]]; then 27 | echo "192.168.56.$((n+10)) node$n" >>/etc/hosts 28 | fi 29 | done 30 | 31 | # cat < /etc/systemd/system/simpleproxy$n.service 32 | # [Service] 33 | # Type=simple 34 | # ExecStart=/usr/bin/simpleproxy $proxy 35 | # Restart=always 36 | 37 | # [Install] 38 | # WantedBy=multi-user.target 39 | 40 | # [Unit] 41 | # Description=Simple Proxy 42 | # After=network.target 43 | # EOF 44 | 45 | 46 | # systemctl daemon-reload 47 | # systemctl start simpleproxy$n 48 | # systemctl enable simpleproxy$n 49 | # done 50 | 51 | hostnamectl set-hostname node$node 52 | -------------------------------------------------------------------------------- /experiments/era5-vs-oper.py: -------------------------------------------------------------------------------- 1 | # flake8: noqa 2 | from tensorflow.keras.layers import Dense 3 | from tensorflow.keras.layers import Flatten 4 | from tensorflow.keras.layers import Input 5 | from tensorflow.keras.layers import Reshape 6 | from tensorflow.keras.models import Sequential 7 | 8 | import climetlab as cml 9 | 10 | request = dict( 11 | date="20220101/to/20220131", 12 | levtype="sfc", 13 | param="2t", 14 | grid="1/1", 15 | ) 16 | 17 | ds1 = cml.load_source("mars", **request) 18 | ds2 = cml.load_source( 19 | "cds", 20 | dataset="reanalysis-era5-single-levels", 21 | product_type="reanalysis", 22 | **request, 23 | ) 24 | 25 | shape = ds1[0].shape 26 | 27 | 28 | def match_other(i): 29 | return ds2[i].to_numpy() - ds1[i].to_numpy() 30 | 31 | 32 | def ds1_(i): 33 | return ds1_[i].to_numpy() 34 | 35 | 36 | input = ds1.to_tfdataset(features=ds1_, target=match_other) 37 | 38 | print(shape) 39 | # shape = tf1.element_spec.shape 40 | # shape=(36,19) 41 | 42 | model = Sequential() 43 | model.add(Input(shape=(shape[-2], shape[-1]))) 44 | model.add(Flatten()) 45 | model.add(Dense(64, activation="sigmoid")) 46 | model.add(Dense(64, activation="sigmoid")) 47 | model.add(Dense(shape[-2] * shape[-1])) 48 | model.add(Reshape(target_shape=(shape[-2], shape[-1]))) 49 | 50 | model.compile( 51 | optimizer="adam", 52 | loss="mean_squared_error", 53 | metrics=["mean_squared_error"], 54 | ) 55 | 56 | 57 | print(model.summary()) 58 | model.fit( 59 | input, 60 | epochs=10, 61 | verbose=1, 62 | use_multiprocessing=True, 63 | workers=10, 64 | ) 65 | -------------------------------------------------------------------------------- /experiments/fdb.py: -------------------------------------------------------------------------------- 1 | # flake8: noqa 2 | # 3 | # ret,param=t,grid=1/1,date=-2,target=data.grib, 4 | # stream=enfo,number=1/to/50,type=pf 5 | # FDB_ROOT_DIRECTORY=/tmp/fdb FDB_SCHEMA_FILE=~/build/fdb5/etc/fdb/schema /usr/local/bin/fdb-write data.grib 6 | # 7 | 8 | import time 9 | 10 | import climetlab as cml 11 | 12 | now = time.time() 13 | ds = cml.load_source( 14 | "fdb", 15 | root="/tmp/fdb", 16 | request={ 17 | "class": "od", 18 | "type": "pf", 19 | }, 20 | ) 21 | 22 | print(ds.to_xarray()) 23 | print(time.time() - now) 24 | 25 | 26 | print(ds.to_xarray().mean(dim="number")) 27 | -------------------------------------------------------------------------------- /experiments/gaussian.py: -------------------------------------------------------------------------------- 1 | # (C) Copyright 2022 ECMWF. 2 | # 3 | # This software is licensed under the terms of the Apache Licence Version 2.0 4 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 5 | # In applying this licence, ECMWF does not waive the privileges and immunities 6 | # granted to it by virtue of its status as an intergovernmental organisation 7 | # nor does it submit to any jurisdiction. 8 | # 9 | # flake8: noqa 10 | 11 | import time 12 | 13 | import climetlab as cml 14 | from climetlab.grids import lookup 15 | from climetlab.grids import unstructed_to_structed 16 | 17 | ds = cml.load_source("mars", param="2t", date=20220907, levtype="sfc") 18 | tree = unstructed_to_structed(ds[0], 15) 19 | 20 | now = time.time() 21 | print(lookup(tree, 51.0, -1.0)) 22 | print("----", time.time() - now) 23 | -------------------------------------------------------------------------------- /experiments/grib-caching.py: -------------------------------------------------------------------------------- 1 | # (C) Copyright 2020 ECMWF. 2 | # 3 | # This software is licensed under the terms of the Apache Licence Version 2.0 4 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 5 | # In applying this licence, ECMWF does not waive the privileges and immunities 6 | # granted to it by virtue of its status as an intergovernmental organisation 7 | # nor does it submit to any jurisdiction. 8 | # 9 | # flake8: noqa 10 | 11 | 12 | from climetlab import load_source 13 | from climetlab.profiling import timer 14 | 15 | years = list(range(1979, 2021)) 16 | years = list(range(1979, 1979 + 3)) 17 | 18 | 19 | with timer("load_source"): 20 | s = load_source( 21 | "cds", 22 | "reanalysis-era5-single-levels-monthly-means", 23 | variable="all", 24 | year=years, 25 | month=list(range(1, 13)), 26 | time=0, 27 | product_type="monthly_averaged_reanalysis", 28 | # grid=[0.25, 0.25], 29 | grid=[1, 1], 30 | split_on="year", 31 | ) 32 | 33 | with timer("len"): 34 | print(len(s)) 35 | -------------------------------------------------------------------------------- /experiments/grib-coding.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import numpy as np 4 | 5 | import climetlab as cml 6 | import climetlab.debug 7 | 8 | f = cml.new_grib_output("test.grib") 9 | 10 | f.write( 11 | np.random.rand(181, 360), 12 | metadata=dict( 13 | param="msl", 14 | date="1990-01-01T12:00", 15 | expver="xxxx", 16 | ), 17 | ) 18 | 19 | print(os.path.getsize("test.grib")) 20 | 21 | 22 | f = cml.new_grib_output("test2.grib") 23 | 24 | f.write( 25 | np.random.rand(180 * 2, 360 * 2), 26 | metadata=dict( 27 | param="msl", 28 | date="1990-01-01T12:00", 29 | expver="xxxx", 30 | ), 31 | ) 32 | 33 | print(os.path.getsize("test2.grib")) 34 | 35 | f = cml.new_grib_output("test3.grib") 36 | 37 | f.write( 38 | np.random.rand(40320), 39 | metadata=dict( 40 | param="msl", 41 | date="1990-01-01T12:00", 42 | expver="xxxx", 43 | ), 44 | ) 45 | 46 | print(os.path.getsize("test3.grib")) 47 | -------------------------------------------------------------------------------- /experiments/load-zarr-fourcastnet.yaml: -------------------------------------------------------------------------------- 1 | input: 2 | inherit: true 3 | source: 4 | - name: cds 5 | dataset: era5-complete 6 | class: ea 7 | date: '2000-01-01/to/2000-01-02' 8 | time: ['0000', '0600', '1200', '1800'] 9 | param: [10u, 10v, 2t, sp, msl, tcwv] 10 | levtype: sfc 11 | grid: [0.25, 0.25] 12 | 13 | - name: cds 14 | param: [u, v] 15 | level: [1000, 850, 500] 16 | levtype: pl 17 | 18 | - name: cds 19 | param: z 20 | level: [1000, 850, 500, 50] 21 | 22 | - name: cds 23 | param: [t, r] 24 | level: [850, 500] 25 | 26 | 27 | output: 28 | remapping: 29 | param_level: '{param}{level}' 30 | 31 | chunking: 32 | date: 1 33 | order: 34 | - datetime 35 | - param_level: 36 | - t850 37 | - u1000 38 | - v1000 39 | - z1000 40 | - u850 41 | - v850 42 | - z850 43 | - u500 44 | - v500 45 | - z500 46 | - t500 47 | - z50 48 | - r500 49 | - r850 50 | - 10u 51 | - 10v 52 | - 2t 53 | - sp 54 | - msl 55 | - tcwv 56 | dtype: float32 57 | -------------------------------------------------------------------------------- /experiments/load-zarr-from-db-pl-and-sfc.yaml: -------------------------------------------------------------------------------- 1 | input: 2 | dataset: 3 | - name: era5-for-ai 4 | date: '2000-01-01/to/2000-01-31' 5 | time: [0, 1200] 6 | param: [t, z, u,v,q] 7 | level: [500, 600, 850] 8 | 9 | - name: era5-for-ai 10 | date: '2000-01-01/to/2000-01-31' 11 | time: [0, 1200] 12 | param: [2t] 13 | 14 | 15 | output: 16 | chunking: 17 | date: 1 18 | order: 19 | - datetime 20 | - param_level 21 | dtype: float32 22 | -------------------------------------------------------------------------------- /experiments/load-zarr.yaml: -------------------------------------------------------------------------------- 1 | loop: 2 | dates: 3 | daily: 4 | start: '2015-04-18' 5 | end: '2015-04-19' 6 | input: 7 | inherit: true 8 | source: 9 | - name: file 10 | path: test-data/uvz-20150418.grib 11 | 12 | 13 | output: 14 | remapping: 15 | param_level: '{param}_{levelist}' 16 | statistics: param_level 17 | chunking: 18 | valid_datetime: 1 19 | order_by: 20 | - valid_datetime 21 | - param_level: 22 | - u_500 23 | - u_850 24 | - v_500 25 | - v_850 26 | - z_500 27 | - z_850 28 | 29 | dtype: float32 30 | -------------------------------------------------------------------------------- /experiments/order.py: -------------------------------------------------------------------------------- 1 | # flake8: noqa 2 | import os 3 | 4 | import climetlab as cml 5 | 6 | here = os.path.dirname(__file__) 7 | 8 | ds = cml.load_source( 9 | "cds", 10 | "reanalysis-era5-single-levels", 11 | variable=["msl", "2t"], 12 | product_type="reanalysis", 13 | area=[50, -50, 20, 50], 14 | date="2012-12-12", 15 | time=list(reversed(range(24))), 16 | ) 17 | for x in ds: 18 | print(x) 19 | print("====") 20 | 21 | for x in ds.order_by("param", "time"): 22 | print(x) 23 | -------------------------------------------------------------------------------- /experiments/scaling.py: -------------------------------------------------------------------------------- 1 | # flake8: noqa 2 | import climetlab as cml 3 | 4 | request = dict( 5 | date="20220101/to/20220131", 6 | levtype="sfc", 7 | param="2t", 8 | grid="1/1", 9 | ) 10 | 11 | ds = cml.load_source("mars", **request) 12 | 13 | print(ds.statistics()) 14 | print(ds.to_bounding_box()) 15 | print(dir(ds)) 16 | 17 | s = ds.scaled(method="minmax") 18 | 19 | print(s[0].to_numpy()) 20 | -------------------------------------------------------------------------------- /experiments/start_dask.py: -------------------------------------------------------------------------------- 1 | #!/user/bin/env python3 2 | # flake8: noqa 3 | 4 | import sys 5 | import time 6 | 7 | import climetlab as cml 8 | from climetlab.utils.dask import start 9 | 10 | 11 | def main(): 12 | if len(sys.argv) > 1: 13 | kind = sys.argv[1] 14 | deploy = start(kind, start_client=False) 15 | else: 16 | deploy = start(start_client=False) 17 | print(f"Running dask server {deploy}") 18 | while True: 19 | time.sleep(10) 20 | 21 | 22 | if __name__ == "__main__": 23 | main() 24 | -------------------------------------------------------------------------------- /experiments/test-cubes.py: -------------------------------------------------------------------------------- 1 | import climetlab as cml 2 | 3 | # import climetlab.debug # noqa 4 | 5 | t850 = cml.load_source("mars", param="t", level=850, grid=[1, 1], time=[1200, 1800]) 6 | z500 = cml.load_source("mars", param="z", level=500, grid=[1, 1], time=[1200, 1800]) 7 | 8 | ds = cml.load_source("multi", t850, z500) 9 | ds = ds.full("param", "levelist") 10 | print(len(ds)) 11 | 12 | for i in range(len(ds)): 13 | try: 14 | print("---", ds[i]) 15 | except Exception as e: 16 | print(e) 17 | 18 | ############ 19 | 20 | ds = cml.load_source("multi", t850, z500) 21 | c = ds.cube("time", "param+levelist") 22 | # print(c) 23 | print(c[0]) 24 | print(c[1]) 25 | -------------------------------------------------------------------------------- /experiments/test-cubes2.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from climetlab.loaders import ZarrLoader 4 | from climetlab.loaders import load 5 | 6 | load( 7 | ZarrLoader("out.zarr"), 8 | os.path.join(os.path.dirname(__file__), "load-zarr-fourcastnet.yaml"), 9 | ) 10 | -------------------------------------------------------------------------------- /experiments/timeseries.py: -------------------------------------------------------------------------------- 1 | # flake8: noqa 2 | # 3 | # ret,param=t,grid=1/1,date=-2,target=data.grib, 4 | # stream=enfo,number=1/to/50,type=pf 5 | # FDB_ROOT_DIRECTORY=/tmp/fdb FDB_SCHEMA_FILE=~/build/fdb5/etc/fdb/schema /usr/local/bin/fdb-write data.grib 6 | # 7 | 8 | import time 9 | 10 | import climetlab as cml 11 | 12 | now = time.time() 13 | ds = cml.load_source( 14 | "cds", 15 | dataset="reanalysis-era5-single-levels", 16 | product_type="reanalysis", 17 | param="2t", 18 | grid="10/10", 19 | date="19590101/to/19590201", 20 | time=12, 21 | ) 22 | 23 | df = ds.to_pandas(latitude=0.0, longitude=0.0) 24 | print(df) 25 | -------------------------------------------------------------------------------- /experiments/virtual1.py: -------------------------------------------------------------------------------- 1 | # flake8: noqa 2 | import os 3 | import time 4 | 5 | import dask 6 | 7 | import climetlab as cml 8 | 9 | ds = cml.load_source("virtual", param="2t") 10 | now = time.time() 11 | print("a", len(ds), now) 12 | x = ds.to_xarray() 13 | print(x) 14 | 15 | y = x.t2m 16 | print(y.mean(dim="time").compute()) 17 | -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | ; addopts=-s --cov climetlab --verbose --cov-report xml --cov-report html 3 | ; addopts=--no-cov 4 | addopts=-s --verbose -E release 5 | ; log_cli = 1 6 | markers = 7 | long_test: a test which is long to run. Typically more that 5 sec. 8 | download: a test downloading some data. 9 | external_download: a test downloading data from an external location (not related to climetlab). 10 | small_download: a test downloading a very small amount of data (very small overhead). 11 | large_download: a test downloading a very small amount of data (very small overhead). 12 | ftp: test that used FTP. FTP is an old protocol and is not supported by most recent firewalls. 13 | documentation: testing documentation can be slow (especially notebooks with long download). But need to be performed on release to ensure that the documention is valid. 14 | testpaths = tests 15 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | # These are the requirements for Binder and Deepnote (?) 2 | climetlab 3 | -------------------------------------------------------------------------------- /schemas/magics.json: -------------------------------------------------------------------------------- 1 | { 2 | "$id": "https://example.com/ecmwf-magics.schema.json", 3 | "$schema": "http://json-schema.org/draft-07/schema#", 4 | "title": "Magics", 5 | "type": "object", 6 | "properties": { 7 | "magics": { 8 | "type": "object", 9 | "description": "TODO", 10 | "minProperties": 1, 11 | "maxProperties": 1, 12 | "properties": { 13 | "mcoast": { 14 | "$ref": "mcoast.json" 15 | }, 16 | "mmap": { 17 | "$ref": "mmap.json" 18 | }, 19 | "mcont": { 20 | "$ref": "mcont.json" 21 | }, 22 | "msymb": { 23 | "$ref": "msymb.json" 24 | } 25 | }, 26 | "additionalProperties": false 27 | }, 28 | "plot_map": { 29 | "type": "object", 30 | "properties": { 31 | "margin": { 32 | "type": "number" 33 | } 34 | } 35 | }, 36 | "gallery": { 37 | "type": "object", 38 | "properties": { 39 | "sample": { 40 | "type": "object", 41 | "minProperties": 1, 42 | "maxProperties": 1, 43 | "properties": { 44 | "dataset": { 45 | "type": "object", 46 | "properties": { 47 | "name": { 48 | "type": "string" 49 | } 50 | }, 51 | "required": ["name"] 52 | }, 53 | "source": { 54 | "type": "object", 55 | "properties": { 56 | "name": { 57 | "type": "string" 58 | } 59 | }, 60 | "required": ["name"] 61 | } 62 | } 63 | } 64 | } 65 | } 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /src/climetlab/__main__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # (C) Copyright 2022 ECMWF. 4 | # 5 | # This software is licensed under the terms of the Apache Licence Version 2.0 6 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 7 | # In applying this licence, ECMWF does not waive the privileges and immunities 8 | # granted to it by virtue of its status as an intergovernmental organisation 9 | # nor does it submit to any jurisdiction. 10 | # 11 | 12 | from climetlab.scripts.main import main 13 | 14 | main() 15 | -------------------------------------------------------------------------------- /src/climetlab/aaa.py: -------------------------------------------------------------------------------- 1 | # (C) Copyright 2023 ECMWF. 2 | # 3 | # This software is licensed under the terms of the Apache Licence Version 2.0 4 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 5 | # In applying this licence, ECMWF does not waive the privileges and immunities 6 | # granted to it by virtue of its status as an intergovernmental organisation 7 | # nor does it submit to any jurisdiction. 8 | # 9 | 10 | # This module is called aaa so isort keeps it at the top on the imports 11 | # as it needs to run first 12 | 13 | import inspect 14 | import os 15 | import sys 16 | 17 | LOADED_MODULES = {} 18 | CLIMETLAB_DEBUG_IMPORTS = int(os.environ.get("CLIMETLAB_DEBUG_IMPORTS", 0)) 19 | 20 | 21 | class Requested: 22 | def find_spec(self, name, path=None, target=None): 23 | LOADED_MODULES[name] = [] 24 | if CLIMETLAB_DEBUG_IMPORTS: 25 | for f in inspect.stack(): 26 | if f.filename == __file__: 27 | continue 28 | 29 | if "importlib._bootstrap" in f.filename: 30 | continue 31 | 32 | LOADED_MODULES[name].append(f"{f.filename}:{f.lineno}") 33 | 34 | 35 | class NotFound: 36 | def find_spec(self, name, path=None, target=None): 37 | LOADED_MODULES.pop(name, None) 38 | 39 | 40 | def loaded_modules(): 41 | # global LOADED_MODULES 42 | # return LOADED_MODULES 43 | return set(sys.modules.keys()) 44 | 45 | 46 | sys.meta_path.insert(0, Requested()) 47 | sys.meta_path.append(NotFound()) 48 | -------------------------------------------------------------------------------- /src/climetlab/arguments/__init__.py: -------------------------------------------------------------------------------- 1 | # (C) Copyright 2021 ECMWF. 2 | # 3 | # This software is licensed under the terms of the Apache Licence Version 2.0 4 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 5 | # In applying this licence, ECMWF does not waive the privileges and immunities 6 | # granted to it by virtue of its status as an intergovernmental organisation 7 | # nor does it submit to any jurisdiction. 8 | # 9 | 10 | from .argument import Argument 11 | from .input_manager import InputManager 12 | 13 | __all__ = ["InputManager", "normaliser"] 14 | 15 | 16 | def normaliser(*args, **kwargs): 17 | return Argument("no-name", *args, **kwargs) 18 | -------------------------------------------------------------------------------- /src/climetlab/arguments/guess.py: -------------------------------------------------------------------------------- 1 | # (C) Copyright 2021 ECMWF. 2 | # 3 | # This software is licensed under the terms of the Apache Licence Version 2.0 4 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 5 | # In applying this licence, ECMWF does not waive the privileges and immunities 6 | # granted to it by virtue of its status as an intergovernmental organisation 7 | # nor does it submit to any jurisdiction. 8 | # 9 | import logging 10 | 11 | LOG = logging.getLogger(__name__) 12 | 13 | 14 | def guess_type_list(lst): 15 | assert isinstance(lst, (tuple, list)) 16 | 17 | for typ in [int, float, str]: 18 | if all([isinstance(x, typ) for x in lst]): 19 | return typ 20 | 21 | return None 22 | -------------------------------------------------------------------------------- /src/climetlab/config/units.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | # This file contains convertion for units that are not cf-compliants 3 | 4 | - from: kelvin 5 | to: degC 6 | scaling: 1 7 | offset: -273.15 8 | 9 | - from: kelvin 10 | to: K 11 | scaling: 1 12 | offset: 0 13 | 14 | - from: Celcius 15 | to: degC 16 | scaling: 1 17 | offset: 0 18 | -------------------------------------------------------------------------------- /src/climetlab/core/constants.py: -------------------------------------------------------------------------------- 1 | # (C) Copyright 2020 ECMWF. 2 | # 3 | # This software is licensed under the terms of the Apache Licence Version 2.0 4 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 5 | # In applying this licence, ECMWF does not waive the privileges and immunities 6 | # granted to it by virtue of its status as an intergovernmental organisation 7 | # nor does it submit to any jurisdiction. 8 | # 9 | 10 | DATETIME = "valid_datetime" 11 | -------------------------------------------------------------------------------- /src/climetlab/core/initialise.py: -------------------------------------------------------------------------------- 1 | # (C) Copyright 2020 ECMWF. 2 | # 3 | # This software is licensed under the terms of the Apache Licence Version 2.0 4 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 5 | # In applying this licence, ECMWF does not waive the privileges and immunities 6 | # granted to it by virtue of its status as an intergovernmental organisation 7 | # nor does it submit to any jurisdiction. 8 | # 9 | import mimetypes 10 | 11 | from .metadata import init_metadata 12 | 13 | mimetypes.add_type("application/x-netcdf", ".nc") 14 | mimetypes.add_type("application/x-netcdf", ".nc4") 15 | mimetypes.add_type("application/x-netcdf", ".cdf") 16 | mimetypes.add_type("application/x-netcdf", ".netcdf") 17 | 18 | mimetypes.add_type("application/x-grib", ".grib") 19 | mimetypes.add_type("application/x-grib", ".grib1") 20 | mimetypes.add_type("application/x-grib", ".grib2") 21 | 22 | 23 | def initialise(): 24 | init_metadata() 25 | -------------------------------------------------------------------------------- /src/climetlab/core/select.py: -------------------------------------------------------------------------------- 1 | # (C) Copyright 2023 ECMWF. 2 | # 3 | # This software is licensed under the terms of the Apache Licence Version 2.0 4 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 5 | # In applying this licence, ECMWF does not waive the privileges and immunities 6 | # granted to it by virtue of its status as an intergovernmental organisation 7 | # nor does it submit to any jurisdiction. 8 | # 9 | 10 | import datetime 11 | import logging 12 | 13 | LOG = logging.getLogger(__name__) 14 | 15 | 16 | def normalize_selection(*args, **kwargs): 17 | from climetlab.arguments.transformers import ALL 18 | 19 | _kwargs = {} 20 | for a in args: 21 | if a is None: 22 | continue 23 | if isinstance(a, dict): 24 | _kwargs.update(a) 25 | continue 26 | raise ValueError(f"Cannot make a selection with {a}") 27 | 28 | _kwargs.update(kwargs) 29 | 30 | for k, v in _kwargs.items(): 31 | assert ( 32 | v is None 33 | or v is ALL 34 | or callable(v) 35 | or isinstance(v, (list, tuple, set)) 36 | or isinstance(v, (str, int, float, datetime.datetime)) 37 | ), f"Unsupported type: {type(v)} for key {k}" 38 | return _kwargs 39 | -------------------------------------------------------------------------------- /src/climetlab/dask/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ecmwf/climetlab/a55f9c8590c2ede64a2b0fdc1b163501c81a34f9/src/climetlab/dask/__init__.py -------------------------------------------------------------------------------- /src/climetlab/data/css/table.css: -------------------------------------------------------------------------------- 1 | table.climetlab td { 2 | vertical-align: top; 3 | text-align: left !important; 4 | } 5 | -------------------------------------------------------------------------------- /src/climetlab/data/dask/local.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | dask: 3 | cluster_cls: local 4 | scale: null 5 | -------------------------------------------------------------------------------- /src/climetlab/data/dask/slurm.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | dask: 3 | cluster_cls: slurm 4 | cluster_kwargs: 5 | cores: 4 6 | memory: 8G 7 | walltime: "00:30:00" 8 | # job_extra will be renamed job_extra_directives in next version of dask_jobqueue 9 | job_extra: 10 | - "--qos=np" 11 | - "--cpus-per-task=256" 12 | - "--ntasks=1" 13 | scheduler_options: 14 | dashboard_address: 12367 15 | scale: 8 16 | -------------------------------------------------------------------------------- /src/climetlab/data/dask/ssh.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | dask: 3 | cluster_cls: ssh 4 | cluster_kwargs: 5 | hosts: 6 | - localhost 7 | - localhost 8 | scale: null 9 | -------------------------------------------------------------------------------- /src/climetlab/data/layers/default-background.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | magics: 3 | mcoast: 4 | map_grid: off 5 | map_grid_colour: tan 6 | map_label: off 7 | map_boundaries: on 8 | map_coastline_land_shade: on 9 | map_coastline_land_shade_colour: cream 10 | map_coastline_colour: tan 11 | map_grid_frame: on 12 | map_grid_frame_thickness: 5 13 | -------------------------------------------------------------------------------- /src/climetlab/data/layers/default-foreground.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | magics: 3 | mcoast: 4 | map_grid: off 5 | map_label: off 6 | map_grid_frame: on 7 | map_grid_frame_thickness: 5 8 | -------------------------------------------------------------------------------- /src/climetlab/data/layers/example-foreground.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | hidden: true 3 | magics: 4 | mcoast: 5 | map_grid: off 6 | map_label: off 7 | map_grid_frame: on 8 | map_grid_frame_thickness: 5 9 | -------------------------------------------------------------------------------- /src/climetlab/data/layers/land-sea.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | magics: 3 | mcoast: 4 | map_grid: off 5 | map_grid_colour: tan 6 | map_label: off 7 | map_boundaries: off 8 | map_coastline_land_shade: on 9 | map_coastline_land_shade_colour: cream 10 | map_coastline_sea_shade: on 11 | map_coastline_sea_shade_colour: sky 12 | map_coastline_colour: tan 13 | map_grid_frame: off 14 | map_grid_frame_thickness: 5 15 | -------------------------------------------------------------------------------- /src/climetlab/data/projections/africa.yaml: -------------------------------------------------------------------------------- 1 | magics: 2 | mmap: 3 | subpage_lower_left_latitude: -40 4 | subpage_lower_left_longitude: -45 5 | subpage_map_projection: cylindrical 6 | subpage_upper_right_latitude: 40 7 | subpage_upper_right_longitude: 75 8 | -------------------------------------------------------------------------------- /src/climetlab/data/projections/asia.yaml: -------------------------------------------------------------------------------- 1 | magics: 2 | mmap: 3 | subpage_lower_left_latitude: 0 4 | subpage_lower_left_longitude: 55 5 | subpage_map_projection: cylindrical 6 | subpage_upper_right_latitude: 80 7 | subpage_upper_right_longitude: 175 8 | -------------------------------------------------------------------------------- /src/climetlab/data/projections/bonne.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | magics: 3 | mmap: 4 | subpage_map_projection: bonne 5 | -------------------------------------------------------------------------------- /src/climetlab/data/projections/collignon.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | magics: 3 | mmap: 4 | subpage_map_projection: collignon 5 | -------------------------------------------------------------------------------- /src/climetlab/data/projections/euro-atlantic.yaml: -------------------------------------------------------------------------------- 1 | magics: 2 | mmap: 3 | subpage_lower_left_latitude: 15 4 | subpage_lower_left_longitude: -40 5 | subpage_map_hemisphere: north 6 | subpage_map_projection: polar_stereographic 7 | subpage_upper_right_latitude: 50 8 | subpage_upper_right_longitude: 70 9 | -------------------------------------------------------------------------------- /src/climetlab/data/projections/europe-cylindrical.yaml: -------------------------------------------------------------------------------- 1 | magics: 2 | mmap: 3 | subpage_lower_left_latitude: 34.5 4 | subpage_lower_left_longitude: -10.67 5 | subpage_map_projection: cylindrical 6 | subpage_upper_right_latitude: 71.05 7 | subpage_upper_right_longitude: 31.55 8 | -------------------------------------------------------------------------------- /src/climetlab/data/projections/europe.yaml: -------------------------------------------------------------------------------- 1 | magics: 2 | mmap: 3 | subpage_lower_left_latitude: 21.51 4 | subpage_lower_left_longitude: -37.27 5 | subpage_map_projection: polar_stereographic 6 | subpage_upper_right_latitude: 51.28 7 | subpage_upper_right_longitude: 65 8 | subpage_map_vertical_longitude: 0 9 | -------------------------------------------------------------------------------- /src/climetlab/data/projections/global.yaml: -------------------------------------------------------------------------------- 1 | magics: 2 | mmap: 3 | subpage_lower_left_latitude: -90 4 | subpage_lower_left_longitude: -180 5 | subpage_map_projection: cylindrical 6 | subpage_upper_right_latitude: 90 7 | subpage_upper_right_longitude: 180 8 | -------------------------------------------------------------------------------- /src/climetlab/data/projections/goode.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | magics: 3 | mmap: 4 | subpage_map_projection: goode 5 | -------------------------------------------------------------------------------- /src/climetlab/data/projections/mercator.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | magics: 3 | mmap: 4 | subpage_map_projection: mercator 5 | -------------------------------------------------------------------------------- /src/climetlab/data/projections/mollweide.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | magics: 3 | mmap: 4 | subpage_map_projection: mollweide 5 | -------------------------------------------------------------------------------- /src/climetlab/data/projections/north-america.yaml: -------------------------------------------------------------------------------- 1 | magics: 2 | mmap: 3 | subpage_lower_left_latitude: -5 4 | subpage_lower_left_longitude: -140 5 | subpage_map_projection: polar_stereographic 6 | subpage_upper_right_latitude: 30 7 | subpage_upper_right_longitude: -15 8 | subpage_map_vertical_longitude: -100 9 | -------------------------------------------------------------------------------- /src/climetlab/data/projections/north-america1.yaml: -------------------------------------------------------------------------------- 1 | magics: 2 | mmap: 3 | subpage_lower_left_latitude: -5 4 | subpage_lower_left_longitude: -140 5 | subpage_map_projection: cylindrical 6 | subpage_upper_right_latitude: 80 7 | subpage_upper_right_longitude: -15 8 | -------------------------------------------------------------------------------- /src/climetlab/data/projections/north-atlantic.yaml: -------------------------------------------------------------------------------- 1 | magics: 2 | mmap: 3 | subpage_lower_left_latitude: 5 4 | subpage_lower_left_longitude: -48 5 | subpage_map_hemisphere: north 6 | subpage_map_projection: polar_stereographic 7 | subpage_upper_right_latitude: 40 8 | subpage_upper_right_longitude: 87 9 | -------------------------------------------------------------------------------- /src/climetlab/data/projections/north-hemisphere.yaml: -------------------------------------------------------------------------------- 1 | magics: 2 | mmap: 3 | subpage_lower_left_latitude: 0.0 4 | subpage_lower_left_longitude: -45.0 5 | subpage_map_hemisphere: north 6 | subpage_map_projection: polar_stereographic 7 | subpage_upper_right_latitude: 0.0 8 | subpage_upper_right_longitude: 135.0 9 | -------------------------------------------------------------------------------- /src/climetlab/data/projections/polar-north.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | magics: 3 | mmap: 4 | # subpage_lower_left_longitude: -37.27 5 | # subpage_upper_right_longitude: 65.00 6 | # subpage_upper_right_latitude: 51.28 7 | subpage_map_projection: polar_stereographic 8 | # subpage_lower_left_latitude: 21.51 9 | -------------------------------------------------------------------------------- /src/climetlab/data/projections/robinson.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | magics: 3 | mmap: 4 | subpage_map_projection: robinson 5 | -------------------------------------------------------------------------------- /src/climetlab/data/projections/south-america.yaml: -------------------------------------------------------------------------------- 1 | magics: 2 | mmap: 3 | subpage_lower_left_latitude: -65 4 | subpage_lower_left_longitude: -125 5 | subpage_map_projection: cylindrical 6 | subpage_upper_right_latitude: 20 7 | subpage_upper_right_longitude: 5 8 | -------------------------------------------------------------------------------- /src/climetlab/data/projections/south-atlantic.yaml: -------------------------------------------------------------------------------- 1 | magics: 2 | mmap: 3 | subpage_lower_left_latitude: -70 4 | subpage_lower_left_longitude: -90 5 | subpage_map_projection: mercator 6 | subpage_upper_right_latitude: -20 7 | subpage_upper_right_longitude: 50 8 | -------------------------------------------------------------------------------- /src/climetlab/data/projections/south-hemisphere.yaml: -------------------------------------------------------------------------------- 1 | magics: 2 | mmap: 3 | subpage_lower_left_latitude: 0.0 4 | subpage_lower_left_longitude: 44.9 5 | subpage_map_hemisphere: south 6 | subpage_map_projection: polar_stereographic 7 | subpage_upper_right_latitude: 0.0 8 | subpage_upper_right_longitude: -134.9 9 | -------------------------------------------------------------------------------- /src/climetlab/data/projections/south-pacific.yaml: -------------------------------------------------------------------------------- 1 | magics: 2 | mmap: 3 | subpage_lower_left_latitude: -70 4 | subpage_lower_left_longitude: -190 5 | subpage_map_projection: mercator 6 | subpage_upper_right_latitude: -20 7 | subpage_upper_right_longitude: -50 8 | -------------------------------------------------------------------------------- /src/climetlab/data/projections/tapestry.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | hidden: true 3 | magics: 4 | mmap: 5 | subpage_map_area_definition_polar: corners 6 | subpage_map_projection: polar_stereographic 7 | subpage_lower_left_latitude: 13.646 8 | subpage_lower_left_longitude: -44.725 9 | subpage_upper_right_latitude: 38.2711 10 | subpage_upper_right_longitude: 123.288 11 | -------------------------------------------------------------------------------- /src/climetlab/data/projections/tropics-east.yaml: -------------------------------------------------------------------------------- 1 | magics: 2 | mmap: 3 | subpage_lower_left_latitude: -35 4 | subpage_lower_left_longitude: 0 5 | subpage_map_projection: cylindrical 6 | subpage_upper_right_latitude: 35 7 | subpage_upper_right_longitude: 180 8 | -------------------------------------------------------------------------------- /src/climetlab/data/projections/tropics-west.yaml: -------------------------------------------------------------------------------- 1 | magics: 2 | mmap: 3 | subpage_lower_left_latitude: -35 4 | subpage_lower_left_longitude: -180 5 | subpage_map_projection: cylindrical 6 | subpage_upper_right_latitude: 35 7 | subpage_upper_right_longitude: 0 8 | -------------------------------------------------------------------------------- /src/climetlab/data/projections/web-mercator.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | magics: 3 | mmap: 4 | subpage_map_projection: "EPSG:3857" 5 | subpage_lower_left_latitude: -20048966.10 6 | subpage_lower_left_longitude: -20026376.39 7 | subpage_upper_right_latitude: 20048966.10 8 | subpage_upper_right_longitude: 20026376.39 9 | subpage_coordinates_system: projection 10 | -------------------------------------------------------------------------------- /src/climetlab/data/styles/cyclone-track.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | magics: 3 | msymb: 4 | legend: off 5 | symbol_type: marker 6 | symbol_marker_index: 28 7 | symbol_colour: red 8 | symbol_height: 0.20 9 | symbol_text_font_colour: black 10 | symbol_connect_line: on 11 | 12 | plot_map: 13 | margins: 0.01 14 | 15 | gallery: 16 | sample: 17 | dataset: 18 | name: hurricane-database 19 | args: 20 | bassin: atlantic 21 | to_pandas: 22 | name: irma 23 | year: 2017 24 | -------------------------------------------------------------------------------- /src/climetlab/data/styles/default-style-fields.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | magics: 3 | mcont: 4 | contour_automatic_setting: ecmwf 5 | legend: off 6 | -------------------------------------------------------------------------------- /src/climetlab/data/styles/default-style-observations.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | magics: 3 | msymb: 4 | legend: off 5 | symbol_type: marker 6 | symbol_colour: red 7 | symbol_height: 0.08 8 | symbol_marker_index: 15 # Circle 9 | plot_map: 10 | margins: 0.01 11 | -------------------------------------------------------------------------------- /src/climetlab/data/styles/land-sea-mask.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | magics: 3 | mcont: 4 | contour: off 5 | contour_grid_value_marker_colour: brown 6 | contour_grid_value_marker_height: 0.1 7 | contour_grid_value_marker_index: 5 8 | contour_grid_value_max: 1.2 9 | contour_grid_value_min: 0.5 10 | contour_grid_value_plot: on 11 | contour_grid_value_plot_type: marker 12 | contour_hilo: off 13 | contour_label: off 14 | contour_legend_text: Land grid points (lsm > 0.5) 15 | -------------------------------------------------------------------------------- /src/climetlab/data/styles/no-style.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | magics: 3 | mcont: 4 | legend: off 5 | -------------------------------------------------------------------------------- /src/climetlab/data/styles/rainbow-markers.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | magics: 3 | msymb: 4 | contour_interval: 1.0 5 | contour_reference_level: 0.0 6 | symbol_advanced_table_colour_direction: clockwise 7 | symbol_advanced_table_interval: 1.0 8 | symbol_advanced_table_max_level_colour: red 9 | symbol_advanced_table_min_level_colour: blue 10 | symbol_advanced_table_reference_level: 0.0 11 | symbol_advanced_table_selection_type: interval 12 | symbol_marker_index: 15 13 | symbol_table_mode: advanced 14 | symbol_type: marker 15 | symbol_advanced_table_height_list: [0.08] 16 | gallery: 17 | plot_map: 18 | background: false 19 | plot_map: 20 | margins: 0.01 21 | -------------------------------------------------------------------------------- /src/climetlab/data/styles/tapestry.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | hidden: true 3 | magics: 4 | mcont: 5 | contour_line_thickness: 4 6 | contour_line_colour: rgb(0.7,0.41,0.54) 7 | contour_reference_level: 556 8 | contour_highlight_colour: rgb(0.73,0.2,0.4) 9 | contour_highlight_thickness: 8 10 | contour_level_selection_type: interval 11 | contour_interval: 5 12 | contour_label: off 13 | -------------------------------------------------------------------------------- /src/climetlab/datasets/era5_precipitations.py: -------------------------------------------------------------------------------- 1 | # (C) Copyright 2020 ECMWF. 2 | # 3 | # This software is licensed under the terms of the Apache Licence Version 2.0 4 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 5 | # In applying this licence, ECMWF does not waive the privileges and immunities 6 | # granted to it by virtue of its status as an intergovernmental organisation 7 | # nor does it submit to any jurisdiction. 8 | # 9 | 10 | from .era5_single_levels import Era5SingleLevels 11 | 12 | 13 | class Era5Precipitations(Era5SingleLevels): 14 | def __init__(self, *args, **kwargs): 15 | super().__init__("tp", *args, **kwargs) 16 | 17 | 18 | dataset = Era5Precipitations 19 | -------------------------------------------------------------------------------- /src/climetlab/datasets/era5_temperature.py: -------------------------------------------------------------------------------- 1 | # (C) Copyright 2020 ECMWF. 2 | # 3 | # This software is licensed under the terms of the Apache Licence Version 2.0 4 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 5 | # In applying this licence, ECMWF does not waive the privileges and immunities 6 | # granted to it by virtue of its status as an intergovernmental organisation 7 | # nor does it submit to any jurisdiction. 8 | # 9 | 10 | from .era5_single_levels import Era5SingleLevels 11 | 12 | 13 | class Era5Temperature(Era5SingleLevels): 14 | def __init__(self, *args, **kwargs): 15 | super().__init__("2t", *args, **kwargs) 16 | 17 | 18 | dataset = Era5Temperature 19 | -------------------------------------------------------------------------------- /src/climetlab/datasets/example-dataset.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | hidden: True 3 | dataset: 4 | source: url 5 | args: 6 | url: https://github.com/ecmwf/climetlab/raw/main/docs/examples/test.grib 7 | 8 | metadata: 9 | documentation: Sample GRIB file 10 | -------------------------------------------------------------------------------- /src/climetlab/datasets/meteonet_samples/__init__.py: -------------------------------------------------------------------------------- 1 | # (C) Copyright 2020 ECMWF. 2 | # 3 | # This software is licensed under the terms of the Apache Licence Version 2.0 4 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 5 | # In applying this licence, ECMWF does not waive the privileges and immunities 6 | # granted to it by virtue of its status as an intergovernmental organisation 7 | # nor does it submit to any jurisdiction. 8 | # 9 | 10 | from climetlab import Dataset 11 | 12 | 13 | class Meteonet(Dataset): 14 | URL = "https://github.com/meteofrance/meteonet/raw/master/data_samples" 15 | 16 | home_page = "https://meteonet.umr-cnrm.fr" 17 | 18 | licence = "https://meteonet.umr-cnrm.fr/dataset/LICENCE.md" 19 | 20 | documentation = "https://meteofrance.github.io/meteonet/" 21 | -------------------------------------------------------------------------------- /src/climetlab/datasets/meteonet_samples/dataset.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | dataset: 3 | home_page: https://meteofrance.github.io/meteonet/ 4 | licence: https://meteonet.umr-cnrm.fr/dataset/LICENCE.md 5 | -------------------------------------------------------------------------------- /src/climetlab/datasets/meteonet_samples/ground_stations.py: -------------------------------------------------------------------------------- 1 | # (C) Copyright 2020 ECMWF. 2 | # 3 | # This software is licensed under the terms of the Apache Licence Version 2.0 4 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 5 | # In applying this licence, ECMWF does not waive the privileges and immunities 6 | # granted to it by virtue of its status as an intergovernmental organisation 7 | # nor does it submit to any jurisdiction. 8 | # 9 | 10 | import pandas as pd 11 | 12 | from climetlab.utils import download_and_cache 13 | 14 | from . import Meteonet 15 | 16 | 17 | class MeteonetGroundStations(Meteonet): 18 | """ 19 | See https://github.com/meteofrance/meteonet 20 | """ 21 | 22 | def __init__(self, domain="NW", date="20160101"): 23 | url = "{url}/ground_stations/{domain}_{date}.csv".format(url=self.URL, domain=domain, date=date) 24 | 25 | self.path = download_and_cache(url) 26 | self._pandas = pd.read_csv(self.path, parse_dates=[4], infer_datetime_format=True) 27 | 28 | def to_pandas(self): 29 | return self._pandas 30 | 31 | def plot_map(self, backend): 32 | north, east = self._pandas[["lat", "lon"]].max() 33 | south, west = self._pandas[["lat", "lon"]].min() 34 | 35 | backend.bounding_box(north, west, south, east) 36 | backend.plot_pandas(self._pandas, "lat", "lon", "t") 37 | 38 | 39 | dataset = MeteonetGroundStations 40 | -------------------------------------------------------------------------------- /src/climetlab/datasets/meteonet_samples/masks.py: -------------------------------------------------------------------------------- 1 | # (C) Copyright 2020 ECMWF. 2 | # 3 | # This software is licensed under the terms of the Apache Licence Version 2.0 4 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 5 | # In applying this licence, ECMWF does not waive the privileges and immunities 6 | # granted to it by virtue of its status as an intergovernmental organisation 7 | # nor does it submit to any jurisdiction. 8 | # 9 | 10 | 11 | from climetlab import load_source 12 | 13 | from . import Meteonet 14 | 15 | 16 | class MeteonetMasks(Meteonet): 17 | """ 18 | See https://github.com/meteofrance/meteonet 19 | """ 20 | 21 | def __init__( 22 | self, 23 | domain="NW", 24 | ): 25 | url = "{url}/masks/{domain}_masks.grib".format( 26 | url=self.URL, 27 | domain=domain, 28 | ) 29 | self.source = load_source("url", url) 30 | 31 | 32 | dataset = MeteonetMasks 33 | -------------------------------------------------------------------------------- /src/climetlab/datasets/meteonet_samples/styles/meteonet-radar-rainfall.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | hidden: true 3 | magics: 4 | mcont: 5 | contour: off 6 | contour_highlight: off 7 | contour_hilo: off 8 | contour_label: off 9 | contour_level_list: 10 | - -1.0 11 | - 0.0 12 | - 2.0 13 | - 4.0 14 | - 6.0 15 | - 8.0 16 | - 10.0 17 | - 15.0 18 | - 20.0 19 | - 25.0 20 | - 30.0 21 | - 35.0 22 | - 40.0 23 | - 45.0 24 | - 50.0 25 | - 55.0 26 | - 60.0 27 | - 65.0 28 | - 75.0 29 | contour_shade: on 30 | contour_shade_colour_list: 31 | - silver 32 | - none 33 | - darkslateblue 34 | - mediumblue 35 | - dodgerblue 36 | - skyblue 37 | - olive 38 | - mediumseagreen 39 | - cyan 40 | - lime 41 | - yellow 42 | - khaki 43 | - burlywood 44 | - orange 45 | - brown 46 | - pink 47 | - red 48 | - plum 49 | contour_shade_colour_method: list 50 | contour_shade_technique: grid_shading 51 | -------------------------------------------------------------------------------- /src/climetlab/datasets/meteonet_samples/weather_models.py: -------------------------------------------------------------------------------- 1 | # (C) Copyright 2020 ECMWF. 2 | # 3 | # This software is licensed under the terms of the Apache Licence Version 2.0 4 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 5 | # In applying this licence, ECMWF does not waive the privileges and immunities 6 | # granted to it by virtue of its status as an intergovernmental organisation 7 | # nor does it submit to any jurisdiction. 8 | # 9 | 10 | 11 | from climetlab import load_source 12 | 13 | from . import Meteonet 14 | 15 | 16 | class MeteonetWeatherModels(Meteonet): 17 | """ 18 | See https://github.com/meteofrance/meteonet 19 | """ 20 | 21 | def __init__(self, model="arome", variable="2m", domain="NW", date="20180501", time="0000"): 22 | url = "{url}/weather_models/{model}_{variable}_{domain}_{date}{time}00.grib".format( 23 | url=self.URL, 24 | variable=variable, 25 | model=model, 26 | domain=domain, 27 | date=date, 28 | time=time, 29 | ) 30 | self.source = load_source("url", url) 31 | 32 | 33 | dataset = MeteonetWeatherModels 34 | -------------------------------------------------------------------------------- /src/climetlab/datasets/sample-bufr-data.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | dataset: 3 | source: url 4 | args: 5 | url: http://get.ecmwf.int/test-data/metview/gallery/temp.bufr 6 | 7 | metadata: 8 | documentation: Sample BUFR file containing TEMP messages 9 | -------------------------------------------------------------------------------- /src/climetlab/datasets/sample-grib-data.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | dataset: 3 | source: url 4 | args: 5 | url: https://github.com/ecmwf/climetlab/raw/main/docs/examples/test.grib 6 | 7 | metadata: 8 | documentation: Sample GRIB file 9 | -------------------------------------------------------------------------------- /src/climetlab/debug.py: -------------------------------------------------------------------------------- 1 | # (C) Copyright 2020 ECMWF. 2 | # 3 | # This software is licensed under the terms of the Apache Licence Version 2.0 4 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 5 | # In applying this licence, ECMWF does not waive the privileges and immunities 6 | # granted to it by virtue of its status as an intergovernmental organisation 7 | # nor does it submit to any jurisdiction. 8 | # 9 | 10 | 11 | def debug(status=True, level="DEBUG"): 12 | if status in ["on", "ON", True]: 13 | import logging 14 | 15 | logging.basicConfig(level=level) 16 | 17 | 18 | debug(status=True, level="DEBUG") 19 | -------------------------------------------------------------------------------- /src/climetlab/distributed/__init__.py: -------------------------------------------------------------------------------- 1 | # (C) Copyright 2021 ECMWF. 2 | # 3 | # This software is licensed under the terms of the Apache Licence Version 2.0 4 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 5 | # In applying this licence, ECMWF does not waive the privileges and immunities 6 | # granted to it by virtue of its status as an intergovernmental organisation 7 | # nor does it submit to any jurisdiction. 8 | # 9 | -------------------------------------------------------------------------------- /src/climetlab/exceptions.py: -------------------------------------------------------------------------------- 1 | # (C) Copyright 2023 ECMWF. 2 | # 3 | # This software is licensed under the terms of the Apache Licence Version 2.0 4 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 5 | # In applying this licence, ECMWF does not waive the privileges and immunities 6 | # granted to it by virtue of its status as an intergovernmental organisation 7 | # nor does it submit to any jurisdiction. 8 | # 9 | 10 | 11 | class NotIndexedDirectoryError(ValueError): 12 | pass 13 | -------------------------------------------------------------------------------- /src/climetlab/indexing/__init__.py: -------------------------------------------------------------------------------- 1 | # (C) Copyright 2021 ECMWF. 2 | # 3 | # This software is licensed under the terms of the Apache Licence Version 2.0 4 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 5 | # In applying this licence, ECMWF does not waive the privileges and immunities 6 | # granted to it by virtue of its status as an intergovernmental organisation 7 | # nor does it submit to any jurisdiction. 8 | # 9 | 10 | import warnings 11 | 12 | 13 | class GlobalIndex: 14 | def __init__(self, index_location, baseurl) -> None: 15 | warnings.warn("GlobalIndex is obsolete. Please update your code and use the 'directory' source") 16 | raise NotImplementedError("GlobalIndex is obsolete. Please update your code and use the 'directory' source") 17 | """The GloblaIndex has one index managing multiple urls/files. 18 | This unique index is found at "index_location" 19 | The path of each file is written in the index as a relative path. 20 | It is relative to a base url: "baseurl". 21 | """ 22 | 23 | 24 | class PerUrlIndex: 25 | def __init__( 26 | self, 27 | pattern, 28 | ) -> None: 29 | # warnings.warn( "PerUrlIndex is obsolete.") 30 | 31 | self.pattern = pattern 32 | -------------------------------------------------------------------------------- /src/climetlab/mergers/pandas.py: -------------------------------------------------------------------------------- 1 | # (C) Copyright 2020 ECMWF. 2 | # 3 | # This software is licensed under the terms of the Apache Licence Version 2.0 4 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 5 | # In applying this licence, ECMWF does not waive the privileges and immunities 6 | # granted to it by virtue of its status as an intergovernmental organisation 7 | # nor does it submit to any jurisdiction. 8 | # 9 | 10 | import pandas as pd 11 | 12 | 13 | def merge( 14 | sources=None, 15 | paths=None, 16 | reader_class=None, 17 | **kwargs, 18 | ): 19 | options = dict(ignore_index=True) # Renumber all indices 20 | options.update(kwargs) 21 | return pd.concat([s.to_pandas(**kwargs) for s in sources], **options) 22 | -------------------------------------------------------------------------------- /src/climetlab/mergers/tfdataset.py: -------------------------------------------------------------------------------- 1 | # (C) Copyright 2020 ECMWF. 2 | # 3 | # This software is licensed under the terms of the Apache Licence Version 2.0 4 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 5 | # In applying this licence, ECMWF does not waive the privileges and immunities 6 | # granted to it by virtue of its status as an intergovernmental organisation 7 | # nor does it submit to any jurisdiction. 8 | # 9 | 10 | 11 | def _concat(sources, *args, **kwargs): 12 | ds = sources[0].to_tfdataset(**kwargs) 13 | for s in sources[1:]: 14 | ds = ds.concatenate(s.to_tfdataset(**kwargs)) 15 | return ds 16 | 17 | 18 | def _zip(sources, *args, **kwargs): 19 | import tensorflow as tf 20 | 21 | return tf.data.Dataset.zip(tuple(s.to_tfdataset(**kwargs) for s in sources)) 22 | 23 | 24 | def merge( 25 | sources=None, 26 | paths=None, 27 | reader_class=None, 28 | **kwargs, 29 | ): 30 | if paths is not None: 31 | if reader_class is not None and hasattr(reader_class, "to_tfdataset_multi"): 32 | return reader_class.to_tfdataset_multi(paths, **kwargs) 33 | 34 | method = "_" + kwargs.get("method", "concat") 35 | return globals()[method](sources, paths, reader_class, **kwargs) 36 | -------------------------------------------------------------------------------- /src/climetlab/metview/__init__.py: -------------------------------------------------------------------------------- 1 | # (C) Copyright 2021 ECMWF. 2 | # 3 | # This software is licensed under the terms of the Apache Licence Version 2.0 4 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 5 | # In applying this licence, ECMWF does not waive the privileges and immunities 6 | # granted to it by virtue of its status as an intergovernmental organisation 7 | # nor does it submit to any jurisdiction. 8 | # 9 | 10 | import metview as mv 11 | 12 | from climetlab.core.ipython import ipython_active 13 | 14 | if ipython_active: 15 | try: 16 | import ipywidgets # noqa 17 | 18 | mv.setoutput("jupyter", plot_widget=True) 19 | except ImportError: 20 | mv.setoutput("jupyter", plot_widget=False) 21 | 22 | 23 | def mv_read(*args, **kwargs): 24 | return mv.read(*args, **kwargs) 25 | 26 | 27 | def mv_read_table(*args, **kwargs): 28 | return mv.read_table(*args, **kwargs) 29 | -------------------------------------------------------------------------------- /src/climetlab/ml/__init__.py: -------------------------------------------------------------------------------- 1 | # (C) Copyright 2020 ECMWF. 2 | # 3 | # This software is licensed under the terms of the Apache Licence Version 2.0 4 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 5 | # In applying this licence, ECMWF does not waive the privileges and immunities 6 | # granted to it by virtue of its status as an intergovernmental organisation 7 | # nor does it submit to any jurisdiction. 8 | # 9 | -------------------------------------------------------------------------------- /src/climetlab/ml/tf.py: -------------------------------------------------------------------------------- 1 | # (C) Copyright 2022 ECMWF. 2 | # 3 | # This software is licensed under the terms of the Apache Licence Version 2.0 4 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 5 | # In applying this licence, ECMWF does not waive the privileges and immunities 6 | # granted to it by virtue of its status as an intergovernmental organisation 7 | # nor does it submit to any jurisdiction. 8 | # 9 | 10 | import logging 11 | 12 | import tensorflow as tf 13 | from tensorflow.keras.layers import Conv2D 14 | 15 | LOG = logging.getLogger(__name__) 16 | 17 | 18 | class PeriodicConv2D(Conv2D): 19 | def __init__(self, *args, **kwargs): 20 | super().__init__(*args, **kwargs) 21 | # self.kernel_size 22 | 23 | def call(self, inputs): 24 | w, h = self.kernel_size 25 | inputs = tf.concat([inputs, inputs[:, :, :w, :]], axis=2) 26 | inputs = tf.pad(inputs, [[0, 0], [h // 2, h // 2], [0, 0], [0, 0]], constant_values=0) 27 | return super().call(inputs) 28 | -------------------------------------------------------------------------------- /src/climetlab/ml/torch.py: -------------------------------------------------------------------------------- 1 | # (C) Copyright 2022 ECMWF. 2 | # 3 | # This software is licensed under the terms of the Apache Licence Version 2.0 4 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 5 | # In applying this licence, ECMWF does not waive the privileges and immunities 6 | # granted to it by virtue of its status as an intergovernmental organisation 7 | # nor does it submit to any jurisdiction. 8 | # 9 | 10 | import logging 11 | 12 | LOG = logging.getLogger(__name__) 13 | 14 | 15 | def _find_device(): 16 | try: 17 | import torch 18 | except ImportError: 19 | LOG.debug("Torch not found") 20 | return None 21 | 22 | if torch.backends.mps.is_available() and torch.backends.mps.is_built(): 23 | return "mps" 24 | if torch.cuda.is_available() and torch.backends.cuda.is_built(): 25 | return "cuda" 26 | LOG.debug("Found no GPU, using CPU") 27 | return "cpu" 28 | 29 | 30 | device = _find_device() 31 | 32 | 33 | def to_pytorch_dataloader(dataset, **kwargs): 34 | import torch 35 | 36 | default_kwargs = dict( 37 | batch_size=128, 38 | # multi-process data loading 39 | # use as many workers as you have cores on your machine 40 | num_workers=1, 41 | # default: no shuffle, so need to explicitly set it here 42 | shuffle=True, 43 | # uses pinned memory to speed up CPU-to-GPU data transfers 44 | # see https://pytorch.org/docs/stable/notes/cuda.html#cuda-memory-pinning 45 | pin_memory=True, 46 | # function used to collate samples into batches 47 | # if None then Pytorch uses the default collate_fn (see below) 48 | collate_fn=None, 49 | ) 50 | merged_kwargs = {k: v for k, v in default_kwargs.items()} 51 | if kwargs: 52 | merged_kwargs.update(kwargs) 53 | 54 | return torch.utils.data.DataLoader(dataset, **merged_kwargs) 55 | -------------------------------------------------------------------------------- /src/climetlab/normalize.py: -------------------------------------------------------------------------------- 1 | # (C) Copyright 2020 ECMWF. 2 | # 3 | # This software is licensed under the terms of the Apache Licence Version 2.0 4 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 5 | # In applying this licence, ECMWF does not waive the privileges and immunities 6 | # granted to it by virtue of its status as an intergovernmental organisation 7 | # nor does it submit to any jurisdiction. 8 | # 9 | 10 | import warnings 11 | 12 | from climetlab.decorators import normalize 13 | 14 | 15 | class normalize_args: 16 | def __init__(self, **dic): 17 | warnings.warn("Deprecated decorator @normalize_arg. Use @normalise on each argument instead.") 18 | self.decorators = [] 19 | for name, values in dic.items(): 20 | if isinstance(values, list): 21 | self.decorators.append(normalize(name, values, multiple=True)) 22 | continue 23 | 24 | if isinstance(values, tuple): 25 | warnings.warn( 26 | "Using tuple to set multiple=False is deprecated in @normalize_arg. Use with multiple=False." 27 | ) 28 | self.decorators.append(normalize(name, values, multiple=False)) 29 | continue 30 | 31 | self.decorators.append(normalize(name, values)) 32 | 33 | def __call__(self, func): 34 | for d in self.decorators: 35 | func = d(func) 36 | return func 37 | -------------------------------------------------------------------------------- /src/climetlab/notebook/__init__.py: -------------------------------------------------------------------------------- 1 | # (C) Copyright 2021- ECMWF. 2 | # 3 | # This software is licensed under the terms of the Apache Licence Version 2.0 4 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 5 | # In applying this licence, ECMWF does not waive the privileges and immunities 6 | # granted to it by virtue of its status as an intergovernmental organisation 7 | # nor does it submit to any jurisdiction. 8 | # 9 | -------------------------------------------------------------------------------- /src/climetlab/notebook/table.py: -------------------------------------------------------------------------------- 1 | # (C) Copyright 2021- ECMWF. 2 | # 3 | # This software is licensed under the terms of the Apache Licence Version 2.0 4 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 5 | # In applying this licence, ECMWF does not waive the privileges and immunities 6 | # granted to it by virtue of its status as an intergovernmental organisation 7 | # nor does it submit to any jurisdiction. 8 | # 9 | 10 | 11 | class Table: 12 | def __init__(self, rows, columns): 13 | self.rows = rows 14 | self.columns = columns 15 | self.elements = {} 16 | 17 | def __setitem__(self, key, value): 18 | self.elements[key] = value 19 | 20 | def __getitem__(self, key): 21 | return self.elements[key] 22 | 23 | def _repr_html_(self): 24 | result = [] 25 | result.append('') 26 | for r in range(self.rows): 27 | result.append('') 28 | for c in range(self.columns): 29 | result.append('") 35 | result.append("") 36 | result.append("
') 30 | if (r, c) in self.elements: 31 | result.append(self.render(self.elements[(r, c)])) 32 | else: 33 | result.append(" ") 34 | result.append("
") 37 | return "\n".join(str(s) for s in result) 38 | 39 | def render(self, element): 40 | e = element.render() 41 | src, _ = e._repr_png_() 42 | return f'' 43 | -------------------------------------------------------------------------------- /src/climetlab/plotting/backends/__init__.py: -------------------------------------------------------------------------------- 1 | # (C) Copyright 2020 ECMWF. 2 | # 3 | # This software is licensed under the terms of the Apache Licence Version 2.0 4 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 5 | # In applying this licence, ECMWF does not waive the privileges and immunities 6 | # granted to it by virtue of its status as an intergovernmental organisation 7 | # nor does it submit to any jurisdiction. 8 | # 9 | -------------------------------------------------------------------------------- /src/climetlab/plotting/backends/magics/__init__.py: -------------------------------------------------------------------------------- 1 | # (C) Copyright 2020 ECMWF. 2 | # 3 | # This software is licensed under the terms of the Apache Licence Version 2.0 4 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 5 | # In applying this licence, ECMWF does not waive the privileges and immunities 6 | # granted to it by virtue of its status as an intergovernmental organisation 7 | # nor does it submit to any jurisdiction. 8 | # 9 | import os 10 | from collections import defaultdict 11 | 12 | import yaml 13 | 14 | from climetlab.decorators import locked 15 | 16 | MAGICS_KEYS = None 17 | MAGICS_DEF = None 18 | MAGICS_PARAMS = None 19 | _inited = False 20 | 21 | 22 | @locked 23 | def init(): 24 | global _inited, MAGICS_KEYS, MAGICS_DEF, MAGICS_PARAMS 25 | 26 | if not _inited: 27 | MAGICS_KEYS = defaultdict(set) 28 | MAGICS_PARAMS = defaultdict(dict) 29 | with open(os.path.join(os.path.dirname(__file__), "magics.yaml")) as f: 30 | MAGICS_DEF = yaml.load(f, Loader=yaml.SafeLoader) 31 | for action, params in MAGICS_DEF.items(): 32 | for param in params: 33 | name = param["name"] 34 | MAGICS_KEYS[name].add(action) 35 | MAGICS_PARAMS[action][name] = param 36 | 37 | _inited = True 38 | 39 | 40 | def magics_keys_to_actions(): 41 | init() 42 | return MAGICS_KEYS 43 | 44 | 45 | def magics_keys_definitions(): 46 | init() 47 | return MAGICS_DEF 48 | 49 | 50 | def magics_keys_parameters(name): 51 | init() 52 | return MAGICS_PARAMS[name] 53 | -------------------------------------------------------------------------------- /src/climetlab/plotting/options.py: -------------------------------------------------------------------------------- 1 | # (C) Copyright 2020 ECMWF. 2 | # 3 | # This software is licensed under the terms of the Apache Licence Version 2.0 4 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 5 | # In applying this licence, ECMWF does not waive the privileges and immunities 6 | # granted to it by virtue of its status as an intergovernmental organisation 7 | # nor does it submit to any jurisdiction. 8 | # 9 | 10 | 11 | class Options: 12 | def __init__(self, options=None): 13 | self._options = {} if options is None else options 14 | self._used_options = set() 15 | 16 | def __getitem__(self, name: str): 17 | self._used_options.add(name) 18 | return self._options[name] 19 | 20 | def __call__(self, name: str, default): 21 | self._used_options.add(name) 22 | return self._options.get(name, default) 23 | 24 | def provided(self, name: str) -> bool: 25 | return name in self._options 26 | 27 | def check_unused(self): 28 | unused = set(self._options.keys()) - self._used_options 29 | if unused: 30 | raise TypeError( 31 | "".join( 32 | [ 33 | "Unused argument%s: " % ("s" if len(unused) > 1 else "",), 34 | ", ".join("%s=%s" % (x, self._options[x]) for x in unused), 35 | ] 36 | ) 37 | ) 38 | 39 | def update_if_not_set(self, **kwargs): 40 | for k, v in kwargs.items(): 41 | if k not in self._options: 42 | self._options[k] = v 43 | 44 | def __repr__(self) -> str: 45 | return f"Options({str(self._options)})" 46 | -------------------------------------------------------------------------------- /src/climetlab/plotting/wms/__init__.py: -------------------------------------------------------------------------------- 1 | from climetlab import new_plot 2 | from climetlab.core.temporary import temp_file 3 | 4 | from ._folium import make_map 5 | 6 | 7 | def interactive_map(obj, **kwargs): 8 | h = 100 * 3 9 | 10 | tmp = temp_file(".svg") 11 | p = new_plot( 12 | projection="web-mercator", 13 | width=6 * 1024, 14 | width_cm=h, 15 | height_cm=h, 16 | frame=False, 17 | foreground=False, 18 | background=False, 19 | ) 20 | p.plot_map(obj) 21 | bbox = p.save(tmp.path) 22 | 23 | return make_map(tmp.path, bbox) 24 | -------------------------------------------------------------------------------- /src/climetlab/plotting/wms/_ipyleafet.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ecmwf/climetlab/a55f9c8590c2ede64a2b0fdc1b163501c81a34f9/src/climetlab/plotting/wms/_ipyleafet.py -------------------------------------------------------------------------------- /src/climetlab/plotting/wms/wms.j2: -------------------------------------------------------------------------------- 1 | {% macro script(this, kwargs) %} 2 | 3 | // https://leafletjs.com/examples/extending/extending-2-layers.html 4 | 5 | if(!L.DirectWMS) { 6 | 7 | 8 | L.DirectWMS = L.TileLayer.WMS.extend({ 9 | 10 | createTile: function (coords, done) { 11 | var tile = document.createElement('img'); 12 | 13 | var event = new CustomEvent('direct-wms', { 14 | 'detail': { 15 | 'tile': tile, 16 | 'url': this.getTileUrl(coords), 17 | 'done': done 18 | } }); 19 | 20 | window.parent.document.dispatchEvent(event); 21 | 22 | return tile; 23 | } 24 | 25 | }); 26 | 27 | } 28 | 29 | 30 | 31 | var {{ this.get_name() }} = new L.DirectWMS( 32 | {{ this.url|tojson }}, 33 | {{ this.options|tojson }} 34 | ).addTo({{ this._parent.get_name() }}); 35 | 36 | {% endmacro %} 37 | -------------------------------------------------------------------------------- /src/climetlab/plotting/wms/wms.js: -------------------------------------------------------------------------------- 1 | /* 2 | Colab: 3 | See https://gist.github.com/korakot/d10a43490f3da17d4915cdc1f200b180 4 | */ 5 | 6 | 7 | window.document.addEventListener('direct-wms', function(e) { 8 | console.log('direct-wms', e.detail); 9 | 10 | 11 | 12 | var url = e.detail.url; 13 | 14 | var callbacks = { 15 | iopub: { 16 | output: function(data) { 17 | var error = null; 18 | 19 | if (data.msg_type == 'error') { 20 | error = data.content.ename + ': ' + data.content.evalue; 21 | } 22 | 23 | if (data.msg_type == 'stream') { 24 | e.detail.tile.src = data.content.text; 25 | } 26 | 27 | e.detail.done(error, e.detail.tile); 28 | } 29 | } 30 | }; 31 | 32 | 33 | if (Jupyter) { 34 | var python = 35 | `import climetlab.plotting.wms\nclimetlab.plotting.wms.direct_wms('${ 36 | url}')\n`; 37 | console.log('python', python); 38 | Jupyter.notebook.kernel.execute(python, callbacks); 39 | } 40 | 41 | if (google) { 42 | var result = await google.colab.kernel.invokeFunction('direct_wms', [url], {}); 43 | console.log('result', result); 44 | } 45 | 46 | }, false); 47 | -------------------------------------------------------------------------------- /src/climetlab/prompt.py: -------------------------------------------------------------------------------- 1 | import code 2 | import inspect 3 | 4 | 5 | def get_locals(callers): 6 | for c in callers[1:]: 7 | if not c.filename.startswith(" 1 33 | 34 | return GribMultiFieldSet(readers) 35 | 36 | def mutate_source(self): 37 | # A GRIBReader is a source itself 38 | return self 39 | -------------------------------------------------------------------------------- /src/climetlab/readers/matlab.py: -------------------------------------------------------------------------------- 1 | # (C) Copyright 2020 ECMWF. 2 | # 3 | # This software is licensed under the terms of the Apache Licence Version 2.0 4 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 5 | # In applying this licence, ECMWF does not waive the privileges and immunities 6 | # granted to it by virtue of its status as an intergovernmental organisation 7 | # nor does it submit to any jurisdiction. 8 | # 9 | 10 | from . import Reader 11 | 12 | 13 | class MatlabReader(Reader): 14 | def __init__(self, source, path): 15 | super().__init__(source, path) 16 | 17 | def to_numpy(self, key=None, scipy_io_loadmat_kwargs={}): 18 | from scipy.io import loadmat 19 | 20 | ds = loadmat(self.path, **scipy_io_loadmat_kwargs) 21 | if key is not None: 22 | ds = ds[key] 23 | return ds 24 | 25 | def __iter__(self): 26 | return iter([self]) 27 | 28 | 29 | def reader(source, path, magic=None, deeper_check=False): 30 | if magic[:7] == b"MATLAB ": 31 | return MatlabReader(source, path) 32 | -------------------------------------------------------------------------------- /src/climetlab/readers/netcdf/__init__.py: -------------------------------------------------------------------------------- 1 | # (C) Copyright 2020 ECMWF. 2 | # 3 | # This software is licensed under the terms of the Apache Licence Version 2.0 4 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 5 | # In applying this licence, ECMWF does not waive the privileges and immunities 6 | # granted to it by virtue of its status as an intergovernmental organisation 7 | # nor does it submit to any jurisdiction. 8 | # 9 | 10 | 11 | from .. import Reader 12 | from .fieldset import NetCDFFieldSetFromFile 13 | 14 | 15 | class NetCDFReader(Reader, NetCDFFieldSetFromFile): 16 | def __init__(self, source, path): 17 | Reader.__init__(self, source, path) 18 | NetCDFFieldSetFromFile.__init__(self, path) 19 | 20 | 21 | def reader(source, path, magic=None, deeper_check=False): 22 | if magic is None or magic[:4] in (b"\x89HDF", b"CDF\x01", b"CDF\x02"): 23 | return NetCDFReader(source, path) 24 | -------------------------------------------------------------------------------- /src/climetlab/readers/numpy.py: -------------------------------------------------------------------------------- 1 | # (C) Copyright 2020 ECMWF. 2 | # 3 | # This software is licensed under the terms of the Apache Licence Version 2.0 4 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 5 | # In applying this licence, ECMWF does not waive the privileges and immunities 6 | # granted to it by virtue of its status as an intergovernmental organisation 7 | # nor does it submit to any jurisdiction. 8 | # 9 | 10 | import os 11 | 12 | import numpy as np 13 | 14 | from . import Reader 15 | 16 | 17 | class NumpyReader(Reader): 18 | def __init__(self, source, path): 19 | super().__init__(source, path) 20 | 21 | def to_numpy(self, numpy_load_kwargs={}): 22 | return np.load(self.path, **numpy_load_kwargs) 23 | 24 | def __iter__(self): 25 | return iter([self]) 26 | 27 | 28 | class NumpyZipReader(Reader): 29 | def __init__(self, source, path): 30 | super().__init__(source, path) 31 | 32 | def to_numpy(self, numpy_load_kwargs={}): 33 | return np.load(self.path, **numpy_load_kwargs) 34 | 35 | 36 | def reader(source, path, magic=None, deeper_check=False): 37 | if magic is None: # Bypass check and force 38 | return NumpyReader(source, path) 39 | 40 | if magic[:6] == b"\x93NUMPY": 41 | return NumpyReader(source, path) 42 | 43 | _, extension = os.path.splitext(path) 44 | if magic[:4] == b"PK\x03\x04" and extension == ".npz": 45 | return NumpyZipReader(source, path) 46 | -------------------------------------------------------------------------------- /src/climetlab/readers/odb.py: -------------------------------------------------------------------------------- 1 | # (C) Copyright 2020 ECMWF. 2 | # 3 | # This software is licensed under the terms of the Apache Licence Version 2.0 4 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 5 | # In applying this licence, ECMWF does not waive the privileges and immunities 6 | # granted to it by virtue of its status as an intergovernmental organisation 7 | # nor does it submit to any jurisdiction. 8 | # 9 | 10 | import logging 11 | 12 | from . import Reader 13 | 14 | LOG = logging.getLogger(__name__) 15 | 16 | 17 | class ODBReader(Reader): 18 | def to_pandas(self, **kwargs): 19 | try: 20 | import codc as odc 21 | except Exception: 22 | import pyodc as odc 23 | 24 | LOG.debug("Using pure Python odc decoder.") 25 | 26 | odc_read_odb_kwargs = kwargs.get("odc_read_odb_kwargs", {}) 27 | return odc.read_odb(self.path, single=True, *odc_read_odb_kwargs) 28 | 29 | 30 | def reader(source, path, magic=None, deeper_check=False): 31 | if magic is None or magic[:5] == b"\xff\xffODA": 32 | return ODBReader(source, path) 33 | -------------------------------------------------------------------------------- /src/climetlab/readers/tar.py: -------------------------------------------------------------------------------- 1 | # (C) Copyright 2020 ECMWF. 2 | # 3 | # This software is licensed under the terms of the Apache Licence Version 2.0 4 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 5 | # In applying this licence, ECMWF does not waive the privileges and immunities 6 | # granted to it by virtue of its status as an intergovernmental organisation 7 | # nor does it submit to any jurisdiction. 8 | # 9 | 10 | import logging 11 | import mimetypes 12 | import tarfile 13 | 14 | from .archive import ArchiveReader 15 | 16 | LOG = logging.getLogger(__name__) 17 | 18 | 19 | class TarReader(ArchiveReader): 20 | def __init__(self, source, path, compression=None): 21 | super().__init__(source, path) 22 | 23 | with tarfile.open(path) as tar: 24 | self.expand( 25 | tar, 26 | tar.getmembers(), 27 | set_attrs=False, 28 | ) 29 | 30 | 31 | def reader(source, path, magic=None, deeper_check=False): 32 | # We don't use tarfile.is_tarfile() because is 33 | # returns true given a file of zeros 34 | 35 | kind, compression = mimetypes.guess_type(path) 36 | 37 | if magic is None or kind == "application/x-tar": 38 | return TarReader(source, path, compression) 39 | -------------------------------------------------------------------------------- /src/climetlab/readers/text.py: -------------------------------------------------------------------------------- 1 | # (C) Copyright 2020 ECMWF. 2 | # 3 | # This software is licensed under the terms of the Apache Licence Version 2.0 4 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 5 | # In applying this licence, ECMWF does not waive the privileges and immunities 6 | # granted to it by virtue of its status as an intergovernmental organisation 7 | # nor does it submit to any jurisdiction. 8 | # 9 | 10 | 11 | from . import Reader 12 | from .csv import CSVReader 13 | from .csv import is_csv 14 | 15 | 16 | def is_text(path, prob_lines=1000, probe_size=4096): 17 | try: 18 | with open(path, "rb") as f: 19 | if 0x0 in f.read(probe_size): 20 | return False 21 | 22 | with open(path, "r", encoding="utf-8") as f: 23 | for i, _ in enumerate(f): 24 | if i > prob_lines: 25 | break 26 | return True 27 | except UnicodeDecodeError: 28 | return False 29 | 30 | 31 | class TextReader(Reader): 32 | def __init__(self, source, path): 33 | super().__init__(source, path) 34 | 35 | def ignore(self): 36 | # Used by multi-source 37 | return True 38 | 39 | def mutate(self): 40 | if is_csv(self.path): 41 | return CSVReader(self.source, self.path) 42 | 43 | return self 44 | 45 | 46 | def reader(source, path, magic=None, deeper_check=False): 47 | if magic is None: # Bypass check and force 48 | return TextReader(source, path) 49 | 50 | if deeper_check: 51 | if is_text(path): 52 | return TextReader(source, path) 53 | -------------------------------------------------------------------------------- /src/climetlab/readers/tfrecord.py: -------------------------------------------------------------------------------- 1 | # (C) Copyright 2021 ECMWF. 2 | # 3 | # This software is licensed under the terms of the Apache Licence Version 2.0 4 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 5 | # In applying this licence, ECMWF does not waive the privileges and immunities 6 | # granted to it by virtue of its status as an intergovernmental organisation 7 | # nor does it submit to any jurisdiction. 8 | # 9 | 10 | import logging 11 | 12 | from . import Reader 13 | 14 | LOG = logging.getLogger(__name__) 15 | 16 | 17 | class TfRecordReader(Reader): 18 | _tfrecord = None 19 | _len = None 20 | 21 | @property 22 | def tfrecord(self): 23 | return type(self).to_tfdataset_multi([self.path]) 24 | 25 | def to_tfdataset(self, **kwargs): 26 | return self.tfrecord 27 | 28 | @classmethod 29 | def to_tfdataset_multi(cls, paths, **kwargs): 30 | import tensorflow as tf 31 | 32 | files_ds = tf.data.Dataset.list_files(paths) 33 | options = tf.data.Options() 34 | # options.experimental_deterministic = False 35 | files_ds = files_ds.with_options(options) 36 | return tf.data.TFRecordDataset( 37 | files_ds, 38 | num_parallel_reads=tf.data.experimental.AUTOTUNE, 39 | ) 40 | 41 | def __len__(self): 42 | if self._len is None: 43 | record = self.tfrecord 44 | try: 45 | self._len = len(record) 46 | except TypeError: 47 | self._len = 0 48 | for _ in record: 49 | self._len += 1 50 | return self._len 51 | 52 | 53 | def reader(source, path, magic=None, deeper_check=False): 54 | if magic is None or path.endswith(".tfrecord"): 55 | return TfRecordReader(source, path) 56 | -------------------------------------------------------------------------------- /src/climetlab/readers/unknown.py: -------------------------------------------------------------------------------- 1 | # (C) Copyright 2021 ECMWF. 2 | # 3 | # This software is licensed under the terms of the Apache Licence Version 2.0 4 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 5 | # In applying this licence, ECMWF does not waive the privileges and immunities 6 | # granted to it by virtue of its status as an intergovernmental organisation 7 | # nor does it submit to any jurisdiction. 8 | 9 | 10 | import logging 11 | 12 | from . import Reader 13 | 14 | LOG = logging.getLogger(__name__) 15 | 16 | 17 | class Unknown(Reader): 18 | def __init__(self, source, path, magic): 19 | super().__init__(source, path) 20 | self.magic = magic 21 | LOG.warning("Unknown file type %s (%s), ignoring", path, magic) 22 | 23 | def ignore(self): 24 | # Used by multi-source 25 | return True 26 | 27 | def __len__(self): 28 | return 0 29 | -------------------------------------------------------------------------------- /src/climetlab/scripts/__init__.py: -------------------------------------------------------------------------------- 1 | # (C) Copyright 2021 ECMWF. 2 | # 3 | # This software is licensed under the terms of the Apache Licence Version 2.0 4 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 5 | # In applying this licence, ECMWF does not waive the privileges and immunities 6 | # granted to it by virtue of its status as an intergovernmental organisation 7 | # nor does it submit to any jurisdiction. 8 | # 9 | 10 | from .main import main 11 | 12 | if __name__ == "__main__": 13 | main() 14 | -------------------------------------------------------------------------------- /src/climetlab/scripts/benchmarks/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ecmwf/climetlab/a55f9c8590c2ede64a2b0fdc1b163501c81a34f9/src/climetlab/scripts/benchmarks/__init__.py -------------------------------------------------------------------------------- /src/climetlab/scripts/test_data.py: -------------------------------------------------------------------------------- 1 | # (C) Copyright 2021 ECMWF. 2 | # 3 | # This software is licensed under the terms of the Apache Licence Version 2.0 4 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 5 | # In applying this licence, ECMWF does not waive the privileges and immunities 6 | # granted to it by virtue of its status as an intergovernmental organisation 7 | # nor does it submit to any jurisdiction. 8 | # 9 | 10 | from .tools import parse_args 11 | 12 | 13 | class TestDataCmd: 14 | @parse_args( 15 | directory=( 16 | None, 17 | dict( 18 | metavar="DIRECTORY", 19 | help="Shell to use for autocompletion. Must be zsh or bash.", 20 | nargs="?", 21 | ), 22 | ), 23 | ) 24 | def do_test_data(self, args): 25 | """ 26 | Create a directory with data used to test climetlab. 27 | 28 | """ 29 | from climetlab.testing import build_testdata 30 | 31 | directory = args.directory 32 | if not directory: 33 | directory = "./test-data" 34 | 35 | print(f"Adding testdata in {directory}") 36 | build_testdata(directory) 37 | print(f"Added testdata in {directory}") 38 | -------------------------------------------------------------------------------- /src/climetlab/sources/dummy.grib: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ecmwf/climetlab/a55f9c8590c2ede64a2b0fdc1b163501c81a34f9/src/climetlab/sources/dummy.grib -------------------------------------------------------------------------------- /src/climetlab/sources/ecmwf_data_server.py: -------------------------------------------------------------------------------- 1 | # (C) Copyright 2020 ECMWF. 2 | # 3 | # This software is licensed under the terms of the Apache Licence Version 2.0 4 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 5 | # In applying this licence, ECMWF does not waive the privileges and immunities 6 | # granted to it by virtue of its status as an intergovernmental organisation 7 | # nor does it submit to any jurisdiction. 8 | # 9 | 10 | from .ecmwf_data_server_base import DatasetRetrieverBase 11 | 12 | 13 | class DatasetRetriever(DatasetRetrieverBase): 14 | pass 15 | 16 | 17 | source = DatasetRetriever 18 | -------------------------------------------------------------------------------- /src/climetlab/sources/ecmwf_data_server_base.py: -------------------------------------------------------------------------------- 1 | # (C) Copyright 2020 ECMWF. 2 | # 3 | # This software is licensed under the terms of the Apache Licence Version 2.0 4 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 5 | # In applying this licence, ECMWF does not waive the privileges and immunities 6 | # granted to it by virtue of its status as an intergovernmental organisation 7 | # nor does it submit to any jurisdiction. 8 | # 9 | 10 | import ecmwfapi 11 | 12 | from .ecmwf_api import ECMWFApi 13 | 14 | 15 | class Wrapper(ecmwfapi.ECMWFDataServer): 16 | def __init__(self, dataset): 17 | super().__init__() 18 | self.dataset = dataset 19 | 20 | def execute(self, request, target): 21 | request = dict(**request) 22 | request["dataset"] = self.dataset 23 | request["target"] = target 24 | print(request) 25 | return self.retrieve(request) 26 | 27 | 28 | class DatasetRetrieverBase(ECMWFApi): 29 | def __init__(self, dataset, *args, **kwargs): 30 | self.ecmwf_dataset = dataset 31 | super().__init__(*args, **kwargs) 32 | 33 | def service(self): 34 | return Wrapper(self.ecmwf_dataset) 35 | -------------------------------------------------------------------------------- /src/climetlab/sources/ecmwf_open_data.py: -------------------------------------------------------------------------------- 1 | # (C) Copyright 2020 ECMWF. 2 | # 3 | # This software is licensed under the terms of the Apache Licence Version 2.0 4 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 5 | # In applying this licence, ECMWF does not waive the privileges and immunities 6 | # granted to it by virtue of its status as an intergovernmental organisation 7 | # nor does it submit to any jurisdiction. 8 | # 9 | 10 | import ecmwf.opendata 11 | 12 | from .file import FileSource 13 | 14 | 15 | class EODRetriever(FileSource): 16 | sphinxdoc = """ 17 | EODRetriever 18 | """ 19 | 20 | def __init__(self, source="ecmwf", *args, **kwargs): 21 | if len(args): 22 | assert len(args) == 1 23 | assert isinstance(args[0], dict) 24 | assert not kwargs 25 | kwargs = args[0] 26 | 27 | self.source_kwargs = self.request(**kwargs) 28 | 29 | self.client = ecmwf.opendata.Client(source=source, preserve_request_order=True) 30 | 31 | self.path = self._retrieve(self.source_kwargs) 32 | 33 | def connect_to_mirror(self, mirror): 34 | return mirror.connection_for_eod(self) 35 | 36 | def _retrieve(self, request): 37 | def retrieve(target, request): 38 | self.client.retrieve(request, target) 39 | 40 | return self.cache_file( 41 | retrieve, 42 | request, 43 | ) 44 | 45 | # @normalize("date", "date-list(%Y-%m-%d)") 46 | # @normalize("area", "bounding-box(list)") 47 | def request(self, **request): 48 | return request 49 | 50 | 51 | source = EODRetriever 52 | -------------------------------------------------------------------------------- /src/climetlab/sources/ecmwf_research_experiment.py: -------------------------------------------------------------------------------- 1 | # (C) Copyright 2020 ECMWF. 2 | # 3 | # This software is licensed under the terms of the Apache Licence Version 2.0 4 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 5 | # In applying this licence, ECMWF does not waive the privileges and immunities 6 | # granted to it by virtue of its status as an intergovernmental organisation 7 | # nor does it submit to any jurisdiction. 8 | # 9 | 10 | from .ecmwf_data_server_base import DatasetRetrieverBase 11 | 12 | 13 | class ExperimentRetriever(DatasetRetrieverBase): 14 | def __init__(self, experiment, *args, **kwargs): 15 | extra = {"class": "rd", "expver": experiment} 16 | super().__init__("research", *args, **kwargs, **extra) 17 | 18 | 19 | source = ExperimentRetriever 20 | -------------------------------------------------------------------------------- /src/climetlab/sources/empty.py: -------------------------------------------------------------------------------- 1 | # (C) Copyright 2020 ECMWF. 2 | # 3 | # This software is licensed under the terms of the Apache Licence Version 2.0 4 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 5 | # In applying this licence, ECMWF does not waive the privileges and immunities 6 | # granted to it by virtue of its status as an intergovernmental organisation 7 | # nor does it submit to any jurisdiction. 8 | # 9 | 10 | from climetlab.indexing.fieldset import FieldSet 11 | 12 | 13 | class EmptySource(FieldSet): 14 | def ignore(self): 15 | # Used by multi-source 16 | return True 17 | 18 | def __len__(self): 19 | return 0 20 | 21 | def __repr__(self): 22 | return "EmptySource" 23 | 24 | 25 | source = EmptySource 26 | -------------------------------------------------------------------------------- /src/climetlab/sources/fdb.py: -------------------------------------------------------------------------------- 1 | # (C) Copyright 2020 ECMWF. 2 | # 3 | # This software is licensed under the terms of the Apache Licence Version 2.0 4 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 5 | # In applying this licence, ECMWF does not waive the privileges and immunities 6 | # granted to it by virtue of its status as an intergovernmental organisation 7 | # nor does it submit to any jurisdiction. 8 | # 9 | 10 | import glob 11 | import logging 12 | import os 13 | import time 14 | 15 | import pyfdb 16 | import yaml 17 | 18 | from climetlab.readers.grib.index import FieldSetInFiles 19 | from climetlab.utils.parts import Part 20 | 21 | LOG = logging.getLogger(__name__) 22 | 23 | 24 | class FDB(FieldSetInFiles): 25 | def __init__(self, root=None, schema=None, request={}): 26 | super().__init__(db=None) 27 | 28 | if schema is None and root is not None: 29 | for n in glob.iglob(f"{root}/*/schema"): 30 | schema = n 31 | break 32 | 33 | config = { 34 | "spaces": [{"roots": [{"path": root}]}], 35 | "schema": schema, 36 | } 37 | os.environ["FDB5_CONFIG"] = yaml.dump(config) 38 | 39 | now = time.time() 40 | 41 | fdb = pyfdb.FDB() 42 | self.parts = list(fdb.list(request)) 43 | print("pyfdb.list", time.time() - now) 44 | 45 | def number_of_parts(self): 46 | return len(self.parts) 47 | 48 | def part(self, i): 49 | f = self.parts[i] 50 | return Part(f["path"], f["offset"], f["length"]) 51 | 52 | 53 | source = FDB 54 | -------------------------------------------------------------------------------- /src/climetlab/sources/file_pattern.py: -------------------------------------------------------------------------------- 1 | # (C) Copyright 2020 ECMWF. 2 | # 3 | # This software is licensed under the terms of the Apache Licence Version 2.0 4 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 5 | # In applying this licence, ECMWF does not waive the privileges and immunities 6 | # granted to it by virtue of its status as an intergovernmental organisation 7 | # nor does it submit to any jurisdiction. 8 | # 9 | 10 | from climetlab.sources.file import File 11 | from climetlab.sources.multi import MultiSource 12 | from climetlab.utils.patterns import Pattern 13 | 14 | 15 | class FilePattern(MultiSource): 16 | def __init__(self, pattern, *args, filter=None, merger=None, **kwargs): 17 | files = Pattern(pattern).substitute(*args, **kwargs) 18 | if not isinstance(files, list): 19 | files = [files] 20 | 21 | sources = [File(file) for file in sorted(files)] 22 | super().__init__(sources, filter=filter, merger=merger) 23 | 24 | 25 | source = FilePattern 26 | -------------------------------------------------------------------------------- /src/climetlab/sources/indexed_url.py: -------------------------------------------------------------------------------- 1 | # (C) Copyright 2020 ECMWF. 2 | # 3 | # This software is licensed under the terms of the Apache Licence Version 2.0 4 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 5 | # In applying this licence, ECMWF does not waive the privileges and immunities 6 | # granted to it by virtue of its status as an intergovernmental organisation 7 | # nor does it submit to any jurisdiction. 8 | # 9 | 10 | 11 | from climetlab.readers.grib.index.sql import FieldsetInFilesWithSqlIndex 12 | from climetlab.sources.indexed import IndexedSource 13 | from climetlab.sources.indexed_urls import add_path 14 | from climetlab.sources.indexed_urls import get_index_url 15 | 16 | 17 | class IndexedUrl(IndexedSource): 18 | def __init__( 19 | self, 20 | url, 21 | substitute_extension=False, 22 | index_extension=".index", 23 | **kwargs, 24 | ): 25 | index = FieldsetInFilesWithSqlIndex.from_url( 26 | get_index_url(url, substitute_extension, index_extension), 27 | patch_entry=add_path(url), 28 | ) 29 | super().__init__(index, **kwargs) 30 | 31 | 32 | source = IndexedUrl 33 | -------------------------------------------------------------------------------- /src/climetlab/sources/indexed_url_with_json_index.py: -------------------------------------------------------------------------------- 1 | # (C) Copyright 2020 ECMWF. 2 | # 3 | # This software is licensed under the terms of the Apache Licence Version 2.0 4 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 5 | # In applying this licence, ECMWF does not waive the privileges and immunities 6 | # granted to it by virtue of its status as an intergovernmental organisation 7 | # nor does it submit to any jurisdiction. 8 | # 9 | 10 | 11 | from climetlab.readers.grib.index import FieldsetInFilesWithJsonIndex 12 | from climetlab.sources.indexed import IndexedSource 13 | 14 | 15 | class IndexedUrlWithJsonIndex(IndexedSource): 16 | # This class is mostly for testing purposes 17 | # sql indexing is usually better than json indexing. 18 | def __init__(self, url, **kwargs): 19 | index = FieldsetInFilesWithJsonIndex.from_url(url) 20 | super().__init__(index, **kwargs) 21 | 22 | 23 | source = IndexedUrlWithJsonIndex 24 | -------------------------------------------------------------------------------- /src/climetlab/sources/metview.py: -------------------------------------------------------------------------------- 1 | # (C) Copyright 2020 ECMWF. 2 | # 3 | # This software is licensed under the terms of the Apache Licence Version 2.0 4 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 5 | # In applying this licence, ECMWF does not waive the privileges and immunities 6 | # granted to it by virtue of its status as an intergovernmental organisation 7 | # nor does it submit to any jurisdiction. 8 | # 9 | 10 | import logging 11 | 12 | from climetlab.sources.file import FileSource 13 | 14 | LOG = logging.getLogger(__name__) 15 | 16 | 17 | class Metview(FileSource): 18 | def __init__(self, metview_object): 19 | super().__init__(path=metview_object.url()) 20 | 21 | 22 | source = Metview 23 | -------------------------------------------------------------------------------- /src/climetlab/sources/multi_url.py: -------------------------------------------------------------------------------- 1 | # (C) Copyright 2020 ECMWF. 2 | # 3 | # This software is licensed under the terms of the Apache Licence Version 2.0 4 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 5 | # In applying this licence, ECMWF does not waive the privileges and immunities 6 | # granted to it by virtue of its status as an intergovernmental organisation 7 | # nor does it submit to any jurisdiction. 8 | # 9 | 10 | from climetlab import load_source 11 | 12 | from .multi import MultiSource 13 | 14 | 15 | class MultiUrl(MultiSource): 16 | def __init__(self, urls, *args, filter=None, merger=None, force=None, **kwargs): 17 | if not isinstance(urls, (list, tuple)): 18 | urls = [urls] 19 | 20 | # if filter is not None: 21 | # urls = [url for url in urls if filter(url)] 22 | 23 | assert len(urls) 24 | 25 | sources = [ 26 | load_source( 27 | "url", 28 | url, 29 | filter=filter, 30 | merger=merger, 31 | force=force, 32 | # Load lazily so we can do parallel downloads 33 | lazily=True, 34 | ) 35 | for url in sorted(urls) 36 | ] 37 | 38 | super().__init__(sources, filter=filter, merger=merger) 39 | -------------------------------------------------------------------------------- /src/climetlab/sources/opendap.py: -------------------------------------------------------------------------------- 1 | # (C) Copyright 2021 ECMWF. 2 | # 3 | # This software is licensed under the terms of the Apache Licence Version 2.0 4 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 5 | # In applying this licence, ECMWF does not waive the privileges and immunities 6 | # granted to it by virtue of its status as an intergovernmental organisation 7 | # nor does it submit to any jurisdiction. 8 | # 9 | 10 | from climetlab.readers.netcdf.fieldset import NetCDFFieldSetFromURL 11 | from climetlab.sources import Source 12 | 13 | 14 | class OpenDAP(Source): 15 | def __init__(self, url): 16 | self.url = url 17 | 18 | def mutate(self): 19 | return NetCDFFieldSetFromURL(self.url) 20 | 21 | 22 | source = OpenDAP 23 | -------------------------------------------------------------------------------- /src/climetlab/sources/sentinel_hub.py: -------------------------------------------------------------------------------- 1 | # (C) Copyright 2020 ECMWF. 2 | # 3 | # This software is licensed under the terms of the Apache Licence Version 2.0 4 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 5 | # In applying this licence, ECMWF does not waive the privileges and immunities 6 | # granted to it by virtue of its status as an intergovernmental organisation 7 | # nor does it submit to any jurisdiction. 8 | # 9 | 10 | # WIP. 11 | 12 | import json 13 | import os 14 | 15 | from oauthlib.oauth2 import BackendApplicationClient 16 | from requests_oauthlib import OAuth2Session 17 | 18 | # import climetlab as cml 19 | 20 | # Your client credentials 21 | with open(os.path.expanduser("~/.sentinelhubrc")) as f: 22 | credentials = json.loads(f.read()) 23 | client_id = credentials["client_id"] 24 | client_secret = credentials["client_secret"] 25 | 26 | # Create a session 27 | client = BackendApplicationClient(client_id=client_id) 28 | oauth = OAuth2Session(client=client) 29 | 30 | 31 | # Get token for the session 32 | token = oauth.fetch_token( 33 | token_url="https://services.sentinel-hub.com/oauth/token", 34 | client_id=client_id, 35 | client_secret=client_secret, 36 | ) 37 | 38 | # All requests using this session will have an access token automatically added 39 | resp = oauth.get("https://services.sentinel-hub.com/oauth/tokeninfo") 40 | print(resp.content) 41 | -------------------------------------------------------------------------------- /src/climetlab/sources/url_pattern.py: -------------------------------------------------------------------------------- 1 | # (C) Copyright 2020 ECMWF. 2 | # 3 | # This software is licensed under the terms of the Apache Licence Version 2.0 4 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 5 | # In applying this licence, ECMWF does not waive the privileges and immunities 6 | # granted to it by virtue of its status as an intergovernmental organisation 7 | # nor does it submit to any jurisdiction. 8 | # 9 | 10 | from climetlab.sources.multi_url import MultiUrl 11 | from climetlab.utils.patterns import Pattern 12 | 13 | 14 | class UrlPattern(MultiUrl): 15 | def __init__(self, pattern, *args, filter=None, merger=None, force=False, **kwargs): 16 | urls = Pattern(pattern).substitute(*args, **kwargs) 17 | super().__init__(urls, *args, filter=filter, merger=merger, force=force, **kwargs) 18 | 19 | 20 | source = UrlPattern 21 | -------------------------------------------------------------------------------- /src/climetlab/sphinxext/__init__.py: -------------------------------------------------------------------------------- 1 | # (C) Copyright 2020 ECMWF. 2 | # 3 | # This software is licensed under the terms of the Apache Licence Version 2.0 4 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 5 | # In applying this licence, ECMWF does not waive the privileges and immunities 6 | # granted to it by virtue of its status as an intergovernmental organisation 7 | # nor does it submit to any jurisdiction. 8 | # 9 | -------------------------------------------------------------------------------- /src/climetlab/sphinxext/command_output.py: -------------------------------------------------------------------------------- 1 | # (C) Copyright 2020 ECMWF. 2 | # 3 | # This software is licensed under the terms of the Apache Licence Version 2.0 4 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 5 | # In applying this licence, ECMWF does not waive the privileges and immunities 6 | # granted to it by virtue of its status as an intergovernmental organisation 7 | # nor does it submit to any jurisdiction. 8 | # 9 | 10 | import os 11 | import subprocess 12 | import traceback 13 | from shlex import split 14 | 15 | from docutils import statemachine 16 | from docutils.parsers.rst import Directive 17 | 18 | # Examples at https://github.com/docutils-mirror/docutils 19 | 20 | 21 | class CommandOutput(Directive): 22 | has_content = True 23 | 24 | def run(self): 25 | self.assert_has_content() 26 | 27 | here = os.getcwd() 28 | 29 | try: 30 | # Get current file 31 | current_rst_file = self.state_machine.input_lines.source(self.lineno - self.state_machine.input_offset - 1) 32 | 33 | os.chdir(os.path.dirname(current_rst_file)) 34 | 35 | cmd = [x for x in self.content if x != ""][0] 36 | 37 | out = subprocess.check_output(split(cmd)).decode("utf-8") 38 | 39 | # Parse output 40 | rst_lines = statemachine.string2lines(out) 41 | # Insert in place 42 | self.state_machine.insert_input(rst_lines, current_rst_file) 43 | 44 | except Exception: 45 | # rst_lines = statemachine.string2lines(str(e)) 46 | rst_lines = statemachine.string2lines(traceback.format_exc()) 47 | self.state_machine.insert_input(rst_lines, current_rst_file) 48 | 49 | finally: 50 | os.chdir(here) 51 | 52 | return [] 53 | 54 | 55 | def setup(app): 56 | app.add_directive("command-output", CommandOutput) 57 | -------------------------------------------------------------------------------- /src/climetlab/sphinxext/file_content.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # (C) Copyright 2020 ECMWF. 3 | # 4 | # This software is licensed under the terms of the Apache Licence Version 2.0 5 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 6 | # In applying this licence, ECMWF does not waive the privileges and immunities 7 | # granted to it by virtue of its status as an intergovernmental organisation 8 | # nor does it submit to any jurisdiction. 9 | # 10 | 11 | import os 12 | 13 | TOP = os.path.dirname(os.path.dirname(__file__)) 14 | 15 | LANGUAGES = {".py": "python", ".yaml": "yaml"} 16 | 17 | 18 | def execute(path): 19 | _, ext = os.path.splitext(path) 20 | 21 | print() 22 | print(".. code-block::", LANGUAGES[ext]) 23 | print() 24 | 25 | with open(os.path.join(TOP, path)) as f: 26 | for line in f: 27 | print(" ", line.rstrip()) 28 | 29 | print() 30 | print() 31 | 32 | 33 | if __name__ == "__main__": 34 | execute("data/projections/global.yaml") 35 | -------------------------------------------------------------------------------- /src/climetlab/sphinxext/generate_settings_rst.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # (C) Copyright 2020 ECMWF. 3 | # 4 | # This software is licensed under the terms of the Apache Licence Version 2.0 5 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 6 | # In applying this licence, ECMWF does not waive the privileges and immunities 7 | # granted to it by virtue of its status as an intergovernmental organisation 8 | # nor does it submit to any jurisdiction. 9 | # 10 | 11 | import getpass 12 | import os 13 | import re 14 | 15 | from climetlab.core.settings import SETTINGS_AND_HELP 16 | 17 | HOME = os.path.expanduser("~/") 18 | USER = getpass.getuser() 19 | 20 | 21 | def tidy(x): 22 | if isinstance(x, (list, tuple)): 23 | return [tidy(y) for y in x] 24 | 25 | if isinstance(x, dict): 26 | d = {} 27 | for k, v in x.items(): 28 | d[k] = tidy(v) 29 | return d 30 | 31 | if isinstance(x, str): 32 | if x.startswith(HOME): 33 | n = len(HOME) 34 | return tidy("~/{}".format(x[n:])) 35 | 36 | if "-" + USER in x: 37 | return tidy(x.replace("-" + USER, "-${USER}")) 38 | 39 | return x 40 | 41 | 42 | def execute(*args): 43 | print() 44 | print(".. list-table::") 45 | print(" :header-rows: 1") 46 | print(" :widths: 10 10 70") 47 | print() 48 | print(" * - | Name") 49 | print(" - | Default") 50 | print(" - | Description") 51 | print() 52 | for k, v in sorted(tidy(SETTINGS_AND_HELP).items()): 53 | if len(args) and not any(re.match(arg, k) for arg in args): 54 | continue 55 | print(" * - |", k.replace("-", "\u2011")) # Non-breaking hyphen 56 | print(" - |", repr(v.default).replace("-", "\u2011")) 57 | print(" - |", v.description) 58 | print() 59 | 60 | 61 | if __name__ == "__main__": 62 | execute() 63 | -------------------------------------------------------------------------------- /src/climetlab/utils/conventions.py: -------------------------------------------------------------------------------- 1 | # (C) Copyright 2021 ECMWF. 2 | # 3 | # This software is licensed under the terms of the Apache Licence Version 2.0 4 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 5 | # In applying this licence, ECMWF does not waive the privileges and immunities 6 | # granted to it by virtue of its status as an intergovernmental organisation 7 | # nor does it submit to any jurisdiction. 8 | # 9 | 10 | import logging 11 | 12 | from climetlab.vocabularies import VOCABULARIES 13 | 14 | LOG = logging.getLogger(__name__) 15 | 16 | 17 | def normalise_string(key, convention="cf"): 18 | vocabulary = VOCABULARIES[convention] 19 | 20 | new = vocabulary.normalise(key) 21 | LOG.debug(f"Normalising '{key}' into '{new}' ({convention} convention)") 22 | return new 23 | -------------------------------------------------------------------------------- /src/climetlab/utils/html.py: -------------------------------------------------------------------------------- 1 | # (C) Copyright 2020 ECMWF. 2 | # 3 | # This software is licensed under the terms of the Apache Licence Version 2.0 4 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 5 | # In applying this licence, ECMWF does not waive the privileges and immunities 6 | # granted to it by virtue of its status as an intergovernmental organisation 7 | # nor does it submit to any jurisdiction. 8 | # 9 | 10 | import os 11 | import re 12 | 13 | 14 | def urlify(text): 15 | return re.sub(r"(https?://.*\S)", r'\1', text) 16 | 17 | 18 | def css(name): 19 | path = os.path.join(os.path.dirname(os.path.dirname(__file__)), "data", "css", name) 20 | with open(path + ".css") as f: 21 | return "" % (f.read(),) 22 | 23 | 24 | def table(obj): 25 | style = css("table") 26 | 27 | table = """ 28 |

{name}

29 | 30 | 31 | 32 | 33 | 34 |
Home page{home_page}
Documentation{documentation}
Citation
{citation}
Licence{licence}
35 | """.format( 36 | name=obj.name, 37 | home_page=urlify(obj.home_page), 38 | licence=urlify(obj.licence), 39 | citation=obj.citation, 40 | documentation=urlify(obj.documentation), 41 | ) 42 | 43 | return style + table 44 | -------------------------------------------------------------------------------- /src/climetlab/utils/parts.py: -------------------------------------------------------------------------------- 1 | # (C) Copyright 2022 ECMWF. 2 | # 3 | # This software is licensed under the terms of the Apache Licence Version 2.0 4 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 5 | # In applying this licence, ECMWF does not waive the privileges and immunities 6 | # granted to it by virtue of its status as an intergovernmental organisation 7 | # nor does it submit to any jurisdiction. 8 | 9 | import os 10 | from collections import defaultdict 11 | 12 | from climetlab.utils import download_and_cache 13 | 14 | 15 | class Part: 16 | def __init__(self, path, offset, length): 17 | assert path is not None 18 | self.path = path 19 | self.offset = offset 20 | self.length = length 21 | 22 | def __eq__(self, other): 23 | return self.path == other.path and self.offset == other.offset and self.length == other.length 24 | 25 | @classmethod 26 | def resolve(cls, parts, directory=None): 27 | paths = defaultdict(list) 28 | for i, part in enumerate(parts): 29 | paths[part.path].append(part) 30 | 31 | for path, bits in paths.items(): 32 | if path.startswith("http://") or path.startswith("https://") or path.startswith("ftp://"): 33 | newpath = download_and_cache(path, parts=[(p.offset, p.length) for p in bits]) 34 | newoffset = 0 35 | for p in bits: 36 | p.path = newpath 37 | p.offset = newoffset 38 | newoffset += p.length 39 | 40 | elif directory and not os.path.isabs(path): 41 | for p in bits: 42 | p.path = os.path.join(directory, path) 43 | 44 | return parts 45 | 46 | def __repr__(self): 47 | return f"Part[{self.path},{self.offset},{self.length}]" 48 | -------------------------------------------------------------------------------- /src/climetlab/utils/serialise.py: -------------------------------------------------------------------------------- 1 | # (C) Copyright 2022 ECMWF. 2 | # 3 | # This software is licensed under the terms of the Apache Licence Version 2.0 4 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 5 | # In applying this licence, ECMWF does not waive the privileges and immunities 6 | # granted to it by virtue of its status as an intergovernmental organisation 7 | # nor does it submit to any jurisdiction. 8 | # 9 | 10 | import logging 11 | from importlib import import_module 12 | 13 | LOG = logging.getLogger(__name__) 14 | 15 | SERIALISATION = {} 16 | 17 | 18 | def register_serialisation(cls, serialise, create): 19 | fullname = (cls.__module__, cls.__qualname__) 20 | SERIALISATION[fullname] = (serialise, create) 21 | 22 | 23 | def serialise_state(obj): 24 | fullname = (obj.__class__.__module__, obj.__class__.__qualname__) 25 | LOG.info("serialise %s", fullname) 26 | return (fullname, SERIALISATION[fullname][0](obj)) 27 | 28 | 29 | def deserialise_state(state): 30 | fullname, args = state 31 | LOG.info("deserialise %s %s", fullname, args) 32 | # warnings.warn('deserialise %s %s', fullname, args) 33 | # TODO: if KeyError, maybe the function is not imported 34 | module, _ = fullname 35 | import_module(module) 36 | 37 | create = SERIALISATION[fullname][1] 38 | return create(args) 39 | -------------------------------------------------------------------------------- /src/climetlab/vocabularies/aliases.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # (C) Copyright 2021- ECMWF. 4 | # 5 | # This software is licensed under the terms of the Apache Licence Version 2.0 6 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 7 | # In applying this licence, ECMWF does not waive the privileges and immunities 8 | # granted to it by virtue of its status as an intergovernmental organisation nor 9 | # does it submit to any jurisdiction. 10 | # 11 | import csv 12 | import logging 13 | import os 14 | 15 | from climetlab.utils import load_json_or_yaml 16 | 17 | LOG = logging.getLogger(__name__) 18 | 19 | ALIASES = {} 20 | 21 | 22 | def load_csv(path): 23 | result = {} 24 | with open(path) as f: 25 | for row in csv.reader(f): 26 | result[row[0]] = row[1] 27 | 28 | return result 29 | 30 | 31 | def _find_aliases(name): 32 | if name not in ALIASES: 33 | path = os.path.join(os.path.dirname(__file__), name) 34 | if os.path.exists(path + ".csv"): 35 | ALIASES[name] = load_csv(path + ".csv") 36 | else: 37 | ALIASES[name] = load_json_or_yaml(path + ".yaml") 38 | return ALIASES[name] 39 | 40 | 41 | def unalias(name, value): 42 | return _find_aliases(name).get(value, value) 43 | 44 | 45 | if __name__ == "__main__": 46 | print(unalias("grib-paramid", "2t")) 47 | -------------------------------------------------------------------------------- /src/climetlab/vocabularies/cf.py: -------------------------------------------------------------------------------- 1 | # (C) Copyright 2021 ECMWF. 2 | # 3 | # This software is licensed under the terms of the Apache Licence Version 2.0 4 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 5 | # In applying this licence, ECMWF does not waive the privileges and immunities 6 | # granted to it by virtue of its status as an intergovernmental organisation 7 | # nor does it submit to any jurisdiction. 8 | # 9 | 10 | from xml.dom import minidom 11 | 12 | from climetlab.utils import download_and_cache 13 | 14 | URL = "https://cfconventions.org/Data/cf-standard-names/{version}/src/cf-standard-name-table.xml" 15 | 16 | 17 | def cf_standard_names(version=78): 18 | path = download_and_cache(URL.format(version=version)) 19 | xmldoc = minidom.parse(path) 20 | entries = xmldoc.getElementsByTagName("entry") 21 | for entry in entries: 22 | yield entry.attributes["id"].value 23 | 24 | 25 | if __name__ == "__main__": 26 | for n in cf_standard_names(): 27 | print(n) 28 | -------------------------------------------------------------------------------- /src/climetlab/vocabularies/grib.py: -------------------------------------------------------------------------------- 1 | # (C) Copyright 2021 ECMWF. 2 | # 3 | # This software is licensed under the terms of the Apache Licence Version 2.0 4 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 5 | # In applying this licence, ECMWF does not waive the privileges and immunities 6 | # granted to it by virtue of its status as an intergovernmental organisation 7 | # nor does it submit to any jurisdiction. 8 | # 9 | 10 | import json 11 | 12 | from climetlab.utils import download_and_cache 13 | 14 | URL = "https://apps.ecmwf.int/codes/grib/json/" 15 | 16 | PARAMS = {} 17 | 18 | 19 | def _param_id_dict(): 20 | if not PARAMS: 21 | path = download_and_cache(URL) 22 | with open(path) as f: 23 | entries = json.load(f) 24 | 25 | for entry in entries["parameters"]: 26 | PARAMS[int(entry["param_id"])] = entry 27 | 28 | return PARAMS 29 | 30 | 31 | def param_id_to_dict(param_id): 32 | dic = _param_id_dict() 33 | return dic[param_id] 34 | 35 | 36 | def param_id_to_short_name(param_id): 37 | entry = param_id_to_dict(param_id) 38 | return entry["param_shortName"] 39 | 40 | 41 | if __name__ == "__main__": 42 | import sys 43 | 44 | for i in sys.argv[1:]: 45 | print(param_id_to_short_name(int(i))) 46 | -------------------------------------------------------------------------------- /src/climetlab/wrappers/__init__.py: -------------------------------------------------------------------------------- 1 | # (C) Copyright 2020 ECMWF. 2 | # 3 | # This software is licensed under the terms of the Apache Licence Version 2.0 4 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 5 | # In applying this licence, ECMWF does not waive the privileges and immunities 6 | # granted to it by virtue of its status as an intergovernmental organisation 7 | # nor does it submit to any jurisdiction. 8 | # 9 | 10 | import logging 11 | import os 12 | from importlib import import_module 13 | 14 | from climetlab.core import Base 15 | from climetlab.decorators import locked 16 | 17 | LOG = logging.getLogger(__name__) 18 | 19 | 20 | class Wrapper(Base): 21 | pass 22 | 23 | 24 | _HELPERS = {} 25 | 26 | 27 | # TODO: Add plugins 28 | @locked 29 | def _wrappers(): 30 | if not _HELPERS: 31 | here = os.path.dirname(__file__) 32 | for path in os.listdir(here): 33 | if path.endswith(".py") and path[0] not in ("_", "."): 34 | name, _ = os.path.splitext(path) 35 | try: 36 | _HELPERS[name] = import_module(f".{name}", package=__name__).wrapper 37 | except Exception as e: 38 | LOG.warning(f"Error loading wrapper '{name}': {e}") 39 | return _HELPERS 40 | 41 | 42 | def get_wrapper(data, *args, **kwargs): 43 | """ 44 | Returns an object that wraps classes from other packages 45 | to support 46 | """ 47 | 48 | if isinstance(data, Base): 49 | return data 50 | 51 | for name, h in _wrappers().items(): 52 | wrapper = h(data, *args, **kwargs) 53 | if wrapper is not None: 54 | return wrapper.mutate() 55 | 56 | fullname = ".".join([data.__class__.__module__, data.__class__.__qualname__]) 57 | 58 | raise ValueError(f"Cannot find a wrapper for class {fullname}") 59 | -------------------------------------------------------------------------------- /src/climetlab/wrappers/date.py: -------------------------------------------------------------------------------- 1 | # (C) Copyright 2023 ECMWF. 2 | # 3 | # This software is licensed under the terms of the Apache Licence Version 2.0 4 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 5 | # In applying this licence, ECMWF does not waive the privileges and immunities 6 | # granted to it by virtue of its status as an intergovernmental organisation 7 | # nor does it submit to any jurisdiction. 8 | # 9 | import datetime 10 | 11 | from climetlab.wrappers import Wrapper 12 | 13 | 14 | class DateWrapper(Wrapper): 15 | def __init__(self, data): 16 | self.data = data 17 | 18 | def to_datetime(self): 19 | return datetime.datetime(self.data.year, self.data.month, self.data.day) 20 | 21 | def to_datetime_list(self): 22 | return [self.to_datetime()] 23 | 24 | 25 | def wrapper(data, *args, **kwargs): 26 | if isinstance(data, datetime.date): 27 | return DateWrapper(data) 28 | return None 29 | -------------------------------------------------------------------------------- /src/climetlab/wrappers/integer.py: -------------------------------------------------------------------------------- 1 | # (C) Copyright 2020 ECMWF. 2 | # 3 | # This software is licensed under the terms of the Apache Licence Version 2.0 4 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 5 | # In applying this licence, ECMWF does not waive the privileges and immunities 6 | # granted to it by virtue of its status as an intergovernmental organisation 7 | # nor does it submit to any jurisdiction. 8 | # 9 | import datetime 10 | 11 | from climetlab.wrappers import Wrapper 12 | 13 | 14 | class IntWrapper(Wrapper): 15 | def __init__(self, data): 16 | self.data = data 17 | 18 | def to_datetime(self): 19 | if self.data <= 0: 20 | date = datetime.datetime.utcnow() + datetime.timedelta(days=self.data) 21 | return datetime.datetime(date.year, date.month, date.day) 22 | else: 23 | return datetime.datetime(self.data // 10000, self.data % 10000 // 100, self.data % 100) 24 | 25 | def to_datetime_list(self): 26 | return [self.to_datetime()] 27 | 28 | 29 | def wrapper(data, *args, **kwargs): 30 | if isinstance(data, int): 31 | return IntWrapper(data) 32 | return None 33 | -------------------------------------------------------------------------------- /src/climetlab/wrappers/ndarray.py: -------------------------------------------------------------------------------- 1 | # (C) Copyright 2020 ECMWF. 2 | # 3 | # This software is licensed under the terms of the Apache Licence Version 2.0 4 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 5 | # In applying this licence, ECMWF does not waive the privileges and immunities 6 | # granted to it by virtue of its status as an intergovernmental organisation 7 | # nor does it submit to any jurisdiction. 8 | # 9 | import datetime 10 | 11 | from climetlab.wrappers import Wrapper 12 | from climetlab.wrappers import get_wrapper 13 | 14 | 15 | class NumpyArrayWrapper(Wrapper): 16 | def __init__(self, data, *args, **kwargs): 17 | self.data = data 18 | 19 | def plot_map(self, backend): 20 | wrapper = get_wrapper(backend.option("metadata")) 21 | 22 | if hasattr(wrapper, "plot_numpy"): 23 | return wrapper.plot_numpy(backend, self.data) 24 | 25 | metadata = wrapper.field_metadata() 26 | 27 | backend.bounding_box( 28 | north=metadata["north"], 29 | south=metadata["south"], 30 | west=metadata["west"], 31 | east=metadata["east"], 32 | ) 33 | 34 | backend.plot_numpy( 35 | self.data.reshape(metadata.get("shape", self.data.shape)), 36 | metadata=metadata, 37 | ) 38 | 39 | def to_datetime_list(self): 40 | return [datetime.datetime.fromtimestamp(x * 1e-9, tz=datetime.timezone.utc) for x in self.data.tolist()] 41 | 42 | 43 | def wrapper(data, *args, **kwargs): 44 | import numpy as np 45 | 46 | if isinstance(data, np.ndarray): 47 | return NumpyArrayWrapper(data, *args, **kwargs) 48 | return None 49 | -------------------------------------------------------------------------------- /src/climetlab/wrappers/none.py: -------------------------------------------------------------------------------- 1 | # (C) Copyright 2020 ECMWF. 2 | # 3 | # This software is licensed under the terms of the Apache Licence Version 2.0 4 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 5 | # In applying this licence, ECMWF does not waive the privileges and immunities 6 | # granted to it by virtue of its status as an intergovernmental organisation 7 | # nor does it submit to any jurisdiction. 8 | # 9 | from climetlab.wrappers import Wrapper 10 | 11 | 12 | class NoneWrapper(Wrapper): 13 | def __init__(self, data): 14 | pass 15 | 16 | def plot_map(self, backend): 17 | pass 18 | 19 | def field_metadata(self): 20 | return {} 21 | 22 | 23 | def wrapper(data, *args, **kwargs): 24 | if data is None: 25 | return NoneWrapper(data) 26 | return None 27 | -------------------------------------------------------------------------------- /src/climetlab/wrappers/tensor.py: -------------------------------------------------------------------------------- 1 | # (C) Copyright 2023 ECMWF. 2 | # 3 | # This software is licensed under the terms of the Apache Licence Version 2.0 4 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 5 | # In applying this licence, ECMWF does not waive the privileges and immunities 6 | # granted to it by virtue of its status as an intergovernmental organisation 7 | # nor does it submit to any jurisdiction. 8 | # 9 | 10 | 11 | from climetlab.utils import module_loaded 12 | from climetlab.wrappers import get_wrapper 13 | 14 | 15 | def wrapper(data, *args, **kwargs): 16 | if not module_loaded("torch"): 17 | return None 18 | 19 | import torch # noqa 20 | 21 | if isinstance(data, torch.Tensor): 22 | return get_wrapper(data.detach().cpu().numpy(), *args, **kwargs) 23 | 24 | return None 25 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | SKIP = { 4 | "short": ["documentation", "download", "external_download", "ftp", "long_test"], 5 | "long": ["documentation", "ftp"], 6 | "release": [], 7 | "documentation": None, 8 | # pytest.mark.short_download not used 9 | } 10 | 11 | 12 | def pytest_addoption(parser): 13 | help_str = "NAME: short, long, release. Runs a subset of tests.\n" 14 | for k, v in SKIP.items(): 15 | if v: 16 | help_str += f"'{k}': skip tests marked as {','.join(v)}.\n" 17 | else: 18 | help_str += f"'{k}': do not skip tests.\n" 19 | 20 | parser.addoption( 21 | "-E", 22 | action="store", 23 | metavar="NAME", 24 | default="short", 25 | help=help_str, 26 | ) 27 | 28 | 29 | def pytest_runtest_setup(item): 30 | flag = item.config.getoption("-E") 31 | marks_to_skip = SKIP[flag] 32 | 33 | marks_in_items = list([m.name for m in item.iter_markers()]) 34 | 35 | if marks_to_skip is None: 36 | if flag not in marks_in_items: 37 | pytest.skip(f"test is skipped because custom pytest option : -E {flag}") 38 | return 39 | 40 | for m in marks_in_items: 41 | if m in marks_to_skip: 42 | pytest.skip(f"test is skipped because custom pytest option: -E {flag}") 43 | -------------------------------------------------------------------------------- /tests/converters/test_metview.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # (C) Copyright 2021 ECMWF. 4 | # 5 | # This software is licensed under the terms of the Apache Licence Version 2.0 6 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 7 | # In applying this licence, ECMWF does not waive the privileges and immunities 8 | # granted to it by virtue of its status as an intergovernmental organisation 9 | # nor does it submit to any jurisdiction. 10 | # 11 | 12 | import logging 13 | 14 | import pytest 15 | 16 | import climetlab as cml 17 | from climetlab.testing import MISSING 18 | from climetlab.testing import climetlab_file 19 | 20 | LOG = logging.getLogger(__name__) 21 | 22 | 23 | @pytest.mark.skipif(MISSING("metview"), reason="Metview not installed") 24 | def test_metview_grib(): 25 | s = cml.load_source("file", climetlab_file("docs/examples/test.grib")) 26 | fs = s.to_metview() 27 | 28 | assert fs.url() == s.path 29 | 30 | 31 | @pytest.mark.skipif(MISSING("metview"), reason="Metview not installed") 32 | def test_metview_netcdf(): 33 | s = cml.load_source("file", climetlab_file("docs/examples/test.nc")) 34 | fs = s.to_metview() 35 | 36 | assert fs.url() == s.path 37 | 38 | 39 | @pytest.mark.skipif(MISSING("metview"), reason="Metview not installed") 40 | def test_metview_csv(): 41 | s = cml.load_source( 42 | "climetlab-testing", 43 | "csv", 44 | headers=["a", "b", "c"], 45 | lines=[ 46 | [1, 2, 3], 47 | [4, 5, 6], 48 | [7, 8, 9], 49 | ], 50 | ) 51 | fs = s.to_metview() 52 | 53 | assert fs.url() == s.path 54 | 55 | 56 | if __name__ == "__main__": 57 | from climetlab.testing import main 58 | 59 | main(__file__) 60 | -------------------------------------------------------------------------------- /tests/converters/test_numpy.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # (C) Copyright 2020 ECMWF. 4 | # 5 | # This software is licensed under the terms of the Apache Licence Version 2.0 6 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 7 | # In applying this licence, ECMWF does not waive the privileges and immunities 8 | # granted to it by virtue of its status as an intergovernmental organisation 9 | # nor does it submit to any jurisdiction. 10 | # 11 | 12 | import pytest 13 | 14 | import climetlab as cml 15 | from climetlab.testing import climetlab_file 16 | 17 | """ 18 | Test if a numpy array can be plotted using another field as metadata, 19 | i.e. for bounding box, style, etc. 20 | """ 21 | 22 | 23 | def test_numpy_grib(): 24 | s = cml.load_source("file", climetlab_file("docs/examples/test.grib")) 25 | x = s.to_xarray() 26 | cml.plot_map(x.msl.values, metadata=s[1]) 27 | 28 | 29 | @pytest.mark.skip("Not implemented yet") 30 | def test_numpy_netcdf(): 31 | s = cml.load_source("file", climetlab_file("docs/examples/test.nc")) 32 | x = s.to_xarray() 33 | cml.plot_map(x.msl.values, metadata=s[1]) 34 | 35 | 36 | def test_numpy_xarray(): 37 | s = cml.load_source("file", climetlab_file("docs/examples/test.grib")) 38 | x = s.to_xarray() 39 | cml.plot_map(x.msl.values, metadata=x.msl) 40 | 41 | 42 | if __name__ == "__main__": 43 | from climetlab.testing import main 44 | 45 | main(__file__) 46 | -------------------------------------------------------------------------------- /tests/core/test_plugins.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # (C) Copyright 2021 ECMWF. 4 | # 5 | # This software is licensed under the terms of the Apache Licence Version 2.0 6 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 7 | # In applying this licence, ECMWF does not waive the privileges and immunities 8 | # granted to it by virtue of its status as an intergovernmental organisation 9 | # nor does it submit to any jurisdiction. 10 | # 11 | 12 | import numpy as np 13 | 14 | import climetlab as cml 15 | from climetlab.datasets import Dataset 16 | from climetlab.datasets import register as register_dataset 17 | from climetlab.sources import Source 18 | from climetlab.sources import register as register_source 19 | 20 | 21 | class RegisteredSource(Source): 22 | def __init__(self, shape): 23 | self.shape = shape 24 | 25 | def to_numpy(self): 26 | return np.zeros(self.shape) 27 | 28 | 29 | def test_register_source(): 30 | register_source("test-register-source", RegisteredSource) 31 | ds = cml.load_source("test-register-source", shape=(3, 3)) 32 | 33 | assert ds.to_numpy().shape == (3, 3) 34 | 35 | 36 | class RegisteredDataset(Dataset): 37 | def __init__(self, shape): 38 | self.shape = shape 39 | 40 | def to_numpy(self): 41 | return np.zeros(self.shape) 42 | 43 | 44 | def test_register_dataset(): 45 | register_dataset("test-register-dataset", RegisteredDataset) 46 | ds = cml.load_dataset("test-register-dataset", shape=(3, 3)) 47 | assert ds.to_numpy().shape == (3, 3) 48 | 49 | 50 | if __name__ == "__main__": 51 | from climetlab.testing import main 52 | 53 | main(__file__) 54 | -------------------------------------------------------------------------------- /tests/core/test_version.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # (C) Copyright 2020 ECMWF. 4 | # 5 | # This software is licensed under the terms of the Apache Licence Version 2.0 6 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 7 | # In applying this licence, ECMWF does not waive the privileges and immunities 8 | # granted to it by virtue of its status as an intergovernmental organisation 9 | # nor does it submit to any jurisdiction. 10 | # 11 | 12 | 13 | def test_version(): 14 | from climetlab import __version__ 15 | 16 | assert __version__ is not None 17 | 18 | 19 | if __name__ == "__main__": 20 | from climetlab.testing import main 21 | 22 | main(__file__) 23 | -------------------------------------------------------------------------------- /tests/documentation/test_examples.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # (C) Copyright 2020 ECMWF. 4 | # 5 | # This software is licensed under the terms of the Apache Licence Version 2.0 6 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 7 | # In applying this licence, ECMWF does not waive the privileges and immunities 8 | # granted to it by virtue of its status as an intergovernmental organisation 9 | # nor does it submit to any jurisdiction. 10 | # 11 | 12 | import os 13 | 14 | import pytest 15 | 16 | from climetlab.testing import IN_GITHUB 17 | from climetlab.testing import climetlab_file 18 | 19 | # import pytest 20 | 21 | 22 | IGNORE = [ 23 | "conf.py", 24 | "xml2rst.py", 25 | "actions.py", 26 | "generate-examples-maps.py", 27 | "settings-2-set.py", 28 | ] 29 | 30 | EXAMPLES = climetlab_file("docs") 31 | 32 | 33 | def example_list(): 34 | # return [] 35 | 36 | examples = [] 37 | for root, _, files in os.walk(EXAMPLES): 38 | for file in files: 39 | path = os.path.join(root, file) 40 | if path.endswith(".py") and file not in IGNORE: 41 | n = len(EXAMPLES) + 1 42 | examples.append(path[n:]) 43 | 44 | return sorted(examples) 45 | 46 | 47 | @pytest.mark.skipif(not IN_GITHUB, reason="Not on GITHUB") 48 | @pytest.mark.parametrize("path", example_list()) 49 | @pytest.mark.documentation 50 | def test_example(path): 51 | full = os.path.join(EXAMPLES, path) 52 | with open(full) as f: 53 | exec(f.read(), dict(__file__=full), {}) 54 | 55 | 56 | if __name__ == "__main__": 57 | from climetlab.testing import main 58 | 59 | main(__file__) 60 | -------------------------------------------------------------------------------- /tests/documentation/test_generate_doc.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # (C) Copyright 2020 ECMWF. 4 | # 5 | # This software is licensed under the terms of the Apache Licence Version 2.0 6 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 7 | # In applying this licence, ECMWF does not waive the privileges and immunities 8 | # granted to it by virtue of its status as an intergovernmental organisation 9 | # nor does it submit to any jurisdiction. 10 | # 11 | 12 | 13 | import pytest 14 | 15 | 16 | @pytest.mark.documentation 17 | def test_generate_cmdline_help(): 18 | from climetlab.sphinxext.generate_cmdline_help import execute 19 | 20 | execute() 21 | 22 | 23 | if __name__ == "__main__": 24 | from climetlab.testing import main 25 | 26 | main(__file__) 27 | -------------------------------------------------------------------------------- /tests/example.json: -------------------------------------------------------------------------------- 1 | [ 2 | {"level": "500", "param": "Z", "step": "24"}, 3 | {"level": "500", "param": "Z", "step": "36"}, 4 | {"level": "500", "param": "Z", "step": "48"}, 5 | {"level": "500", "param": "T", "step": "24"}, 6 | {"level": "500", "param": "T", "step": "36"}, 7 | {"level": "500", "param": "T", "step": "48"}, 8 | {"level": "850", "param": "T", "step": "36"}, 9 | {"level": "850", "param": "T", "step": "48"}, 10 | {"level": "1000", "param": "Z", "step": "24"}, 11 | {"level": "1000", "param": "Z", "step": "48"} 12 | ] 13 | -------------------------------------------------------------------------------- /tests/example.ya_ml: -------------------------------------------------------------------------------- 1 | title: 2 | - [string:with:colon, string with spaces, string] 3 | - - string:with:colon 4 | - string with spaces 5 | - string 6 | -------------------------------------------------------------------------------- /tests/example.yaml: -------------------------------------------------------------------------------- 1 | a: 2 2 | -------------------------------------------------------------------------------- /tests/example.yml: -------------------------------------------------------------------------------- 1 | a: 2 2 | -------------------------------------------------------------------------------- /tests/normalize/aliases.json: -------------------------------------------------------------------------------- 1 | {"y": "b", "z": "c"} 2 | -------------------------------------------------------------------------------- /tests/normalize/availability.json: -------------------------------------------------------------------------------- 1 | [ 2 | {"level": "500", "param": "Z", "step": "24"}, 3 | {"level": "500", "param": "Z", "step": "36"}, 4 | {"level": "500", "param": "Z", "step": "48"}, 5 | {"level": "500", "param": "T", "step": "24"}, 6 | {"level": "500", "param": "T", "step": "36"}, 7 | {"level": "500", "param": "T", "step": "48"}, 8 | {"level": "850", "param": "T", "step": "36"}, 9 | {"level": "850", "param": "T", "step": "48"}, 10 | {"level": "1000", "param": "Z", "step": "24"}, 11 | {"level": "1000", "param": "Z", "step": "48"} 12 | ] 13 | -------------------------------------------------------------------------------- /tests/normalize/test_normalize_args.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # (C) Copyright 2020 ECMWF. 4 | # 5 | # This software is licensed under the terms of the Apache Licence Version 2.0 6 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 7 | # In applying this licence, ECMWF does not waive the privileges and immunities 8 | # granted to it by virtue of its status as an intergovernmental organisation 9 | # nor does it submit to any jurisdiction. 10 | # 11 | 12 | 13 | from climetlab.normalize import normalize_args 14 | 15 | 16 | def a_b_no_default(a, b): 17 | return a, b 18 | 19 | 20 | def test_normalize_args(): 21 | @normalize_args( 22 | dates="date-list(%Y.%m.%d)", 23 | names=["a", "b", "c"], 24 | name=("A", "B", "C"), 25 | ) 26 | def f(dates, names, name): 27 | return dates, names, name 28 | 29 | assert f("20200101", "A", "b") == (["2020.01.01"], ["a"], "B") 30 | 31 | 32 | if __name__ == "__main__": 33 | from climetlab.testing import main 34 | 35 | main(__file__) 36 | -------------------------------------------------------------------------------- /tests/normalize/test_normalize_int_list.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # (C) Copyright 2020 ECMWF. 4 | # 5 | # This software is licensed under the terms of the Apache Licence Version 2.0 6 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 7 | # In applying this licence, ECMWF does not waive the privileges and immunities 8 | # granted to it by virtue of its status as an intergovernmental organisation 9 | # nor does it submit to any jurisdiction. 10 | # 11 | 12 | from climetlab.decorators import normalize 13 | 14 | 15 | def x_no_default(x): 16 | return x 17 | 18 | 19 | def test_int_list_normalizers(): 20 | g = x_no_default 21 | g = normalize("x", "int-list")(g) 22 | assert g(x=1) == [1] 23 | assert g(x=(1, 2)) == [1, 2] 24 | assert g(x=("1", 2)) == [1, 2] 25 | assert g(x="1") == [1] 26 | assert g(x="1/to/3") == [1, 2, 3] 27 | assert g(x="1/to/3/by/2") == [1, 3] 28 | 29 | 30 | if __name__ == "__main__": 31 | from climetlab.testing import main 32 | 33 | main(__file__) 34 | -------------------------------------------------------------------------------- /tests/readers/little_endian.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ecmwf/climetlab/a55f9c8590c2ede64a2b0fdc1b163501c81a34f9/tests/readers/little_endian.mat -------------------------------------------------------------------------------- /tests/readers/test_fwf_reader.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # (C) Copyright 2020 ECMWF. 4 | # 5 | # This software is licensed under the terms of the Apache Licence Version 2.0 6 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 7 | # In applying this licence, ECMWF does not waive the privileges and immunities 8 | # granted to it by virtue of its status as an intergovernmental organisation 9 | # nor does it submit to any jurisdiction. 10 | # 11 | 12 | import pytest 13 | 14 | import climetlab as cml 15 | 16 | 17 | @pytest.mark.external_download 18 | @pytest.mark.download 19 | def test_fwf(): 20 | url = "https://www.cpc.ncep.noaa.gov/products/precip/CWlink/daily_ao_index/monthly.ao.index.b50.current.ascii" 21 | 22 | s = cml.load_source( 23 | "url", 24 | url, 25 | reader="fix_width_format", 26 | ) 27 | 28 | print(s.to_pandas()) 29 | 30 | 31 | if __name__ == "__main__": 32 | from climetlab.testing import main 33 | 34 | main(__file__) 35 | -------------------------------------------------------------------------------- /tests/readers/test_matlab_reader.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # (C) Copyright 2020 ECMWF. 4 | # 5 | # This software is licensed under the terms of the Apache Licence Version 2.0 6 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 7 | # In applying this licence, ECMWF does not waive the privileges and immunities 8 | # granted to it by virtue of its status as an intergovernmental organisation 9 | # nor does it submit to any jurisdiction. 10 | # 11 | 12 | import os 13 | 14 | import numpy as np 15 | import pytest 16 | 17 | import climetlab as cml 18 | from climetlab.testing import MISSING 19 | 20 | 21 | @pytest.mark.skipif(MISSING("scipy"), reason="scipy not installed") 22 | def test_matlab_1(): 23 | here = os.path.dirname(__file__) 24 | testfile = os.path.join(here, "little_endian.mat") 25 | s = cml.load_source("file", testfile) 26 | arr = s.to_numpy(key="floats") 27 | ref = np.array([[2, 3], [3, 4]]) 28 | assert (arr == ref).all(), (arr, ref) 29 | 30 | 31 | if __name__ == "__main__": 32 | from climetlab.testing import main 33 | 34 | main(__file__) 35 | -------------------------------------------------------------------------------- /tests/readers/test_tar_reader.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # (C) Copyright 2020 ECMWF. 4 | # 5 | # This software is licensed under the terms of the Apache Licence Version 2.0 6 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 7 | # In applying this licence, ECMWF does not waive the privileges and immunities 8 | # granted to it by virtue of its status as an intergovernmental organisation 9 | # nor does it submit to any jurisdiction. 10 | # 11 | 12 | 13 | import mimetypes 14 | 15 | from climetlab.testing import check_unsafe_archives 16 | 17 | 18 | # @pytest.mark.download # not marked as download to make sure we run this every time 19 | def test_tar_safety(): 20 | check_unsafe_archives(".tar") 21 | 22 | 23 | def test_tar_mimetypes(): 24 | assert mimetypes.guess_type("x.tar") == ("application/x-tar", None) 25 | assert mimetypes.guess_type("x.tgz") == ("application/x-tar", "gzip") 26 | 27 | assert mimetypes.guess_type("x.tar.gz") == ("application/x-tar", "gzip") 28 | assert mimetypes.guess_type("x.tar.bz2") == ("application/x-tar", "bzip2") 29 | 30 | 31 | if __name__ == "__main__": 32 | from climetlab.testing import main 33 | 34 | main(__file__) 35 | -------------------------------------------------------------------------------- /tests/readers/test_tfdataset_reader.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # (C) Copyright 2020 ECMWF. 4 | # 5 | # This software is licensed under the terms of the Apache Licence Version 2.0 6 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 7 | # In applying this licence, ECMWF does not waive the privileges and immunities 8 | # granted to it by virtue of its status as an intergovernmental organisation 9 | # nor does it submit to any jurisdiction. 10 | # 11 | 12 | import pytest 13 | 14 | from climetlab import load_source 15 | from climetlab.testing import MISSING 16 | from climetlab.testing import TEST_DATA_URL 17 | 18 | 19 | @pytest.mark.skipif(MISSING("tensorflow"), reason="No tensorflow") 20 | @pytest.mark.download 21 | def test_download_tfdataset(): 22 | ds = load_source( 23 | "url-pattern", 24 | "{url}/fixtures/tfrecord/EWCTest0.{n}.tfrecord", 25 | n=[0, 1], 26 | url=TEST_DATA_URL, 27 | # TODO: move adapt test data creation script 28 | # url=f"{TEST_DATA_URL}/input/", 29 | ) 30 | 31 | ds.graph() 32 | assert len(ds) == 200, len(ds) 33 | 34 | 35 | if __name__ == "__main__": 36 | from climetlab.testing import main 37 | 38 | main(__file__) 39 | -------------------------------------------------------------------------------- /tests/readers/test_unknown_reader.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # (C) Copyright 2020 ECMWF. 4 | # 5 | # This software is licensed under the terms of the Apache Licence Version 2.0 6 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 7 | # In applying this licence, ECMWF does not waive the privileges and immunities 8 | # granted to it by virtue of its status as an intergovernmental organisation 9 | # nor does it submit to any jurisdiction. 10 | # 11 | 12 | import os 13 | 14 | import climetlab as cml 15 | 16 | 17 | def test_unknown_reader(): 18 | s = cml.load_source( 19 | "file", 20 | os.path.join(os.path.dirname(__file__), "unknown_file.unknown_ext"), 21 | ) 22 | print(s) 23 | assert isinstance(s._reader, cml.readers.unknown.Unknown) 24 | 25 | 26 | def test_text_reader(): 27 | s = cml.load_source( 28 | "file", 29 | os.path.join(os.path.dirname(__file__), "unknown_text_file.unknown_ext"), 30 | ) 31 | print(s) 32 | assert isinstance(s._reader, cml.readers.text.TextReader) 33 | 34 | 35 | if __name__ == "__main__": 36 | from climetlab.testing import main 37 | 38 | main(__file__) 39 | -------------------------------------------------------------------------------- /tests/readers/test_zip_reader.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # (C) Copyright 2020 ECMWF. 4 | # 5 | # This software is licensed under the terms of the Apache Licence Version 2.0 6 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 7 | # In applying this licence, ECMWF does not waive the privileges and immunities 8 | # granted to it by virtue of its status as an intergovernmental organisation 9 | # nor does it submit to any jurisdiction. 10 | # 11 | 12 | from climetlab.testing import check_unsafe_archives 13 | 14 | 15 | # @pytest.mark.download # not marked as download to make sure we run this every time 16 | def test_zip_safety(): 17 | check_unsafe_archives(".zip") 18 | 19 | 20 | if __name__ == "__main__": 21 | from climetlab.testing import main 22 | 23 | main(__file__) 24 | -------------------------------------------------------------------------------- /tests/readers/unknown_file.unknown_ext: -------------------------------------------------------------------------------- 1 | unknown This file has 0x0 as first character to simulate an unknown magic header. 2 | -------------------------------------------------------------------------------- /tests/readers/unknown_text_file.unknown_ext: -------------------------------------------------------------------------------- 1 | This is a text file. 2 | -------------------------------------------------------------------------------- /tests/sources/test_mars.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # (C) Copyright 2020 ECMWF. 4 | # 5 | # This software is licensed under the terms of the Apache Licence Version 2.0 6 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 7 | # In applying this licence, ECMWF does not waive the privileges and immunities 8 | # granted to it by virtue of its status as an intergovernmental organisation 9 | # nor does it submit to any jurisdiction. 10 | # 11 | 12 | import pytest 13 | 14 | from climetlab import load_source 15 | from climetlab.testing import NO_MARS 16 | 17 | 18 | @pytest.mark.long_test 19 | @pytest.mark.download 20 | @pytest.mark.skipif(NO_MARS, reason="No access to MARS") 21 | @pytest.mark.skip(reason="No access to MARS for now (DHS move)") # TODO: remove this line 22 | def test_mars_grib_1(): 23 | s = load_source( 24 | "mars", 25 | param=["2t", "msl"], 26 | levtype="sfc", 27 | area=[50, -50, 20, 50], 28 | grid=[1, 1], 29 | date="2012-12-13", 30 | ) 31 | assert len(s) == 2 32 | 33 | 34 | @pytest.mark.long_test 35 | @pytest.mark.download 36 | @pytest.mark.skipif(NO_MARS, reason="No access to MARS") 37 | @pytest.mark.skip(reason="No access to MARS for now (DHS move)") # TODO: remove this line 38 | def test_mars_grib_2(): 39 | s = load_source( 40 | "mars", 41 | param=["2t", "msl"], 42 | levtype="sfc", 43 | area=[50, -50, 20, 50], 44 | grid=[1, 1], 45 | date="2012-12-13", 46 | split_on="param", 47 | ) 48 | assert len(s) == 2 49 | 50 | 51 | if __name__ == "__main__": 52 | from climetlab.testing import main 53 | 54 | main(__file__) 55 | -------------------------------------------------------------------------------- /tests/sources/test_open_data.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # (C) Copyright 2020 ECMWF. 4 | # 5 | # This software is licensed under the terms of the Apache Licence Version 2.0 6 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 7 | # In applying this licence, ECMWF does not waive the privileges and immunities 8 | # granted to it by virtue of its status as an intergovernmental organisation 9 | # nor does it submit to any jurisdiction. 10 | # 11 | 12 | import pytest 13 | 14 | from climetlab import load_source 15 | from climetlab.testing import NO_EOD 16 | 17 | 18 | @pytest.mark.skipif(NO_EOD, reason="No access to Open data") 19 | @pytest.mark.download 20 | def test_open_data(): 21 | s = load_source( 22 | "ecmwf-open-data", 23 | step=240, 24 | type="fc", 25 | param="msl", 26 | ) 27 | print(s.path) 28 | 29 | 30 | if __name__ == "__main__": 31 | # test_open_data() 32 | from climetlab.testing import main 33 | 34 | main(__file__) 35 | -------------------------------------------------------------------------------- /tests/sources/test_url_pattern.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # (C) Copyright 2020 ECMWF. 4 | # 5 | # This software is licensed under the terms of the Apache Licence Version 2.0 6 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 7 | # In applying this licence, ECMWF does not waive the privileges and immunities 8 | # granted to it by virtue of its status as an intergovernmental organisation 9 | # nor does it submit to any jurisdiction. 10 | # 11 | 12 | 13 | import pytest 14 | 15 | from climetlab import load_source 16 | 17 | 18 | @pytest.mark.download 19 | def test_url_pattern_source_3(): 20 | source = load_source( 21 | "url-pattern", 22 | "https://github.com/ecmwf/climetlab/raw/main/docs/examples/test.{format}", 23 | {"format": ["nc", "grib"]}, 24 | ) 25 | # note that both files contain the same data 26 | assert len(source) == 4 27 | for i in source: 28 | print(i) 29 | 30 | 31 | if __name__ == "__main__": 32 | from climetlab.testing import main 33 | 34 | main(__file__) 35 | -------------------------------------------------------------------------------- /tests/sources/zedono-dataset-1.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | dataset: 3 | source: zenodo 4 | args: 5 | record_id: 4707154 6 | file_key: soltau_station_data.zip 7 | filter: '**/reforecasts_2t_2002-01-02*' 8 | -------------------------------------------------------------------------------- /tests/test_domains.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # (C) Copyright 2020 ECMWF. 4 | # 5 | # This software is licensed under the terms of the Apache Licence Version 2.0 6 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 7 | # In applying this licence, ECMWF does not waive the privileges and immunities 8 | # granted to it by virtue of its status as an intergovernmental organisation 9 | # nor does it submit to any jurisdiction. 10 | # 11 | 12 | 13 | from climetlab.utils.domains import domain_to_area 14 | from climetlab.utils.domains import domain_to_area_long_name 15 | 16 | 17 | def test_domains(): 18 | assert domain_to_area("italy") == (50.5, 5.0, 35.0, 20.5) 19 | assert domain_to_area_long_name("italy") is None 20 | 21 | assert domain_to_area("verification.germany") == (55.5, 5.5, 47.0, 15.5) 22 | assert domain_to_area_long_name("verification.germany") == "Germany" 23 | 24 | assert domain_to_area("uk") == (63.5, -10.0, 48.0, 5.5) 25 | assert domain_to_area("verification.uk") == (59.5, -10.5, 49.5, 2.0) 26 | assert domain_to_area_long_name("verification.uk") == "United Kingdom" 27 | 28 | assert domain_to_area("france") == (54.5, -6.0, 39.0, 9.5) 29 | assert domain_to_area("verification.france") == (51.5, -5.0, 42.0, 8.5) 30 | 31 | 32 | if __name__ == "__main__": 33 | from climetlab.testing import main 34 | 35 | main(__file__) 36 | -------------------------------------------------------------------------------- /tests/test_download.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # (C) Copyright 2020 ECMWF. 4 | # 5 | # This software is licensed under the terms of the Apache Licence Version 2.0 6 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 7 | # In applying this licence, ECMWF does not waive the privileges and immunities 8 | # granted to it by virtue of its status as an intergovernmental organisation 9 | # nor does it submit to any jurisdiction. 10 | # 11 | 12 | import os 13 | import pathlib 14 | import time 15 | 16 | import pytest 17 | 18 | from climetlab import settings 19 | from climetlab.utils import download_and_cache 20 | 21 | 22 | def path_to_url(path): 23 | return pathlib.Path(os.path.abspath(path)).as_uri() 24 | 25 | 26 | @pytest.mark.download 27 | @pytest.mark.small_download 28 | def test_download_1(): 29 | url = "https://github.com/ecmwf/climetlab/raw/main/docs/examples/test.grib?_=%s" % (time.time(),) 30 | download_and_cache(url) 31 | 32 | 33 | @pytest.mark.download 34 | @pytest.mark.small_download 35 | def test_download_2(): 36 | url = "https://github.com/ecmwf/climetlab/raw/main/docs/examples/test.grib" 37 | download_and_cache(url) 38 | 39 | 40 | @pytest.mark.download 41 | @pytest.mark.small_download 42 | def test_download_3(): 43 | with settings.temporary("download-out-of-date-urls", True): 44 | url = "https://get.ecmwf.int/test-data/climetlab/input/test.txt" 45 | download_and_cache(url) 46 | 47 | 48 | @pytest.mark.download 49 | @pytest.mark.small_download 50 | def test_download_4(): 51 | url = "https://get.ecmwf.int/test-data/climetlab/input/missing.txt" 52 | r = download_and_cache(url, return_none_on_404=True) 53 | assert r is None, r 54 | 55 | 56 | if __name__ == "__main__": 57 | from climetlab.testing import main 58 | 59 | main(__file__) 60 | -------------------------------------------------------------------------------- /tests/test_imports.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # (C) Copyright 2023 ECMWF. 4 | # 5 | # This software is licensed under the terms of the Apache Licence Version 2.0 6 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 7 | # In applying this licence, ECMWF does not waive the privileges and immunities 8 | # granted to it by virtue of its status as an intergovernmental organisation 9 | # nor does it submit to any jurisdiction. 10 | # 11 | 12 | import os 13 | 14 | import pytest 15 | 16 | 17 | # Make sure all these modules are loaded lazily 18 | # To find the culprit, rerun with: 19 | # CLIMETLAB_DEBUG_IMPORTS=1 pytest tests/test_imports.py 20 | @pytest.mark.parametrize( 21 | "module", 22 | [ 23 | "pandas", 24 | "xarray", 25 | "eccodes", 26 | "Magics", 27 | "IPython", 28 | "jupyter", 29 | "torch", 30 | "tensorflow", 31 | ], 32 | ) 33 | @pytest.mark.skipif( 34 | int(os.environ.get("SKIP_TEST_IMPORTS", "1")), 35 | reason="Only activated if SKIP_TEST_IMPORTS=0", 36 | ) 37 | def test_imports(module): 38 | import climetlab as cml 39 | from climetlab.aaa import loaded_modules 40 | 41 | # This will trigger the loading of wrappers 42 | cml.load_source("file", __file__) 43 | 44 | modules = loaded_modules() 45 | 46 | assert module not in modules, "\n".join(modules[module]) 47 | 48 | 49 | if __name__ == "__main__": 50 | from climetlab.testing import main 51 | 52 | main(__file__) 53 | -------------------------------------------------------------------------------- /tests/test_kwargs.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # (C} Copyright 2022 ECMWF. 4 | # 5 | # This software is licensed under the terms of the Apache Licence Version 2.0 6 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 7 | # In applying this licence, ECMWF does not waive the privileges and immunities 8 | # granted to it by virtue of its status as an intergovernmental organisation 9 | # nor does it submit to any jurisdiction. 10 | # 11 | from climetlab.utils.kwargs import merge_dicts 12 | 13 | 14 | def test_merge_dicts(): 15 | assert merge_dicts({"a": 1, "b": 2}, {"a": 1, "b": 6}) == {"a": 1, "b": 6} 16 | assert merge_dicts({"a": 1}, {"a": 1, "b": 6}) == {"a": 1, "b": 6} 17 | assert merge_dicts({"a": 1, "b": 2}, {"a": 3}) == {"a": 3, "b": 2} 18 | assert merge_dicts({"a": 1, "b": 2}, {"c": 3}) == {"a": 1, "b": 2, "c": 3} 19 | 20 | assert merge_dicts({"a": {"b": 2}}, {"c": 3}) == {"a": {"b": 2}, "c": 3} 21 | assert merge_dicts({"a": {"b": 2}}, {"a": {"c": 4}}) == {"a": {"b": 2, "c": 4}} 22 | 23 | 24 | if __name__ == "__main__": 25 | from climetlab.testing import main 26 | 27 | main(__file__) 28 | -------------------------------------------------------------------------------- /tests/test_magics.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # (C) Copyright 2020 ECMWF. 4 | # 5 | # This software is licensed under the terms of the Apache Licence Version 2.0 6 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 7 | # In applying this licence, ECMWF does not waive the privileges and immunities 8 | # granted to it by virtue of its status as an intergovernmental organisation 9 | # nor does it submit to any jurisdiction. 10 | # 11 | 12 | import pytest 13 | from Magics.Magics import MagicsError 14 | 15 | import climetlab as cml 16 | from climetlab.utils.bbox import BoundingBox 17 | 18 | 19 | def plot(): 20 | bbox = BoundingBox(north=90, west=0, east=360, south=-90) 21 | cml.plot_map(bounding_box=bbox, projection="polar-north") 22 | 23 | 24 | def test_exception(): 25 | with pytest.raises(MagicsError): 26 | plot() 27 | 28 | 29 | if __name__ == "__main__": 30 | from climetlab.testing import main 31 | 32 | main(__file__) 33 | -------------------------------------------------------------------------------- /tests/test_meteonet.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # (C) Copyright 2020 ECMWF. 4 | # 5 | # This software is licensed under the terms of the Apache Licence Version 2.0 6 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 7 | # In applying this licence, ECMWF does not waive the privileges and immunities 8 | # granted to it by virtue of its status as an intergovernmental organisation 9 | # nor does it submit to any jurisdiction. 10 | # 11 | 12 | import logging 13 | 14 | import pytest 15 | 16 | import climetlab as cml 17 | 18 | LOG = logging.getLogger(__name__) 19 | 20 | 21 | @pytest.mark.long_test 22 | @pytest.mark.external_download 23 | def test_load_dataset_meteonet_sample_masks(): 24 | cml.load_dataset("meteonet-samples-masks", domain="SE") 25 | -------------------------------------------------------------------------------- /tests/test_patterns.py: -------------------------------------------------------------------------------- 1 | # (C) Copyright 2020 ECMWF. 2 | # 3 | # This software is licensed under the terms of the Apache Licence Version 2.0 4 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 5 | # In applying this licence, ECMWF does not waive the privileges and immunities 6 | # granted to it by virtue of its status as an intergovernmental organisation 7 | # nor does it submit to any jurisdiction. 8 | # 9 | 10 | import pytest 11 | 12 | from climetlab.utils.patterns import Pattern 13 | 14 | 15 | def test_patterns(): 16 | p = Pattern("{date:date(%Y%m%d)}-{param}-{level:int}-{level:int(%03d)}") 17 | 18 | assert p.names == ["date", "level", "param"], p.names 19 | 20 | assert p.substitute(dict(date="2000-01-01", param="2t", level=12)) == "20000101-2t-12-012" 21 | 22 | p = Pattern("{variable:enum(2t,tp)}.{type:enum(rt,hc)}.{date:date(%Y%m%d)}.grib") 23 | assert p.substitute(dict(date="2000-01-01", variable="tp", type="rt")) == "tp.rt.20000101.grib" 24 | 25 | assert p.substitute(dict(date="2000-01-01", variable=["tp", "2t"], type="rt")) == [ 26 | "tp.rt.20000101.grib", 27 | "2t.rt.20000101.grib", 28 | ] 29 | 30 | 31 | def test_patterns_missing_key(): 32 | p = Pattern("{date}-{param}") 33 | with pytest.raises(ValueError, match=".*level.*"): 34 | p.substitute(dict(date="20000101", param="2t", level=12)) 35 | 36 | p = Pattern("{date}-{param}", ignore_missing_keys=False) 37 | with pytest.raises(ValueError, match=".*level.*"): 38 | p.substitute(dict(date="20000101", param="2t", level=12)) 39 | 40 | p = Pattern("{date}-{param}", ignore_missing_keys=True) 41 | assert p.substitute(dict(date="20000101", param="2t", level=12)) == "20000101-2t" 42 | 43 | 44 | if __name__ == "__main__": 45 | from climetlab.testing import main 46 | 47 | main(__file__) 48 | -------------------------------------------------------------------------------- /tests/test_plotting.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # (C) Copyright 2021 ECMWF. 4 | # 5 | # This software is licensed under the terms of the Apache Licence Version 2.0 6 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 7 | # In applying this licence, ECMWF does not waive the privileges and immunities 8 | # granted to it by virtue of its status as an intergovernmental organisation 9 | # nor does it submit to any jurisdiction. 10 | # 11 | 12 | import pytest 13 | 14 | import climetlab as cml 15 | from climetlab.testing import climetlab_file 16 | 17 | 18 | def test_plot_1(): 19 | s = cml.load_source("file", climetlab_file("docs/examples/test.grib")) 20 | assert cml.plot_map(s) is None 21 | 22 | assert cml.plot_map(s[0]) is None 23 | 24 | p = cml.new_plot() 25 | p.plot_map(s[0]) 26 | p.plot_map(s[1]) 27 | 28 | assert p.show() is None 29 | 30 | with pytest.raises(TypeError): 31 | cml.plot_map(s, unknown=42) 32 | 33 | 34 | if __name__ == "__main__": 35 | from climetlab.testing import main 36 | 37 | main(__file__) 38 | -------------------------------------------------------------------------------- /tests/test_thread.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # (C) Copyright 2020 ECMWF. 4 | # 5 | # This software is licensed under the terms of the Apache Licence Version 2.0 6 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 7 | # In applying this licence, ECMWF does not waive the privileges and immunities 8 | # granted to it by virtue of its status as an intergovernmental organisation 9 | # nor does it submit to any jurisdiction. 10 | # 11 | 12 | 13 | import time 14 | from datetime import datetime 15 | from datetime import timedelta 16 | 17 | from climetlab.core.thread import SoftThreadPool 18 | 19 | 20 | def test_thread(): 21 | pool = SoftThreadPool() 22 | 23 | def f(x, t): 24 | time.sleep(t) 25 | return x * x 26 | 27 | start = datetime.now() 28 | 29 | futures = [] 30 | futures.append(pool.submit(f, 0, 2)) 31 | futures.append(pool.submit(f, 1, 2)) 32 | futures.append(pool.submit(f, 2, 2)) 33 | futures.append(pool.submit(f, 3, 1)) 34 | 35 | duration = datetime.now() - start 36 | assert duration > timedelta(seconds=-0.000001) 37 | assert duration < timedelta(seconds=0.5) 38 | 39 | assert futures[3].result() == 9 40 | 41 | duration = datetime.now() - start 42 | assert duration > timedelta(seconds=0.5) 43 | assert duration < timedelta(seconds=1.5) 44 | 45 | assert futures[0].result() == 0 46 | 47 | duration = datetime.now() - start 48 | assert duration > timedelta(seconds=1.5) 49 | assert duration < timedelta(seconds=2.5) 50 | 51 | assert futures[0].result() == 0 52 | assert futures[1].result() == 1 53 | assert futures[2].result() == 4 54 | assert futures[3].result() == 9 55 | 56 | 57 | if __name__ == "__main__": 58 | from climetlab.testing import main 59 | 60 | main(__file__) 61 | -------------------------------------------------------------------------------- /tests/test_yaml.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # (C) Copyright 2020 ECMWF. 4 | # 5 | # This software is licensed under the terms of the Apache Licence Version 2.0 6 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 7 | # In applying this licence, ECMWF does not waive the privileges and immunities 8 | # granted to it by virtue of its status as an intergovernmental organisation 9 | # nor does it submit to any jurisdiction. 10 | # 11 | 12 | import os 13 | 14 | import yaml 15 | 16 | 17 | def test_yaml(): 18 | yfile = os.path.dirname(__file__) + "/example.yaml" 19 | with open(yfile) as f: 20 | yaml.load(f.read(), Loader=yaml.SafeLoader) 21 | 22 | 23 | if __name__ == "__main__": 24 | from climetlab.testing import main 25 | 26 | main(__file__) 27 | -------------------------------------------------------------------------------- /tools/imports.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # (C) Copyright 2023 ECMWF. 3 | # 4 | # This software is licensed under the terms of the Apache Licence Version 2.0 5 | # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 6 | # In applying this licence, ECMWF does not waive the privileges and immunities 7 | # granted to it by virtue of its status as an intergovernmental organisation 8 | # nor does it submit to any jurisdiction. 9 | # 10 | 11 | 12 | import inspect 13 | import sys 14 | 15 | show_stack = "--stack" in sys.argv 16 | show_first = "--first" in sys.argv 17 | show_fail_if_loaded = "--fail" in sys.argv 18 | 19 | 20 | def call_stack(): 21 | for f in inspect.stack(): 22 | if f.filename == __file__: 23 | continue 24 | 25 | if "importlib._bootstrap" in f.filename: 26 | continue 27 | 28 | yield (f.filename, f.lineno) 29 | 30 | 31 | class SpecFinder: 32 | def find_spec(self, name, path=None, target=None): 33 | if show_fail_if_loaded: 34 | if name in sys.argv[1:]: 35 | filename = "?" 36 | lineno = 0 37 | for filename, lineno in call_stack(): 38 | break 39 | print(f"{name} loaded from {filename}:{lineno}") 40 | sys.exit(1) 41 | return 42 | if show_first: 43 | for filename, lineno in call_stack(): 44 | print("=", name, f"{filename}:{lineno}") 45 | return 46 | 47 | print("=", name) 48 | if show_stack: 49 | for filename, lineno in call_stack(): 50 | print( 51 | f" {filename}:{lineno}", 52 | ) 53 | 54 | 55 | sys.meta_path.insert(0, SpecFinder()) 56 | 57 | 58 | import climetlab as cml # noqa 59 | 60 | # This should trigger a lot of wrappers 61 | cml.load_source("file", __file__) 62 | -------------------------------------------------------------------------------- /tools/test-data/.gitignore: -------------------------------------------------------------------------------- 1 | *.grib 2 | data 3 | *.tar 4 | *.tgz 5 | *.zip 6 | *.gz 7 | *.bin 8 | *.txt 9 | --------------------------------------------------------------------------------