├── xmitgcm ├── test │ ├── __init__.py │ ├── global_oce_cs32.tar.gz │ ├── test_file_utils.py │ ├── test_llcreader.py │ ├── test_xmitgcm_common.py │ └── test_mds_store.py ├── __init__.py ├── llcreader │ ├── duck_array_ops.py │ ├── __init__.py │ ├── llcutils.py │ ├── stores.py │ └── known_models.py └── file_utils.py ├── doc ├── aste_pad.png ├── pleiades_tests │ ├── compute_sum.png │ ├── load_data.png │ ├── extract_chunk.png │ ├── compute_sum_3D.png │ ├── extract_chunk_3D.png │ └── load_data_normalized.png ├── llc_cartoon │ └── llc_cartoon.001.png ├── environment.yml ├── utils.rst ├── index.rst ├── installation.rst ├── examples.rst ├── development.rst ├── Makefile ├── make.bat ├── conf.py ├── quick_start.rst ├── performance.rst ├── usage.rst └── demo_writing_binary_file.ipynb ├── .stickler.yml ├── setup.py ├── .bumpversion.cfg ├── readthedocs.yml ├── ci ├── environment-py3.11.yml ├── environment-py3.12.yml ├── environment-py3.13.yml └── environment-xarraymaster.yml ├── codecov.yml ├── .github └── workflows │ ├── pythonpublish.yaml │ └── ci.yaml ├── .gitignore ├── LICENSE ├── pyproject.toml └── README.rst /xmitgcm/test/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /doc/aste_pad.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MITgcm/xmitgcm/HEAD/doc/aste_pad.png -------------------------------------------------------------------------------- /.stickler.yml: -------------------------------------------------------------------------------- 1 | linters: 2 | flake8: 3 | fixer: true 4 | fixers: 5 | enable: true 6 | -------------------------------------------------------------------------------- /doc/pleiades_tests/compute_sum.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MITgcm/xmitgcm/HEAD/doc/pleiades_tests/compute_sum.png -------------------------------------------------------------------------------- /doc/pleiades_tests/load_data.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MITgcm/xmitgcm/HEAD/doc/pleiades_tests/load_data.png -------------------------------------------------------------------------------- /doc/llc_cartoon/llc_cartoon.001.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MITgcm/xmitgcm/HEAD/doc/llc_cartoon/llc_cartoon.001.png -------------------------------------------------------------------------------- /doc/pleiades_tests/extract_chunk.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MITgcm/xmitgcm/HEAD/doc/pleiades_tests/extract_chunk.png -------------------------------------------------------------------------------- /xmitgcm/test/global_oce_cs32.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MITgcm/xmitgcm/HEAD/xmitgcm/test/global_oce_cs32.tar.gz -------------------------------------------------------------------------------- /doc/pleiades_tests/compute_sum_3D.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MITgcm/xmitgcm/HEAD/doc/pleiades_tests/compute_sum_3D.png -------------------------------------------------------------------------------- /doc/pleiades_tests/extract_chunk_3D.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MITgcm/xmitgcm/HEAD/doc/pleiades_tests/extract_chunk_3D.png -------------------------------------------------------------------------------- /doc/pleiades_tests/load_data_normalized.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MITgcm/xmitgcm/HEAD/doc/pleiades_tests/load_data_normalized.png -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from setuptools import setup 3 | 4 | setup( 5 | use_scm_version={"fallback_version": "9999"}, 6 | ) 7 | -------------------------------------------------------------------------------- /.bumpversion.cfg: -------------------------------------------------------------------------------- 1 | [bumpversion] 2 | current_version = 0.2.2 3 | commit = True 4 | tag = True 5 | 6 | [bumpversion:file:setup.py] 7 | 8 | [bumpversion:file:xmitgcm/__init__.py] 9 | 10 | -------------------------------------------------------------------------------- /doc/environment.yml: -------------------------------------------------------------------------------- 1 | name: xmitgcm 2 | dependencies: 3 | - python=3.6 4 | - xarray 5 | - dask 6 | - numpydoc=0.7 7 | - sphinx=1.7.5 8 | - pip 9 | - pip: 10 | - zarr 11 | - fsspec 12 | - nbsphinx 13 | -------------------------------------------------------------------------------- /readthedocs.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | conda: 4 | environment: doc/environment.yml 5 | 6 | python: 7 | version: 2.7 8 | install: 9 | - method: setuptools 10 | path: . 11 | system_packages: true 12 | -------------------------------------------------------------------------------- /ci/environment-py3.11.yml: -------------------------------------------------------------------------------- 1 | name: test_env 2 | channels: 3 | - conda-forge 4 | dependencies: 5 | - python=3.11 6 | - xarray 7 | - dask 8 | - numpy 9 | - pytest 10 | - future 11 | - zarr 12 | - cftime 13 | - aiohttp 14 | - pytest-cov 15 | - codecov 16 | - fsspec 17 | - cachetools 18 | -------------------------------------------------------------------------------- /doc/utils.rst: -------------------------------------------------------------------------------- 1 | Low Level Utilities 2 | =================== 3 | 4 | These functions can be imported from ``xmitgcm.utils``. They are meant for 5 | internal use in xmitgcm, and users are recommended to use 6 | ``open_mdsdataset`` wherever possible (see :doc:`/usage`). 7 | 8 | .. automodule:: xmitgcm.llcreader 9 | :members: 10 | -------------------------------------------------------------------------------- /xmitgcm/__init__.py: -------------------------------------------------------------------------------- 1 | from .mds_store import open_mdsdataset 2 | 3 | from importlib.metadata import version as _version 4 | try: 5 | __version__ = _version("xmitgcm") 6 | except Exception: 7 | # Local copy or not installed with setuptools. 8 | # Disable minimum version checks on downstream libraries. 9 | __version__ = "9999" 10 | -------------------------------------------------------------------------------- /ci/environment-py3.12.yml: -------------------------------------------------------------------------------- 1 | name: test_env 2 | channels: 3 | - conda-forge 4 | dependencies: 5 | - python=3.12 6 | - xarray 7 | - dask 8 | - numpy 9 | - pytest 10 | - future 11 | - zarr >=3.1.3 12 | - cftime 13 | - aiohttp 14 | - pytest-cov 15 | - codecov 16 | - fsspec 17 | - cachetools 18 | - setuptools >=68 19 | - wheel 20 | - pip >=24 21 | -------------------------------------------------------------------------------- /ci/environment-py3.13.yml: -------------------------------------------------------------------------------- 1 | name: test_env 2 | channels: 3 | - conda-forge 4 | dependencies: 5 | - python=3.13 6 | - xarray 7 | - dask 8 | - numpy 9 | - pytest 10 | - future 11 | - zarr >=3.1.3 12 | - cftime 13 | - aiohttp 14 | - pytest-cov 15 | - codecov 16 | - fsspec 17 | - cachetools 18 | - setuptools >=68 19 | - wheel 20 | - pip >=24 21 | -------------------------------------------------------------------------------- /codecov.yml: -------------------------------------------------------------------------------- 1 | codecov: 2 | require_ci_to_pass: false 3 | 4 | coverage: 5 | precision: 2 6 | round: down 7 | status: 8 | project: 9 | default: 10 | target: 90% 11 | informational: true 12 | ignore: 13 | - "xmitgcm/test/*" 14 | - "xmitgcm/__init__.py" 15 | - "setup.py" 16 | - "xmitgcm/llcreader/*" 17 | 18 | comment: false 19 | -------------------------------------------------------------------------------- /ci/environment-xarraymaster.yml: -------------------------------------------------------------------------------- 1 | name: test_env 2 | channels: 3 | - conda-forge 4 | dependencies: 5 | - python=3.12 6 | - dask 7 | - numpy 8 | - pytest 9 | - future 10 | - zarr >=3.1.3 11 | - cftime 12 | - aiohttp 13 | - pytest-cov 14 | - codecov 15 | - fsspec 16 | - cachetools 17 | - git 18 | - setuptools >=68 19 | - wheel 20 | - pip >=24 21 | - pip: 22 | - git+https://github.com/pydata/xarray.git 23 | -------------------------------------------------------------------------------- /.github/workflows/pythonpublish.yaml: -------------------------------------------------------------------------------- 1 | name: Upload Python Package 2 | 3 | on: 4 | release: 5 | types: [created] 6 | 7 | jobs: 8 | deploy: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - uses: actions/checkout@v1 12 | - name: Set up Python 13 | uses: actions/setup-python@v1 14 | with: 15 | python-version: '3.x' 16 | - name: Install dependencies 17 | run: | 18 | python -m pip install --upgrade pip 19 | pip install setuptools setuptools-scm wheel twine 20 | - name: Build and publish 21 | env: 22 | TWINE_USERNAME: "__token__" 23 | TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} 24 | run: | 25 | python setup.py sdist bdist_wheel 26 | twine upload dist/* 27 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | 5 | # C extensions 6 | *.so 7 | 8 | # Distribution / packaging 9 | .Python 10 | env/ 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | *.egg-info/ 23 | .installed.cfg 24 | *.egg 25 | 26 | # PyInstaller 27 | # Usually these files are written by a python script from a template 28 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 29 | *.manifest 30 | *.spec 31 | 32 | # Installer logs 33 | pip-log.txt 34 | pip-delete-this-directory.txt 35 | 36 | # Unit test / coverage reports 37 | htmlcov/ 38 | .tox/ 39 | .coverage 40 | .coverage.* 41 | .cache 42 | nosetests.xml 43 | coverage.xml 44 | *,cover 45 | 46 | # Translations 47 | *.mo 48 | *.pot 49 | 50 | # Django stuff: 51 | *.log 52 | 53 | # Sphinx documentation 54 | docs/_build/ 55 | 56 | # PyBuilder 57 | target/ 58 | 59 | # notebook 60 | .ipynb_checkpoints/* 61 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016 Ryan Abernathey 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /xmitgcm/llcreader/duck_array_ops.py: -------------------------------------------------------------------------------- 1 | """Compatibility module defining operations on duck numpy-arrays. 2 | Shamelessly copied from xarray.""" 3 | 4 | from __future__ import division 5 | from __future__ import print_function 6 | 7 | import numpy as np 8 | 9 | try: 10 | import dask.array as dsa 11 | has_dask = True 12 | except ImportError: 13 | has_dask = False 14 | 15 | 16 | def _dask_or_eager_func(name, eager_module=np, list_of_args=False, n_array_args=1): 17 | """Create a function that dispatches to dask for dask array inputs.""" 18 | if has_dask: 19 | 20 | def f(*args, **kwargs): 21 | dispatch_args = args[0] if list_of_args else args 22 | if any(isinstance(a, dsa.Array) for a in dispatch_args[:n_array_args]): 23 | module = dsa 24 | else: 25 | module = eager_module 26 | return getattr(module, name)(*args, **kwargs) 27 | 28 | else: 29 | 30 | def f(data, *args, **kwargs): 31 | return getattr(eager_module, name)(data, *args, **kwargs) 32 | 33 | return f 34 | 35 | 36 | concatenate = _dask_or_eager_func("concatenate", list_of_args=True) 37 | -------------------------------------------------------------------------------- /doc/index.rst: -------------------------------------------------------------------------------- 1 | xmitgcm 2 | ======= 3 | 4 | xmitgcm is a python package for reading MITgcm_ binary MDS files into 5 | xarray_ data structures. By storing data in dask_ arrays, xmitgcm enables 6 | parallel, out-of-core_ analysis of MITgcm output data. 7 | 8 | .. toctree:: 9 | :maxdepth: 2 10 | 11 | installation 12 | quick_start 13 | usage 14 | examples 15 | demo_read_input_grid 16 | demo_writing_binary_file 17 | performance 18 | llcreader 19 | utils 20 | development 21 | 22 | 23 | .. _dask: http://dask.pydata.org 24 | .. _xarray: http://xarray.pydata.org 25 | .. _Comodo: http://pycomodo.forge.imag.fr/norm.html 26 | .. _issues: https://github.com/xgcm/xmitgcm/issues 27 | .. _`pull requests`: https://github.com/xgcm/xmitgcm/pulls 28 | .. _MITgcm: http://mitgcm.org/public/r2_manual/latest/online_documents/node277.html 29 | .. _out-of-core: https://en.wikipedia.org/wiki/Out-of-core_algorithm 30 | .. _Anaconda: https://www.continuum.io/downloads 31 | .. _`CF conventions`: http://cfconventions.org/Data/cf-conventions/cf-conventions-1.7/build/ch04s04.html 32 | .. _gcmfaces: http://mitgcm.org/viewvc/*checkout*/MITgcm/MITgcm_contrib/gael/matlab_class/gcmfaces.pdf 33 | -------------------------------------------------------------------------------- /xmitgcm/llcreader/__init__.py: -------------------------------------------------------------------------------- 1 | from packaging import version 2 | 3 | try: 4 | import fsspec 5 | assert version.parse(fsspec.__version__) >= version.parse("0.4.4") 6 | except (ImportError, AssertionError): # pramga: no cover 7 | raise ImportError('The llcreader module requires fsspec version 0.4.4 or ' 8 | 'or greater to be installed. ' 9 | 'To install it, run `pip install fsspec`. See ' 10 | 'https://filesystem-spec.readthedocs.io/en/latest/ ' 11 | 'for more info.') 12 | 13 | try: 14 | import zarr 15 | assert version.parse(zarr.__version__) >= version.parse("2.3.1") 16 | except (ImportError, AssertionError): # pramga: no cover 17 | raise ImportError('The llcreader module requires zarr version 2.3.1 or ' 18 | 'or greater to be installed. ' 19 | 'To install it, run `pip install zarr`. See ' 20 | 'https://zarr.readthedocs.io/en/stable/index.html#installation ' 21 | 'for more info.') 22 | 23 | 24 | from .known_models import * 25 | from .stores import * 26 | from .llcmodel import BaseLLCModel, faces_dataset_to_latlon 27 | -------------------------------------------------------------------------------- /xmitgcm/file_utils.py: -------------------------------------------------------------------------------- 1 | import cachetools.func 2 | import os 3 | import fnmatch 4 | 5 | cache_maxsize = 100 6 | cache_ttl = 600 # tem minutes 7 | 8 | @cachetools.func.ttl_cache(maxsize=cache_maxsize, ttl=cache_ttl) 9 | def listdir(path): 10 | return os.listdir(path) 11 | 12 | @cachetools.func.ttl_cache(maxsize=cache_maxsize, ttl=cache_ttl) 13 | def listdir_startswith(path, pattern): 14 | files = listdir(path) 15 | return [f for f in files if f.startswith(pattern)] 16 | 17 | @cachetools.func.ttl_cache(maxsize=cache_maxsize, ttl=cache_ttl) 18 | def listdir_endswith(path, pattern): 19 | files = listdir(path) 20 | return [f for f in files if f.endswith(pattern)] 21 | 22 | @cachetools.func.ttl_cache(maxsize=cache_maxsize, ttl=cache_ttl) 23 | def listdir_startsandendswith(path, start, end): 24 | files = listdir(path) 25 | return [f for f in files if f.endswith(end) and f.startswith(start)] 26 | 27 | @cachetools.func.ttl_cache(maxsize=cache_maxsize, ttl=cache_ttl) 28 | def listdir_fnmatch(path, pattern): 29 | files = listdir(path) 30 | return [f for f in files if fnmatch.fnmatch(f, pattern)] 31 | 32 | def clear_cache(): 33 | listdir.cache_clear() 34 | listdir_startswith.cache_clear() 35 | listdir_endswith.cache_clear() 36 | listdir_startsandendswith.cache_clear() 37 | listdir_fnmatch.cache_clear() 38 | -------------------------------------------------------------------------------- /xmitgcm/test/test_file_utils.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from xmitgcm import file_utils 4 | 5 | @pytest.fixture(scope="session") 6 | def directory_with_files(tmpdir_factory): 7 | temppath = tmpdir_factory.mktemp("xmitgcm_test_data") 8 | temppath.join('bar.0000000001.meta').ensure(file=True) 9 | temppath.join('baz.data').ensure(file=True) 10 | return temppath 11 | 12 | def test_listdir(directory_with_files): 13 | path = str(directory_with_files) 14 | assert sorted(file_utils.listdir(path)) == sorted(['bar.0000000001.meta', 'baz.data']) 15 | 16 | def test_listdir_startswith(directory_with_files): 17 | path = str(directory_with_files) 18 | assert file_utils.listdir_startswith(path, 'bar') == ['bar.0000000001.meta'] 19 | 20 | def test_listdir_endswith(directory_with_files): 21 | path = str(directory_with_files) 22 | assert file_utils.listdir_endswith(path, '.data') == ['baz.data'] 23 | 24 | def test_listdir_startsandendswith(directory_with_files): 25 | path = str(directory_with_files) 26 | assert file_utils.listdir_startsandendswith(path, 'bar', '.meta') == ['bar.0000000001.meta'] 27 | 28 | def test_listdir_fnmatch(directory_with_files): 29 | path = str(directory_with_files) 30 | assert file_utils.listdir_fnmatch(path, '*.??????????.meta') == ['bar.0000000001.meta'] 31 | 32 | def test_clear_cache(): 33 | file_utils.clear_cache() 34 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | # TODO fully switch to PEP 621: get rid of versioneer 2 | [project] 3 | authors = [{ name = "xmitgcm Developers", email = "rpa@ldeo.colombia.edu" }] 4 | name = "xmitgcm" 5 | requires-python = ">=3.11" 6 | description = "Read MITgcm MDS binary files using xarray" 7 | license = { text = "Apache-2.0" } 8 | dependencies = [ 9 | "xarray >= 0.14.1", 10 | "dask >= 1.0", 11 | "cachetools" 12 | ] 13 | classifiers = [ 14 | "Development Status :: 4 - Beta", 15 | "License :: OSI Approved :: Apache Software License", 16 | "Operating System :: OS Independent", 17 | "Intended Audience :: Science/Research", 18 | "Programming Language :: Python", 19 | "Programming Language :: Python :: 3", 20 | "Programming Language :: Python :: 3.11", 21 | "Programming Language :: Python :: 3.12", 22 | "Programming Language :: Python :: 3.13", 23 | ] 24 | dynamic = ['version'] 25 | 26 | #[project.optional-dependencies] 27 | #... 28 | 29 | [project.urls] 30 | Homepage = "https://github.com/MITgcm/xmitgcm" 31 | Documentation = "https://xmitgcm.readthedocs.io/" 32 | 33 | [build-system] 34 | build-backend = "setuptools.build_meta" 35 | requires = ["setuptools>=42", "setuptools-scm", "wheel"] 36 | 37 | [tool.setuptools] 38 | packages = ["xmitgcm"] 39 | 40 | [tool.pytest.ini_options] 41 | minversion= "6.0" 42 | testpaths = [ 43 | "xmitgcm/test", 44 | ] 45 | markers = [ 46 | "slow: slow tests" 47 | ] 48 | -------------------------------------------------------------------------------- /xmitgcm/llcreader/llcutils.py: -------------------------------------------------------------------------------- 1 | """ 2 | Weird one-off functions that don't have a good home. 3 | """ 4 | import xarray 5 | import numpy as np 6 | 7 | def load_masks_from_mds(grid_dir): 8 | import xmitgcm 9 | ds = xmitgcm.open_mdsdataset(grid_dir, iters=None, geometry='llc') 10 | 11 | points = ['C', 'W', 'S'] 12 | masks = [ds['hfac' + point].reset_coords(drop=True).rename('mask' + point) 13 | for point in points] 14 | ds_mask = xr.merge(masks) 15 | for c in ds_mask.coords: 16 | ds_mask[c] = ds_mask[c].astype('i2') 17 | return ds_mask.chunk({'k': 1, 'face': -1}) 18 | 19 | 20 | def write_masks_to_zarr(ds_mask, output_path): 21 | import lzma 22 | lzma_filters = [dict(id=lzma.FILTER_DELTA, dist=1), 23 | dict(id=lzma.FILTER_LZMA2, preset=1)] 24 | from numcodecs import LZMA 25 | compressor = LZMA(filters=lzma_filters, format=lzma.FORMAT_RAW) 26 | encoding = {vname: {'compressor': compressor} for vname in ds_mask.data_vars} 27 | return ds_mask.to_zarr(output_path, encoding=encoding) 28 | 29 | 30 | def copy_masks_from_mds_to_zarr(grid_dir, output_path): 31 | ds_mask = load_masks_from_mds(grid_dir) 32 | write_masks_to_zarr(ds_mask, output_path) 33 | 34 | 35 | _facet_strides = ((0,3), (3,6), (6,7), (7,10), (10,13)) 36 | def face_mask_to_facet_index_list(mask): 37 | nk, nf, nx, ny = mask.shape 38 | assert nf == 13 39 | index = np.asarray(mask.sum(axis=(2, 3))) 40 | index_facets = np.array([list(index[:, slice(*stride)].sum(axis=1)) 41 | for stride in _facet_strides]).transpose() 42 | return [0] + list(index_facets.ravel().cumsum()) 43 | -------------------------------------------------------------------------------- /doc/installation.rst: -------------------------------------------------------------------------------- 1 | 2 | Installation 3 | ------------ 4 | 5 | Installation via conda 6 | ^^^^^^^^^^^^^^^^^^^^^^ 7 | 8 | If you just want to use xmitgcm, anaconda users can install with:: 9 | 10 | conda install -c conda-forge xmitgcm 11 | 12 | This will install the latest conda-forge build. 13 | 14 | Installation via pip 15 | ^^^^^^^^^^^^^^^^^^^^ 16 | 17 | Alternatively, xmitgcm can be installed via pip:: 18 | 19 | pip install xmitgcm 20 | 21 | This will automatically install the latest release from 22 | `pypi `_. 23 | 24 | Installation from github 25 | ^^^^^^^^^^^^^^^^^^^^^^^^ 26 | 27 | xmitgcm is under active development. To obtain the latest development version, 28 | you may clone the `source repository `_ 29 | and install it:: 30 | 31 | git clone https://github.com/MITgcm/xmitgcm.git 32 | cd xmitgcm 33 | python setup.py install 34 | 35 | Users are encouraged to `fork `_ 36 | xmitgcm and submit issues_ _ and `pull requests`_. 37 | 38 | .. _dask: http://dask.pydata.org 39 | .. _xarray: http://xarray.pydata.org 40 | .. _Comodo: http://pycomodo.forge.imag.fr/norm.html 41 | .. _issues: https://github.com/MITgcm/xmitgcm/issues 42 | .. _`pull requests`: https://github.com/MITgcm/xmitgcm/pulls 43 | .. _MITgcm: http://mitgcm.org/public/r2_manual/latest/online_documents/node277.html 44 | .. _out-of-core: https://en.wikipedia.org/wiki/Out-of-core_algorithm 45 | .. _Anaconda: https://www.continuum.io/downloads 46 | .. _`CF conventions`: http://cfconventions.org/Data/cf-conventions/cf-conventions-1.7/build/ch04s04.html 47 | .. _gcmfaces: http://mitgcm.org/viewvc/*checkout*/MITgcm/MITgcm_contrib/gael/matlab_class/gcmfaces.pdf 48 | .. _figshare: https://figshare.com/account/home#/collections/4362224 49 | -------------------------------------------------------------------------------- /doc/examples.rst: -------------------------------------------------------------------------------- 1 | Example Usage 2 | ============= 3 | 4 | Once you have loaded your data, you can analyze it using all the capabilities 5 | available in xarray_. Here are a few quick examples. 6 | 7 | Land Masks 8 | ---------- 9 | 10 | xmitgcm simply reads the MDS data directly for the disk; it does not attempt to 11 | mask land points, which are usually filled with zero values. To mask land, you 12 | can use xarray's ``where`` function together the the ``hFac`` variables related 13 | to MITgcm's 14 | `partially filled cells `_. 15 | For example, with the ``global_oce_latlon`` dataset, an unmasked average of 16 | salinity gives:: 17 | 18 | ds.S.mean() 19 | >>> 20 | array(18.85319709777832, dtype=float32) 21 | 22 | This value is unrealistically low because it includes all of the zeros inside 23 | the land which shold be masked. To take the masked average, instead do:: 24 | 25 | ds.S.where(ds.hFacC>0).mean() 26 | >>> 27 | array(34.73611831665039) 28 | 29 | This is a more correct value. 30 | 31 | Volume Weighting 32 | ---------------- 33 | 34 | However, it is still not properly volume weighted. 35 | To take a volume-weighted average, you can do:: 36 | 37 | volume = ds.hFacC * ds.drF * ds.rA 38 | (ds.S * volume).sum() / volume.sum() 39 | >>> 40 | array(34.779126627139945) 41 | 42 | This represents the correct mean ocean salinity. 43 | A different land mask and volume weighting is required for variables located at 44 | the u and v points. 45 | 46 | netCDF conversion 47 | ----------------- 48 | 49 | Thanks to xarray_, it is trivial to convert our dataset to netCDF:: 50 | 51 | ds.to_netcdf('myfile.nc') 52 | 53 | It can then be reloaded directly with xarray:: 54 | 55 | import xarray as xr 56 | ds = xr.open_dataset('myfile.nc') 57 | 58 | This is an attractive option for archiving MDS data in a self-contained way. 59 | 60 | .. _dask: http://dask.pydata.org 61 | .. _xarray: http://xarray.pydata.org 62 | .. _Comodo: http://pycomodo.forge.imag.fr/norm.html 63 | .. _issues: https://github.com/xgcm/xmitgcm/issues 64 | .. _`pull requests`: https://github.com/xgcm/xmitgcm/pulls 65 | .. _MITgcm: http://mitgcm.org/public/r2_manual/latest/online_documents/node277.html 66 | .. _out-of-core: https://en.wikipedia.org/wiki/Out-of-core_algorithm 67 | .. _Anaconda: https://www.continuum.io/downloads 68 | .. _`CF conventions`: http://cfconventions.org/Data/cf-conventions/cf-conventions-1.7/build/ch04s04.html 69 | .. _gcmfaces: http://mitgcm.org/viewvc/*checkout*/MITgcm/MITgcm_contrib/gael/matlab_class/gcmfaces.pdf 70 | -------------------------------------------------------------------------------- /.github/workflows/ci.yaml: -------------------------------------------------------------------------------- 1 | # This uses actios: 2 | # checkout: https://github.com/actions/checkout 3 | # cache: https://github.com/actions/cache 4 | # codecov-action: https://github.com/codecov/codecov-action 5 | 6 | name: CI 7 | on: 8 | push: 9 | branches: 10 | - "*" 11 | pull_request: 12 | branches: 13 | - "*" 14 | # Nothing scheduled, just PRs and pushes 15 | # schedule: 16 | # - cron: "0 0 * * *" # Daily at midnight 17 | 18 | jobs: 19 | build: 20 | name: Build (${{ matrix.python-version }} | ${{ matrix.os }}) 21 | runs-on: ${{ matrix.os }} 22 | strategy: 23 | fail-fast: false 24 | matrix: 25 | os: ["ubuntu-latest"] 26 | python-version: ["3.11", "3.12", "3.13"] 27 | steps: 28 | - uses: actions/checkout@v4 29 | - name: Cache conda 30 | uses: actions/cache@v4 31 | env: 32 | # Increase this value to reset cache if ci/environment.yml has not changed 33 | CACHE_NUMBER: 0 34 | with: 35 | path: ~/conda_pkgs_dir 36 | key: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ hashFiles('ci/environment.yml') }} 37 | - uses: conda-incubator/setup-miniconda@v3 38 | with: 39 | miniconda-version: "latest" 40 | activate-environment: test_env # Defined in ci/environment*.yml 41 | auto-update-conda: false 42 | python-version: ${{ matrix.python-version }} 43 | environment-file: ci/environment-py${{ matrix.python-version }}.yml 44 | use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly! 45 | - name: Set up conda environment 46 | shell: bash -l {0} 47 | run: | 48 | python -m pip install -e . 49 | conda list 50 | - name: Run Tests 51 | shell: bash -l {0} 52 | run: | 53 | py.test --cov=./ --cov-report=xml 54 | - name: Upload code coverage to Codecov 55 | uses: codecov/codecov-action@v4 56 | with: 57 | token: ${{ secrets.CODECOV_TOKEN }} 58 | file: ./coverage.xml 59 | flags: unittests 60 | name: codecov-umbrella 61 | fail_ci_if_error: false 62 | 63 | xarray-master: 64 | name: Build xarray-master 65 | runs-on: ubuntu-latest 66 | steps: 67 | - uses: actions/checkout@v4 68 | - name: Cache conda 69 | uses: actions/cache@v4 70 | env: 71 | # Increase this value to reset cache if ci/environment-upstream-dev.yml has not changed 72 | CACHE_NUMBER: 0 73 | with: 74 | path: ~/conda_pkgs_dir 75 | key: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ hashFiles('ci/environment-upstream-dev.yml') }} 76 | - uses: conda-incubator/setup-miniconda@v3 77 | with: 78 | miniconda-version: "latest" 79 | activate-environment: test_env # Defined in ci/environment-*.yml 80 | auto-update-conda: false 81 | environment-file: ci/environment-xarraymaster.yml 82 | use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly! 83 | - name: Set up conda environment 84 | shell: bash -l {0} 85 | run: | 86 | python -m pip install -e . 87 | conda list 88 | - name: Run Tests 89 | shell: bash -l {0} 90 | run: | 91 | py.test --cov=./ --cov-report=xml 92 | - name: Upload code coverage to Codecov 93 | uses: codecov/codecov-action@v4 94 | with: 95 | token: ${{ secrets.CODECOV_TOKEN }} 96 | file: ./coverage.xml 97 | flags: unittests 98 | name: codecov-umbrella 99 | fail_ci_if_error: false 100 | -------------------------------------------------------------------------------- /xmitgcm/llcreader/stores.py: -------------------------------------------------------------------------------- 1 | import fsspec 2 | import os 3 | import zarr 4 | 5 | 6 | class BaseStore: 7 | """Basic storage class for LLC data. 8 | 9 | Parameters 10 | ---------- 11 | fs : fsspec.AbstractFileSystem 12 | base_path : str, optional 13 | Where to find the data within the filesystem 14 | shrunk : bool, optional 15 | Whether the data files have been tagged with `.shrunk` 16 | mask_fs, grid_fs : fsspec.AbstractFileSystem, optional 17 | Where to find the mask or grid datasets to decode the compression 18 | mask_path, grid_path : str, optional 19 | Path to the mask or grid datasets on the ``mask_fs`` or ``grid_fs`` filesystem 20 | shrunk_grid : bool, optional 21 | Whether the grid files have been tagged with `.shrunk` 22 | not always the same as for data variables 23 | join_char : str or None 24 | Character to use to join paths. Falls back on os.path.join if None. 25 | """ 26 | 27 | def __init__(self, fs, base_path='/', shrunk=False, 28 | mask_fs=None, mask_path=None, 29 | grid_fs=None, grid_path=None, 30 | shrunk_grid=False, join_char=None): 31 | self.base_path = base_path 32 | self.fs = fs 33 | self.shrunk = shrunk 34 | self.mask_fs = mask_fs or self.fs 35 | self.mask_path = mask_path 36 | self.grid_fs = grid_fs or self.fs 37 | self.grid_path = grid_path 38 | self.shrunk_grid = shrunk_grid 39 | self.join_char = join_char 40 | if shrunk and (mask_path is None): 41 | raise ValueError("`mask_path` can't be None if `shrunk` is True") 42 | 43 | 44 | def _directory(self, varname, iternum): 45 | if iternum is not None: 46 | return self.base_path 47 | else: 48 | return self.grid_path 49 | 50 | def _fname(self, varname, iternum): 51 | 52 | if iternum is not None: 53 | fname = varname + '.%010d.data' % iternum 54 | if self.shrunk: 55 | fname += '.shrunk' 56 | else: 57 | fname = varname + '.data' 58 | if self.shrunk_grid: 59 | fname = varname + '.shrunk' 60 | 61 | return fname 62 | 63 | def _join(self, *args): 64 | if self.join_char: 65 | return self.join_char.join(args) 66 | else: 67 | return os.path.join(*args) 68 | 69 | def _full_path(self, varname, iternum): 70 | return self._join(self._directory(varname, iternum), 71 | self._fname(varname, iternum)) 72 | 73 | def get_fs_and_full_path(self, varname, iternum): 74 | """Return references to a filesystem and path within it for a specific 75 | variable and iteration number. 76 | 77 | Parameters 78 | ---------- 79 | varname : str 80 | iternum : int 81 | 82 | Returns 83 | ------- 84 | fs : fsspec.AbstractFileSystem 85 | The filesytem where the file can be found 86 | path : str 87 | The path to open 88 | """ 89 | return self.fs, self._full_path(varname, iternum) 90 | 91 | def open_data_file(self, varname, iternum): 92 | """Open the file for a specific variable and iteration number. 93 | 94 | Parameters 95 | ---------- 96 | varname : str 97 | iternum : int 98 | 99 | Returns 100 | ------- 101 | fobj : file-like object 102 | """ 103 | fs, path = self.get_fs_and_full_path(varname, iternum) 104 | return fs.open(path) 105 | 106 | def open_mask_group(self): 107 | """Open the zarr group that contains the masks 108 | 109 | Returns 110 | ------- 111 | mask_group : zarr.Group 112 | """ 113 | 114 | mapper = self.mask_fs.get_mapper(self.mask_path) 115 | zgroup = zarr.open_consolidated(mapper) 116 | return zgroup 117 | 118 | 119 | class NestedStore(BaseStore): 120 | """Store where the variable are stored in subdirectories according to 121 | iteration number.""" 122 | 123 | def _directory(self, varname, iternum): 124 | if iternum is not None: 125 | return self._join(self.base_path, '%010d' % iternum) 126 | else: 127 | return self.grid_path 128 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | xmitgcm: Read MITgcm mds binary files into xarray 2 | ================================================= 3 | 4 | |pypi| |Build Status| |codecov| |docs| |DOI| 5 | 6 | xmitgcm is a python package for reading MITgcm_ binary MDS files into 7 | xarray_ data structures. By storing data in dask_ arrays, xmitgcm enables 8 | parallel, out-of-core_ analysis of MITgcm output data. 9 | 10 | Links 11 | ----- 12 | 13 | - HTML documentation: https://xmitgcm.readthedocs.org 14 | - Issue tracker: https://github.com/MITgcm/xmitgcm/issues 15 | - Source code: https://github.com/MITgcm/xmitgcm 16 | 17 | Installation 18 | ------------ 19 | 20 | Requirements 21 | ^^^^^^^^^^^^ 22 | 23 | xmitgcm is compatible with python >=3.7. It requires xarray_ 24 | (>= version 0.14.1) and dask_ (>= version 1.0). 25 | These packages are most reliably installed via the 26 | `conda `_ environment management 27 | system, which is part of the Anaconda_ python distribution. Assuming you have 28 | conda available on your system, the dependencies can be installed with the 29 | command:: 30 | 31 | conda install xarray dask 32 | 33 | If you are using earlier versions of these packages, you should update before 34 | installing xmitgcm. 35 | 36 | Installation via pip 37 | ^^^^^^^^^^^^^^^^^^^^ 38 | 39 | If you just want to use xmitgcm, the easiest way is to install via pip:: 40 | 41 | pip install xmitgcm 42 | 43 | This will automatically install the latest release from 44 | `pypi `_. 45 | 46 | Installation from github 47 | ^^^^^^^^^^^^^^^^^^^^^^^^ 48 | 49 | xmitgcm is under active development. To obtain the latest development version, 50 | you may clone the `source repository `_ 51 | and install it:: 52 | 53 | git clone https://github.com/MITgcm/xmitgcm.git 54 | cd xmitgcm 55 | python setup.py install 56 | 57 | Users are encouraged to `fork `_ 58 | xmitgcm and submit issues_ and `pull requests`_. 59 | 60 | Quick Start 61 | ----------- 62 | 63 | First make sure you understand what an xarray_ Dataset object is. Then find 64 | some MITgcm MDS data. If you don't have any data of your own, you can download 65 | the xmitgcm 66 | `test repositories `_ 67 | To download the some test data, run the shell commands:: 68 | 69 | $ curl -L -J -O https://ndownloader.figshare.com/files/6494718 70 | $ tar -xvzf global_oce_latlon.tar.gz 71 | 72 | This will create a directory called ``global_oce_latlon`` which we will use 73 | for the rest of these examples. If you have your own data, replace this with 74 | the path to your mitgcm files. 75 | 76 | To open MITgcm MDS data as an xarray.Dataset, do the following in python:: 77 | 78 | from xmitgcm import open_mdsdataset 79 | data_dir = './global_oce_latlon' 80 | ds = open_mdsdataset(data_dir) 81 | 82 | ``data_dir``, should be the path (absolute or relative) to an 83 | MITgcm run directory. xmitgcm will automatically scan this directory and 84 | try to determine the file prefixes and iteration numbers to read. In some 85 | configurations, the ``open_mdsdataset`` function may work without further 86 | keyword arguments. In most cases, you will have to specify further details. 87 | 88 | Consult the `online documentation `_ for 89 | more details. 90 | 91 | .. |DOI| image:: https://zenodo.org/badge/70649781.svg 92 | :target: https://zenodo.org/badge/latestdoi/70649781 93 | .. |Build Status| image:: https://travis-ci.org/MITgcm/xmitgcm.svg?branch=master 94 | :target: https://travis-ci.org/MITgcm/xmitgcm 95 | :alt: travis-ci build status 96 | .. |codecov| image:: https://codecov.io/github/MITgcm/xmitgcm/coverage.svg?branch=master 97 | :target: https://codecov.io/github/MITgcm/xmitgcm?branch=master 98 | :alt: code coverage 99 | .. |pypi| image:: https://badge.fury.io/py/xmitgcm.svg 100 | :target: https://badge.fury.io/py/xmitgcm 101 | :alt: pypi package 102 | .. |docs| image:: https://readthedocs.org/projects/xmitgcm/badge/?version=stable 103 | :target: https://xmitgcm.readthedocs.org/en/stable/?badge=stable 104 | :alt: documentation status 105 | 106 | .. _dask: https://dask.pydata.org 107 | .. _xarray: https://xarray.pydata.org 108 | .. _Comodo: https://pycomodo.forge.imag.fr/norm.html 109 | .. _issues: https://github.com/MITgcm/xmitgcm/issues 110 | .. _`pull requests`: https://github.com/MITgcm/xmitgcm/pulls 111 | .. _MITgcm: http://mitgcm.org/public/r2_manual/latest/online_documents/node277.html 112 | .. _out-of-core: https://en.wikipedia.org/wiki/Out-of-core_algorithm 113 | .. _Anaconda: https://www.continuum.io/downloads 114 | -------------------------------------------------------------------------------- /doc/development.rst: -------------------------------------------------------------------------------- 1 | Development 2 | =========== 3 | 4 | Release History 5 | --------------- 6 | 7 | 8 | v0.5.0 (2020-24-11) 9 | ~~~~~~~~~~~~~~~~~~~ 10 | 11 | - Add mask override option to llcreader from Ryan Abernathey (:issue:`191`) 12 | - Python 3.8 compatibility from Takaya Uchida (:issue:`194`) 13 | - llcreader: get grid info from ECCO portal from Tim Smith (:issue:`158`, :issue:`161`, :issue:`166`) 14 | - Fix python 3.6 master build from Tim Smith (:issue:`200`) 15 | - ECCO portal new iter_stop from Antonio Quintana (:issue:`193`) 16 | - Added missing grid variables to llcreader known_models from Ryan Abernathey (:issue:`207`) 17 | - Migrated to GitHub Actions from Tim Smith (:issue:`223`) 18 | - Dropped python 2 from Tim Smith (:issue:`226`) 19 | - llcreader klevels bugfix from Ryan Abernathey (:issue:`224`) 20 | - Incorporated llcreader for ASTE release 1 from Tim Smith (:issue:`231`) 21 | - Fixed typo for 'coordinate' entry in dimensions dictionary from Ian Fenty (:issue:`236`) 22 | - Lazy open_mdsdataset from Pascal Bourgault (:issue:`229`) 23 | - Implemented checking for variable mates from Fraser Goldsworth (:issue:`234`) 24 | - Added metadata to llcreader dimensions from Ryan Abernathey (:issue:`239`) 25 | - Cehck iter_start and iter_stop from Fraser Goldsworth (:issue:`235`) 26 | - Automated release to pypi from from Ryan Abernathey (:issue:`241`) 27 | 28 | v0.4.1 (2019-07-11) 29 | ~~~~~~~~~~~~~~~~~~~ 30 | 31 | - Incorporated llcreader bugfix from Spencer Jones (:issue:`154`) 32 | 33 | v0.4.0 (2019-07-11) 34 | ~~~~~~~~~~~~~~~~~~~ 35 | 36 | - New :doc:`llcreader` module (see 37 | `blog post `_ 38 | for more details.) 39 | 40 | 41 | v0.3.0 (2019-05-19) 42 | ~~~~~~~~~~~~~~~~~~~~ 43 | - Ability to read ASTE grids 44 | - Ability to read seaice and thsice native output 45 | - Reading of optional grid files 46 | - Moved test data to figshare 47 | - Writing of binary files 48 | - Xarray 0.12 compatibility 49 | - Ability to read 2D slice diagnostics of 3D fields 50 | 51 | 52 | v.0.2.2 (2018-07-18) 53 | ~~~~~~~~~~~~~~~~~~~~ 54 | - Extend capabilities of read_raw_data (:issue:`84`) 55 | - Fix the problem with testing type of prefix (:issue:`83`) 56 | - Cast prefix to list if it isn't already one (:issue:`79`) 57 | - Generalizes _get_all_iternums in order to handle compressed data (:issue:`77`) 58 | - Extract version number from git tag (:issue:`72`) 59 | - Adding .stickler.yml (:issue:`70`) 60 | - Added functionality to read PTRtave files (:issue:`63`) 61 | - Update examples.rst (:issue:`65`) 62 | - fix time encoding (:issue:`61`) 63 | - Fix llc chunking (:issue:`60`) 64 | - Test refactor (:issue:`54`) 65 | - Kpp added properly (:issue:`55`) 66 | - Tests for ref_date issue (:issue:`53`) 67 | - Add python 3.6 testing (:issue:`52`) 68 | - Added layers axis attribute (:issue:`47`) 69 | 70 | v.0.2.1 (2017-05-31) 71 | ~~~~~~~~~~~~~~~~~~~~ 72 | - Fix to ensure that grid indices are always interger dtype. 73 | - Fix to keep proper Comodo metadata when swapping dimensions. 74 | 75 | v0.2.0 (2017-02-14) 76 | ~~~~~~~~~~~~~~~~~~~ 77 | 78 | This release contains the following feature enhancements: 79 | - Files are not read until the data are accessed. This helps overcome a common 80 | "too many open files issue" (:issue:`11`). 81 | - A workaround for missing ``.meta`` files (:issue:`12`). 82 | - Option for a separate ``grid_dir`` in case it is different from ``data_dir`` 83 | (:issue:`13`). 84 | - Refactor of the way LLC data is read which allows for more efficient chunking 85 | and lower memory usage (:issue:`20`) 86 | - Bug fix related to the handling of `default_dtype` parameter (:issue:`34`). 87 | By `Guillaume Sérazin `_. 88 | - Support for older MITgcm versions that write a different lenght ``drC`` 89 | variable (:issue:`8`). By `Liam Brannigan `_. 90 | - Support for cartesian curvilinear grids. By 91 | `Andrea Cimatoribus `_. 92 | - Expanded and improved documentation. 93 | 94 | Unless otherwise noted, all updates are by 95 | `Ryan Abernathey `_. 96 | 97 | v0.1.0 (2016-10-15) 98 | ~~~~~~~~~~~~~~~~~~~ 99 | 100 | Initial release. 101 | 102 | Develpment Workflow 103 | ------------------- 104 | 105 | Anyone interested in helping to develop xmitgcm needs to create their own fork 106 | of our `git repository`. (Follow the github `forking instructions`_. You 107 | will need a github account.) 108 | 109 | .. _git repository: https://github.com/MITgcm/xmitgcm 110 | .. _forking instructions: https://help.github.com/articles/fork-a-repo/ 111 | 112 | Clone your fork on your local machine. 113 | 114 | .. code-block:: bash 115 | 116 | $ git clone git@github.com:USERNAME/xmitgcm 117 | 118 | (In the above, replace USERNAME with your github user name.) 119 | 120 | Then set your fork to track the upstream xmitgcm repo. 121 | 122 | .. code-block:: bash 123 | 124 | $ cd xmitgcm 125 | $ git remote add upstream git://github.com/MITgcm/xmitgcm.git 126 | 127 | You will want to periodically sync your master branch with the upstream master. 128 | 129 | .. code-block:: bash 130 | 131 | $ git fetch upstream 132 | $ git rebase upstream/master 133 | 134 | Never make any commits on your local master branch. Instead open a feature 135 | branch for every new development task. 136 | 137 | .. code-block:: bash 138 | 139 | $ git checkout -b cool_new_feature 140 | 141 | (Replace `cool_new_feature` with an appropriate description of your feature.) 142 | At this point you work on your new feature, using `git add` to add your 143 | changes. When your feature is complete and well tested, commit your changes 144 | 145 | .. code-block:: bash 146 | 147 | $ git commit -m 'did a bunch of great work' 148 | 149 | and push your branch to github. 150 | 151 | .. code-block:: bash 152 | 153 | $ git push origin cool_new_feature 154 | 155 | At this point, you go find your fork on github.com and create a `pull 156 | request`_. Clearly describe what you have done in the comments. If your 157 | pull request fixes an issue or adds a useful new feature, the team will 158 | gladly merge it. 159 | 160 | .. _pull request: https://help.github.com/articles/using-pull-requests/ 161 | 162 | After your pull request is merged, you can switch back to the master branch, 163 | rebase, and delete your feature branch. You will find your new feature 164 | incorporated into xmitgcm. 165 | 166 | .. code-block:: bash 167 | 168 | $ git checkout master 169 | $ git fetch upstream 170 | $ git rebase upstream/master 171 | $ git branch -d cool_new_feature 172 | 173 | Virtual Environment 174 | ------------------- 175 | 176 | This is how to create a virtual environment into which to test-install xmitgcm, 177 | install it, check the version, and tear down the virtual environment. 178 | 179 | .. code-block:: bash 180 | 181 | $ conda create --yes -n test_env python=3.5 xarray dask numpy pytest future 182 | $ source activate test_env 183 | $ pip install xmitgcm 184 | $ python -c 'import xmitgcm; print(xmitgcm.__version__);' 185 | $ source deactivate 186 | $ conda env remove --yes -n test_env 187 | -------------------------------------------------------------------------------- /doc/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = _build 9 | 10 | # User-friendly check for sphinx-build 11 | ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) 12 | $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) 13 | endif 14 | 15 | # Internal variables. 16 | PAPEROPT_a4 = -D latex_paper_size=a4 17 | PAPEROPT_letter = -D latex_paper_size=letter 18 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 19 | # the i18n builder cannot share the environment and doctrees with the others 20 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 21 | 22 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext 23 | 24 | help: 25 | @echo "Please use \`make ' where is one of" 26 | @echo " html to make standalone HTML files" 27 | @echo " dirhtml to make HTML files named index.html in directories" 28 | @echo " singlehtml to make a single large HTML file" 29 | @echo " pickle to make pickle files" 30 | @echo " json to make JSON files" 31 | @echo " htmlhelp to make HTML files and a HTML help project" 32 | @echo " qthelp to make HTML files and a qthelp project" 33 | @echo " devhelp to make HTML files and a Devhelp project" 34 | @echo " epub to make an epub" 35 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 36 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 37 | @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" 38 | @echo " text to make text files" 39 | @echo " man to make manual pages" 40 | @echo " texinfo to make Texinfo files" 41 | @echo " info to make Texinfo files and run them through makeinfo" 42 | @echo " gettext to make PO message catalogs" 43 | @echo " changes to make an overview of all changed/added/deprecated items" 44 | @echo " xml to make Docutils-native XML files" 45 | @echo " pseudoxml to make pseudoxml-XML files for display purposes" 46 | @echo " linkcheck to check all external links for integrity" 47 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 48 | 49 | clean: 50 | rm -rf $(BUILDDIR)/* 51 | 52 | html: 53 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 54 | @echo 55 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 56 | 57 | dirhtml: 58 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 59 | @echo 60 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 61 | 62 | singlehtml: 63 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 64 | @echo 65 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 66 | 67 | pickle: 68 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 69 | @echo 70 | @echo "Build finished; now you can process the pickle files." 71 | 72 | json: 73 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 74 | @echo 75 | @echo "Build finished; now you can process the JSON files." 76 | 77 | htmlhelp: 78 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 79 | @echo 80 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 81 | ".hhp project file in $(BUILDDIR)/htmlhelp." 82 | 83 | qthelp: 84 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 85 | @echo 86 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 87 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 88 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/xgcm.qhcp" 89 | @echo "To view the help file:" 90 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/xgcm.qhc" 91 | 92 | devhelp: 93 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 94 | @echo 95 | @echo "Build finished." 96 | @echo "To view the help file:" 97 | @echo "# mkdir -p $$HOME/.local/share/devhelp/xgcm" 98 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/xgcm" 99 | @echo "# devhelp" 100 | 101 | epub: 102 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 103 | @echo 104 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 105 | 106 | latex: 107 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 108 | @echo 109 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 110 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 111 | "(use \`make latexpdf' here to do that automatically)." 112 | 113 | latexpdf: 114 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 115 | @echo "Running LaTeX files through pdflatex..." 116 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 117 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 118 | 119 | latexpdfja: 120 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 121 | @echo "Running LaTeX files through platex and dvipdfmx..." 122 | $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja 123 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 124 | 125 | text: 126 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 127 | @echo 128 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 129 | 130 | man: 131 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 132 | @echo 133 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 134 | 135 | texinfo: 136 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 137 | @echo 138 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 139 | @echo "Run \`make' in that directory to run these through makeinfo" \ 140 | "(use \`make info' here to do that automatically)." 141 | 142 | info: 143 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 144 | @echo "Running Texinfo files through makeinfo..." 145 | make -C $(BUILDDIR)/texinfo info 146 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 147 | 148 | gettext: 149 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 150 | @echo 151 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 152 | 153 | changes: 154 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 155 | @echo 156 | @echo "The overview file is in $(BUILDDIR)/changes." 157 | 158 | linkcheck: 159 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 160 | @echo 161 | @echo "Link check complete; look for any errors in the above output " \ 162 | "or in $(BUILDDIR)/linkcheck/output.txt." 163 | 164 | doctest: 165 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 166 | @echo "Testing of doctests in the sources finished, look at the " \ 167 | "results in $(BUILDDIR)/doctest/output.txt." 168 | 169 | xml: 170 | $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml 171 | @echo 172 | @echo "Build finished. The XML files are in $(BUILDDIR)/xml." 173 | 174 | pseudoxml: 175 | $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml 176 | @echo 177 | @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." 178 | -------------------------------------------------------------------------------- /doc/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | REM Command file for Sphinx documentation 4 | 5 | if "%SPHINXBUILD%" == "" ( 6 | set SPHINXBUILD=sphinx-build 7 | ) 8 | set BUILDDIR=_build 9 | set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . 10 | set I18NSPHINXOPTS=%SPHINXOPTS% . 11 | if NOT "%PAPER%" == "" ( 12 | set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% 13 | set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% 14 | ) 15 | 16 | if "%1" == "" goto help 17 | 18 | if "%1" == "help" ( 19 | :help 20 | echo.Please use `make ^` where ^ is one of 21 | echo. html to make standalone HTML files 22 | echo. dirhtml to make HTML files named index.html in directories 23 | echo. singlehtml to make a single large HTML file 24 | echo. pickle to make pickle files 25 | echo. json to make JSON files 26 | echo. htmlhelp to make HTML files and a HTML help project 27 | echo. qthelp to make HTML files and a qthelp project 28 | echo. devhelp to make HTML files and a Devhelp project 29 | echo. epub to make an epub 30 | echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter 31 | echo. text to make text files 32 | echo. man to make manual pages 33 | echo. texinfo to make Texinfo files 34 | echo. gettext to make PO message catalogs 35 | echo. changes to make an overview over all changed/added/deprecated items 36 | echo. xml to make Docutils-native XML files 37 | echo. pseudoxml to make pseudoxml-XML files for display purposes 38 | echo. linkcheck to check all external links for integrity 39 | echo. doctest to run all doctests embedded in the documentation if enabled 40 | goto end 41 | ) 42 | 43 | if "%1" == "clean" ( 44 | for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i 45 | del /q /s %BUILDDIR%\* 46 | goto end 47 | ) 48 | 49 | 50 | %SPHINXBUILD% 2> nul 51 | if errorlevel 9009 ( 52 | echo. 53 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 54 | echo.installed, then set the SPHINXBUILD environment variable to point 55 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 56 | echo.may add the Sphinx directory to PATH. 57 | echo. 58 | echo.If you don't have Sphinx installed, grab it from 59 | echo.http://sphinx-doc.org/ 60 | exit /b 1 61 | ) 62 | 63 | if "%1" == "html" ( 64 | %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html 65 | if errorlevel 1 exit /b 1 66 | echo. 67 | echo.Build finished. The HTML pages are in %BUILDDIR%/html. 68 | goto end 69 | ) 70 | 71 | if "%1" == "dirhtml" ( 72 | %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml 73 | if errorlevel 1 exit /b 1 74 | echo. 75 | echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. 76 | goto end 77 | ) 78 | 79 | if "%1" == "singlehtml" ( 80 | %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml 81 | if errorlevel 1 exit /b 1 82 | echo. 83 | echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. 84 | goto end 85 | ) 86 | 87 | if "%1" == "pickle" ( 88 | %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle 89 | if errorlevel 1 exit /b 1 90 | echo. 91 | echo.Build finished; now you can process the pickle files. 92 | goto end 93 | ) 94 | 95 | if "%1" == "json" ( 96 | %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json 97 | if errorlevel 1 exit /b 1 98 | echo. 99 | echo.Build finished; now you can process the JSON files. 100 | goto end 101 | ) 102 | 103 | if "%1" == "htmlhelp" ( 104 | %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp 105 | if errorlevel 1 exit /b 1 106 | echo. 107 | echo.Build finished; now you can run HTML Help Workshop with the ^ 108 | .hhp project file in %BUILDDIR%/htmlhelp. 109 | goto end 110 | ) 111 | 112 | if "%1" == "qthelp" ( 113 | %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp 114 | if errorlevel 1 exit /b 1 115 | echo. 116 | echo.Build finished; now you can run "qcollectiongenerator" with the ^ 117 | .qhcp project file in %BUILDDIR%/qthelp, like this: 118 | echo.^> qcollectiongenerator %BUILDDIR%\qthelp\xgcm.qhcp 119 | echo.To view the help file: 120 | echo.^> assistant -collectionFile %BUILDDIR%\qthelp\xgcm.ghc 121 | goto end 122 | ) 123 | 124 | if "%1" == "devhelp" ( 125 | %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp 126 | if errorlevel 1 exit /b 1 127 | echo. 128 | echo.Build finished. 129 | goto end 130 | ) 131 | 132 | if "%1" == "epub" ( 133 | %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub 134 | if errorlevel 1 exit /b 1 135 | echo. 136 | echo.Build finished. The epub file is in %BUILDDIR%/epub. 137 | goto end 138 | ) 139 | 140 | if "%1" == "latex" ( 141 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 142 | if errorlevel 1 exit /b 1 143 | echo. 144 | echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. 145 | goto end 146 | ) 147 | 148 | if "%1" == "latexpdf" ( 149 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 150 | cd %BUILDDIR%/latex 151 | make all-pdf 152 | cd %BUILDDIR%/.. 153 | echo. 154 | echo.Build finished; the PDF files are in %BUILDDIR%/latex. 155 | goto end 156 | ) 157 | 158 | if "%1" == "latexpdfja" ( 159 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 160 | cd %BUILDDIR%/latex 161 | make all-pdf-ja 162 | cd %BUILDDIR%/.. 163 | echo. 164 | echo.Build finished; the PDF files are in %BUILDDIR%/latex. 165 | goto end 166 | ) 167 | 168 | if "%1" == "text" ( 169 | %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text 170 | if errorlevel 1 exit /b 1 171 | echo. 172 | echo.Build finished. The text files are in %BUILDDIR%/text. 173 | goto end 174 | ) 175 | 176 | if "%1" == "man" ( 177 | %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man 178 | if errorlevel 1 exit /b 1 179 | echo. 180 | echo.Build finished. The manual pages are in %BUILDDIR%/man. 181 | goto end 182 | ) 183 | 184 | if "%1" == "texinfo" ( 185 | %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo 186 | if errorlevel 1 exit /b 1 187 | echo. 188 | echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. 189 | goto end 190 | ) 191 | 192 | if "%1" == "gettext" ( 193 | %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale 194 | if errorlevel 1 exit /b 1 195 | echo. 196 | echo.Build finished. The message catalogs are in %BUILDDIR%/locale. 197 | goto end 198 | ) 199 | 200 | if "%1" == "changes" ( 201 | %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes 202 | if errorlevel 1 exit /b 1 203 | echo. 204 | echo.The overview file is in %BUILDDIR%/changes. 205 | goto end 206 | ) 207 | 208 | if "%1" == "linkcheck" ( 209 | %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck 210 | if errorlevel 1 exit /b 1 211 | echo. 212 | echo.Link check complete; look for any errors in the above output ^ 213 | or in %BUILDDIR%/linkcheck/output.txt. 214 | goto end 215 | ) 216 | 217 | if "%1" == "doctest" ( 218 | %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest 219 | if errorlevel 1 exit /b 1 220 | echo. 221 | echo.Testing of doctests in the sources finished, look at the ^ 222 | results in %BUILDDIR%/doctest/output.txt. 223 | goto end 224 | ) 225 | 226 | if "%1" == "xml" ( 227 | %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml 228 | if errorlevel 1 exit /b 1 229 | echo. 230 | echo.Build finished. The XML files are in %BUILDDIR%/xml. 231 | goto end 232 | ) 233 | 234 | if "%1" == "pseudoxml" ( 235 | %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml 236 | if errorlevel 1 exit /b 1 237 | echo. 238 | echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml. 239 | goto end 240 | ) 241 | 242 | :end 243 | -------------------------------------------------------------------------------- /doc/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # xmitgcm documentation build configuration file, created by 4 | # sphinx-quickstart on Sat Aug 29 00:18:20 2015. 5 | # 6 | # This file is execfile()d with the current directory set to its 7 | # containing dir. 8 | # 9 | # Note that not all possible configuration values are present in this 10 | # autogenerated file. 11 | # 12 | # All configuration values have a default; values that are commented out 13 | # serve to show the default. 14 | 15 | import sys 16 | import os 17 | import xmitgcm 18 | from subprocess import check_output, CalledProcessError 19 | 20 | # If extensions (or modules to document with autodoc) are in another directory, 21 | # add these directories to sys.path here. If the directory is relative to the 22 | # documentation root, use os.path.abspath to make it absolute, like shown here. 23 | #sys.path.insert(0, os.path.abspath('.')) 24 | #sys.path.insert(os.path.abspath('..')) 25 | 26 | print("python exec:", sys.executable) 27 | print("sys.path:", sys.path) 28 | 29 | 30 | # -- General configuration ------------------------------------------------ 31 | 32 | # If your documentation needs a minimal Sphinx version, state it here. 33 | #needs_sphinx = '1.0' 34 | 35 | # Add any Sphinx extension module names here, as strings. They can be 36 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 37 | # ones. 38 | extensions = [ 39 | 'sphinx.ext.mathjax', 'sphinx.ext.autodoc', 40 | 'sphinx.ext.autosummary', 41 | 'sphinx.ext.extlinks', 42 | 'sphinx.ext.viewcode', 43 | 'numpydoc', 44 | 'nbsphinx', 45 | ] 46 | 47 | # link to github issues 48 | extlinks = {'issue': ('https://github.com/xgcm/xmitgcm/issues/%s', 'GH')} 49 | 50 | autosummary_generate = True 51 | numpydoc_class_members_toctree = True 52 | numpydoc_show_class_members = False 53 | 54 | # Add any paths that contain templates here, relative to this directory. 55 | templates_path = ['_templates'] 56 | 57 | # The suffix of source filenames. 58 | source_suffix = '.rst' 59 | 60 | # The encoding of source files. 61 | #source_encoding = 'utf-8-sig' 62 | 63 | # The master toctree document. 64 | master_doc = 'index' 65 | 66 | # General information about the project. 67 | project = u'xmitgcm' 68 | copyright = u'2016, xmitgcm developers' 69 | 70 | # The version info for the project you're documenting, acts as replacement for 71 | # |version| and |release|, also used in various other places throughout the 72 | # built documents. 73 | # 74 | 75 | 76 | def get_version(): 77 | """ 78 | Return the latest tag (checkpoint) and, if there have 79 | been commits since the version was tagged, the commit hash. 80 | 81 | To get just the release tag use: 82 | version = version.split('-')[0] 83 | """ 84 | 85 | try: 86 | version = check_output(['git', 'describe', '--tags', '--always'], 87 | universal_newlines=True) 88 | except CalledProcessError: 89 | return 'unknown version' 90 | 91 | return version.rstrip() 92 | 93 | 94 | # "version" is used for html build 95 | version = get_version() 96 | # "release" is used for LaTeX build 97 | release = version 98 | 99 | 100 | # The language for content autogenerated by Sphinx. Refer to documentation 101 | # for a list of supported languages. 102 | #language = None 103 | 104 | # There are two options for replacing |today|: either, you set today to some 105 | # non-false value, then it is used: 106 | #today = '' 107 | # Else, today_fmt is used as the format for a strftime call. 108 | #today_fmt = '%B %d, %Y' 109 | 110 | # List of patterns, relative to source directory, that match files and 111 | # directories to ignore when looking for source files. 112 | exclude_patterns = ['_build'] 113 | 114 | # The reST default role (used for this markup: `text`) to use for all 115 | # documents. 116 | #default_role = None 117 | 118 | # If true, '()' will be appended to :func: etc. cross-reference text. 119 | #add_function_parentheses = True 120 | 121 | # If true, the current module name will be prepended to all description 122 | # unit titles (such as .. function::). 123 | #add_module_names = True 124 | 125 | # If true, sectionauthor and moduleauthor directives will be shown in the 126 | # output. They are ignored by default. 127 | #show_authors = False 128 | 129 | # The name of the Pygments (syntax highlighting) style to use. 130 | pygments_style = 'sphinx' 131 | 132 | # A list of ignored prefixes for module index sorting. 133 | #modindex_common_prefix = [] 134 | 135 | # If true, keep warnings as "system message" paragraphs in the built documents. 136 | #keep_warnings = False 137 | 138 | 139 | # -- Options for HTML output ---------------------------------------------- 140 | 141 | on_rtd = os.environ.get('READTHEDOCS', None) == 'True' 142 | 143 | if not on_rtd: # only import and set the theme if we're building docs locally 144 | import sphinx_rtd_theme 145 | html_theme = 'sphinx_rtd_theme' 146 | html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] 147 | 148 | # The theme to use for HTML and HTML Help pages. See the documentation for 149 | # a list of builtin themes. 150 | html_theme = 'default' 151 | 152 | # Theme options are theme-specific and customize the look and feel of a theme 153 | # further. For a list of options available for each theme, see the 154 | # documentation. 155 | #html_theme_options = {} 156 | 157 | # Add any paths that contain custom themes here, relative to this directory. 158 | #html_theme_path = [] 159 | 160 | # The name for this set of Sphinx documents. If None, it defaults to 161 | # " v documentation". 162 | #html_title = None 163 | 164 | # A shorter title for the navigation bar. Default is the same as html_title. 165 | #html_short_title = None 166 | 167 | # The name of an image file (relative to this directory) to place at the top 168 | # of the sidebar. 169 | #html_logo = None 170 | 171 | # The name of an image file (within the static path) to use as favicon of the 172 | # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 173 | # pixels large. 174 | #html_favicon = None 175 | 176 | # Add any paths that contain custom static files (such as style sheets) here, 177 | # relative to this directory. They are copied after the builtin static files, 178 | # so a file named "default.css" will overwrite the builtin "default.css". 179 | html_static_path = ['_static'] 180 | 181 | # Add any extra paths that contain custom files (such as robots.txt or 182 | # .htaccess) here, relative to this directory. These files are copied 183 | # directly to the root of the documentation. 184 | #html_extra_path = [] 185 | 186 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, 187 | # using the given strftime format. 188 | #html_last_updated_fmt = '%b %d, %Y' 189 | 190 | # If true, SmartyPants will be used to convert quotes and dashes to 191 | # typographically correct entities. 192 | #html_use_smartypants = True 193 | 194 | # Custom sidebar templates, maps document names to template names. 195 | #html_sidebars = {} 196 | 197 | # Additional templates that should be rendered to pages, maps page names to 198 | # template names. 199 | #html_additional_pages = {} 200 | 201 | # If false, no module index is generated. 202 | #html_domain_indices = True 203 | 204 | # If false, no index is generated. 205 | #html_use_index = True 206 | 207 | # If true, the index is split into individual pages for each letter. 208 | #html_split_index = False 209 | 210 | # If true, links to the reST sources are added to the pages. 211 | #html_show_sourcelink = True 212 | 213 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 214 | #html_show_sphinx = True 215 | 216 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. 217 | #html_show_copyright = True 218 | 219 | # If true, an OpenSearch description file will be output, and all pages will 220 | # contain a tag referring to it. The value of this option must be the 221 | # base URL from which the finished HTML is served. 222 | #html_use_opensearch = '' 223 | 224 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 225 | #html_file_suffix = None 226 | 227 | # Output file base name for HTML help builder. 228 | htmlhelp_basename = 'xmitgcmdoc' 229 | 230 | 231 | # -- Options for LaTeX output --------------------------------------------- 232 | 233 | latex_elements = { 234 | # The paper size ('letterpaper' or 'a4paper'). 235 | #'papersize': 'letterpaper', 236 | 237 | # The font size ('10pt', '11pt' or '12pt'). 238 | #'pointsize': '10pt', 239 | 240 | # Additional stuff for the LaTeX preamble. 241 | #'preamble': '', 242 | } 243 | 244 | # Grouping the document tree into LaTeX files. List of tuples 245 | # (source start file, target name, title, 246 | # author, documentclass [howto, manual, or own class]). 247 | latex_documents = [ 248 | ('index', 'xmitgcm.tex', u'xmitgcm Documentation', 249 | u'Ryan Abernathey', 'manual'), 250 | ] 251 | 252 | # The name of an image file (relative to this directory) to place at the top of 253 | # the title page. 254 | #latex_logo = None 255 | 256 | # For "manual" documents, if this is true, then toplevel headings are parts, 257 | # not chapters. 258 | #latex_use_parts = False 259 | 260 | # If true, show page references after internal links. 261 | #latex_show_pagerefs = False 262 | 263 | # If true, show URL addresses after external links. 264 | #latex_show_urls = False 265 | 266 | # Documents to append as an appendix to all manuals. 267 | #latex_appendices = [] 268 | 269 | # If false, no module index is generated. 270 | #latex_domain_indices = True 271 | 272 | 273 | # -- Options for manual page output --------------------------------------- 274 | 275 | # One entry per manual page. List of tuples 276 | # (source start file, name, description, authors, manual section). 277 | man_pages = [ 278 | ('index', 'xmitgcm', u'xmitgcm Documentation', 279 | [u'xmitgcm developers'], 1) 280 | ] 281 | 282 | # If true, show URL addresses after external links. 283 | #man_show_urls = False 284 | 285 | 286 | # -- Options for Texinfo output ------------------------------------------- 287 | 288 | # Grouping the document tree into Texinfo files. List of tuples 289 | # (source start file, target name, title, author, 290 | # dir menu entry, description, category) 291 | texinfo_documents = [ 292 | ('index', 'xmitgcm', u'xmitgcm Documentation', 293 | u'Ryan Abernathey', 'xmitgcm', 'Read MITgcm mds binary files into xarray.', 294 | 'Miscellaneous'), 295 | ] 296 | 297 | # Documents to append as an appendix to all manuals. 298 | #texinfo_appendices = [] 299 | 300 | # If false, no module index is generated. 301 | #texinfo_domain_indices = True 302 | 303 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 304 | #texinfo_show_urls = 'footnote' 305 | 306 | # If true, do not generate a @detailmenu in the "Top" node's menu. 307 | #texinfo_no_detailmenu = False 308 | 309 | 310 | # Example configuration for intersphinx: refer to the Python standard library. 311 | intersphinx_mapping = {'http://docs.python.org/': None} 312 | -------------------------------------------------------------------------------- /doc/quick_start.rst: -------------------------------------------------------------------------------- 1 | Quick Start 2 | =========== 3 | 4 | First make sure you understand what an xarray_ Dataset object is. Then find 5 | some MITgcm MDS data. If you don't have any data of your own, you can download 6 | the xmitgcm 7 | `test repositories `_ 8 | To download the some test data, run the shell commands:: 9 | 10 | $ curl -L -J -O https://ndownloader.figshare.com/files/6494718 11 | $ tar -xvzf global_oce_latlon.tar.gz 12 | 13 | This will create a directory called ``global_oce_latlon`` which we will use 14 | for the rest of these examples. If you have your own data, replace this with 15 | the path to your mitgcm files. 16 | 17 | To opean MITgcm MDS data as an xarray.Dataset, do the following in python:: 18 | 19 | from xmitgcm import open_mdsdataset 20 | data_dir = './global_oce_latlon' 21 | ds = open_mdsdataset(data_dir) 22 | # display the contents of the Dataset 23 | print(ds) 24 | >>> 25 | Dimensions: (XC: 90, XG: 90, YC: 40, YG: 40, Z: 15, Zl: 15, Zp1: 16, Zu: 15, layer_1RHO_bounds: 31, layer_1RHO_center: 30, layer_1RHO_interface: 29, time: 1) 26 | Coordinates: 27 | iter (time) int64 39600 28 | * time (time) int64 39600 29 | * XC (XC) >f4 2.0 6.0 10.0 14.0 18.0 22.0 26.0 30.0 ... 30 | * YC (YC) >f4 -78.0 -74.0 -70.0 -66.0 -62.0 -58.0 -54.0 ... 31 | * XG (XG) >f4 0.0 4.0 8.0 12.0 16.0 20.0 24.0 28.0 32.0 ... 32 | * YG (YG) >f4 -80.0 -76.0 -72.0 -68.0 -64.0 -60.0 -56.0 ... 33 | * Zl (Zl) >f4 0.0 -50.0 -120.0 -220.0 -360.0 -550.0 ... 34 | * Zu (Zu) >f4 -50.0 -120.0 -220.0 -360.0 -550.0 -790.0 ... 35 | * Z (Z) >f4 -25.0 -85.0 -170.0 -290.0 -455.0 -670.0 ... 36 | * Zp1 (Zp1) >f4 0.0 -50.0 -120.0 -220.0 -360.0 -550.0 ... 37 | dxC (YC, XG) >f4 92460.4 92460.4 92460.4 92460.4 ... 38 | rAs (YG, XC) >f4 3.43349e+10 3.43349e+10 3.43349e+10 ... 39 | rAw (YC, XG) >f4 4.11097e+10 4.11097e+10 4.11097e+10 ... 40 | Depth (YC, XC) >f4 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 ... 41 | rA (YC, XC) >f4 4.11097e+10 4.11097e+10 4.11097e+10 ... 42 | dxG (YG, XC) >f4 77223.1 77223.1 77223.1 77223.1 ... 43 | dyG (YC, XG) >f4 444710.0 444710.0 444710.0 444710.0 ... 44 | rAz (YG, XG) >f4 3.43349e+10 3.43349e+10 3.43349e+10 ... 45 | dyC (YG, XC) >f4 444710.0 444710.0 444710.0 444710.0 ... 46 | PHrefC (Z) >f4 245.25 833.85 1667.7 2844.9 4463.55 6572.7 ... 47 | drC (Zp1) >f4 25.0 60.0 85.0 120.0 165.0 215.0 265.0 ... 48 | PHrefF (Zp1) >f4 0.0 490.5 1177.2 2158.2 3531.6 5395.5 ... 49 | drF (Z) >f4 50.0 70.0 100.0 140.0 190.0 240.0 290.0 ... 50 | hFacS (Z, YG, XC) >f4 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 ... 51 | hFacC (Z, YC, XC) >f4 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 ... 52 | hFacW (Z, YC, XG) >f4 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 ... 53 | * layer_1RHO_center (layer_1RHO_center) float32 20.1999 20.6922 21.169 ... 54 | * layer_1RHO_interface (layer_1RHO_interface) >f4 20.4499 20.9345 21.4034 ... 55 | * layer_1RHO_bounds (layer_1RHO_bounds) >f4 19.9499 20.4499 20.9345 ... 56 | Data variables: 57 | tFluxtave (time, YC, XC) >f4 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 ... 58 | PHLtave (time, YC, XC) >f4 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 ... 59 | Stave (time, Z, YC, XC) >f4 0.0 0.0 0.0 0.0 0.0 0.0 0.0 ... 60 | UUtave (time, Z, YC, XG) >f4 0.0 0.0 0.0 0.0 0.0 0.0 0.0 ... 61 | LaHw1RHO (time, layer_1RHO_center, YC, XG) >f4 0.0 0.0 0.0 ... 62 | LaPs1RHO (time, layer_1RHO_center, YG, XC) >f4 0.0 0.0 0.0 ... 63 | LaHs1RHO (time, layer_1RHO_center, YG, XC) >f4 0.0 0.0 0.0 ... 64 | LaUH1RHO (time, layer_1RHO_center, YC, XG) >f4 0.0 0.0 0.0 ... 65 | LaVH1RHO (time, layer_1RHO_center, YG, XC) >f4 0.0 0.0 0.0 ... 66 | LaPw1RHO (time, layer_1RHO_center, YC, XG) >f4 0.0 0.0 0.0 ... 67 | UVtave (time, Z, YG, XG) >f4 0.0 0.0 0.0 0.0 0.0 0.0 0.0 ... 68 | uFluxtave (time, YC, XG) >f4 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 ... 69 | VStave (time, Z, YG, XC) >f4 0.0 0.0 0.0 0.0 0.0 0.0 0.0 ... 70 | VTtave (time, Z, YG, XC) >f4 0.0 0.0 0.0 0.0 0.0 0.0 0.0 ... 71 | TTtave (time, Z, YC, XC) >f4 0.0 0.0 0.0 0.0 0.0 0.0 0.0 ... 72 | PhHytave (time, Z, YC, XC) >f4 -8.30019 -8.30019 -8.30019 ... 73 | sFluxtave (time, YC, XC) >f4 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 ... 74 | W (time, Zl, YC, XC) >f4 -0.0 -0.0 -0.0 -0.0 -0.0 ... 75 | ETAtave (time, YC, XC) >f4 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 ... 76 | VVtave (time, Z, YG, XC) >f4 0.0 0.0 0.0 0.0 0.0 0.0 0.0 ... 77 | Ttave (time, Z, YC, XC) >f4 0.0 0.0 0.0 0.0 0.0 0.0 0.0 ... 78 | PH (time, Z, YC, XC) >f4 -8.30019 -8.30019 -8.30019 ... 79 | vVeltave (time, Z, YG, XC) >f4 0.0 0.0 0.0 0.0 0.0 0.0 0.0 ... 80 | UTtave (time, Z, YC, XG) >f4 0.0 0.0 0.0 0.0 0.0 0.0 0.0 ... 81 | PHL2tave (time, YC, XC) >f4 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 ... 82 | UStave (time, Z, YC, XG) >f4 0.0 0.0 0.0 0.0 0.0 0.0 0.0 ... 83 | uVeltave (time, Z, YC, XG) >f4 0.0 0.0 0.0 0.0 0.0 0.0 0.0 ... 84 | S (time, Z, YC, XC) >f4 0.0 0.0 0.0 0.0 0.0 0.0 0.0 ... 85 | Eta (time, YC, XC) >f4 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 ... 86 | Eta2tave (time, YC, XC) >f4 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 ... 87 | DFxE_TH (time, Z, YC, XG) >f4 0.0 0.0 0.0 0.0 0.0 0.0 0.0 ... 88 | ADVy_TH (time, Z, YG, XC) >f4 0.0 0.0 0.0 0.0 0.0 0.0 0.0 ... 89 | VTHMASS (time, Z, YG, XC) >f4 0.0 0.0 0.0 0.0 0.0 0.0 0.0 ... 90 | DFrE_TH (time, Zl, YC, XC) >f4 0.0 0.0 0.0 0.0 0.0 0.0 0.0 ... 91 | WTHMASS (time, Zl, YC, XC) >f4 0.0 0.0 0.0 0.0 0.0 0.0 0.0 ... 92 | TOTTTEND (time, Z, YC, XC) >f4 0.0 0.0 0.0 0.0 0.0 0.0 0.0 ... 93 | ADVr_TH (time, Zl, YC, XC) >f4 0.0 0.0 0.0 0.0 0.0 0.0 0.0 ... 94 | DFyE_TH (time, Z, YG, XC) >f4 0.0 0.0 0.0 0.0 0.0 0.0 0.0 ... 95 | UTHMASS (time, Z, YC, XG) >f4 0.0 0.0 0.0 0.0 0.0 0.0 0.0 ... 96 | DFrI_TH (time, Zl, YC, XC) >f4 0.0 0.0 0.0 0.0 0.0 0.0 0.0 ... 97 | ADVx_TH (time, Z, YC, XG) >f4 0.0 0.0 0.0 0.0 0.0 0.0 0.0 ... 98 | surForcT (time, YC, XC) >f4 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 ... 99 | TFLUX (time, YC, XC) >f4 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 ... 100 | surForcS (time, YC, XC) >f4 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 ... 101 | SFLUX (time, YC, XC) >f4 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 ... 102 | V (time, Z, YG, XC) >f4 -0.0 -0.0 -0.0 -0.0 -0.0 0.0 ... 103 | LaTs1RHO (time, layer_1RHO_interface, YC, XC) >f4 0.0 0.0 ... 104 | LTha1RHO (time, layer_1RHO_interface, YC, XC) >f4 0.0 0.0 ... 105 | LaSz1RHO (time, layer_1RHO_interface, YC, XC) >f4 0.0 0.0 ... 106 | LSto1RHO (time, layer_1RHO_interface, YC, XC) >f4 0.0 0.0 ... 107 | LSha1RHO (time, layer_1RHO_interface, YC, XC) >f4 0.0 0.0 ... 108 | LaTz1RHO (time, layer_1RHO_interface, YC, XC) >f4 0.0 0.0 ... 109 | LaSs1RHO (time, layer_1RHO_interface, YC, XC) >f4 0.0 0.0 ... 110 | LaTh1RHO (time, layer_1RHO_interface, YC, XC) >f4 0.0 0.0 ... 111 | LTto1RHO (time, layer_1RHO_interface, YC, XC) >f4 0.0 0.0 ... 112 | LTza1RHO (time, layer_1RHO_interface, YC, XC) >f4 0.0 0.0 ... 113 | LaSh1RHO (time, layer_1RHO_interface, YC, XC) >f4 0.0 0.0 ... 114 | LSza1RHO (time, layer_1RHO_interface, YC, XC) >f4 0.0 0.0 ... 115 | vFluxtave (time, YG, XC) >f4 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 ... 116 | WTtave (time, Zl, YC, XC) >f4 0.0 0.0 0.0 0.0 0.0 0.0 0.0 ... 117 | WStave (time, Zl, YC, XC) >f4 0.0 0.0 0.0 0.0 0.0 0.0 0.0 ... 118 | GM_Kwz-T (time, Zl, YC, XC) >f4 0.0 0.0 0.0 0.0 0.0 0.0 0.0 ... 119 | PHL (time, YC, XC) >f4 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 ... 120 | THETA (time, Z, YC, XC) >f4 0.0 0.0 0.0 0.0 0.0 0.0 0.0 ... 121 | UVEL (time, Z, YC, XG) >f4 0.0 0.0 0.0 0.0 0.0 0.0 0.0 ... 122 | VVEL (time, Z, YG, XC) >f4 0.0 0.0 0.0 0.0 0.0 0.0 0.0 ... 123 | WVEL (time, Zl, YC, XC) >f4 0.0 0.0 0.0 0.0 0.0 0.0 0.0 ... 124 | SALT (time, Z, YC, XC) >f4 0.0 0.0 0.0 0.0 0.0 0.0 0.0 ... 125 | DFrI_SLT (time, Zl, YC, XC) >f4 0.0 0.0 0.0 0.0 0.0 0.0 0.0 ... 126 | WSLTMASS (time, Zl, YC, XC) >f4 0.0 0.0 0.0 0.0 0.0 0.0 0.0 ... 127 | ADVx_SLT (time, Z, YC, XG) >f4 0.0 0.0 0.0 0.0 0.0 0.0 0.0 ... 128 | ADVr_SLT (time, Zl, YC, XC) >f4 0.0 0.0 0.0 0.0 0.0 0.0 0.0 ... 129 | TOTSTEND (time, Z, YC, XC) >f4 0.0 0.0 0.0 0.0 0.0 0.0 0.0 ... 130 | USLTMASS (time, Z, YC, XG) >f4 0.0 0.0 0.0 0.0 0.0 0.0 0.0 ... 131 | DFxE_SLT (time, Z, YC, XG) >f4 0.0 0.0 0.0 0.0 0.0 0.0 0.0 ... 132 | DFyE_SLT (time, Z, YG, XC) >f4 0.0 0.0 0.0 0.0 0.0 0.0 0.0 ... 133 | DFrE_SLT (time, Zl, YC, XC) >f4 0.0 0.0 0.0 0.0 0.0 0.0 0.0 ... 134 | ADVy_SLT (time, Z, YG, XC) >f4 0.0 0.0 0.0 0.0 0.0 0.0 0.0 ... 135 | VSLTMASS (time, Z, YG, XC) >f4 0.0 0.0 0.0 0.0 0.0 0.0 0.0 ... 136 | Convtave (time, Zl, YC, XC) >f4 0.0 0.0 0.0 0.0 0.0 0.0 0.0 ... 137 | wVeltave (time, Zl, YC, XC) >f4 0.0 0.0 0.0 0.0 0.0 0.0 0.0 ... 138 | GM_Kwy-T (time, Zl, YC, XC) >f4 0.0 0.0 0.0 0.0 0.0 0.0 0.0 ... 139 | U (time, Z, YC, XG) >f4 0.0 0.0 0.0 0.0 0.0 0.0 0.0 ... 140 | Tdiftave (time, Zl, YC, XC) >f4 0.0 0.0 0.0 0.0 0.0 0.0 0.0 ... 141 | GM_Kwx-T (time, Zl, YC, XC) >f4 0.0 0.0 0.0 0.0 0.0 0.0 0.0 ... 142 | T (time, Z, YC, XC) >f4 0.0 0.0 0.0 0.0 0.0 0.0 0.0 ... 143 | 144 | ``data_dir``, should be the path (absolute or relative) to an 145 | MITgcm run directory. xmitgcm will automatically scan this directory and 146 | try to determine the file prefixes and iteration numbers to read. In some 147 | configurations, the ``open_mdsdataset`` function may work without further 148 | keyword arguments. In most cases, you will have to specify further details. 149 | 150 | For more details about the options and internals, consult :doc:`/usage`. 151 | 152 | .. _dask: http://dask.pydata.org 153 | .. _xarray: http://xarray.pydata.org 154 | .. _Comodo: http://pycomodo.forge.imag.fr/norm.html 155 | .. _issues: https://github.com/xgcm/xmitgcm/issues 156 | .. _`pull requests`: https://github.com/xgcm/xmitgcm/pulls 157 | .. _MITgcm: http://mitgcm.org/public/r2_manual/latest/online_documents/node277.html 158 | .. _out-of-core: https://en.wikipedia.org/wiki/Out-of-core_algorithm 159 | .. _Anaconda: https://www.continuum.io/downloads 160 | .. _`CF conventions`: http://cfconventions.org/Data/cf-conventions/cf-conventions-1.7/build/ch04s04.html 161 | .. _gcmfaces: http://mitgcm.org/viewvc/*checkout*/MITgcm/MITgcm_contrib/gael/matlab_class/gcmfaces.pdf 162 | -------------------------------------------------------------------------------- /doc/performance.rst: -------------------------------------------------------------------------------- 1 | Performance Issues 2 | ================== 3 | 4 | A major goal of xmitgcm is to achieve scalable performance with very large 5 | datasets. We were motivated by the new 6 | `LLC4320 `_ 7 | simulations run by 8 | `Dimitris Menemenlis `_ 9 | and 10 | `Chris Hill `_ on NASA's 11 | Pleiades_ supercomputer. 12 | 13 | This page documents ongoing research into the performance of xmitgcm. 14 | 15 | 16 | LLC Reading Strategies 17 | ---------------------- 18 | 19 | The physical layout of the LLC MDS files creates a challenge for performance. 20 | Some of the cube facets are written with C order, while others are written 21 | with Fortran order. This means that complex logic is required to translate the 22 | raw data on disk to the desired logical array layout within xarray. Because of 23 | this complication, the raw data cannot be accessed using the numpy 24 | `ndarray data model `_. 25 | 26 | .. figure:: llc_cartoon/llc_cartoon.001.png 27 | :scale: 100 % 28 | :alt: LLC data layout 29 | 30 | The physical layout of a single level of LLC data in terms of facets (top) 31 | and the translation by xmitgcm to faces (bottom). 32 | 33 | Two different approaches to reading LLC data have been developed. This option 34 | is specified via the ``llc_method`` keyword in ``open_mdsdataset``. 35 | 36 | smallchunks 37 | ~~~~~~~~~~~ 38 | 39 | ``method="smallchunks"`` creates a large number of small dask chunks to 40 | represent the 41 | array. A chunk (of size nx x nx) is created for each face of each vertical 42 | level. The total number of chunks is consequently 13 * nz. This appears 43 | to perform better when you want to access a small subset of a very large model, 44 | since only the required data will be read. It has a much lower memory footprint 45 | than bigchunks. No memmaps are used withsmallchunks, since that implies leaving 46 | a huge number of files open. Instead, each chunk is read directly by 47 | ``numpy.fromfile``. 48 | 49 | 50 | bigchunks 51 | ~~~~~~~~~ 52 | 53 | ``method="bigchunks"`` loads the entire raw data on disk as either a 54 | ``numpy.memmap`` (default) or directly as a numpy array. It then slices this 55 | array into facets, reshapes them as necessary, slices each facet into faces, 56 | and concatenates the faces along a new dimension using 57 | ``dask.array.concatentate``. This approach can be more efficient if the goal is 58 | to read all of the array data into memory. Any attempt to read data from the 59 | reshaped faces (faces 8-13 in the cartoon above) will trigger the 60 | *entire facet* to be loaded into memory. For this reason, the bigchunks method 61 | is impractical for very large LLC datasets that don't easily fit into memory 62 | 63 | comparison 64 | ~~~~~~~~~~ 65 | 66 | A `test script `_ 67 | was developed to evaluate the two strategies for reading LLC4320 data on 68 | Pleiades_. Files were selected for analysis randomly from over 10000 files on 69 | disk in order to avoid any caching from the filesystem. The data were read 70 | using the low level routine ``read_3d_llc_data`` (see :doc:`/utils`). Tests 71 | were performed on both 2D data (4320 x 56160 32-bit floats) and 3D data 72 | (4320 x 56160 x 90 32-bit floats). 73 | 74 | The first task was a reduction: computing the sum of the array. For 2D data, 75 | the smallchunks method performed marginally better in terms of speed and 76 | memory usage. 77 | 78 | .. figure:: pleiades_tests/compute_sum.png 79 | :scale: 100 80 | :alt: compute sum 2D 81 | 82 | Walltime and memory usage for ``compute_sum`` on 2D data as a function of 83 | number of dask workers 84 | 85 | However, a dramatic difference was evident for 3D data. The inefficient memory 86 | usage of bigchunks is especially evident for large numbers of dask workers, 87 | since each worker repeatedly triggers the loading of whole array facets. 88 | 89 | .. figure:: pleiades_tests/compute_sum_3D.png 90 | :scale: 100 91 | :alt: compute sum 3D 92 | 93 | Walltime and memory usage for ``compute_sum`` on 3D data as a function of 94 | number of dask workers 95 | 96 | For this sort of reduction workflow, smallchunks with a large number of dask 97 | workers is the clear winner. 98 | 99 | A second common workflow is subsetting. In the test script, we load into memory 100 | 1080 x 1080 region from chunk 2. 101 | 102 | .. figure:: pleiades_tests/extract_chunk.png 103 | :scale: 100 104 | :alt: extract chunk 2D 105 | 106 | Walltime and memory usage for ``extract_chunk`` on 2D data as a function of 107 | number of dask workers 108 | 109 | Again, smallchunks is the clear winner here, with much faster execution and 110 | lower memory usage. Interestingly, there is little speedup using multiple 111 | multiple workers. All the same conclusions are true for 3D data. 112 | 113 | .. figure:: pleiades_tests/extract_chunk_3D.png 114 | :scale: 100 115 | :alt: extract chunk 3D 116 | 117 | Walltime and memory usage for ``extract_chunk`` on 3D data as a function of 118 | number of dask workers 119 | 120 | A final workload is simply loading the whole array into memory. (This turned out 121 | to be impossible for 3D data, since the compute nodes ran out of memory in the 122 | process.) This is the only workload where bigchunks has some advantages. Here 123 | a tradeoff between speed and memory usage is clear: bigchunks goes faster 124 | because it reads the data in bigger chunks, but it also uses much more memory. 125 | 126 | .. figure:: pleiades_tests/load_data.png 127 | :scale: 100 128 | :alt: load data 2D 129 | 130 | Walltime and memory usage for ``load_data_in_memory`` on 2D data as a 131 | function of number of dask workers 132 | 133 | It is useful to compare these numbers to the speed of a raw ``numpy.fromfile`` 134 | read of the data. This measures the overhead associated with chunking and 135 | reshaping the data from its physical layout on disk to the desired logical 136 | layout. Reading with smallchunks takes about 150 times the raw read time, while 137 | for bigchunks it is more like 10 times. Here there is a *disadvantage* to using 138 | multiple dask workers; while there is no speed improvement, the memory usage 139 | increases with number of workers for bigchunks. 140 | 141 | .. figure:: pleiades_tests/load_data_normalized.png 142 | :scale: 100 143 | :alt: load data 2D normalized 144 | 145 | Walltime and memory usage for ``load_data_in_memory`` on 2D data as a 146 | function of number of dask workers, normalized against loading the same 147 | data directly using ``numpy.fromfile``. 148 | 149 | Running xmitgcm on Pleaides 150 | --------------------------- 151 | 152 | These instructions describe how to get a working xmitgcm environment on a 153 | cluster such as Pleiades_. (See related 154 | `blog post `_) 155 | 156 | Step 1: Install miniconda in user space 157 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 158 | 159 | Miniconda is a mini version of Anaconda that includes just conda and its 160 | dependencies. It is a very small download. If you want to use python 3 161 | (recommended) you can call:: 162 | 163 | wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh 164 | 165 | or for python 2.7:: 166 | 167 | wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh -O miniconda.sh 168 | 169 | Step 2: Run Miniconda 170 | ~~~~~~~~~~~~~~~~~~~~~ 171 | 172 | Now you actually run miniconda to install the package manager. The trick is to 173 | specify the install directory within your home directory, rather in the default 174 | system-wide installation (which you won’t have permissions to do). 175 | You then have to add this directory to your path:: 176 | 177 | bash miniconda.sh -b -p $HOME/miniconda export PATH="$HOME/miniconda/bin:$PATH" 178 | 179 | Step 3: Create a custom conda environment specification 180 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 181 | You now have to define what packages you actually want to install. A good way 182 | to do this is with a custom conda environment file. The contents of this file 183 | will differ for each project. Below is an environment.yml suitable for 184 | xmitgcm:: 185 | 186 | name: xmitgcm 187 | dependencies: 188 | - numpy 189 | - scipy 190 | - xarray 191 | - netcdf4 192 | - dask 193 | - jupyter 194 | - matplotlib 195 | - pip: 196 | - pytest 197 | - xmitgcm 198 | 199 | Create a similar file and save it as environment.yml. 200 | 201 | Step 4: Create the conda environment 202 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 203 | 204 | You should now be able to run the following command:: 205 | 206 | conda env create --file environment.yml 207 | 208 | This will download and install all the packages and their dependencies. 209 | 210 | Step 5: Activate The environment 211 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 212 | 213 | The environment you created needs to be activated before you can actually use 214 | it. To do this, you call:: 215 | 216 | source activate xmitgcm 217 | 218 | This step needs to be repeated whenever you want to use the environment (i.e. 219 | every time you launch an interactive job or call python from within a batch 220 | job). 221 | 222 | Step 6: Use xmitgcm 223 | ~~~~~~~~~~~~~~~~~~~ 224 | You can now call ipython on the command line or launch a jupyter notebook and 225 | import xmitgcm. This should be done from a compute node, rather than the head 226 | node. 227 | 228 | .. _Pleiades: https://www.nas.nasa.gov/hecc/resources/pleiades.html 229 | 230 | 231 | Example Pleiades Scripts 232 | ------------------------ 233 | 234 | Below is an example python which extracts a subset from the LLC4320 simulation 235 | on Pleiades_ and saves it a sequence of netCDF files. 236 | 237 | .. code-block:: python 238 | 239 | import os 240 | import sys 241 | import numpy as np 242 | import xarray as xr 243 | import dask 244 | from multiprocessing.pool import ThreadPool 245 | from xmitgcm import open_mdsdataset 246 | 247 | # By default, dask will use one worker for each core available. 248 | # This can be changed by uncommenting below 249 | #dask.set_options(pool=ThreadPool(4)) 250 | 251 | # where the data lives 252 | data_dir = '/u/dmenemen/llc_4320/MITgcm/run/' 253 | grid_dir = '/u/dmenemen/llc_4320/grid/' 254 | # where to save the subsets 255 | outdir_base = '/nobackup/rpaberna/LLC/tile_data/' 256 | 257 | dtype = np.dtype('>f4') 258 | 259 | # can complete 300 files in < 12 hours 260 | nfiles = 300 261 | 262 | # the first available iteration is iter0=10368 263 | # we start from an iteration number specified on the command line 264 | iter0 = int(sys.argv[1]) 265 | 266 | delta = 144 # iters 267 | delta_t = 25. # seconds 268 | all_iters = iter0 + delta*np.arange(nfiles) 269 | 270 | region_name = 'agulhas' 271 | region_slice = {'face': 1, 272 | 'i': slice(1080,3240), 'i_g': slice(1080,3240), 273 | 'j': slice(0,2160), 'j_g': slice(0,2160)} 274 | 275 | fprefix = 'llc_4320_%s' % region_name 276 | outdir = os.path.join(outdir_base, fprefix) 277 | 278 | ds = open_mdsdataset(data_dir, grid_dir=grid_dir, 279 | iters=list(all_iters), geometry='llc', read_grid=False, 280 | default_dtype=np.dtype('>f4'), delta_t=delta_t, 281 | ignore_unknown_vars=True) 282 | 283 | region = ds.isel(**region_slice) 284 | 285 | # group for writing 286 | iters, datasets = zip(*region.groupby('iter')) 287 | paths = [os.path.join(outdir, '%s.%010d.nc' % (fprefix, d)) 288 | for d in iters] 289 | 290 | # write the data...takes a long time and executes in parallel 291 | xr.save_mfdataset(datasets, paths, engine='netcdf4') 292 | 293 | Here is a batch job which calls the script 294 | 295 | .. code-block:: bash 296 | 297 | #!/bin/bash 298 | #PBS -N read_llc 299 | #PBS -l select=1:ncpus=28:model=bro 300 | #PBS -l walltime=12:00:00 301 | 302 | source activate xmitgcm 303 | cd $PBS_O_WORKDIR 304 | # the first available iteration 305 | iter0=10368 306 | python -u write_by_iternum.py $iter0 307 | 308 | .. _Pleiades: https://www.nas.nasa.gov/hecc/resources/pleiades.html 309 | -------------------------------------------------------------------------------- /xmitgcm/llcreader/known_models.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from .llcmodel import BaseLLCModel 4 | from . import stores 5 | 6 | 7 | def _requires_pleiades(func): 8 | def wrapper(*args, **kwargs): 9 | # is there a better choice 10 | test_path = '/home6/dmenemen' 11 | if not os.path.exists(test_path): 12 | raise OSError("Can't find %s. We must not be on Pleiades." % test_path) 13 | func(*args, **kwargs) 14 | return wrapper 15 | 16 | def _requires_sverdrup(func): 17 | def wrapper(*args,**kwargs): 18 | test_path = '/scratch2/heimbach' 19 | if not os.path.exists(test_path): 20 | raise OSError("Can't find %s. We must not be on Sverdrup." % test_path) 21 | func(*args, **kwargs) 22 | return wrapper 23 | 24 | 25 | def _make_http_filesystem(): 26 | import fsspec 27 | from fsspec.implementations.http import HTTPFileSystem 28 | return HTTPFileSystem() 29 | 30 | class LLC90Model(BaseLLCModel): 31 | nx = 90 32 | nz = 50 33 | delta_t = 3600 34 | time_units = 'seconds since 1948-01-01 12:00:00' 35 | calendar = 'gregorian' 36 | 37 | 38 | class LLC2160Model(BaseLLCModel): 39 | nx = 2160 40 | nz = 90 41 | delta_t = 45 42 | iter_start = 92160 43 | iter_stop = 1586400 + 1 44 | iter_step = 80 45 | time_units='seconds since 2011-01-17' 46 | calendar = 'gregorian' 47 | varnames = ['Eta', 'KPPhbl', 'oceFWflx', 'oceQnet', 'oceQsw', 'oceSflux', 48 | 'oceTAUX', 'oceTAUY', 'PhiBot', 'Salt', 'SIarea', 'SIheff', 49 | 'SIhsalt', 'SIhsnow', 'SIuice', 'SIvice', 'Theta', 'U', 'V', 'W'] 50 | grid_varnames = ['AngleCS','AngleSN','Depth', 51 | 'DRC','DRF', 52 | 'DXC','DXF','DXG', 53 | 'DYC','DYF','DYG', 54 | 'hFacC','hFacS','hFacW','PHrefC','PHrefF', 55 | 'RAC','RAS','RAW', 56 | 'RC','RF','RhoRef','rLowC','rLowS','rLowW', 57 | 'rSurfC','rSurfS','rSurfW','XC','YC', 58 | 'RAZ','XG','YG','DXV','DYU'] 59 | mask_override = {'oceTAUX': 'c', 'oceTAUY': 'c', 60 | 'SIuice': 'c', 'SIvice': 'c'} 61 | 62 | class LLC4320Model(BaseLLCModel): 63 | nx = 4320 64 | nz = 90 65 | delta_t = 25 66 | iter_start = 10368 67 | iter_stop = 1495152 + 1 68 | iter_step = 144 69 | 70 | 71 | time_units='seconds since 2011-09-10' 72 | calendar = 'gregorian' 73 | varnames = ['Eta', 'KPPhbl', 'oceFWflx', 'oceQnet', 'oceQsw', 'oceSflux', 74 | 'oceTAUX', 'oceTAUY', 'PhiBot', 'Salt', 'SIarea', 'SIheff', 75 | 'SIhsalt', 'SIhsnow', 'SIuice', 'SIvice', 'Theta', 'U', 'V', 'W'] 76 | grid_varnames = ['AngleCS','AngleSN','Depth', 77 | 'DRC','DRF', 78 | 'DXC','DXF','DXG', 79 | 'DYC','DYF','DYG', 80 | 'hFacC','hFacS','hFacW','PHrefC','PHrefF', 81 | 'RAC','RAS','RAW','RC','RF', 82 | 'RhoRef','XC','YC','RAZ','XG','YG','DXV','DYU'] 83 | mask_override = {'oceTAUX': 'c', 'oceTAUY': 'c', 84 | 'SIuice': 'c', 'SIvice': 'c'} 85 | 86 | class ASTE270Model(BaseLLCModel): 87 | nface = 6 88 | nx = 270 89 | nz = 50 90 | domain = 'aste' 91 | pad_before = [90, 0, 0, 0, 0] 92 | pad_after = [0, 0, 0, 90, 90] 93 | delta_t = 600 94 | time_units='seconds since 2002-01-01' 95 | calendar = 'gregorian' 96 | 97 | iters=[4464, 8496, 12960, 17280, 21744, 98 | 26064, 30528, 34992, 39312, 43776, 99 | 48096, 52560, 57024, 61056, 65520, 100 | 69840, 74304, 78624, 83088, 87552, 101 | 91872, 96336, 100656, 105120, 109584, 102 | 113760, 118224, 122544, 127008, 131328, 103 | 135792, 140256, 144576, 149040, 153360, 104 | 157824, 162288, 166320, 170784, 175104, 105 | 179568, 183888, 188352, 192816, 197136, 106 | 201600, 205920, 210384, 214848, 218880, 107 | 223344, 227664, 232128, 236448, 240912, 108 | 245376, 249696, 254160, 258480, 262944, 109 | 267408, 271440, 275904, 280224, 284688, 110 | 289008, 293472, 297936, 302256, 306720, 111 | 311040, 315504, 319968, 324144, 328608, 112 | 332928, 337392, 341712, 346176, 350640, 113 | 354960, 359424, 363744, 368208, 372672, 114 | 376704, 381168, 385488, 389952, 394272, 115 | 398736, 403200, 407520, 411984, 416304, 116 | 420768, 425232, 429264, 433728, 438048, 117 | 442512, 446832, 451296, 455760, 460080, 118 | 464544, 468864, 473328, 477792, 481824, 119 | 486288, 490608, 495072, 499392, 503856, 120 | 508320, 512640, 517104, 521424, 525888, 121 | 530352, 534528, 538992, 543312, 547776, 122 | 552096, 556560, 561024, 565344, 569808, 123 | 574128, 578592, 583056, 587088, 591552, 124 | 595872, 600336, 604656, 609120, 613584, 125 | 617904, 622368, 626688, 631152, 635616, 126 | 639648, 644112, 648432, 652896, 657216, 127 | 661680, 666144, 670464, 674928, 679248, 128 | 683712, 688176, 692208, 696672, 700992, 129 | 705456, 709776, 714240, 718704, 723024, 130 | 727488, 731808, 736272, 740736, 744912, 131 | 749376, 753696, 758160, 762480, 766944, 132 | 771408, 775728, 780192, 784512, 788976, 133 | 793440, 797472, 801936, 806256, 810720, 134 | 815040, 819504, 823968, 828288, 832752, 135 | 837072, 841536, 841544] 136 | varnames = ['ADVr_SLT', 'ADVr_TH', 'ADVxHEFF', 'ADVxSNOW', 'ADVx_SLT', 137 | 'ADVx_TH', 'ADVyHEFF', 'ADVySNOW', 'ADVy_SLT', 'ADVy_TH', 138 | 'DETADT2', 'DFrE_SLT', 'DFrE_TH', 'DFrI_SLT', 'DFrI_TH', 139 | 'DFxEHEFF', 'DFxESNOW', 'DFxE_SLT', 'DFxE_TH', 'DFyEHEFF', 140 | 'DFyESNOW', 'DFyE_SLT', 'DFyE_TH', 'ETAN', 'ETANSQ', 141 | 'GM_PsiX', 'GM_PsiY', 'KPPg_SLT', 'KPPg_TH', 'MXLDEPTH', 142 | 'PHIBOT', 'SALT', 'SFLUX', 'SIaaflux', 'SIacSubl', 143 | 'SIarea', 'SIatmFW', 'SIatmQnt', 'SIheff', 'SIhsnow', 144 | 'SIsnPrcp', 'SItflux', 'SIuice', 'SIvice', 'SRELAX', 145 | 'TFLUX', 'THETA', 'TRELAX', 'UVELMASS', 'VVELMASS', 146 | 'WSLTMASS', 'WTHMASS', 'WVELMASS', 'oceFWflx', 'oceQnet', 147 | 'oceQsw', 'oceSPDep', 'oceSPflx', 'oceSPtnd', 'oceSflux', 148 | 'oceTAUX', 'oceTAUY', 'sIceLoad'] 149 | 150 | grid_varnames = ['AngleCS', 'AngleSN', 'DRC', 'DRF', 'DXC', 151 | 'DXG', 'DYC', 'DYG', 'Depth', 'PHrefC', 152 | 'PHrefF', 'RAC', 'RAS', 'RAW', 'RAZ', 153 | 'RC', 'RF', 'RhoRef', 'XC', 'XG', 154 | 'YC', 'YG', 'hFacC', 'hFacS', 'hFacW', 155 | 'maskC', 'maskCtrlC', 'maskCtrlS', 'maskCtrlW', 'maskInC', 156 | 'maskInS', 'maskInW', 'maskS', 'maskW'] 157 | 158 | dtype={"ADVr_SLT":">f8", "ADVr_TH":">f8", "ADVxHEFF":">f8", 159 | "ADVxSNOW":">f8", "ADVx_SLT":">f8", "ADVx_TH":">f8", 160 | "ADVyHEFF":">f8", "ADVySNOW":">f8", "ADVy_SLT":">f8", 161 | "ADVy_TH":">f8", "AngleCS":">f8", "AngleSN":">f8", 162 | "DETADT2":">f4", "DFrE_SLT":">f8", "DFrE_TH":">f8", 163 | "DFrI_SLT":">f8", "DFrI_TH":">f8", "DFxEHEFF":">f8", 164 | "DFxESNOW":">f8", "DFxE_SLT":">f8", "DFxE_TH":">f8", 165 | "DFyEHEFF":">f8", "DFyESNOW":">f8", "DFyE_SLT":">f8", 166 | "DFyE_TH":">f8", "DRC":">f8", "DRF":">f8", 167 | "DXC":">f8", "DXG":">f8", "DYC":">f8", 168 | "DYG":">f8", "Depth":">f8", "ETAN":">f4", 169 | "ETANSQ":">f4", "GM_PsiX":">f4", "GM_PsiY":">f4", 170 | "KPPg_SLT":">f8", "KPPg_TH":">f8", "MXLDEPTH":">f4", 171 | "PHIBOT":">f4", "PHrefC":">f8", "PHrefF":">f8", 172 | "RAC":">f8", "RAS":">f8", "RAW":">f8", 173 | "RAZ":">f8", "RC":">f8", "RF":">f8", 174 | "RhoRef":">f8", "SALT":">f4", "SFLUX":">f8", 175 | "SIaaflux":">f8", "SIacSubl":">f8", "SIarea":">f4", 176 | "SIatmFW":">f8", "SIatmQnt":">f8", "SIheff":">f4", 177 | "SIhsnow":">f4", "SIsnPrcp":">f8", "SItflux":">f8", 178 | "SIuice":">f4", "SIvice":">f4", "SRELAX":">f8", 179 | "TFLUX":">f8", "THETA":">f4", "TRELAX":">f8", 180 | "UVELMASS":">f8", "VVELMASS":">f8", "WSLTMASS":">f8", 181 | "WTHMASS":">f8", "WVELMASS":">f8", "XC":">f8", 182 | "XG":">f8", "YC":">f8", "YG":">f8", 183 | "hFacC":">f8", "hFacS":">f8", "hFacW":">f8", 184 | "maskC":">f8", "maskCtrlC":">f8", "maskCtrlS":">f8", 185 | "maskCtrlW":">f8", "maskInC":">f8", "maskInS":">f8", 186 | "maskInW":">f8", "maskS":">f8", "maskW":">f8", 187 | "oceFWflx":">f8", "oceQnet":">f8", "oceQsw":">f8", 188 | "oceSPDep":">f4", "oceSPflx":">f8", "oceSPtnd":">f8", 189 | "oceSflux":">f8", "oceTAUX":">f4", "oceTAUY":">f4", 190 | "sIceLoad":">f4"} 191 | 192 | class ECCOPortalLLC2160Model(LLC2160Model): 193 | 194 | def __init__(self): 195 | fs = _make_http_filesystem() 196 | # base_path = 'https://data.nas.nasa.gov/ecco/download_data.php?file=/eccodata/llc_2160/compressed' 197 | # grid_path = 'https://data.nas.nasa.gov/ecco/download_data.php?file=/eccodata/llc_2160/grid' 198 | mask_path = 'https://storage.googleapis.com/pangeo-ecco/llc/masks/llc_2160_masks.zarr/' 199 | base_path = None 200 | grid_path = None 201 | store = stores.NestedStore(fs, base_path=base_path, mask_path=mask_path, 202 | grid_path=grid_path, shrunk=True, join_char='/') 203 | super(ECCOPortalLLC2160Model, self).__init__(store) 204 | 205 | 206 | class ECCOPortalLLC4320Model(LLC4320Model): 207 | 208 | def __init__(self): 209 | fs = _make_http_filesystem() 210 | # base_path = 'https://data.nas.nasa.gov/ecco/download_data.php?file=/eccodata/llc_4320/compressed' 211 | # grid_path = 'https://data.nas.nasa.gov/ecco/download_data.php?file=/eccodata/llc_4320/grid' 212 | mask_path = 'https://storage.googleapis.com/pangeo-ecco/llc/masks/llc_4320_masks.zarr/' 213 | base_path = None 214 | grid_path = None 215 | store = stores.NestedStore(fs, base_path=base_path, mask_path=mask_path, 216 | grid_path=grid_path, shrunk=True, join_char='/') 217 | super(ECCOPortalLLC4320Model, self).__init__(store) 218 | 219 | 220 | class PleiadesLLC2160Model(LLC2160Model): 221 | 222 | @_requires_pleiades 223 | def __init__(self): 224 | from fsspec.implementations.local import LocalFileSystem 225 | fs = LocalFileSystem() 226 | base_path = '/home6/dmenemen/llc_2160/compressed' 227 | grid_path = '/home6/dmenemen/llc_2160/grid' 228 | mask_path = '/nobackup/rpaberna/llc/masks/llc_2160_masks.zarr' 229 | store = stores.NestedStore(fs, base_path=base_path, mask_path=mask_path, 230 | shrunk=True,grid_path=grid_path) 231 | super(PleiadesLLC2160Model, self).__init__(store) 232 | 233 | 234 | class PleiadesLLC4320Model(LLC4320Model): 235 | 236 | @_requires_pleiades 237 | def __init__(self): 238 | from fsspec.implementations.local import LocalFileSystem 239 | fs = LocalFileSystem() 240 | base_path = '/home6/dmenemen/llc_4320/compressed' 241 | grid_path = '/home6/dmenemen/llc_4320/grid' 242 | mask_path = '/nobackup/rpaberna/llc/masks/llc_4320_masks.zarr' 243 | store = stores.NestedStore(fs, base_path=base_path, mask_path=mask_path, 244 | shrunk=True,grid_path=grid_path) 245 | super(PleiadesLLC4320Model, self).__init__(store) 246 | 247 | class CRIOSPortalASTE270Model(ASTE270Model): 248 | 249 | def __init__(self): 250 | fs = _make_http_filesystem() 251 | base_path = 'https://aste-release1.s3.us-east-2.amazonaws.com/diags' 252 | grid_path = 'https://aste-release1.s3.us-east-2.amazonaws.com/grid' 253 | mask_path = 'https://aste-release1.s3.us-east-2.amazonaws.com/masks.zarr' 254 | store = stores.NestedStore(fs, base_path=base_path, grid_path=grid_path, 255 | mask_path=mask_path, 256 | shrunk=True, join_char='/') 257 | 258 | super(CRIOSPortalASTE270Model, self).__init__(store) 259 | 260 | class SverdrupASTE270Model(ASTE270Model): 261 | 262 | @_requires_sverdrup 263 | def __init__(self): 264 | from fsspec.implementations.local import LocalFileSystem 265 | fs = LocalFileSystem() 266 | base_path = '/scratch2/shared/aste-release1/diags' 267 | grid_path = '/scratch2/shared/aste-release1/grid' 268 | mask_path = '/scratch2/shared/aste-release1/masks.zarr' 269 | store = stores.NestedStore(fs, base_path=base_path, grid_path=grid_path, 270 | mask_path=mask_path, 271 | shrunk=True, join_char='/') 272 | 273 | super(SverdrupASTE270Model, self).__init__(store) 274 | -------------------------------------------------------------------------------- /xmitgcm/test/test_llcreader.py: -------------------------------------------------------------------------------- 1 | import os 2 | import re 3 | import pytest 4 | import warnings 5 | from dask.array.core import Array as dsa 6 | from dask.array.core import PerformanceWarning 7 | 8 | llcreader = pytest.importorskip("xmitgcm.llcreader") 9 | 10 | from .test_xmitgcm_common import llc_mds_datadirs 11 | 12 | EXPECTED_VARS = ['Eta', 'KPPhbl', 'oceFWflx', 'oceQnet', 'oceQsw', 'oceSflux', 13 | 'oceTAUX', 'oceTAUY', 'PhiBot', 'Salt', 'SIarea', 'SIheff', 14 | 'SIhsalt', 'SIhsnow', 'SIuice', 'SIvice', 'Theta', 'U', 'V', 'W'] 15 | GRID_VARNAMES = ['AngleCS', 'AngleSN', 'DRC', 'DRF', 'DXC', 'DXG', 'DYC', 'DYG', 16 | 'Depth', 'PHrefC', 'PHrefF', 'RAC', 'RAS', 'RAW', 'RAZ', 'RC', 'RF', 17 | 'RhoRef', 'XC', 'XG', 'YC', 'YG', 'hFacC', 'hFacS', 'hFacW'] 18 | 19 | 20 | EXPECTED_COORDS = {2160: ['CS','SN','Depth', 21 | 'drC','drF','dxC','dxF','dxG','dyC','dyF','dyG', 22 | 'hFacC','hFacS','hFacW','PHrefC','PHrefF','rA','rAs','rAw', 23 | 'Z','Zp1','Zl','Zu','rhoRef','rLowC','rLowS','rLowW', 24 | 'rSurfC','rSurfS','rSurfW','XC','YC','rAz','XG','YG', 25 | 'dxV','dyU'], 26 | 4320: ['CS','SN','Depth', 27 | 'drC','drF','dxC','dxF','dxG','dyC','dyF','dyG', 28 | 'hFacC','hFacS','hFacW','PHrefC','PHrefF', 29 | 'rA','rAs','rAw','rhoRef','Z','Zp1','Zl','Zu','XC','YC', 30 | 'rAz','XG','YG','dxV','dyU'], 31 | 'aste_270': ["CS", "Depth", "PHrefC", "PHrefF", "SN", 32 | "XC", "XG", "YC", "YG", "Z", 33 | "Zl", "Zp1", "Zu", "drC", "drF", 34 | "dxC", "dxG", "dyC", "dyG", "hFacC", 35 | "hFacS", "hFacW", "maskC", "maskCtrlC", "maskCtrlS", 36 | "maskCtrlW", "maskInC", "maskInS", "maskInW", "maskS", 37 | "maskW", "niter", "rA", "rAs", "rAw", 38 | "rAz", "rhoRef"]} 39 | 40 | ########### Generic llcreader tests on local data ############################## 41 | 42 | @pytest.fixture(scope='module') 43 | def local_llc90_store(llc_mds_datadirs): 44 | from fsspec.implementations.local import LocalFileSystem 45 | dirname, expected = llc_mds_datadirs 46 | fs = LocalFileSystem() 47 | store = llcreader.BaseStore(fs, base_path=dirname, grid_path=dirname) 48 | return store 49 | 50 | @pytest.fixture(scope='module') 51 | def llc90_kwargs(): 52 | return dict(varnames=['S', 'T', 'U', 'V', 'Eta'], 53 | iter_start=0, iter_stop=9, iter_step=8) 54 | 55 | def test_llc90_local_faces(local_llc90_store, llc90_kwargs): 56 | store = local_llc90_store 57 | model = llcreader.LLC90Model(store) 58 | ds_faces = model.get_dataset(**llc90_kwargs) 59 | assert set(llc90_kwargs['varnames']) == set(ds_faces.data_vars) 60 | assert ds_faces.sizes == {'face': 13, 'i': 90, 'i_g': 90, 'j': 90, 'j_g': 90, 61 | 'k': 50, 'k_u': 50, 'k_l': 50, 'k_p1': 51, 'time': 2} 62 | 63 | def test_llc90_dim_metadata(local_llc90_store, llc90_kwargs): 64 | store = local_llc90_store 65 | model = llcreader.LLC90Model(store) 66 | ds_faces = model.get_dataset(**llc90_kwargs) 67 | assert ds_faces.i.attrs['axis'] == 'X' 68 | 69 | def test_llc90_local_latlon(local_llc90_store, llc90_kwargs): 70 | store = local_llc90_store 71 | model = llcreader.LLC90Model(store) 72 | ds_latlon = model.get_dataset(type='latlon', **llc90_kwargs) 73 | assert set(llc90_kwargs['varnames']) == set(ds_latlon.data_vars) 74 | assert ds_latlon.sizes == {'i': 360, 'time': 2, 'k_p1': 51, 'face': 13, 75 | 'i_g': 360, 'k_u': 50, 'k': 50, 'k_l': 50, 76 | 'j_g': 270, 'j': 270} 77 | 78 | 79 | # includes regression test for https://github.com/MITgcm/xmitgcm/issues/233 80 | @pytest.mark.parametrize('rettype', ['faces', 'latlon']) 81 | @pytest.mark.parametrize('k_levels, kp1_levels, k_chunksize', 82 | [(None, None, 1), 83 | ([1], [1, 2], 1), 84 | ([0, 2, 7, 9, 10, 20], 85 | [0,1,2,3,7,8,9,10,11,20,21], 1), 86 | ([0, 2, 7, 9, 10, 20], 87 | [0,1,2,3,7,8,9,10,11,20,21], 2) 88 | ]) 89 | @pytest.mark.parametrize('read_grid', [False, True] 90 | ) 91 | def test_llc90_local_faces_load(local_llc90_store, llc90_kwargs, rettype, k_levels, 92 | kp1_levels, k_chunksize, read_grid): 93 | store = local_llc90_store 94 | model = llcreader.LLC90Model(store) 95 | model.grid_varnames = GRID_VARNAMES 96 | ds = model.get_dataset(k_levels=k_levels, k_chunksize=k_chunksize, 97 | type=rettype, read_grid=read_grid, **llc90_kwargs) 98 | if read_grid: 99 | # doesn't work because the variables change name 100 | # assert set(GRID_VARNAMES).issubset(set(ds.coords)) 101 | pass 102 | if k_levels is None: 103 | assert list(ds.k.values) == list(range(50)) 104 | assert list(ds.k_p1.values) == list(range(51)) 105 | else: 106 | assert list(ds.k.values) == k_levels 107 | assert list(ds.k_p1.values) == kp1_levels 108 | assert all([cs==k_chunksize for cs in ds['T'].data.chunks[1]]) 109 | 110 | ds.load() 111 | 112 | 113 | @pytest.mark.parametrize('varname', [['U'], ['V']]) 114 | def test_vector_mate_error(local_llc90_store, varname): 115 | store = local_llc90_store 116 | model = llcreader.LLC90Model(store) 117 | with pytest.raises(ValueError, match=r".* must also be .*"): 118 | ds_latlon = model.get_dataset(type='latlon', varnames=varname, iter_start=0, iter_stop=9, iter_step=8) 119 | 120 | 121 | ########### ECCO Portal Tests ################################################## 122 | 123 | @pytest.fixture(scope='module', params=[('portal', 2160), ('portal', 4320), 124 | ('pleiades',2160), ('pleiades',4320)]) 125 | @pytest.mark.skip(reason="ECCO Portal is depracated") 126 | def llc_global_model(request): 127 | if request.param[0]=='portal': 128 | if request.param[1]==2160: 129 | return llcreader.ECCOPortalLLC2160Model() 130 | else: 131 | return llcreader.ECCOPortalLLC4320Model() 132 | else: 133 | 134 | if not os.path.exists('/home6/dmenemen'): 135 | pytest.skip("Not on Pleiades") 136 | else: 137 | if request.param[1]==2160: 138 | return llcreader.PleiadesLLC2160Model() 139 | else: 140 | return llcreader.PleiadesLLC4320Model() 141 | 142 | @pytest.mark.skip(reason="ECCO Data path is depracated") 143 | def test_ecco_portal_faces(llc_global_model): 144 | # just get three timesteps 145 | iter_stop = llc_global_model.iter_start + 2 * llc_global_model.iter_step + 1 146 | ds_faces = llc_global_model.get_dataset(iter_stop=iter_stop) 147 | nx = llc_global_model.nx 148 | assert ds_faces.sizes == {'face': 13, 'i': nx, 'i_g': nx, 'j': nx, 149 | 'j_g': nx, 'k': 90, 'k_u': 90, 'k_l': 90, 150 | 'k_p1': 91, 'time': 3} 151 | assert set(EXPECTED_VARS) == set(ds_faces.data_vars) 152 | assert set(EXPECTED_COORDS[nx]).issubset(set(ds_faces.coords)) 153 | 154 | # make sure vertical coordinates are in one single chunk 155 | for fld in ds_faces[['Z','Zl','Zu','Zp1']].coords: 156 | if isinstance(ds_faces[fld].data,dsa): 157 | assert len(ds_faces[fld].data.chunks)==1 158 | assert (len(ds_faces[fld]),)==ds_faces[fld].data.chunks[0] 159 | 160 | 161 | @pytest.mark.skip(reason="ECCO Portal is depracated") 162 | def test_ecco_portal_iterations(llc_global_model): 163 | with pytest.warns(RuntimeWarning, match=r"Iteration .* may not exist, you may need to change 'iter_start'"): 164 | llc_global_model.get_dataset(varnames=['Eta'], iter_start=llc_global_model.iter_start + 1, read_grid=False) 165 | 166 | with pytest.warns(RuntimeWarning, match=r"'iter_step' is not a multiple of .*, meaning some expected timesteps may not be returned"): 167 | llc_global_model.get_dataset(varnames=['Eta'], iter_step=llc_global_model.iter_step - 1, read_grid=False) 168 | 169 | with pytest.warns(RuntimeWarning, match=r"Some requested iterations may not exist, you may need to change 'iters'"): 170 | iters = [llc_global_model.iter_start, llc_global_model.iter_start + 1] 171 | llc_global_model.get_dataset(varnames=['Eta'], iters=iters, read_grid=False) 172 | 173 | with warnings.catch_warnings(record=True) as record: 174 | # Make sure only deprecation warnings are triggered 175 | warnings.simplefilter("ignore", category=DeprecationWarning) 176 | warnings.simplefilter("ignore", category=PerformanceWarning) 177 | llc_global_model.get_dataset(varnames=['Eta'], read_grid=False) 178 | assert not record 179 | 180 | 181 | #@pytest.mark.slow 182 | @pytest.mark.skip(reason="ECCO Data path is depracated") 183 | def test_ecco_portal_load(llc_global_model): 184 | # an expensive test because it actually loads data 185 | iter_stop = llc_global_model.iter_start + 2 * llc_global_model.iter_step + 1 186 | ds_faces = llc_global_model.get_dataset(varnames=['Eta'], iter_stop=iter_stop) 187 | # a lookup table 188 | expected = {2160: -1.3054643869400024, 4320: -1.262018084526062} 189 | assert ds_faces.Eta[0, 0, -1, -1].values.item() == expected[llc_global_model.nx] 190 | 191 | @pytest.mark.skip(reason="ECCO Data path is depracated") 192 | def test_ecco_portal_latlon(llc_global_model): 193 | iter_stop = llc_global_model.iter_start + 2 * llc_global_model.iter_step + 1 194 | ds_ll = llc_global_model.get_dataset(iter_stop=iter_stop, type='latlon') 195 | nx = llc_global_model.nx 196 | assert ds_ll.sizes == {'i': 4*nx, 'k_u': 90, 'k_l': 90, 'time': 3, 197 | 'k': 90, 'j_g': 3*nx, 'i_g': 4*nx, 'k_p1': 91, 198 | 'j': 3*nx, 'face': 13} 199 | assert set(EXPECTED_VARS) == set(ds_ll.data_vars) 200 | assert set(EXPECTED_COORDS[nx]).issubset(set(ds_ll.coords)) 201 | 202 | # make sure vertical coordinates are in one single chunk 203 | for fld in ds_ll[['Z','Zl','Zu','Zp1']].coords: 204 | if isinstance(ds_ll[fld].data,dsa): 205 | assert len(ds_ll[fld].data.chunks)==1 206 | assert (len(ds_ll[fld]),)==ds_ll[fld].data.chunks[0] 207 | 208 | 209 | ########### ASTE Portal Tests ################################################## 210 | #@pytest.fixture(scope='module', params=['portal','sverdrup']) 211 | @pytest.fixture(scope='module', params=['sverdrup']) 212 | def aste_model(request): 213 | if request.param == 'portal': 214 | return llcreader.CRIOSPortalASTE270Model() 215 | else: 216 | if not os.path.exists('/scratch2/heimbach'): 217 | pytest.skip("Not on Sverdrup") 218 | else: 219 | return llcreader.SverdrupASTE270Model() 220 | 221 | def test_aste_portal_faces(aste_model): 222 | # just get three timesteps 223 | iters = aste_model.iters[:3] 224 | ds_faces = aste_model.get_dataset(iters=iters) 225 | nx = aste_model.nx 226 | assert ds_faces.sizes == {'face': 6, 'i': nx, 'i_g': nx, 'j': nx, 227 | 'j_g': nx, 'k': 50, 'k_u': 50, 'k_l': 50, 228 | 'k_p1': 51, 'time': 3} 229 | assert set(aste_model.varnames) == set(ds_faces.data_vars) 230 | assert set(EXPECTED_COORDS['aste_270']).issubset(set(ds_faces.coords)) 231 | 232 | # make sure vertical coordinates are in one single chunk 233 | for fld in ds_faces[['Z','Zl','Zu','Zp1']].coords: 234 | if isinstance(ds_faces[fld].data,dsa): 235 | assert len(ds_faces[fld].data.chunks)==1 236 | assert (len(ds_faces[fld]),)==ds_faces[fld].data.chunks[0] 237 | 238 | 239 | def test_aste_portal_iterations(aste_model): 240 | with pytest.warns(RuntimeWarning, match=r"Some requested iterations may not exist, you may need to change 'iters'"): 241 | iters = aste_model.iters[:2] 242 | iters[1] = iters[1] + 1 243 | aste_model.get_dataset(varnames=['ETAN'], iters=iters, read_grid=False) 244 | 245 | with warnings.catch_warnings(record=True) as record: 246 | # Make sure only deprecation warnings are triggered 247 | warnings.simplefilter("ignore", category=DeprecationWarning) 248 | iters = aste_model.iters[:2] 249 | aste_model.get_dataset(varnames=['ETAN'], iters=iters, read_grid=False) 250 | assert not record 251 | 252 | 253 | @pytest.mark.slow 254 | def test_aste_portal_load(aste_model): 255 | # an expensive test because it actually loads data 256 | iters = aste_model.iters[:3] 257 | ds_faces = aste_model.get_dataset(varnames=['ETAN'], iters=iters) 258 | 259 | try: 260 | values = ds_faces.ETAN[0,1,0,0].compute() 261 | except Exception as e: 262 | pytest.skip(f"Skip test: ASTE model {aste_model} loaded empty") 263 | expected = 0.641869068145752 264 | assert ds_faces.ETAN[0, 1, 0, 0].values.item() == expected 265 | 266 | def test_aste_portal_latlon(aste_model): 267 | iters = aste_model.iters[:3] 268 | with pytest.raises(TypeError): 269 | ds_ll = aste_model.get_dataset(iters=iters,type='latlon') 270 | 271 | 272 | -------------------------------------------------------------------------------- /xmitgcm/test/test_xmitgcm_common.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import os 3 | import sys 4 | import fnmatch 5 | import tarfile 6 | import numpy as np 7 | import dask 8 | from contextlib import contextmanager 9 | import py 10 | import tempfile 11 | import hashlib 12 | try: 13 | import urllib.request as req 14 | except ImportError: 15 | # urllib in python2 has different structure 16 | import urllib as req 17 | 18 | from xmitgcm.file_utils import clear_cache 19 | 20 | @contextmanager 21 | def hide_file(origdir, *basenames): 22 | """Temporarily hide files within the context.""" 23 | # make everything a py.path.local 24 | tmpdir = py.path.local(tempfile.mkdtemp()) 25 | origdir = py.path.local(origdir) 26 | oldpaths = [origdir.join(basename) for basename in basenames] 27 | newpaths = [tmpdir.join(basename) for basename in basenames] 28 | 29 | # move the files 30 | for oldpath, newpath in zip(oldpaths, newpaths): 31 | oldpath.rename(newpath) 32 | # clear the cache if it exists 33 | clear_cache() 34 | try: 35 | yield str(tmpdir) 36 | finally: 37 | # move them back 38 | for oldpath, newpath in zip(oldpaths, newpaths): 39 | newpath.rename(oldpath) 40 | 41 | 42 | dlroot = 'https://ndownloader.figshare.com/files/' 43 | 44 | # parameterized fixture are complicated 45 | # http://docs.pytest.org/en/latest/fixture.html#fixture-parametrize 46 | 47 | # dictionary of archived experiments and some expected properties 48 | _experiments = { 49 | 'global_oce_latlon': {'geometry': 'sphericalpolar', 50 | 'dlink': dlroot + '14066630', 51 | 'md5': '0a846023d01cbcc16bed4963431968cf', 52 | 'shape': (15, 40, 90), 'test_iternum': 39600, 53 | 'expected_values': {'XC': ((0, 0), 2)}, 54 | 'dtype': np.dtype('f4'), 55 | 'layers': {'1RHO': 31}, 56 | 'diagnostics': ('DiagGAD-T', 57 | ['TOTTTEND', 'ADVr_TH', 58 | 'ADVx_TH', 'ADVy_TH', 59 | 'DFrE_TH', 'DFxE_TH', 60 | 'DFyE_TH', 'DFrI_TH', 61 | 'UTHMASS', 'VTHMASS', 'WTHMASS'])}, 62 | 'barotropic_gyre': {'geometry': 'cartesian', 63 | 'dlink': dlroot + '14066618', 64 | 'md5': '5200149791bfd24989ad8b98c18937dc', 65 | 'shape': (1, 60, 60), 'test_iternum': 10, 66 | 'dtype': np.dtype('f4'), 67 | 'expected_values': {'XC': ((0, 0), 10000.0)}, 68 | 'all_iters': [0, 10], 69 | 'delta_t': 3600, 70 | 'prefixes': ['T', 'S', 'Eta', 'U', 'V', 'W']}, 71 | 'internal_wave': {'geometry': 'sphericalpolar', 72 | 'dlink': dlroot + '14066642', 73 | 'md5': 'eedfab1aec365fd8c17d3bc0f86a1431', 74 | 'shape': (20, 1, 30), 'test_iternum': 100, 75 | 'dtype': np.dtype('f8'), 76 | 'expected_values': {'XC': ((0, 0), 109.01639344262296)}, 77 | 'all_iters': [0, 100, 200], 78 | 'ref_date': "1990-1-1", 79 | 'delta_t': 60, 80 | 'expected_time': [ 81 | (0, np.datetime64('1990-01-01T00:00:00.000000000')), 82 | (1, np.datetime64('1990-01-01T01:40:00.000000000'))], 83 | # these diagnostics won't load because not all levels 84 | # were output...no idea how to overcome that bug 85 | # 'diagnostics': ('diagout1', ['UVEL', 'VVEL']), 86 | 'prefixes': ['T', 'S', 'Eta', 'U', 'V', 'W']}, 87 | 'global_oce_llc90': {'geometry': 'llc', 88 | 'dlink': dlroot + '14066567', 89 | 'md5': '6c309416f91ae9baaf1fb21b3dc50e49', 90 | 'ref_date': "1948-01-01 12:00:00", 91 | 'delta_t': 3600, 92 | 'expected_time': [ 93 | (0, np.datetime64('1948-01-01T12:00:00.000000000')), 94 | (1, np.datetime64('1948-01-01T20:00:00.000000000'))], 95 | 'shape': (50, 13, 90, 90), 'test_iternum': 8, 96 | 'dtype': np.dtype('f4'), 97 | 'expected_values': {'XC': ((2, 3, 5), -32.5)}, 98 | 'diagnostics': ('state_2d_set1', ['ETAN', 99 | 'SIarea', 100 | 'SIheff', 101 | 'SIhsnow', 102 | 'DETADT2', 103 | 'PHIBOT', 104 | 'sIceLoad', 105 | 'MXLDEPTH', 106 | 'oceSPDep', 107 | 'SIatmQnt', 108 | 'SIatmFW', 109 | 'oceQnet', 110 | 'oceFWflx', 111 | 'oceTAUX', 112 | 'oceTAUY', 113 | 'ADVxHEFF', 114 | 'ADVyHEFF', 115 | 'DFxEHEFF', 116 | 'DFyEHEFF', 117 | 'ADVxSNOW', 118 | 'ADVySNOW', 119 | 'DFxESNOW', 120 | 'DFyESNOW', 121 | 'SIuice', 122 | 'SIvice'])}, 123 | 'curvilinear_leman': {'geometry': 'curvilinear', 124 | 'dlink': dlroot + '14066621', 125 | 'md5': 'c3203ae1fb0d6a61174dd680b7660894', 126 | 'delta_t': 20, 127 | 'ref_date': "2013-11-12 12:00", 128 | 'shape': (35, 64, 340), 129 | 'test_iternum': 6, 130 | 'dtype': np.dtype('f4'), 131 | 'expected_values': {'XC': ((0, 0), 501919.21875)}, 132 | 'all_iters': [0, 3, 6], 133 | 'expected_time': [ 134 | (0, np.datetime64('2013-11-12T12:00:00.000000000')), 135 | (1, np.datetime64('2013-11-12T12:02:00.000000000'))], 136 | 'prefixes': ['THETA']}, 137 | 138 | 'global_oce_cs32': {'geometry': 'cs', 139 | 'dlink': dlroot + '14066390', 140 | 'md5': '209193f4a183307b89337bb18e2e9451', 141 | 'shape': (15, 32, 6, 32), 'test_iternum': 72020, 142 | 'dtype': np.dtype('f4'), 143 | 'expected_values': {'XC': ((2, 3, 5), -29.98921)}, 144 | 'diagnostics': ('oceDiag', ['DRHODR', 145 | 'RHOAnoma', 146 | 'CONVADJ', 147 | 'GM_Kwx', 148 | 'GM_Kwy', 149 | 'GM_Kwz', 150 | 'GM_PsiX', 151 | 'GM_PsiY'])}, 152 | 153 | 'ideal_2D_oce': {'geometry': 'sphericalpolar', 154 | 'dlink': dlroot + '17288255', 155 | 'md5': 'd8868731ff6a8fd951babefbc5ea69ba', 156 | 'expected_namelistvals': {'eosType': 'LINEAR', 157 | 'viscAh': 12e5, 158 | 'niter0': 36000, 159 | 'delX': [3.], 160 | 'fileName': ['surfDiag', 'dynDiag', 161 | 'oceDiag', 'flxDiag'], 162 | 'levels': [[1.0], [], 163 | [2., 3., 4., 5., 6., 164 | 7., 8., 9., 10., 11., 165 | 12., 13.]], 166 | 'useDiagnostics': True}, 167 | 'diag_levels': {'surfDiag': ([1], (0, -50)), 168 | 'oceDiag': (slice(2, 14), (0, -122.5))}, 169 | 'expected_values': {'XC': ((0, 0), 1.5)}, 170 | 'shape': (15, 56, 1), 171 | 'test_iternum': 36020, 172 | 'dtype': np.dtype('f4')} 173 | } 174 | 175 | 176 | _grids = { 177 | 'grid_llc90': {'geometry': 'llc', 'domain': 'llc', 178 | 'dlink': dlroot + '14072594', 179 | 'md5': 'f66c3195a62790d539debe6ca8f3a851', 180 | 'gridfile': 'tile.mitgrid', 181 | 'nx': 90, 'shape': (13, 90, 90), 182 | 'endianness': '>', 'precision': 'd'}, 183 | 184 | 'grid_aste270': {'geometry': 'llc', 'domain': 'aste', 185 | 'dlink': dlroot + '14072591', 186 | 'md5': '92b28c65e0dfb54b253bfcd0a249359b', 187 | 'gridfile': 'tile.mitgrid', 188 | 'nx': 270, 'shape': (6, 270, 270), 189 | 'endianness': '>', 'precision': 'd'}, 190 | 191 | 'grid_cs32': {'geometry': 'cs', 'domain': 'cs', 192 | 'dlink': dlroot + '14072597', 193 | 'md5': '848cd5b6daab5b069e96a0cff67d4b57', 194 | 'gridfile': 'grid_cs32.face.bin', 195 | 'nx': 32, 'shape': (6, 32, 32), 196 | 'endianness': '>', 'precision': 'd'}, 197 | } 198 | 199 | 200 | def setup_mds_dir(tmpdir_factory, request, db): 201 | """Helper function for setting up test cases.""" 202 | expt_name = request.param 203 | expected_results = db[expt_name] 204 | target_dir = str(tmpdir_factory.mktemp('mdsdata')) 205 | try: 206 | # user-defined directory for test datasets 207 | data_dir = os.environ["XMITGCM_TESTDATA"] 208 | except KeyError: 209 | # default to HOME/.xmitgcm-test-data/ 210 | data_dir = os.environ["HOME"] + '/.xmitgcm-test-data' 211 | # create the directory if it doesn't exixt 212 | if not os.path.exists(data_dir): 213 | os.makedirs(data_dir) 214 | 215 | datafile = os.path.join(data_dir, expt_name + '.tar.gz') 216 | # download if does not exist locally 217 | if not os.path.exists(datafile): 218 | print('File does not exist locally, downloading...') 219 | download_archive(expected_results['dlink'], datafile) 220 | localmd5 = file_md5_checksum(datafile) 221 | if localmd5 != expected_results['md5']: 222 | os.remove(datafile) 223 | msg = """ 224 | MD5 checksum does not match, try downloading dataset again. 225 | """ 226 | raise IOError(msg) 227 | 228 | return untar(data_dir, expt_name, target_dir), expected_results 229 | 230 | 231 | def download_archive(url, filename): 232 | """ download file from url into datafile 233 | 234 | PARAMETERS: 235 | 236 | url: str 237 | url to retrieve 238 | filename: str 239 | file to save on disk 240 | """ 241 | 242 | req.urlretrieve(url, filename) 243 | return None 244 | 245 | 246 | def untar(data_dir, basename, target_dir): 247 | """Unzip a tar file into the target directory. Return path to unzipped 248 | directory.""" 249 | datafile = os.path.join(data_dir, basename + '.tar.gz') 250 | if not os.path.exists(datafile): 251 | raise IOError('Could not find data file %s' % datafile) 252 | tar = tarfile.open(datafile) 253 | if sys.version >= '3.12': 254 | tar.extractall(target_dir,filter='data') 255 | else: 256 | tar.extractall(target_dir) 257 | tar.close() 258 | # subdirectory where file should have been untarred. 259 | # assumes the directory is the same name as the tar file itself. 260 | # e.g. testdata.tar.gz --> testdata/ 261 | fulldir = os.path.join(target_dir, basename) 262 | if not os.path.exists(fulldir): 263 | raise IOError('Could not find tar file output dir %s' % fulldir) 264 | # the actual data lives in a file called testdata 265 | # clean up ugly weird hidden files that mac-os sometimes puts in the archive 266 | # https://unix.stackexchange.com/questions/9665/create-tar-archive-of-a-directory-except-for-hidden-files 267 | # https://superuser.com/questions/259703/get-mac-tar-to-stop-putting-filenames-in-tar-archives 268 | bad_files = [f for f in os.listdir(fulldir) 269 | if fnmatch.fnmatch(f, '._*') ] 270 | for f in bad_files: 271 | os.remove(os.path.join(fulldir, f)) 272 | 273 | return fulldir 274 | 275 | 276 | def file_md5_checksum(fname): 277 | hash_md5 = hashlib.md5() 278 | with open(fname, "rb") as f: 279 | hash_md5.update(f.read()) 280 | return hash_md5.hexdigest() 281 | 282 | 283 | # find the tar archive in the test directory 284 | # http://stackoverflow.com/questions/29627341/pytest-where-to-store-expected-data 285 | @pytest.fixture(scope='module', params=_experiments.keys()) 286 | def all_mds_datadirs(tmpdir_factory, request): 287 | return setup_mds_dir(tmpdir_factory, request, _experiments) 288 | 289 | 290 | @pytest.fixture(scope='module', params=['barotropic_gyre', 'internal_wave']) 291 | def multidim_mds_datadirs(tmpdir_factory, request): 292 | return setup_mds_dir(tmpdir_factory, request, _experiments) 293 | 294 | 295 | @pytest.fixture(scope='module', params=['global_oce_latlon', 296 | 'global_oce_llc90']) 297 | def mds_datadirs_with_diagnostics(tmpdir_factory, request): 298 | return setup_mds_dir(tmpdir_factory, request, _experiments) 299 | 300 | 301 | @pytest.fixture(scope='module', params=['internal_wave', 'global_oce_llc90']) 302 | def mds_datadirs_with_refdate(tmpdir_factory, request): 303 | return setup_mds_dir(tmpdir_factory, request, _experiments) 304 | 305 | 306 | @pytest.fixture(scope='module', params=['global_oce_latlon']) 307 | def layers_mds_datadirs(tmpdir_factory, request): 308 | return setup_mds_dir(tmpdir_factory, request, _experiments) 309 | 310 | 311 | @pytest.fixture(scope='module', params=['global_oce_llc90']) 312 | def llc_mds_datadirs(tmpdir_factory, request): 313 | return setup_mds_dir(tmpdir_factory, request, _experiments) 314 | 315 | 316 | @pytest.fixture(scope='module', params=['global_oce_cs32']) 317 | def cs_mds_datadirs(tmpdir_factory, request): 318 | return setup_mds_dir(tmpdir_factory, request, _experiments) 319 | 320 | 321 | @pytest.fixture(scope='module', params=_grids.keys()) 322 | def all_grid_datadirs(tmpdir_factory, request): 323 | return setup_mds_dir(tmpdir_factory, request, _grids) 324 | 325 | 326 | @pytest.fixture(scope='module', params=['ideal_2D_oce']) 327 | def mds_datadirs_with_inputfiles(tmpdir_factory, request): 328 | return setup_mds_dir(tmpdir_factory, request, _experiments) 329 | -------------------------------------------------------------------------------- /doc/usage.rst: -------------------------------------------------------------------------------- 1 | 2 | Reading MDS Data 3 | ================ 4 | 5 | open_mdsdataset 6 | --------------- 7 | 8 | All loading of data in xmitgcm occurs through the function ``open_mdsdataset``. 9 | Its full documentation below enumerates all the possible options. 10 | 11 | .. autofunction:: xmitgcm.open_mdsdataset 12 | 13 | The optional arguments are explained in more detail in the following sections 14 | and examples. 15 | 16 | Lazy Evaluation and Dask Chunking 17 | --------------------------------- 18 | 19 | ``open_mdsdataset`` does not actually load all of the data into memory 20 | when it is invoked. Rather, it opens the files using 21 | `numpy.memmap `_, 22 | which means that the data is not read until it is actually needed for 23 | a computation. This is a cheap form of 24 | `lazy evaluation `_. 25 | 26 | Additional performance benefits can be achieved with 27 | `xarray dask chunking `_:: 28 | 29 | ds_chunked = open_mdsdataset(data_dir, chunks={'Z':1, 'Zl':1}) 30 | 31 | In the example above, the each horizontal slice of the model is assigned to its 32 | own chunk; dask_ will automatically try to parellize operations across these 33 | chunks using all your CPU cores. For this small example dataset, no performance 34 | boost is expected (due to the overhead of parallelization), but very large 35 | simulations will certainly benefit from chunking. 36 | 37 | When chunking is applied, the data are represetned by `dask`_ arrays, and all 38 | operations are evaluated lazily. No computation actually takes place until you 39 | call ``.load()``:: 40 | 41 | # take the mean of the squared zonal velocity 42 | (ds_chunked.U**2).mean() 43 | >>> 44 | dask.array 45 | # load the value and execute the dask computational graph 46 | >>> 47 | array(0.00020325234800111502, dtype=float32) 48 | 49 | .. note:: 50 | In certain cases, chunking is applied automatically. These cases are 51 | 52 | - If there is more than one timestep to be read (see :ref:`expected-files`) 53 | the time dimension is automatically chunked. 54 | - In llc simulations, (see :ref:`geometries`) the ``face`` dimension is 55 | automatically chunked. 56 | 57 | .. _expected-files: 58 | 59 | Expected Files 60 | -------------- 61 | 62 | MITgcm writes MDS output as pairs of files with the suffixes ``*.data`` and 63 | ``*.meta``. The model timestep iteration number is represented by a 64 | ten-digit number at the end of the filename, e.g. ``T.0000000090.data`` and 65 | ``T.0000000090.meta``. MDS files without an iteration number are grid files. 66 | 67 | xmitgcm has certain expectations regarding the files that will be present in 68 | ``datadir``. In particular, it assumes ``datadir`` is an MITgcm "run" directory. 69 | By default, `open_mdsdataset` will read the grid files which describe the 70 | geometry of the computational domain. If these files are not present in 71 | ``datadir``, this behavior can be turned off by setting ``read_grid=False``. 72 | 73 | In order to determine the dimensions of the model domain ``open_mdsdataset`` 74 | needs to peek at the metadata in two grid files: ``XC.meta`` and ``RC.meta``. 75 | (even when ``read_grid=False``). If these files are not available, you have the 76 | option to manually specify the parameters ``nx``, ``ny``, and ``nz`` as keyword 77 | arguments to ``open_mdsdataset``. (``ny`` is not required for 78 | ``geometry='llc'``). 79 | 80 | By default, ``open_mdsdataset`` attempts to read all the data files in 81 | ``datadir``. The files and iteration numbers to read are determined in the 82 | following way: 83 | 84 | #. First ``datadir`` is scanned to determine all iteration numbers 85 | present in the directory. To override this behavior and manually specify the 86 | iteration numbers to read, use the ``iters`` keyword argument, e.g. 87 | ``iters=[10,20,30]``. 88 | 89 | #. To dertmine the file prefixes to read, ``open_mdsdataset`` looks for all 90 | ``*.data`` filenames which match the *first* iteration number. To override 91 | this behavior and manually specify the file prefixes via the ``prefix`` 92 | keyword argument, e.g. ``prefix=['UTave', 'VTave']``. 93 | 94 | #. ``open_mdsdataset`` then looks for each file prefix at each iteration 95 | number. 96 | 97 | This approach works for the test examples, but perhaps it does not suit your 98 | model configuration. Suggestions are welcome on how to improve the discovery of 99 | files in the form of issues_ and `pull requests`_. 100 | 101 | .. warning:: 102 | If you have certain file prefixes that are present at the first iteration 103 | (e.g. ``T.0000000000.data``) but not at later iterations (e.g 104 | ``iters=[0,10]``) but there is no ``T.0000000010.data`` file, 105 | ``open_mdsdataset`` will raise an error because it can't find the expected 106 | files. To overcome this you need to manually specify ``iters`` and / or 107 | ``prefix``. 108 | 109 | To determine the variable metadata, xmitgcm is able to parse the 110 | model's ``available_diagnostics.log`` file. If you use diagnostic output, 111 | the ``available_diagnostics.log`` file corresponding with your model run should 112 | be present in ``datadir``. 113 | 114 | .. note:: 115 | If the ``available_diagnostics.log`` file can't be found, a 116 | `default version `_ 117 | will be used. This could lead to problems, since you may have custom 118 | diagnostics enabled in your run that are not present in the default. 119 | The default ``available_diagnostics.log`` file was taken from the ECCOv4 120 | ``global_oce_llc90`` experiment. 121 | 122 | For non-diagnostic output (e.g. default "state" or "timeave" output), xmitgcm 123 | assigns the variable metadata based on filenames. The additional metadata 124 | makes the internal represtation of the model variables more verbose and 125 | ensures compliance with `CF Conventions`_. 126 | 127 | Dimensions and Coordinates 128 | -------------------------- 129 | 130 | One major advantage of using xarray_ to represent data is that the variable 131 | dimensions are *labeled*, much like netCDF data structures. This labeling 132 | enables much clearer code. For example, to take a time average of a Dataset, 133 | one just says ``ds.mean(dim='time')`` without having to remember which logical 134 | axis is the time dimension. 135 | 136 | xmitgcm distinguishes between *logical dimensions* and *physical dimensions* or 137 | coordinates. Open ``open_mdsdataset`` will attempt to assign physical 138 | dimensions to the data. The physical dimensions correspond to the axes of 139 | the MITgcm grids in ``cartesian`` or ``sphericalpolar`` coordinates. The 140 | standard names have been assigned according to `CF Conventions`_. 141 | 142 | +----------------------+---------------------------------------+ 143 | | name | standard_name | 144 | +======================+=======================================+ 145 | | time | time | 146 | +----------------------+---------------------------------------+ 147 | | XC | longitude | 148 | +----------------------+---------------------------------------+ 149 | | YC | latitude | 150 | +----------------------+---------------------------------------+ 151 | | XG | longitude_at_f_location | 152 | +----------------------+---------------------------------------+ 153 | | YG | latitude_at_f_location | 154 | +----------------------+---------------------------------------+ 155 | | Zl | depth_at_lower_w_location | 156 | +----------------------+---------------------------------------+ 157 | | Zu | depth_at_upper_w_location | 158 | +----------------------+---------------------------------------+ 159 | | Z | depth | 160 | +----------------------+---------------------------------------+ 161 | | Zp1 | depth_at_w_location | 162 | +----------------------+---------------------------------------+ 163 | | layer_1RHO_center | ocean_layer_coordinate_1RHO_center | 164 | +----------------------+---------------------------------------+ 165 | | layer_1RHO_interface | ocean_layer_coordinate_1RHO_interface | 166 | +----------------------+---------------------------------------+ 167 | | layer_1RHO_bounds | ocean_layer_coordinate_1RHO_bounds | 168 | +----------------------+---------------------------------------+ 169 | 170 | The physical dimensions of typical veriables are:: 171 | 172 | print(ds.THETA.dims) 173 | >>> ('time', 'Z', 'YC', 'XC') 174 | print(ds.UVEL.dims) 175 | >>> ('time', 'Z', 'YC', 'XG') 176 | print(ds.VVEL.dims) 177 | >>> ('time', 'Z', 'YG', 'XC') 178 | print(ds.WVEl.dims) 179 | >>> ('time', 'Zl', 'YC', 'XG') 180 | 181 | In order for physical dimensions to be assigned ``open_mdsdataset`` must be 182 | involved with ``read_grid=True`` (default). For a more minimalistic approach, 183 | one can use ``read_grid=False`` and assign logcial dimensions. Logical 184 | dimensions can also be chosen by explicitly setting ``swap_dims=False``, even 185 | with ``read_grid=False``. 186 | Physical dimension only work with ``geometry=='cartesian'`` or 187 | ``geometry=='sphericalpolar'``. For ``geometry=='llc'`` or ``geometry=='curvilinear'``, 188 | it is not possible to replace the logical dimensions with physical dimensions, and setting 189 | ``swap_dims=False`` will raise an error. 190 | 191 | Logical dimensions follow the naming conventions of the 192 | `MITgcm numerical grid `_. 193 | The dimension names are augmented with metadata attributes from the Comodo_ 194 | conventions. These logical spatial dimensions are 195 | 196 | +------+----------------------------------+------+-------------------+ 197 | | name | standard_name | axis | c_grid_axis_shift | 198 | +======+==================================+======+===================+ 199 | | i | x_grid_index | X | | 200 | +------+----------------------------------+------+-------------------+ 201 | | i_g | x_grid_index_at_u_location | X | -0.5 | 202 | +------+----------------------------------+------+-------------------+ 203 | | j | y_grid_index | Y | | 204 | +------+----------------------------------+------+-------------------+ 205 | | j_g | y_grid_index_at_v_location | Y | -0.5 | 206 | +------+----------------------------------+------+-------------------+ 207 | | k | z_grid_index | Z | | 208 | +------+----------------------------------+------+-------------------+ 209 | | k_l | z_grid_index_at_lower_w_location | Z | -0.5 | 210 | +------+----------------------------------+------+-------------------+ 211 | | k_u | z_grid_index_at_upper_w_location | Z | 0.5 | 212 | +------+----------------------------------+------+-------------------+ 213 | | k_p1 | z_grid_index_at_w_location | Z | (-0.5, 0.5) | 214 | +------+----------------------------------+------+-------------------+ 215 | 216 | As explained in the Comodo_ documentation, the use of different dimensions is 217 | necessary to represent the fact that, in c-grid ocean models, different 218 | variables are staggered in different ways with respect to the model cells. 219 | For example, tracers and velocity components are all have different logical 220 | dimensions:: 221 | 222 | print(ds.THETA.dims) 223 | >>> ('time', 'k', 'j', 'i') 224 | print(ds.UVEL.dims) 225 | >>> ('time', 'k', 'j', 'i_g' 226 | print(ds.VVEL.dims) 227 | >>> ('time', 'k', 'j_g', 'i') 228 | print(ds.WVEl.dims) 229 | >>> ('time', 'k_l', 'j', 'i') 230 | 231 | xarray_ distinguishes between "coordinates" and "data_vars". By default, 232 | ``open_mdsdataset`` will promote all grid variables to coordinates. To turn off 233 | this behavior and treat grid variables as data_vars, use 234 | ``grid_vars_to_coords=False``. 235 | 236 | .. warning:: 237 | For vertical coordinates (``Zl``, ``k_l``, etc.) the notion of "lower" and 238 | "upper" always refers to the *logical* grid position, NOT the *physical* grid position. 239 | This can be a major point of confusion. In array index space, the point ``k_l`` 240 | is indeed lower than ``k_u``; however, for typical MITgcm ocean model configurations, 241 | in physical space the point ``k_l`` is physically *above* ``k_u``. This is because, 242 | the Z axis starts from the ocean surface and increases downward. 243 | In contrast, for atmospheric configurations which start from the ground and increase upwards, 244 | the logical and physical positions are self-consistent. 245 | 246 | Time 247 | ---- 248 | 249 | ``open_mdsdataset`` attemts to determine the time dimension based on ``iters``. 250 | However, addtiional input is required from the user to fully exploit this 251 | capability. If the user specifies ``delta_t``, the numerical timestep used for 252 | the MITgcm simulation, it is used to muliply ``iters`` to determine the time 253 | in seconds. Additionally, if the user specifies ``ref_date`` (an ISO date 254 | string, e.g. ``“1990-1-1 0:0:0”``), the time dimension will be converted into 255 | a datetime index, exposing all sorts of 256 | `useful timeseries functionalty `_ 257 | within xarray. 258 | 259 | .. _geometries: 260 | 261 | Grid Geometries 262 | --------------- 263 | 264 | The grid geometry is not inferred; it must be specified via the ``geometry`` 265 | keyword. xmitgcm currently supports four MITgcm grid geometries: ``cartesian``, 266 | ``sphericalpolar``, ``curvilinear``, and ``llc``. The first two are straightforward. 267 | The ``curvilinear`` is used for curvilinear cartesian grids. The ``llc`` 268 | ("lat-lon-cap") geometry is more complicated. This grid consists of four 269 | distinct faces of the same size plus a smaller north-polar cap. Each face has a 270 | distinct relatioship between its logical dimensions and its physical 271 | coordinates. Because netCDF and xarray.Dataset data structures do not support 272 | this sort of complex geometry (multiple faces of different sizes), our approach, 273 | inspired by nc-tiles, is to split the domain into 13 equal-sized "faces". 274 | ``face`` then becomes an additional dimension of the data. 275 | 276 | To download an example llc dataset, run the following shell commands:: 277 | 278 | $ curl -L -J -O https://ndownloader.figshare.com/files/6494721 279 | $ tar -xvzf global_oce_llc90.tar.gz 280 | 281 | And to read it, in python:: 282 | 283 | ds_llc = open_mdsdataset('./global_oce_llc90/', iters=8, geometry='llc') 284 | print(ds_llc['S'].dims) 285 | >>> ('time', 'k', 'face', 'j', 'i') 286 | 287 | xmitgcm is not nearly as comprehensive as gcmfaces_. It does not offer 288 | sophisticated operations involving exchanges at face boundaries, integrals 289 | across sections, etc. The goal of this package is simply to read the mds data. 290 | However, by outputing an xarray_ data structure, we can use all of xarray's 291 | usual capabilities on the llc data, for example:: 292 | 293 | # calculate the mean squared salinity as a function of depth 294 | (ds_llc.S**2).mean(dim=['face', 'j', 'i']) 295 | >>> 296 | dask.array 297 | Coordinates: 298 | * k (k) int64 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 ... 299 | iter (time) int64 8 300 | * time (time) int64 8 301 | Z (k) >f4 -5.0 -15.0 -25.0 -35.0 -45.0 -55.0 -65.0 -75.005 ... 302 | PHrefC (k) >f4 49.05 147.15 245.25 343.35 441.45 539.55 637.65 735.799 ... 303 | drF (k) >f4 10.0 10.0 10.0 10.0 10.0 10.0 10.0 10.01 10.03 10.11 ... 304 | 305 | (Note that this simple example does not perform correct volume weighting or 306 | land masking in the average.) 307 | 308 | ``open_mdsdataset`` offers two different strategies for reading LLC data, 309 | ``method='smallchunks'`` (the default) and ``method='bigchunks'``. The details 310 | and tradeoffs of these strategies are described in :doc:`/performance`. 311 | 312 | .. _dask: http://dask.pydata.org 313 | .. _xarray: http://xarray.pydata.org 314 | .. _Comodo: http://pycomodo.forge.imag.fr/norm.html 315 | .. _issues: https://github.com/xgcm/xmitgcm/issues 316 | .. _`pull requests`: https://github.com/xgcm/xmitgcm/pulls 317 | .. _MITgcm: http://mitgcm.org/public/r2_manual/latest/online_documents/node277.html 318 | .. _out-of-core: https://en.wikipedia.org/wiki/Out-of-core_algorithm 319 | .. _Anaconda: https://www.continuum.io/downloads 320 | .. _`CF conventions`: http://cfconventions.org/Data/cf-conventions/cf-conventions-1.7/build/ch04s04.html 321 | .. _gcmfaces: http://mitgcm.org/viewvc/*checkout*/MITgcm/MITgcm_contrib/gael/matlab_class/gcmfaces.pdf 322 | -------------------------------------------------------------------------------- /xmitgcm/test/test_mds_store.py: -------------------------------------------------------------------------------- 1 | from xmitgcm.test.test_xmitgcm_common import * 2 | import xmitgcm 3 | import xarray as xr 4 | from glob import glob 5 | from shutil import copyfile 6 | import py 7 | import re 8 | 9 | _TESTDATA_FILENAME = 'testdata.tar.gz' 10 | _TESTDATA_ITERS = [39600, ] 11 | _TESTDATA_DELTAT = 86400 12 | 13 | _EXPECTED_GRID_VARS = ['XC', 'YC', 'XG', 'YG', 'Zl', 'Zu', 'Z', 'Zp1', 'dxC', 14 | 'rAs', 'rAw', 'Depth', 'rA', 'dxG', 'dyG', 'rAz', 'dyC', 15 | 'PHrefC', 'drC', 'PHrefF', 'drF', 16 | 'hFacS', 'hFacC', 'hFacW'] 17 | 18 | 19 | # a meta test 20 | def test_file_hiding(all_mds_datadirs): 21 | dirname, _ = all_mds_datadirs 22 | basenames = ['XC.data', 'XC.meta'] 23 | for basename in basenames: 24 | assert os.path.exists(os.path.join(dirname, basename)) 25 | with hide_file(dirname, *basenames): 26 | for basename in basenames: 27 | assert not os.path.exists(os.path.join(dirname, basename)) 28 | for basename in basenames: 29 | assert os.path.exists(os.path.join(dirname, basename)) 30 | 31 | ######################################################### 32 | ### Below are all tests that actually create datasets ### 33 | ######################################################### 34 | def test_open_mdsdataset_minimal(all_mds_datadirs): 35 | """Create a minimal xarray object with only dimensions in it.""" 36 | 37 | dirname, expected = all_mds_datadirs 38 | 39 | ds = xmitgcm.open_mdsdataset( 40 | dirname, iters=None, read_grid=False, swap_dims=False, 41 | geometry=expected['geometry']) 42 | 43 | # the expected dimensions of the dataset 44 | eshape = expected['shape'] 45 | if len(eshape) == 3: 46 | nz, ny, nx = eshape 47 | nface = None 48 | elif len(eshape) == 4: 49 | if expected['geometry'] == 'cs': 50 | nz, ny, nface, nx = eshape 51 | else: 52 | nz, nface, ny, nx = eshape 53 | else: 54 | raise ValueError("Invalid expected shape") 55 | coords = {'i': np.arange(nx), 56 | 'i_g': np.arange(nx), 57 | # 'i_z': np.arange(nx), 58 | 'j': np.arange(ny), 59 | 'j_g': np.arange(ny), 60 | # 'j_z': np.arange(ny), 61 | 'k': np.arange(nz), 62 | 'k_u': np.arange(nz), 63 | 'k_l': np.arange(nz), 64 | 'k_p1': np.arange(nz+1)} 65 | if nface is not None: 66 | coords['face'] = np.arange(nface) 67 | 68 | if 'layers' in expected: 69 | for layer_name, n_layers in expected['layers'].items(): 70 | for suffix, offset in zip(['bounds', 'center', 'interface'], 71 | [0, -1, -2]): 72 | dimname = 'l' + layer_name[0] + '_' + suffix[0] 73 | index = np.arange(n_layers + offset) 74 | coords[dimname] = index 75 | 76 | ds_expected = xr.Dataset(coords=coords) 77 | assert ds_expected.equals(ds) 78 | 79 | # check that the datatypes are correct 80 | for key in coords: 81 | assert ds[key].dtype == np.int64 82 | 83 | # check for comodo metadata needed by xgcm 84 | assert ds['i'].attrs['axis'] == 'X' 85 | assert ds['i_g'].attrs['axis'] == 'X' 86 | assert ds['i_g'].attrs['c_grid_axis_shift'] == -0.5 87 | assert ds['j'].attrs['axis'] == 'Y' 88 | assert ds['j_g'].attrs['axis'] == 'Y' 89 | assert ds['j_g'].attrs['c_grid_axis_shift'] == -0.5 90 | assert ds['k'].attrs['axis'] == 'Z' 91 | assert ds['k_l'].attrs['axis'] == 'Z' 92 | assert ds['k_l'].attrs['c_grid_axis_shift'] == -0.5 93 | 94 | 95 | def test_read_grid(all_mds_datadirs): 96 | """Make sure we read all the grid variables.""" 97 | dirname, expected = all_mds_datadirs 98 | ds = xmitgcm.open_mdsdataset( 99 | dirname, iters=None, read_grid=True, 100 | geometry=expected['geometry']) 101 | 102 | for vname in _EXPECTED_GRID_VARS: 103 | assert vname in ds.variables 104 | 105 | # make sure angle is present 106 | if expected['geometry'] in ['llc', 'curvilinear']: 107 | assert 'CS' in ds.coords 108 | assert 'SN' in ds.coords 109 | 110 | # actually load the data, to check for dask-related errors 111 | ds.load() 112 | 113 | def test_values_and_endianness(all_mds_datadirs): 114 | """Make sure we read all the grid variables.""" 115 | dirname, expected = all_mds_datadirs 116 | 117 | if expected['geometry']=='llc' and (dask.__version__ < '0.11.2'): 118 | pytest.xfail("LLC value tests require fixed dask") 119 | 120 | # default endianness 121 | ds = xmitgcm.open_mdsdataset( 122 | dirname, iters=None, read_grid=True, swap_dims=False, 123 | geometry=expected['geometry']) 124 | # now reverse endianness 125 | ds_le = xmitgcm.open_mdsdataset( 126 | dirname, iters=None, read_grid=True, endian='<', 127 | swap_dims=False, 128 | geometry=expected['geometry']) 129 | 130 | for vname, (idx, val) in expected['expected_values'].items(): 131 | np.testing.assert_allclose(ds[vname].values[idx], val) 132 | # dask arrays that have been concatenated revert to native endianness 133 | # https://github.com/dask/dask/issues/1647 134 | if ds[vname].dtype.byteorder=='>': 135 | val_le = ds[vname].values.view(ds[vname].values.dtype.newbyteorder('<'))[idx] 136 | np.testing.assert_allclose(ds_le[vname].values[idx], val_le) 137 | 138 | def test_open_dataset_no_meta(all_mds_datadirs): 139 | """Make sure we read variables with no .meta files.""" 140 | dirname, expected = all_mds_datadirs 141 | 142 | shape = expected['shape'] 143 | 144 | nz = shape[0] 145 | ny, nx = shape[-2:] 146 | shape_2d = shape[1:] 147 | dims_2d = ('j', 'i') 148 | if expected['geometry'] == 'llc': 149 | dims_2d = ('face',) + dims_2d 150 | ny = nx*shape[-3] 151 | elif expected['geometry'] == 'cs': 152 | dims_2d = ('j', 'face', 'i') 153 | if len(shape) == 4: 154 | nz, ny, nface, nx = shape 155 | elif len(shape) == 3: 156 | ny, nface, nx = shape 157 | 158 | dims_3d = dims_2d if nz == 1 else ('k',) + dims_2d 159 | dims_2d = ('time',) + dims_2d 160 | dims_3d = ('time',) + dims_3d 161 | 162 | it = expected['test_iternum'] 163 | kwargs = dict(iters=it, geometry=expected['geometry'], read_grid=False, 164 | swap_dims=False, default_dtype=expected['dtype']) 165 | 166 | # a 3D file 167 | to_hide = ['T.%010d.meta' % it, 'Eta.%010d.meta' % it] 168 | with hide_file(dirname, *to_hide): 169 | ds = xmitgcm.open_mdsdataset(dirname, prefix=['T', 'Eta'], **kwargs) 170 | print(ds['T'].dims) 171 | print(dims_3d) 172 | assert ds['T'].dims == dims_3d 173 | assert ds['T'].values.ndim == len(dims_3d) 174 | assert ds['Eta'].dims == dims_2d 175 | assert ds['Eta'].values.ndim == len(dims_2d) 176 | 177 | with pytest.raises(IOError): 178 | xmitgcm.open_mdsdataset(dirname, prefix=['T', 'Eta'], iters=it, 179 | geometry=expected['geometry'], 180 | read_grid=False) 181 | pytest.fail("Expecting IOError when default_dtype " 182 | "is not precised (i.e., None)") 183 | 184 | # now get rid of the variables used to infer dimensions 185 | with hide_file(dirname, 'XC.meta', 'RC.meta'): 186 | with pytest.raises(IOError): 187 | ds = xmitgcm.open_mdsdataset(dirname, prefix=['T', 'Eta'], **kwargs) 188 | if expected['geometry']=='llc': 189 | ds = xmitgcm.open_mdsdataset(dirname, prefix=['T', 'Eta'], 190 | nx=nx, nz=nz, **kwargs) 191 | with hide_file(dirname, *to_hide): 192 | ds = xmitgcm.open_mdsdataset(dirname, prefix=['T', 'Eta'], 193 | nx=nx, nz=nz, **kwargs) 194 | else: 195 | ds = xmitgcm.open_mdsdataset(dirname, prefix=['T', 'Eta'], 196 | nx=nx, ny=ny, nz=nz, **kwargs) 197 | with hide_file(dirname, *to_hide): 198 | ds = xmitgcm.open_mdsdataset(dirname, prefix=['T', 'Eta'], 199 | nx=nx, ny=ny, nz=nz, **kwargs) 200 | 201 | # try just hiding RC 202 | with hide_file(dirname, 'RC.meta'): 203 | if expected['geometry']=='llc': 204 | ds = xmitgcm.open_mdsdataset(dirname, prefix=['T', 'Eta'], 205 | nz=nz, **kwargs) 206 | else: 207 | ds = xmitgcm.open_mdsdataset(dirname, prefix=['T', 'Eta'], 208 | nz=nz, **kwargs) 209 | 210 | def test_open_dataset_2D_diags(all_mds_datadirs): 211 | # convert 3D fields with only 2D diagnostic output 212 | # https://github.com/xgcm/xmitgcm/issues/140 213 | dirname, expected = all_mds_datadirs 214 | 215 | shape = expected['shape'] 216 | 217 | nz = shape[0] 218 | ny, nx = shape[-2:] 219 | shape_2d = shape[1:] 220 | dims_2d = ('j', 'i') 221 | if expected['geometry']=='llc': 222 | dims_2d = ('face',) + dims_2d 223 | ny = nx*shape[-3] 224 | dims_3d = dims_2d if nz==1 else ('k',) + dims_2d 225 | dims_2d = ('time',) + dims_2d 226 | dims_3d = ('time',) + dims_3d 227 | 228 | it = expected['test_iternum'] 229 | kwargs = dict(iters=it, geometry=expected['geometry'], read_grid=False, 230 | swap_dims=False) 231 | 232 | to_hide = ['T.%010d.meta' % it, 'T.%010d.data' % it] 233 | with hide_file(dirname, *to_hide): 234 | 235 | ldir = py.path.local(dirname) 236 | old_prefix = 'Eta.%010d' % it 237 | new_prefix = 'T.%010d' % it 238 | for suffix in ['.data', '.meta']: 239 | lp = ldir.join(old_prefix + suffix) 240 | lp.copy(ldir.join(new_prefix + suffix)) 241 | 242 | ds = xmitgcm.open_mdsdataset(dirname, prefix=['T'], **kwargs) 243 | 244 | def test_swap_dims(all_mds_datadirs): 245 | """See if we can swap dimensions.""" 246 | 247 | dirname, expected = all_mds_datadirs 248 | kwargs = dict(iters=None, read_grid=True, geometry=expected['geometry']) 249 | 250 | expected_dims = ['XC', 'XG', 'YC', 'YG', 'Z', 'Zl', 'Zp1', 'Zu'] 251 | 252 | # make sure we never swap if not reading grid 253 | assert 'i' in xmitgcm.open_mdsdataset(dirname, 254 | iters=None, read_grid=False, geometry=expected['geometry']) 255 | if expected['geometry'] in ('llc', 'cs', 'curvilinear'): 256 | # make sure swapping is not the default 257 | ds = xmitgcm.open_mdsdataset(dirname, **kwargs) 258 | assert 'i' in ds 259 | # and is impossible 260 | with pytest.raises(ValueError) as excinfo: 261 | ds = xmitgcm.open_mdsdataset( 262 | dirname, swap_dims=True, **kwargs) 263 | else: 264 | # make sure swapping *IS* the default 265 | assert 'i' not in xmitgcm.open_mdsdataset(dirname, **kwargs) 266 | ds = xmitgcm.open_mdsdataset( 267 | dirname, geometry=expected['geometry'], 268 | iters=None, read_grid=True, swap_dims=True, 269 | grid_vars_to_coords=True) 270 | 271 | 272 | # check for comodo metadata needed by xgcm 273 | assert ds['XC'].attrs['axis'] == 'X' 274 | assert ds['XG'].attrs['axis'] == 'X' 275 | assert ds['XG'].attrs['c_grid_axis_shift'] == -0.5 276 | assert ds['YC'].attrs['axis'] == 'Y' 277 | assert ds['YG'].attrs['axis'] == 'Y' 278 | assert ds['YG'].attrs['c_grid_axis_shift'] == -0.5 279 | assert ds['Z'].attrs['axis'] == 'Z' 280 | assert ds['Zl'].attrs['axis'] == 'Z' 281 | assert ds['Zl'].attrs['c_grid_axis_shift'] == -0.5 282 | 283 | # add extra layers dimensions if needed 284 | if 'layers' in expected: 285 | for layer_name in expected['layers']: 286 | extra_dims = ['layer_' + layer_name + suffix for suffix in 287 | ['_bounds', '_center', '_interface']] 288 | expected_dims += extra_dims 289 | 290 | assert set(ds.sizes.keys()) == set(expected_dims) 291 | 292 | # make sure swapping works with multiple iters 293 | ds = xmitgcm.open_mdsdataset(dirname, geometry=expected['geometry'], 294 | prefix=['S']) 295 | #print(ds) 296 | ds.load() 297 | assert 'XC' in ds['S'].dims 298 | assert 'YC' in ds['S'].dims 299 | 300 | 301 | 302 | def test_prefixes(all_mds_datadirs): 303 | """Make sure we read all the grid variables.""" 304 | 305 | dirname, expected = all_mds_datadirs 306 | prefixes = ['U', 'V', 'W', 'T', 'S', 'PH'] # , 'PHL', 'Eta'] 307 | iters = [expected['test_iternum']] 308 | ds = xmitgcm.open_mdsdataset( 309 | dirname, iters=iters, prefix=prefixes, 310 | read_grid=False, geometry=expected['geometry']) 311 | 312 | for p in prefixes: 313 | assert p in ds 314 | 315 | def test_separate_grid_dir(all_mds_datadirs): 316 | """Make sure we can have the grid files in a separate directory.""" 317 | 318 | dirname, expected = all_mds_datadirs 319 | prefixes = ['U', 'V', 'W', 'T', 'S', 'PH'] # , 'PHL', 'Eta'] 320 | iters = [expected['test_iternum']] 321 | 322 | with hide_file(dirname, 323 | *['XC.meta', 'XC.data', 'RC.meta', 'RC.data']) as grid_dir: 324 | ds = xmitgcm.open_mdsdataset( 325 | dirname, grid_dir=grid_dir, iters=iters, prefix=prefixes, 326 | read_grid=False, geometry=expected['geometry']) 327 | for p in prefixes: 328 | assert p in ds 329 | 330 | def test_multiple_iters(multidim_mds_datadirs): 331 | """Test ability to load multiple iters into a single dataset.""" 332 | 333 | dirname, expected = multidim_mds_datadirs 334 | # first try specifying the iters 335 | ds = xmitgcm.open_mdsdataset( 336 | dirname, read_grid=False, geometry=expected['geometry'], 337 | iters=expected['all_iters'], 338 | prefix=expected['prefixes']) 339 | assert list(ds.iter.values) == expected['all_iters'] 340 | 341 | # now infer the iters, should be the same 342 | ds2 = xmitgcm.open_mdsdataset( 343 | dirname, read_grid=False, iters='all', geometry=expected['geometry'], 344 | prefix=expected['prefixes']) 345 | assert ds.equals(ds2) 346 | 347 | # In the test datasets, there is no PHL.0000000000.data file. 348 | # By default we infer the prefixes from the first iteration number, so this 349 | # leads to an error. 350 | # (Need to specify iters because there is some diagnostics output with 351 | # weird iterations numbers present in some experiments.) 352 | with pytest.raises(IOError): 353 | ds = xmitgcm.open_mdsdataset( 354 | dirname, read_grid=False, iters=expected['all_iters'], 355 | geometry=expected['geometry']) 356 | 357 | # now hide all the PH and PHL files: should be able to infer prefixes fine 358 | missing_files = [os.path.basename(f) 359 | for f in glob(os.path.join(dirname, 'PH*.0*data'))] 360 | print(missing_files) 361 | with hide_file(dirname, *missing_files): 362 | ds = xmitgcm.open_mdsdataset( 363 | dirname, read_grid=False, iters=expected['all_iters'], 364 | geometry=expected['geometry']) 365 | 366 | 367 | def test_date_parsing(mds_datadirs_with_refdate): 368 | """Verify that time information is decoded properly.""" 369 | dirname, expected = mds_datadirs_with_refdate 370 | 371 | ds = xmitgcm.open_mdsdataset(dirname, iters='all', prefix=['S'], 372 | ref_date=expected['ref_date'], read_grid=False, 373 | delta_t=expected['delta_t'], 374 | geometry=expected['geometry']) 375 | 376 | for i, date in expected['expected_time']: 377 | assert ds.time[i].values == date 378 | 379 | # since time was decoded, this encoding should be removed from attributes 380 | assert 'units' not in ds.time.attrs 381 | assert 'calendar' not in ds.time.attrs 382 | 383 | def test_serialize_nonstandard_calendar(multidim_mds_datadirs, tmp_path): 384 | dirname, expected = multidim_mds_datadirs 385 | ref_date = '2680-01-01 00:00:00' 386 | calendar = '360_day' 387 | ds = xmitgcm.open_mdsdataset(dirname, iters='all', prefix=['S'], 388 | ref_date=ref_date, 389 | calendar=calendar, 390 | read_grid=False, 391 | delta_t=expected['delta_t'], 392 | geometry=expected['geometry']) 393 | ds.to_zarr(tmp_path / 'test.zarr') 394 | 395 | 396 | def test_diagnostics(mds_datadirs_with_diagnostics): 397 | """Try reading dataset with diagnostics output.""" 398 | dirname, expected = mds_datadirs_with_diagnostics 399 | 400 | diag_prefix, expected_diags = expected['diagnostics'] 401 | ds = xmitgcm.open_mdsdataset(dirname, 402 | read_grid=False, 403 | iters=expected['test_iternum'], 404 | prefix=[diag_prefix], 405 | geometry=expected['geometry']) 406 | for diagname in expected_diags: 407 | assert diagname in ds 408 | # check vector mates 409 | if 'mate' in ds[diagname].attrs: 410 | mate = ds[diagname].attrs['mate'] 411 | assert ds[mate].attrs['mate'] == diagname 412 | 413 | def test_default_diagnostics(mds_datadirs_with_diagnostics): 414 | """Try reading dataset with diagnostics output.""" 415 | dirname, expected = mds_datadirs_with_diagnostics 416 | 417 | diag_prefix, expected_diags = expected['diagnostics'] 418 | with hide_file(dirname, 'available_diagnostics.log'): 419 | ds = xmitgcm.open_mdsdataset(dirname, 420 | read_grid=False, 421 | iters=expected['test_iternum'], 422 | prefix=[diag_prefix], 423 | geometry=expected['geometry']) 424 | for diagname in expected_diags: 425 | assert diagname in ds 426 | # check vector mates 427 | if 'mate' in ds[diagname].attrs: 428 | mate = ds[diagname].attrs['mate'] 429 | assert ds[mate].attrs['mate'] == diagname 430 | 431 | def test_drop_uncommon_diagnostics(llc_mds_datadirs): 432 | """Make sure when uncommon diagnostics exist, drop them""" 433 | 434 | dirname, expected = llc_mds_datadirs 435 | diag_prefix, expected_diags = expected['diagnostics'] 436 | iternum = expected['test_iternum'] 437 | 438 | # copy meta/data diagnostics pair 439 | copyfile(os.path.join(dirname, f"{diag_prefix}.{iternum:010d}.data"), 440 | os.path.join(dirname, f"{diag_prefix}.{iternum+1:010d}.data")) 441 | 442 | # modify meta file to "hide" a variable 443 | metafname = os.path.join(dirname, f"{diag_prefix}.{iternum:010d}.meta") 444 | with open(metafname, 'r') as f: 445 | meta = f.read() 446 | 447 | # replace nrecords and nFlds with N-1 in the meta file 448 | for item in ['nrecords', 'nFlds']: 449 | 450 | regex = r"^(\s+%s = \[\s+)(\d+)(\s?\];\n)" % item 451 | matches = re.search(regex, meta, re.MULTILINE) 452 | numrecs = int(matches.groups()[1]) 453 | 454 | substr = f"\\g<1>{numrecs-1}\\g<3>" 455 | meta = re.sub(regex, substr, meta, count=1, flags=re.MULTILINE) 456 | 457 | regex = r"^(\s+timeStepNumber = \[\s+)(\d+)(\s?\];\n)" 458 | substr = f"\\g<1>{iternum+1}\\g<3>" 459 | meta = re.sub(regex, substr, meta, count=1, flags=re.MULTILINE) 460 | 461 | # remove ETAN 462 | meta = meta.replace("'ETAN ' ","") 463 | 464 | # write it out 465 | metafname = os.path.join(dirname, f"{diag_prefix}.{iternum+1:010d}.meta") 466 | with open(metafname, 'w') as f: 467 | f.write(meta) 468 | 469 | with pytest.warns(UserWarning, match="dropped these variables"): 470 | ds = xmitgcm.open_mdsdataset(dirname, 471 | read_grid=False, 472 | iters=[iternum, iternum+1], 473 | prefix=[diag_prefix], 474 | geometry=expected['geometry']) 475 | 476 | assert 'ETAN' not in ds 477 | assert len(ds.data_vars) == numrecs-1 478 | 479 | 480 | 481 | def test_avail_diags_in_grid_dir(mds_datadirs_with_diagnostics): 482 | """Try reading dataset with diagnostics output.""" 483 | dirname, expected = mds_datadirs_with_diagnostics 484 | 485 | diag_prefix, expected_diags = expected['diagnostics'] 486 | iters = expected['test_iternum'] 487 | 488 | with hide_file(dirname, 489 | *['XC.meta', 'XC.data', 'RC.meta', 'RC.data', 490 | 'available_diagnostics.log']) as grid_dir: 491 | ds = xmitgcm.open_mdsdataset( 492 | dirname, grid_dir=grid_dir, iters=iters, prefix=[diag_prefix], 493 | read_grid=False, geometry=expected['geometry']) 494 | 495 | for diagname in expected_diags: 496 | assert diagname in ds 497 | if 'mate' in ds[diagname].attrs: 498 | mate = ds[diagname].attrs['mate'] 499 | assert ds[mate].attrs['mate'] == diagname 500 | 501 | def test_layers_diagnostics(layers_mds_datadirs): 502 | """Try reading dataset with layers output.""" 503 | dirname, expected = layers_mds_datadirs 504 | ds = xmitgcm.open_mdsdataset(dirname, iters='all', swap_dims=False, 505 | geometry=expected['geometry']) 506 | layer_name = list(expected['layers'].keys())[0] 507 | layer_id = 'l' + layer_name[0] 508 | for suf in ['bounds', 'center', 'interface']: 509 | assert ('layer_' + layer_name + '_' + suf) in ds 510 | assert (layer_id + '_' + suf[0]) in ds.dims 511 | 512 | # a few random expected variables 513 | expected_vars = {'LaUH' + layer_name: 514 | ('time', layer_id + '_c', 'j', 'i_g'), 515 | 'LaVH' + layer_name: 516 | ('time', layer_id + '_c', 'j_g', 'i'), 517 | 'LaTs' + layer_name: 518 | ('time', layer_id + '_i', 'j', 'i')} 519 | for var, dims in expected_vars.items(): 520 | assert var in ds 521 | assert ds[var].dims == dims 522 | 523 | @pytest.mark.parametrize("method", ["smallchunks"]) 524 | @pytest.mark.parametrize("with_refdate", [True, False]) 525 | def test_llc_dims(llc_mds_datadirs, method, with_refdate): 526 | """Check that the LLC file dimensions are correct.""" 527 | dirname, expected = llc_mds_datadirs 528 | if with_refdate: 529 | kwargs = {'delta_t': expected['delta_t'], 530 | 'ref_date': expected['ref_date']} 531 | else: 532 | kwargs = {} 533 | ds = xmitgcm.open_mdsdataset(dirname, 534 | iters=expected['test_iternum'], 535 | geometry=expected['geometry'], llc_method=method, 536 | **kwargs) 537 | 538 | nz, nface, ny, nx = expected['shape'] 539 | nt = 1 540 | 541 | assert ds.sizes['face'] == 13 542 | assert ds.rA.dims == ('face', 'j', 'i') 543 | assert ds.rA.values.shape == (nface, ny, nx) 544 | assert ds.U.dims == ('time', 'k', 'face', 'j', 'i_g') 545 | assert ds.U.values.shape == (nt, nz, nface, ny, nx) 546 | assert ds.V.dims == ('time', 'k', 'face', 'j_g', 'i') 547 | assert ds.V.values.shape == (nt, nz, nface, ny, nx) 548 | 549 | print(ds.U.chunks) 550 | if method == "smallchunks": 551 | assert ds.U.chunks == (nt*(1,), nz*(1,), nface*(1,), (ny,), (nx,)) 552 | 553 | 554 | def test_drc_length(all_mds_datadirs): 555 | """Test that open_mdsdataset is adding an extra level to drC if it has length nr""" 556 | dirname, expected = all_mds_datadirs 557 | # Only older versions of the gcm have len(drC) = nr, so force len(drC) = nr for the test 558 | copyfile(os.path.join(dirname, 'DRF.data'), 559 | os.path.join(dirname, 'DRC.data')) 560 | copyfile(os.path.join(dirname, 'DRF.meta'), 561 | os.path.join(dirname, 'DRC.meta')) 562 | ds = xmitgcm.open_mdsdataset( 563 | dirname, iters=None, read_grid=True, 564 | geometry=expected['geometry']) 565 | assert len(ds.drC) == (len(ds.drF)+1) 566 | 567 | 568 | def test_extra_variables(all_mds_datadirs): 569 | """Test that open_mdsdataset reads extra_variables correctly""" 570 | dirname, expected = all_mds_datadirs 571 | 572 | extra_variable_data = dict( 573 | # use U,V to test with attrs (including mate) 574 | testdataU=dict(dims=['k','j','i_g'], attrs=dict( 575 | standard_name='sea_water_x_velocity', mate='testdataV', 576 | long_name='Zonal Component of Velocity', units='m s-1')), 577 | testdataV=dict(dims=['k','j_g','i'], attrs=dict( 578 | standard_name='sea_water_y_velocity', mate='testdataU', 579 | long_name='Meridional Component of Velocity', units='m s-1')), 580 | # use T to test without attrs 581 | testdataT=dict(dims=['k','j','i'], attrs=dict()) 582 | ) 583 | 584 | for var in ["U","V","T"]: 585 | input_dir = os.path.join(dirname, '{}.{:010d}'.format(var, expected['test_iternum'])) 586 | test_dir = os.path.join(dirname, 'testdata{}.{:010d}'.format(var, expected['test_iternum'])) 587 | 588 | if input_dir+".meta" not in os.listdir() or input_dir+".data" not in os.listdir(): 589 | return 590 | 591 | copyfile(input_dir+".meta",test_dir+".meta") 592 | copyfile(input_dir+".data",test_dir+".data") 593 | 594 | assert test_dir + ".meta" not in os.listdir(), f"{var} did not copy meta!" 595 | assert test_dir + ".data" not in os.listdir(), f"{var} did not copy data!" 596 | 597 | ds = xmitgcm.open_mdsdataset( 598 | dirname, 599 | read_grid=False, 600 | iters=expected['test_iternum'], 601 | geometry=expected['geometry'], 602 | prefix=list(extra_variable_data.keys()), 603 | extra_variables=extra_variable_data) 604 | 605 | for var in extra_variable_data.keys(): 606 | assert var in ds 607 | if 'mate' in ds[var].attrs: 608 | mate = ds[var].attrs['mate'] 609 | assert ds[mate].attrs['mate'] == var 610 | 611 | def test_mask_values(all_mds_datadirs): 612 | """Test that open_mdsdataset generates binary masks with correct values""" 613 | 614 | dirname, expected = all_mds_datadirs 615 | ds = xmitgcm.open_mdsdataset( 616 | dirname, iters=None, read_grid=True, 617 | geometry=expected['geometry']) 618 | 619 | hFac_list = ['hFacC', 'hFacW', 'hFacS'] 620 | mask_list = ['maskC', 'maskW', 'maskS'] 621 | 622 | for hFac, mask in zip(hFac_list, mask_list): 623 | xr.testing.assert_equal(ds[hFac] * ds[mask], ds[hFac]) 624 | 625 | # 626 | # Series of tests which try to open a dataset with different combinations of 627 | # of options, to identify if ref_date can trigger an error 628 | # 629 | 630 | 631 | @pytest.mark.parametrize("load", [True, False]) 632 | # can't call swap_dims without read_grid=True 633 | @pytest.mark.parametrize("swap_dims, read_grid", [(True, True), 634 | (False, True), 635 | (False, False)]) 636 | def test_ref_date(mds_datadirs_with_refdate, swap_dims, read_grid, load): 637 | """With ref_date, without grid.""" 638 | dirname, expected = mds_datadirs_with_refdate 639 | 640 | if expected['geometry']=='llc' and swap_dims: 641 | pytest.skip("can't swap_dims with geometry=='llc'") 642 | 643 | ds = xmitgcm.open_mdsdataset(dirname, iters='all', prefix=['S'], 644 | ref_date=expected['ref_date'], 645 | delta_t=expected['delta_t'], 646 | geometry=expected['geometry'], 647 | read_grid=read_grid, swap_dims=swap_dims) 648 | if load: 649 | ds.time.load() 650 | 651 | 652 | @pytest.mark.parametrize("method", ["smallchunks"]) 653 | def test_llc_extra_metadata(llc_mds_datadirs, method): 654 | """Check that the LLC reads properly when using extra_metadata.""" 655 | dirname, expected = llc_mds_datadirs 656 | nz, nface, ny, nx = expected['shape'] 657 | nt = 1 658 | 659 | llc = xmitgcm.utils.get_extra_metadata(domain='llc', nx=nx) 660 | 661 | ds = xmitgcm.open_mdsdataset(dirname, 662 | iters=expected['test_iternum'], 663 | geometry=expected['geometry'], 664 | llc_method=method, 665 | extra_metadata=llc) 666 | 667 | assert ds.sizes['face'] == 13 668 | assert ds.rA.dims == ('face', 'j', 'i') 669 | assert ds.rA.values.shape == (nface, ny, nx) 670 | assert ds.U.dims == ('time', 'k', 'face', 'j', 'i_g') 671 | assert ds.U.values.shape == (nt, nz, nface, ny, nx) 672 | assert ds.V.dims == ('time', 'k', 'face', 'j_g', 'i') 673 | assert ds.V.values.shape == (nt, nz, nface, ny, nx) 674 | 675 | if method == "smallchunks": 676 | assert ds.U.chunks == (nt*(1,), nz*(1,), nface*(1,), (ny,), (nx,)) 677 | 678 | 679 | def test_levels_diagnostics(mds_datadirs_with_inputfiles): 680 | dirname, expected = mds_datadirs_with_inputfiles 681 | 682 | for diagname, (levels, (idx, value)) in expected['diag_levels'].items(): 683 | ds = xmitgcm.open_mdsdataset(dirname, prefix=[diagname], levels=levels, 684 | geometry=expected['geometry']) 685 | 686 | assert ds['Zl'].values[idx] == value 687 | 688 | with pytest.warns(UserWarning, match='nz will be ignored'): 689 | xmitgcm.open_mdsdataset(dirname, prefix=[diagname], levels=levels, 690 | geometry=expected['geometry'], nz=12) 691 | -------------------------------------------------------------------------------- /doc/demo_writing_binary_file.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Use case: writing a binary input file for MITgcm" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "You may want to write binary files to create forcing data, initial condition,... for your MITgcm configuration. Here we show how xmitgcm can help." 15 | ] 16 | }, 17 | { 18 | "cell_type": "markdown", 19 | "metadata": {}, 20 | "source": [ 21 | "## Simple case: a regular grid" 22 | ] 23 | }, 24 | { 25 | "cell_type": "code", 26 | "execution_count": 1, 27 | "metadata": {}, 28 | "outputs": [], 29 | "source": [ 30 | "import numpy as np\n", 31 | "import xmitgcm\n", 32 | "import matplotlib.pylab as plt" 33 | ] 34 | }, 35 | { 36 | "cell_type": "markdown", 37 | "metadata": {}, 38 | "source": [ 39 | "Let's build a regular lat/lon grid with one degree resolution and create a pseudo-field on this regular grid:" 40 | ] 41 | }, 42 | { 43 | "cell_type": "code", 44 | "execution_count": 2, 45 | "metadata": {}, 46 | "outputs": [], 47 | "source": [ 48 | "lon = np.arange(-180,180,1)\n", 49 | "lat = np.arange(-90,90,1)\n", 50 | "\n", 51 | "lon2, lat2 = np.meshgrid(lon,lat)\n", 52 | "\n", 53 | "pseudo = np.cos(2*np.pi*lat2/360) * np.cos(4*np.pi*np.pi*lon2*lon2/360/360)" 54 | ] 55 | }, 56 | { 57 | "cell_type": "code", 58 | "execution_count": 3, 59 | "metadata": {}, 60 | "outputs": [ 61 | { 62 | "data": { 63 | "text/plain": [ 64 | "" 65 | ] 66 | }, 67 | "execution_count": 3, 68 | "metadata": {}, 69 | "output_type": "execute_result" 70 | }, 71 | { 72 | "data": { 73 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xv0XWV95/H3x8RfUMolEC5pEknQ2IhiEX8TmbK0FogExyF0ik6Y6RArTpYdsFbGFhhmoYOlK1g7qKuojRIN1sXFeCFTYzHlUleXcgmIhEBCfgm3nwmEyM1VIDHwnT/2c8LOj3Pf972/r7XOOufsvc/ez37O83zPs5+z935kZjjnnKuX1xSdAOecc+nz4O6cczXkwd0552rIg7tzztWQB3fnnKshD+7OOVdDHtydcy4FklZI2iHpvg7zJelLksYk3Svp+Ni8JZI2h8eSNNLjwd0559LxTWBhl/mnAXPDYynwFQBJhwCfBt4FzAc+LWlq0sR4cHfOuRSY2U+Ap7ossgi42iK3AQdLmg6cCqw1s6fM7GlgLd1/JPoyOekK0jRp//3ttQcfMvDnbCTfq2y1W7lur19554NLzstSZJh82LVtfKeZHZZku+9+73729FMv97XshvW/2QC8GJu03MyWD7C5GcBjsffjYVqn6YmUKri/9uBDOOpj5w/8uV1v2J1Bajqb8uhIrtvrV9754JLzshQZJh8evOT8R5Ju9+mnXua7P5zW17Lz3rD9RTMbTbC5dr9g1mV6It4t45xz+RgHZsXezwS2dZmeiAd355zLx2rg7HDWzAnAs2a2HbgReJ+kqeGP1PeFaYmUqlvGOeeqStI1wHuBaZLGic6AeS2AmX0VWAO8HxgDngf+JMx7StJngTvDqi41s25/zPbFg7tzzqXAzM7qMd+AczvMWwGsSDM93i3jnHM15MHdOedqKJXgLumTkjZIuk/SNZL2kzRH0u3hctrrJJXznC/nnKuhxMFd0gzgz4BRM3sbMAlYDFwOXGFmc4GngXOSbss551x/0uqWmQy8TtJk4PXAduAkYFWYvxI4I6VtOeec6yFxcDezXwKfBx4lCurPAncBz5jZnrBYx8tpJS2VtE7Supf+7d+SJsc55xzpdMtMJbohzhzgt4H9ie5+NlHby2nNbLmZjZrZ6KT990+aHOecc6TTLXMK8JCZPWlmvwG+B/we0R3PWufRp3I5rXPOuf6kEdwfBU6Q9HpJAk4G7gduAc4MyywBbkhhW8455/qQRp/77UR/nN4NrA/rXA5cAJwvaQw4FLgq6bacc871J5XbD5jZp4nuoxC3lWhUkdKZPfPJva8fHk90O2jXQzyvi+bfdba8XpVLKe8tc8Aj0X+vvz4q3YEM2gWa1rS8CmNr3yD9/ctbmQJ3P/pJb9WDUhHlK896dcAjVvl6k5fSBfd44Uzzi+xVsWfPfDLTih3fr3bTqlJgqxbQB1XF1me7shWfnmXZyrNetfan0/66fZXq3jKT2gz+ksYX2W9Ayipw9bMPZS6ws2c+uffRJFXY7yLLVp71qsz1o6xKFdw7qfIXO0jay7afZQ9seSpjXlS5bA2iymkvUiWCexKDVsg0K/AwhbIsBblsgawsypIvRZetIuuV609lgntZgl6/kqS36H31ithd0flT5bI1qKqlt0wqE9yHMWwlLLryQjGFuoxdD2VVVF6VIdh5GamGWgf3opShAjrXSVHls+4/CpIWStokaUzShW3mXyHpnvB4UNIzsXkvxeatTiM9pTsVspumneOa5/7WveJlJetTaOOa1mio0v5KmgRcCSwgugvunZJWm9n9rWXM7JOx5T8OvCO2ihfM7Lg00+Qt9w6aFOyatK9ZaFL+NWlfBzQfGDOzrWa2G7iW6G65nZwFXJNlgmob3IsqhGm3NrJuvXhlrYaqlatOalzeZgCPxd53G8PiKKJbpN8cm7xfGNfiNkmpDGxUqW4Z58oqz+4Zl56nX3o9q547vs+lfzhN0rrYhOVmtjy8btd/2ukXdDGwysxeik17g5ltk3Q0cLOk9Wa2pc+EteXBvcFq3IoqhAf42ttpZqMd5o0Ds2Lvu41hsRg4Nz7BzLaF562SbiXqj08U3CvXLVOlP1nS0sR9dq9o4vdfwX2+E5graY6kEaIA/qqzXiT9DjAV+Fls2lRJU8LracCJRGNiJFK54J6nQVu2FSyQrsEGLa9+pNdZGC/6POBG4AHgejPbIOlSSafHFj0LuNbM4pn/FmCdpF8QDXK0LH6WzbBS6ZaRdDDwdeBtRP1MHwE2AdcBs4GHgQ+Z2dNpbM8l5xU1G94101xmtgZYM2HaJRPef6bN534KHJt2etJquX8R+Cczmwf8LtEv14XATWY2F7gpvM9FHQOXHxU0k3/vbliJg7ukA4H3EIbRM7PdZvYM0TmeK8NiK4FUTu9xydXxx69MPH+78/zJRxrdMkcDTwLfkPS7wF3AJ4AjzGw7gJltl3R4CttKxYIjN+59vfbxeQWmpBni+V0U/56z5/WqXNII7pOB44GPm9ntkr7IAF0wkpYCSwFG9p+6d/pBW3bx7BunpJC8fU0MNAuO3JhKQez38PmgLbv2vs5i/4pUhiDeSae01S0IDVq+0rrFRVb1aqKs4kIdpdHnPg6Mm9nt4f0qomD/hKTpAOF5R7sPm9lyMxs1s9HJr9sfeKWAxgtqGjpV8LyC0sT9OWjLrtT3MW8Ljty491FFVU9/S7uylFfZyqteZRUX6ipxcDezx4HHwvmbACcTnaO5GlgSpi0Bbuhnff18cXn+yZRW/2AaBTLPIQd7qUNAnCjNfSrL0HJpBcI8+8n72W8P8L2ldbbMx4FvS7oXOA74a2AZsEDSZqI7pS0bZsVpfYm9Km2RgapqBbVuQX2iqu1fkeUnr3pVtTpSBqmc525m9wDtLss9eZD1THqxnqd99VMwq9CXWLWgl0RrX8veJ1+XsuXS51eour40KbDHNXW/XfU1Irj3W0GzqMiDHE6W9dCz6QGurPtfdNnKq16VtV6UXSWCu3+56RjmT7GyBra8DZMPfrGOK1IlgvsgylShhvlR6vUZvxy9GXp9z1mUrTyVqZ7WVe2CexHqGnC91b6vuuZHXctv09U+uA9aIetagQfl+dCe50vE61X51T64V1GZDp9d+TSpfDRpX9NWmeBetS+5aumN81ZWd1XOnyqXSzeYygR3l4z/gVUMz3dXFA/ufSiighbVwqpyqzRPReVTEeXCf6CqyYO7c87VUK2D+7Ctq6a2Xpu638Nqan55vWpP0kJJmySNSXrVmBaSPizpSUn3hMdHY/OWSNocHksmfnYYqdw4zO0r60PntAZYcOWU9XnnfiOx9EmaBFxJdAfcceBOSavN7P4Ji15nZudN+OwhwKeJbr5owF3hs08nSVOtW+7OuWqp8AVV84ExM9tqZruBa4nGke7HqcBaM3sqBPS1wMKkCapsy70Jrdc8W1h1P2TOSlbDybXTtNMY89jf5/bsN8D398NpktbFJiw3s+Xh9Qzgsdi8ceBdbVbyR5LeAzwIfNLMHuvw2Rl9JqqjSgV3P5x0zhVop5m1G7cCoF1Lc+JhyP8DrjGzXZI+BqwETurzswNLrVtG0iRJP5f0j+H9HEm3hz8IrpM0kta2nHOuZMaBWbH3M4Ft8QXM7Fdm1joc+Rrwzn4/O4w0W+6fAB4ADgzvLweuMLNrJX0VOAf4SorbG9qZB9699/Wq544vMCXNEM/vovj3nL2G16s7gbmS5gC/BBYD/yW+gKTpZrY9vD2dKF4C3Aj8taSp4f37gIuSJiiV4C5pJvAfgMuA8yWJ6HCjtXMrgc8wQHAf2TjO7nkz00jePiYGmjMPvDvXgjiycXzv6yz2r51eF6Gk3d9ehmA+0cQ0pfmd9+p3nz3zSR4ePyy17XVTRPmC/OpVVnEhKTPbI+k8okA9CVhhZhskXQqsM7PVwJ9JOh3YAzwFfDh89ilJnyX6gQC41MyeSpqmtFruXwD+EjggvD8UeMbM9oT3Hf8gkLQUWAowZcrBwL4FtCjD/lE2yJ9AZS2owypjUO+kldY6tTAHqTfD/n9Vhj/ey1pvzGwNsGbCtEtiry+iQ4vczFYAK9JMT+I+d0kfAHaY2V3xyW0WbfsHgZktN7NRMxsdee3++8xLO8hnEXwGOXVrmP2pyhkSVQrscVVJd69y0K5sDVLesjgFMe28LUOjr0rSaLmfCJwu6f3AfkR97l8ADpY0ObTeU/mDICt5d83ElbUV0q+qBMdu6tiKz1MdykAdJW65m9lFZjbTzGYT/Ylws5n9V+AW4Myw2BLghn7Wl9avcxkOH+Pq2OqoW6Wu2/5A+cpdWvWybPtVRlleoXoB0Z+rY0R98FdluK1KK7KgDlvZ6hgIYfj9KrIx4YHOtZNqcDezW83sA+H1VjObb2ZvMrMPxs7vHEhaBbeuwci5Mkur3vkP2OD83jJBlsG/bgWz7j+Uddu/LMtf3fKqTjy4l0RVfgCaUpmrsp9VKTcuf6UK7npxd9FJcM65WihVcHfttTvHuYhbo1alNZuWIva33fdalWsdXLlULrh3K+hlHOuxzIfNZTtdtOrKnJ9lLoft+A9acpUL7nVW5grYtFZ7S5n3u8zlpR9lbIzVSSOCe14V1FsbrkryKq9l/oGss0oE97xaKF4I22t6vjR9/zvJK1+qfoRSlEoEd+ecc4Px4N5Fmf8gcy5vXh+qxYN7yaR9CJr0TyvvkogkzYe0/zz0rgrXiwf3DHkFdGXi5bFZPLg3lB9iZ6Nu+eqnK1aXB/c+eSF3znUjaaGkTZLGJF3YZv75ku6XdK+kmyQdFZv3kqR7wmN1Gunx4O468v72fXl+uE4kTQKuBE4DjgHOknTMhMV+Doya2duBVcDnYvNeMLPjwuP0NNKUxhiqsyTdIukBSRskfSJMP0TSWkmbw/PU5Ml1zrlSmg+MhXEsdgPXAoviC5jZLWb2fHh7G9Hwo5lJYwzVPcD/NLO7JR0A3CVpLfBh4CYzWxYOUS4kGp3JOdcnv+o5W7t3T+bh8cP6XXyapHWx98vNbHl4PQN4LDZvHHhXl3WdA/wo9n6/sO49wDIz+0G/ieokcXA3s+3A9vD615IeINrRRcB7w2IrgVvx4D60g7bs4tk3Tik6Gc412U4zG+0wT22mtb11q6Q/BkaB349NfoOZbZN0NHCzpPVmtiVJYlPtc5c0G3gHcDtwRAj8rR+Awzt8ZqmkdZLW7X75hTSTAwzeT+r9qs715vXqVcaBWbH3M4FtExeSdApwMXB6fOhRM9sWnrcSNYTfkTRBqQV3Sb8FfBf4czN7rt/PmdlyMxs1s9GR17wureRUWhnOR25AZRxKGfKlDOXDvcqdwFxJcySNAIuBfc56kfQO4O+JAvuO2PSpkqaE19OAE4H7kyYojT53JL2WKLB/28y+FyY/IWm6mW2XNB3Y0XkNzjlXXWa2R9J5wI3AJGCFmW2QdCmwzsxWA38D/BbwHUkAj4YzY94C/L2kl4ka3MvMrPjgriiVVwEPmNn/jc1aDSwBloXnG5Juq191u5DEuTpZcORG1j4+r+hkpM7M1gBrJky7JPb6lA6f+ylwbNrpSaNb5kTgvwEnxU7Cfz9RUF8gaTOwILwfmh+KOtc8Xu+Hl8bZMv9K+3+KAU5Ouv6q8kLpymhk4zi752V6erUrCb9C1TnnasiDu3PO1ZAHd/cqZTjdr8w8f1wVeHB3zrka8uDeQH6qqHP158HduZT5j6crAw/uzjlXQx7cnXOuhjy4O+dcDXlwd865GvLg7pxzNeTB3TnnasiDu3PO1ZAHd+ecqyEP7s5ViN9K2vUr8+AuaaGkTZLGJF2Y9facc85lHNwlTQKuBE4DjgHOknRMltusC2+hOVctvRqykqZIui7Mv13S7Ni8i8L0TZJOTSM9Wbfc5wNjZrbVzHYD1wKLMt6mc87lqs+G7DnA02b2JuAK4PLw2WOAxcBbgYXAl8P6Esk6uM8AHou9Hw/T9pK0VNI6Set2v/xCxslxzrlM9NOQXQSsDK9XASdLUph+rZntMrOHgLGwvkSyDu7txla1fd6YLTezUTMbHXnN6zJOjnPODW1aqyEaHktj83o2ZOPLmNke4Fng0D4/O7DEA2T3MA7Mir2fCWzLeJvOOdcX7RZTHh3pd/GdZjbaaVVtplmfy/Tz2YFl3XK/E5graY6kEaJ+pdUZb7MWfIR65yqln4bs3mUkTQYOAp7q87MDyzS4h0OP84AbgQeA681sQ5bbdM65AvTTkF0NLAmvzwRuNjML0xeHs2nmAHOBO5ImKOtuGcxsDbAm6+041wS7583002RLyMz2SGo1ZCcBK8xsg6RLgXVmthq4CviWpDGiFvvi8NkNkq4H7gf2AOea2UtJ05R5cHfOuSZo15A1s0tir18EPtjhs5cBl6WZHr/9gHPO1ZAHd+ecqyEP7s45V0Me3J1zroY8uDvnXA15cHfOuRry4O6cczXkwb2B1j4+r+gk1JrnrysDD+7OOVdDHtydc66GPLi7V1n13PFFJ6HUPH9cFXhwd865GvLg7pxzNeTBPSM+2IYrIy+XzVGZ4O6F0rnm8Xo/vETBXdLfSNoo6V5J35d0cGzeRZLGJG2SdGrypPbPzzN2rry8fuYjact9LfA2M3s78CBwEYCkY4hGGXkrsBD4sqRJCbflnHOuT4mCu5n9OIyTCnAb0cCuAIuAa81sl5k9BIwB85Nsq0nKcCjqp/u1V4Z8KUP5cOWXZp/7R4AfhdczgMdi88bDtFeRtFTSOknrdr/8QorJiQxaGctQeZ0rO69Xg5F0iKS1kjaH56ltljlO0s8kbQhd3f85Nu+bkh6SdE94HNdrmz2Du6R/lnRfm8ei2DIXEw3s+u3WpDarsnbrN7PlZjZqZqMjr3ldr+Q01rNvnFJ0Epxzw7sQuMnM5gI3hfcTPQ+cbWat7uwvxP/HBP7CzI4Lj3t6bbDnANlmdkq3+ZKWAB8ATjazVgAfB2bFFpsJbOu1Lefcvp594xQO2rKr6GS45BYB7w2vVwK3AhfEFzCzB2Ovt0naARwGPDPMBpOeLbMwJPB0M3s+Nms1sFjSFElzgLnAHUm25ZxzBZvW6kIOj6UDfPYIM9sOEJ4P77awpPnACLAlNvmy0F1zhaSeh/I9W+49/B0wBVgrCeA2M/uYmW2QdD1wP1F3zblm9lLCbbmcrXrueM488O6ik1EaTe83rqNJu+GAR9r2GLez08xGO82U9M/AkW1mXTxImiRNB74FLDGzl8Pki4DHiQL+cqJG9aXd1pMouJvZm7rMuwy4LMn6y+Th8cOKToKrAD+Hu7m6dWFLekLSdDPbHoL3jg7LHQj8EPjfZnZbbN3bw8tdkr4BfKpXeipzhapLlwch1w9v1KRmNbAkvF4C3DBxAUkjwPeBq83sOxPmTQ/PAs4A7uu1QQ/uGfLzkV2ZeHks1DJggaTNwILwHkmjkr4elvkQ8B7gw21Oefy2pPXAemAa8Fe9Npi0z73Wimjdpl0BHx4/jNkznxz6897vHkna3552C3j3vJmMbBxPdZ0uO2b2K+DkNtPXAR8Nr/8B+IcOnz9p0G16y9051xfvyqsWD+7OOVdDlQjuefUV+qlu7TU9X5q+/53klS/+X8FwKhHck8qrEPotAlyV5FVe/cexGI0I7lVR5hZKUytomfd7kPJSxrLlp1lmq3LBvVtro4yFpYyVqsX/IEuX52d6/Cg4ucoF9yZqV9B/fVS7G29mq8yt2CwUsb/tvlcPdG4YpQrutt9I0UlwzrlaKFVwb7Iyd9/ENaX1XpX9rEq5cfnz4B5kWZnrVgGrEviGVbf9y7L81S2v6qT0wT2tgumF0Ln8pVXv6tZAykPpg3sTFFlwhz3Do64/lsPuV5Fnynjgc+2kEtwlfUqSSZoW3kvSlySNhZFD+q4xaRXUsp2WVscKWLcAX7f9gfKVu7TqZdn2q4wSB3dJs4huYflobPJpREPrzQWWAl9Jup0sFVmpq15IVz13fOWDYh32ochyVPW8q6s0Wu5XAH8JxMeqWkR0w3kLo4kc3LrZ/CDSLrBZFMJBzjcfZn+qco5zVSt4VdI9TDkYpLxlcd1E2nlb9YZQ3hLdz13S6cAvzewXYQzVlhnAY7H342HadiYIg8wuBZgy5WCgHPeqHvbwsddo9fF9q1thjVfmMt8DvioBfVCDlK1hGw1rH5/HgiM3DvXZtNSt3mSlZ3DvMejr/wLe1+5jbaa1HYXWzJYTDfjKgQfM3LtMVl/gxMEniqjoeRfOXgN2ZFFhyxbos/yeezUE8rwtRlGBL6965YG9fz2De6dBXyUdC8wBWq32mcDdkuYTtdRnxRafCWxLnNqU1LXlVlae383g33O5DN3nbmbrzexwM5ttZrOJAvrxZvY40WCwZ4ezZk4Ano2N3u2cc40i6RBJayVtDs9TOyz3Umz81NWx6XMk3R4+f10YTLurrM5zXwNsBcaArwH/I42VVuXPRedccjWr7xcCN5nZXOCm8L6dF8zsuPA4PTb9cuCK8PmngXN6bTC14B5a8DvDazOzc83sjWZ2bBgENlVF3BUxb3kW7rJdF1AVeeZbzYJd0ywCVobXK4Ez+v2gon7vk4BVg3zer1B1zpVGyRtt0yStiz2WDvDZI1pd0+H58A7L7RfWfZukVgA/FHjGzPaE962zD7tKdCqka6/X6ZBJlbwCuIR+fZQ44JG2J5elwo8AXjHpRRukru40s9FOM3ucWdivN5jZNklHAzdLWg8812a5ngWk1i33YQ+Zm9pF0dT9HlZT88vrVXtmdoqZva3N4wbgidaFnOF5R4d1bAvPW4FbgXcAO4kuBG01xvs6+7DWwd0550piNbAkvF4C3DBxAUlTJU0Jr6cBJwL3m5kBtwBndvv8RB7c+1DE2KxFHTrXvXWVlqLyqYhyUcaxiStoGbBA0maie3EtA5A0KunrYZm3AOsk/YIomC8zs/vDvAuA8yWNEfXBX9Vrg97n3hC9rlJ12fDA6ADM7FfAyW2mrwM+Gl7/FDi2w+e3AvMH2WZlWu5V+xOoaumN89Z7d1XOn6qVy6qlt0wqE9ybxAu068bLh+tH7YP7oK2sKrfK0uT50J7nS8TrVfnVPrjnoa7nnXuF3Fdd86Ou5bfpahfcy/QH1jCHz70+4xWxGXp9z1mUrTyVqZ7WVSWCe5kKZZUNU6Hq2lod1DD54AHMFakSwT2pfitmFoFskB+msv6INT3Al3X/iy5bedWrstaLsmtEcHfJlTXAZa2p++2qr1QXMb20Xz37k/u5kVgVWietQFf0GJp5qEpQr0vZculL3HKX9HFJmyRtkPS52PSLJI2FeacOu/60CmavylpkZa5a5atK4BtW1favyPKTV72qWh0pg0TBXdIfEN2E/u1m9lbg82H6McBi4K3AQuDLkib1s85+vsQ8zxhJ60+xTvs1SKFNY7/T2p+1j8+rXBDsJc19SiOfB/m+0yhf3eT553A/++3Bvrek3TJ/SnRzm10AZta6jeUi4Now/aFws5v5wM/6WWnrUDPtL3Dt4/PadinkFaRa+9M6jK5DAZ2Yd1Xqsqnbj1O8iybvUbzyqFdZxYW6Shrc3wy8W9JlwIvAp8zsTqJRQm6LLddx5JAwmslSgJH9XxkzNqsvcGJBTKsADjLAQp0LZ6f8LDLo1y2IdzNo2UrrKDirejVRnetO2noG9x6ji0wGpgInAP8OuD6MINKuxLSNfGa2HFgOsP9hs7IbfiamSZW9nSLuENmkPG/q+e1N+o6roGdwN7NTOs2T9KfA98LN5O+Q9DIwjailPiu2aF8jhzjn6q+pP355S3q2zA+IRuVG0puBEaIhoVYDiyVNkTQHmAvckXBbfatj4fHbDjSTf+9uWEn73FcAKyTdB+wGloRW/AZJ1wP3A3uAc83spYTbcinywTuyUceGhaumRMHdzHYDf9xh3mXAZUnWX7RBK2rWo9Y7l6ZBjwq8QVAtlbv9QBMPU5u4z+4VTfz+67bPkg6RtFbS5vA8tc0yfyDpntjjRUlnhHnflPRQbN5xvbZZueDu0uNdCOny/HRdXAjcZGZzgZvC+32Y2S1mdpyZHUf0X+bzwI9ji/xFa76Z3dNrgx7cG84DUjo8H10Pi4CV4fVK4Iwey58J/MjMnh92g7UN7kVVtrQPJ+t2eOqGU5dy1eAfwSPMbDtAeD68x/KLgWsmTLtM0r2SrpDU82quUt0VskyaVAj9j7JkvKxUl17czcjG8X4XnyZpXez98nARZrSu7hd89p8maTpwLHBjbPJFwONEp5svBy4ALu22nkoF96a1YvO+QVqdKm1e8r6hVpPOxirh/u40s9FOM3tc8PmEpOlmtj0E7x2dlgU+BHzfzH4TW/f28HKXpG8An+qV2Np2y0B9umby0qQWaBqqml/eJVOI1cCS8HoJcEOXZc9iQpdM+EFAkoj66+/rtcFaB/dhlaEQegUstyY3HLyMDGUZsEDSZmBBeI+kUUlfby0kaTbRrVv+ZcLnvy1pPbCe6BYvf9Vrg5XplilDoR5EkkPKovfVu2i6Kzq4VblsDaqEXTNDMbNfASe3mb4O+Gjs/cO0uYOumZ006DZr33IftCKmWXGHqUhlqXxFB7CyKku+FF22iqxXrj+VCO5lCXjDGCTtZdvPh8cP80oZlDEvqly2BlHltBepVMH9pZFXT8tzaLmsKm8/+1DmAtwKbGULblmrwn4XWbbyrFdlrh9lVbo+99aXeMAjlvphZLd+5KwrcHy/Jk6rkon5VKe++TIH8W7i5Sjv8pVnvWr1v1ex3hShdMG9JYsvsFXQ4oUx7wpdt4LZT/6V4QegqoF7UEWUrzzrVd3qT5ZKG9yz1JSKXhae383g33O5JOpzl3ScpNvCLSjXSZofpkvSlySNhXshHJ9Ocp1zzvUj6R+qnwP+T7hF5SXhPcBpREPrzQWWAl9JuB3nnHMDSBrcDTgwvD6IVwbBXgRcbZHbgINbl88655zLXtI+9z8HbpT0eaIfit8L02cAj8WWGw/TtuOccy5zPYN7j9tYngx80sy+K+lDwFXAKUC7v7TbXkMsaSlR1w2TD3rVyFPOOeeG0DO497iN5dXAJ8Lb7wCtG+CME938pmUmr3TEI1k5AAAFaElEQVTZTFz/cqL7E7PfjFnVv4mEc86VQNI+923A74fXJwGbw+vVwNnhrJkTgGdj9yN2zjmXsaR97v8d+KKkycCLhO4VYA3wfmCMaJDXP0m4HeeccwNIFNzN7F+Bd7aZbsC5SdbtnHNueKW6cZhzzrl0eHB3zrka8uDunHM15MHdOedqyIO7c85lTNIHJW2Q9LKk0S7LLZS0Kdx08cLY9DmSbpe0WdJ1ktoMbbQvD+7OOZe9+4D/BPyk0wKSJgFXEt148RjgLEnHhNmXA1eY2VzgaeCcXhv04O6ccxkzswfMbFOPxeYDY2a21cx2A9cCiySJ6CLRVWG5lcAZvbZZqsE6dm0b3/ngJec/ktHqpwE7M1r3sDxN/SljmqCc6WpSmo5KuoLn9jx54z89/uVpfS6+n6R1sffLw+1T0tLuhovvAg4FnjGzPbHpM3qtrFTB3cwyG8pF0joz69jXVQRPU3/KmCYoZ7o8TYMxs4VpravbTRbN7IZ+VtFmmnWZ3lWpgrtzzlVVt5ss9qnTDRd3Eo2JMTm03jveiDHO+9ydc64c7gTmhjNjRoDFwOpwO5dbgDPDckuAnkcCTQruafaNpcXT1J8ypgnKmS5PUwlJ+kNJ48C/B34o6cYw/bclrQEIrfLzgBuBB4DrzWxDWMUFwPmSxoj64K/quc3oR8E551ydNKnl7pxzjeHB3Tnnaqh2wb3TZb6SZkt6QdI94fHV2Lx3SlofLvn9UrhoIPM0hXkXhe1uknRqbHrby5CzIukzkn4Zy5/390pjHvLOhy7peDiUkXta5zpLOkTS2nBJ+FpJmQ8CLGmFpB2S7otNa5uOMBLal0Le3Svp+BzTVMry1ChmVqsH8Bbgd4BbgdHY9NnAfR0+cwfRHx0CfgScllOajgF+AUwB5gBbgEnhsQU4GhgJyxyTcb59BvhUm+lt05jTd5l7PnRJy8PAtAnTPgdcGF5fCFyeQzreAxwfL8ud0kE0GtqPQrk+Abg9xzSVrjw17VG7lrv1d5nvXpKmAwea2c8sKn1X08elvSmlaRFwrZntMrOHiIYlnE+Hy5DTTNMAOqUxD2XKh3YWEV0KDn1eEp6Umf0EeKrPdCwCrrbIbUTnSk/PKU2dFFmeGqV2wb2HOZJ+LulfJL07TJtBdPFAS1+X9qak3eXGM7pMz9p54fB9RayLoai0FL3tiQz4saS7JLXGCj7CwsDv4fnwgtLWKR1F51/ZylOjVPIK1SEv890OvMHMfiXpncAPJL2VIS/tTSlNnbbd7kc38Tmr3dIIfAX4bNjOZ4G/BT7SJY15KHLbE51oZtskHQ6slbSxoHQMosj8K2N5apRKBncb4jJfM9sF7Aqv75K0BXgzUcthZmzRvi7tTSNNdL7cmC7Th9ZvGiV9DfjHPtKYtSK3vQ8z2xaed0j6PlFXwhOSppvZ9tDdsaOItHVJR2H5Z2ZPtF6XqDw1SmO6ZSQdpuh+yUg6GpgLbA2Hsb+WdEI4S+Zs+ri0NyWrgcWSpkiaE9J0Bx0uQ84yIRP6Yv+Q6P7T3dKYh9zzoR1J+0s6oPUaeB9R/qwmuhQc+rwkPCOd0rEaODucNXMC8Gyr+yZrJS1PzVL0P7ppP4gK0jhRK/0J4MYw/Y+ADUT/1N8N/MfYZ0aJCt8W4O8IV+5mnaYw7+Kw3U3EztIhOtPhwTDv4hzy7VvAeuBeogo4vVcac/o+c82HDmk4OpSbX4QydHGYfihwE7A5PB+SQ1quIepi/E0oU+d0SgdRF8iVIe/WEztTK4c0lbI8Nenhtx9wzrkaaky3jHPONYkHd+ecqyEP7s45V0Me3J1zroY8uDvnXA15cHfOuRry4O6cczX0/wEZJo+m5ITqIQAAAABJRU5ErkJggg==\n", 74 | "text/plain": [ 75 | "
" 76 | ] 77 | }, 78 | "metadata": { 79 | "needs_background": "light" 80 | }, 81 | "output_type": "display_data" 82 | } 83 | ], 84 | "source": [ 85 | "plt.contourf(lon2, lat2, pseudo)\n", 86 | "plt.colorbar()" 87 | ] 88 | }, 89 | { 90 | "cell_type": "markdown", 91 | "metadata": {}, 92 | "source": [ 93 | "We can write the field as a binary file, to be used as an input file for the model with **xmitgcm.utils.write_to_binary**. Default is single precision, but double precision can be written with corresponding numpy.dtype. Note that here we use a numpy.array but one can use xarray as well using the *DataArray.values*" 94 | ] 95 | }, 96 | { 97 | "cell_type": "code", 98 | "execution_count": 4, 99 | "metadata": {}, 100 | "outputs": [ 101 | { 102 | "data": { 103 | "text/plain": [ 104 | "\u001b[0;31mSignature:\u001b[0m \u001b[0mxmitgcm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mutils\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwrite_to_binary\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mflatdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfileout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'float32'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 105 | "\u001b[0;31mDocstring:\u001b[0m\n", 106 | "write data in binary file\n", 107 | "\n", 108 | "PARAMETERS:\n", 109 | "\n", 110 | "flatdata: numpy.array\n", 111 | " vector of data to write\n", 112 | "fileout: str\n", 113 | " output file name\n", 114 | "dtype: np.dtype\n", 115 | " single/double precision\n", 116 | "\u001b[0;31mFile:\u001b[0m ~/.conda/envs/production2/lib/python3.6/site-packages/xmitgcm-0.2.2-py3.6.egg/xmitgcm/utils.py\n", 117 | "\u001b[0;31mType:\u001b[0m function\n" 118 | ] 119 | }, 120 | "metadata": {}, 121 | "output_type": "display_data" 122 | } 123 | ], 124 | "source": [ 125 | "xmitgcm.utils.write_to_binary?" 126 | ] 127 | }, 128 | { 129 | "cell_type": "code", 130 | "execution_count": 5, 131 | "metadata": {}, 132 | "outputs": [], 133 | "source": [ 134 | "xmitgcm.utils.write_to_binary(pseudo.flatten(), 'file1.bin')" 135 | ] 136 | }, 137 | { 138 | "cell_type": "markdown", 139 | "metadata": {}, 140 | "source": [ 141 | "## More complicated case: a LLC grid" 142 | ] 143 | }, 144 | { 145 | "cell_type": "markdown", 146 | "metadata": {}, 147 | "source": [ 148 | "In this case, let's assume we have a xarray dataarray or dataset well formatted on the llc grid. This dataset can be the result of a regridding onto the LLC grid that we want to use as an initial condition for the model (for example). We need to generate the binary file that MITgcm can read. Here's what we can do:" 149 | ] 150 | }, 151 | { 152 | "cell_type": "code", 153 | "execution_count": 6, 154 | "metadata": {}, 155 | "outputs": [ 156 | { 157 | "name": "stdout", 158 | "output_type": "stream", 159 | "text": [ 160 | "--2019-05-01 21:16:09-- https://ndownloader.figshare.com/files/14066567\n", 161 | "Resolving ndownloader.figshare.com (ndownloader.figshare.com)... 52.48.232.64, 63.32.41.137, 54.229.125.140, ...\n", 162 | "Connecting to ndownloader.figshare.com (ndownloader.figshare.com)|52.48.232.64|:443... connected.\n", 163 | "HTTP request sent, awaiting response... 302 Found\n", 164 | "Location: https://s3-eu-west-1.amazonaws.com/pfigshare-u-files/14066567/global_oce_llc90.tar.gz [following]\n", 165 | "--2019-05-01 21:16:09-- https://s3-eu-west-1.amazonaws.com/pfigshare-u-files/14066567/global_oce_llc90.tar.gz\n", 166 | "Resolving s3-eu-west-1.amazonaws.com (s3-eu-west-1.amazonaws.com)... 52.218.36.2\n", 167 | "Connecting to s3-eu-west-1.amazonaws.com (s3-eu-west-1.amazonaws.com)|52.218.36.2|:443... connected.\n", 168 | "HTTP request sent, awaiting response... 200 OK\n", 169 | "Length: 84966646 (81M) [application/gzip]\n", 170 | "Saving to: ‘14066567.1’\n", 171 | "\n", 172 | "100%[======================================>] 84,966,646 12.9MB/s in 6.6s \n", 173 | "\n", 174 | "2019-05-01 21:16:16 (12.3 MB/s) - ‘14066567.1’ saved [84966646/84966646]\n", 175 | "\n" 176 | ] 177 | } 178 | ], 179 | "source": [ 180 | "# First let's download a sample dataset\n", 181 | "! wget https://ndownloader.figshare.com/files/14066567\n", 182 | "! tar -xf 14066567" 183 | ] 184 | }, 185 | { 186 | "cell_type": "markdown", 187 | "metadata": {}, 188 | "source": [ 189 | "We can load this dataset with xmitgcm:" 190 | ] 191 | }, 192 | { 193 | "cell_type": "code", 194 | "execution_count": 7, 195 | "metadata": {}, 196 | "outputs": [], 197 | "source": [ 198 | "extra_metadata = xmitgcm.utils.get_extra_metadata(domain='llc', nx=90)\n", 199 | "\n", 200 | "ds = xmitgcm.open_mdsdataset('./global_oce_llc90/', iters= [8], geometry='llc', extra_metadata=extra_metadata)" 201 | ] 202 | }, 203 | { 204 | "cell_type": "code", 205 | "execution_count": 8, 206 | "metadata": {}, 207 | "outputs": [ 208 | { 209 | "data": { 210 | "text/plain": [ 211 | "\n", 212 | "Dimensions: (face: 13, i: 90, i_g: 90, j: 90, j_g: 90, k: 50, k_l: 50, k_p1: 51, k_u: 50, time: 1)\n", 213 | "Coordinates:\n", 214 | " * i (i) int64 0 1 2 3 4 5 6 7 8 9 10 ... 80 81 82 83 84 85 86 87 88 89\n", 215 | " * i_g (i_g) int64 0 1 2 3 4 5 6 7 8 9 ... 80 81 82 83 84 85 86 87 88 89\n", 216 | " * j (j) int64 0 1 2 3 4 5 6 7 8 9 10 ... 80 81 82 83 84 85 86 87 88 89\n", 217 | " * j_g (j_g) int64 0 1 2 3 4 5 6 7 8 9 ... 80 81 82 83 84 85 86 87 88 89\n", 218 | " * k (k) int64 0 1 2 3 4 5 6 7 8 9 10 ... 40 41 42 43 44 45 46 47 48 49\n", 219 | " * k_u (k_u) int64 0 1 2 3 4 5 6 7 8 9 ... 40 41 42 43 44 45 46 47 48 49\n", 220 | " * k_l (k_l) int64 0 1 2 3 4 5 6 7 8 9 ... 40 41 42 43 44 45 46 47 48 49\n", 221 | " * k_p1 (k_p1) int64 0 1 2 3 4 5 6 7 8 9 ... 41 42 43 44 45 46 47 48 49 50\n", 222 | " * face (face) int64 0 1 2 3 4 5 6 7 8 9 10 11 12\n", 223 | " iter (time) int64 8\n", 224 | " * time (time) int64 8\n", 225 | " XC (face, j, i) >f4 dask.array\n", 226 | " YC (face, j, i) >f4 dask.array\n", 227 | " XG (face, j_g, i_g) >f4 dask.array\n", 228 | " YG (face, j_g, i_g) >f4 dask.array\n", 229 | " CS (face, j, i) >f4 dask.array\n", 230 | " SN (face, j, i) >f4 dask.array\n", 231 | " Z (k) >f4 -5.0 -15.0 -25.0 -35.0 ... -5039.25 -5461.25 -5906.25\n", 232 | " Zp1 (k_p1) >f4 0.0 -10.0 -20.0 -30.0 ... -5244.5 -5678.0 -6134.5\n", 233 | " Zu (k_u) >f4 -10.0 -20.0 -30.0 -40.0 ... -5244.5 -5678.0 -6134.5\n", 234 | " Zl (k_l) >f4 0.0 -10.0 -20.0 -30.0 ... -4834.0 -5244.5 -5678.0\n", 235 | " rA (face, j, i) >f4 dask.array\n", 236 | " dxG (face, j_g, i) >f4 dask.array\n", 237 | " dyG (face, j, i_g) >f4 dask.array\n", 238 | " Depth (face, j, i) >f4 dask.array\n", 239 | " rAz (face, j_g, i_g) >f4 dask.array\n", 240 | " dxC (face, j, i_g) >f4 dask.array\n", 241 | " dyC (face, j_g, i) >f4 dask.array\n", 242 | " rAw (face, j, i_g) >f4 dask.array\n", 243 | " rAs (face, j_g, i) >f4 dask.array\n", 244 | " drC (k_p1) >f4 5.0 10.0 10.0 10.0 10.0 ... 399.0 422.0 445.0 228.25\n", 245 | " drF (k) >f4 10.0 10.0 10.0 10.0 10.0 ... 364.5 387.5 410.5 433.5 456.5\n", 246 | " PHrefC (k) >f4 49.05 147.15 245.25 ... 49435.043 53574.863 57940.312\n", 247 | " PHrefF (k_p1) >f4 0.0 98.1 196.2 294.3 ... 51448.547 55701.18 60179.445\n", 248 | " hFacC (k, face, j, i) >f4 dask.array\n", 249 | " hFacW (k, face, j, i_g) >f4 dask.array\n", 250 | " hFacS (k, face, j_g, i) >f4 dask.array\n", 251 | " maskC (k, face, j, i) bool dask.array\n", 252 | " maskW (k, face, j, i_g) bool dask.array\n", 253 | " maskS (k, face, j_g, i) bool dask.array\n", 254 | "Data variables:\n", 255 | " W (time, k_l, face, j, i) >f4 dask.array\n", 256 | " PHL (time, face, j, i) >f4 dask.array\n", 257 | " PH (time, k, face, j, i) >f4 dask.array\n", 258 | " V (time, k, face, j_g, i) >f4 dask.array\n", 259 | " Eta (time, face, j, i) >f4 dask.array\n", 260 | " S (time, k, face, j, i) >f4 dask.array\n", 261 | " ETAN (time, face, j, i) >f4 dask.array\n", 262 | " SIarea (time, face, j, i) >f4 dask.array\n", 263 | " SIheff (time, face, j, i) >f4 dask.array\n", 264 | " SIhsnow (time, face, j, i) >f4 dask.array\n", 265 | " DETADT2 (time, face, j, i) >f4 dask.array\n", 266 | " PHIBOT (time, face, j, i) >f4 dask.array\n", 267 | " sIceLoad (time, face, j, i) >f4 dask.array\n", 268 | " MXLDEPTH (time, face, j, i) >f4 dask.array\n", 269 | " oceSPDep (time, face, j, i) >f4 dask.array\n", 270 | " SIatmQnt (time, face, j, i) >f4 dask.array\n", 271 | " SIatmFW (time, face, j, i) >f4 dask.array\n", 272 | " oceQnet (time, face, j, i) >f4 dask.array\n", 273 | " oceFWflx (time, face, j, i) >f4 dask.array\n", 274 | " oceTAUX (time, face, j, i_g) >f4 dask.array\n", 275 | " oceTAUY (time, face, j_g, i) >f4 dask.array\n", 276 | " ADVxHEFF (time, face, j, i_g) >f4 dask.array\n", 277 | " ADVyHEFF (time, face, j_g, i) >f4 dask.array\n", 278 | " DFxEHEFF (time, face, j, i_g) >f4 dask.array\n", 279 | " DFyEHEFF (time, face, j_g, i) >f4 dask.array\n", 280 | " ADVxSNOW (time, face, j, i_g) >f4 dask.array\n", 281 | " ADVySNOW (time, face, j_g, i) >f4 dask.array\n", 282 | " DFxESNOW (time, face, j, i_g) >f4 dask.array\n", 283 | " DFyESNOW (time, face, j_g, i) >f4 dask.array\n", 284 | " SIuice (time, face, j, i_g) >f4 dask.array\n", 285 | " SIvice (time, face, j_g, i) >f4 dask.array\n", 286 | " U (time, k, face, j, i_g) >f4 dask.array\n", 287 | " T (time, k, face, j, i) >f4 dask.array\n", 288 | "Attributes:\n", 289 | " Conventions: CF-1.6\n", 290 | " title: netCDF wrapper of MITgcm MDS binary data\n", 291 | " source: MITgcm\n", 292 | " history: Created by calling `open_mdsdataset(extra_metadata={'has_fa..." 293 | ] 294 | }, 295 | "execution_count": 8, 296 | "metadata": {}, 297 | "output_type": "execute_result" 298 | } 299 | ], 300 | "source": [ 301 | "ds" 302 | ] 303 | }, 304 | { 305 | "cell_type": "markdown", 306 | "metadata": {}, 307 | "source": [ 308 | "Now let's say we want to use the temperature T and make it the initial condition for another simulation. First we need to re-build the facets, concatenate them into the compact form that MITgcm reads/writes and then write the compact to a binary file. We would do this as follows:" 309 | ] 310 | }, 311 | { 312 | "cell_type": "code", 313 | "execution_count": 9, 314 | "metadata": {}, 315 | "outputs": [], 316 | "source": [ 317 | "# temperature\n", 318 | "facets = xmitgcm.utils.rebuild_llc_facets(ds['T'], extra_metadata)\n", 319 | "compact = xmitgcm.utils.llc_facets_3d_spatial_to_compact(facets, 'Z', extra_metadata)\n", 320 | "xmitgcm.utils.write_to_binary(compact, 'T_initial_condition.bin')" 321 | ] 322 | }, 323 | { 324 | "cell_type": "markdown", 325 | "metadata": {}, 326 | "source": [ 327 | "In this case, we already had the binary file to read from so we can compare checksums:" 328 | ] 329 | }, 330 | { 331 | "cell_type": "code", 332 | "execution_count": 10, 333 | "metadata": {}, 334 | "outputs": [ 335 | { 336 | "name": "stdout", 337 | "output_type": "stream", 338 | "text": [ 339 | "045086cc85a66aa59a713ab7d716539e T_initial_condition.bin\n" 340 | ] 341 | } 342 | ], 343 | "source": [ 344 | "!md5sum T_initial_condition.bin" 345 | ] 346 | }, 347 | { 348 | "cell_type": "code", 349 | "execution_count": 11, 350 | "metadata": {}, 351 | "outputs": [ 352 | { 353 | "name": "stdout", 354 | "output_type": "stream", 355 | "text": [ 356 | "045086cc85a66aa59a713ab7d716539e ./global_oce_llc90/T.0000000008.data\n" 357 | ] 358 | } 359 | ], 360 | "source": [ 361 | "!md5sum ./global_oce_llc90/T.0000000008.data" 362 | ] 363 | }, 364 | { 365 | "cell_type": "markdown", 366 | "metadata": {}, 367 | "source": [ 368 | "The file generated is identical to the original file! A similar function exist for 2d files,..." 369 | ] 370 | } 371 | ], 372 | "metadata": { 373 | "kernelspec": { 374 | "display_name": "Python3 (production2)", 375 | "language": "python", 376 | "name": "production2" 377 | }, 378 | "language_info": { 379 | "codemirror_mode": { 380 | "name": "ipython", 381 | "version": 3 382 | }, 383 | "file_extension": ".py", 384 | "mimetype": "text/x-python", 385 | "name": "python", 386 | "nbconvert_exporter": "python", 387 | "pygments_lexer": "ipython3", 388 | "version": "3.6.7" 389 | } 390 | }, 391 | "nbformat": 4, 392 | "nbformat_minor": 2 393 | } 394 | --------------------------------------------------------------------------------