├── .circleci └── config.yml ├── .github └── workflows │ └── python-publish.yml ├── .gitignore ├── .travis.yml ├── COPYING ├── COPYING.LESSER ├── MANIFEST.in ├── README.md ├── bin ├── cis └── cis.lsf ├── cis ├── __init__.py ├── aggregation │ ├── __init__.py │ ├── collapse_kernels.py │ ├── gridded_collapsor.py │ └── ungridded_aggregator.py ├── cis_main.py ├── collocation │ ├── __init__.py │ ├── col.py │ ├── col_framework.py │ ├── col_implementations.py │ ├── data_index.py │ ├── gridded_interpolation.py │ ├── haversinedistancekdtreeindex.py │ └── kdtree.py ├── data_io │ ├── Coord.py │ ├── __init__.py │ ├── aeronet.py │ ├── common_data.py │ ├── data_reader.py │ ├── data_writer.py │ ├── gridded_data.py │ ├── hdf.py │ ├── hdf_sd.py │ ├── hdf_vd.py │ ├── hyperpoint.py │ ├── hyperpoint_view.py │ ├── netcdf.py │ ├── products │ │ ├── AProduct.py │ │ ├── CCI.py │ │ ├── EarthCARE.py │ │ ├── HadGEM.py │ │ ├── MODIS.py │ │ ├── NCAR_NetCDF_RAF.py │ │ ├── __init__.py │ │ ├── caliop.py │ │ ├── cloudsat.py │ │ ├── gridded_NetCDF.py │ │ └── products.py │ ├── ungridded_data.py │ └── write_netcdf.py ├── evaluate.py ├── exceptions.py ├── info.py ├── logging.conf ├── maths.py ├── parse.py ├── parse_datetime.py ├── plotting │ ├── __init__.py │ ├── comparativescatter.py │ ├── contourplot.py │ ├── formatted_plot.py │ ├── genericplot.py │ ├── heatmap.py │ ├── histogram.py │ ├── histogram2d.py │ ├── lineplot.py │ ├── plot.py │ ├── raster │ │ ├── world.topo.bathy.200407.3x1350x675.png │ │ ├── world.topo.bathy.200407.3x2700x1350.png │ │ └── world.topo.bathy.200407.3x5400x2700.png │ ├── scatterplot.py │ └── taylor.py ├── plugin.py ├── stats.py ├── subsetting │ ├── __init__.py │ └── subset.py ├── test │ ├── __init__.py │ ├── integration │ │ ├── __init__.py │ │ ├── base_integration_test.py │ │ ├── cis-test_integration_out.nc │ │ ├── collocated_gassp.nc │ │ ├── test_aggregate.py │ │ ├── test_collapse.py │ │ ├── test_colocate.py │ │ ├── test_eval.py │ │ ├── test_io │ │ │ ├── __init__.py │ │ │ ├── test_aeronet.py │ │ │ ├── test_hdf.py │ │ │ ├── test_hdf_sd.py │ │ │ ├── test_hdf_vd.py │ │ │ ├── test_netcdf.py │ │ │ ├── test_products │ │ │ │ ├── __init__.py │ │ │ │ ├── test_Aproduct.py │ │ │ │ ├── test_NetCDF_CF_Gridded.py │ │ │ │ ├── test_data_products.py │ │ │ │ └── test_ncar_raf.py │ │ │ └── test_write_netcdf.py │ │ ├── test_plot_integration.py │ │ ├── test_read_api.py │ │ ├── test_stats.py │ │ ├── test_subset.py │ │ └── test_version.py │ ├── integration_test_data.py │ ├── plot_tests │ │ ├── __init__.py │ │ ├── idiff.py │ │ ├── reference │ │ │ ├── kitten.png │ │ │ └── visual_tests │ │ │ │ ├── test_plotting.TestPlotAPIVisual.test_can_specify_yaxis_altitude.png │ │ │ │ ├── test_plotting.TestPlotAPIVisual.test_contour_over_bluemarble.png │ │ │ │ ├── test_plotting.TestPlotAPIVisual.test_explicit_comparative_scatter.png │ │ │ │ ├── test_plotting.TestPlotAPIVisual.test_heatmap.png │ │ │ │ ├── test_plotting.TestPlotAPIVisual.test_histogram.png │ │ │ │ ├── test_plotting.TestPlotAPIVisual.test_histogram_2d.png │ │ │ │ ├── test_plotting.TestPlotAPIVisual.test_implicit_comparative_scatter.png │ │ │ │ ├── test_plotting.TestPlotAPIVisual.test_invalid_args.png │ │ │ │ ├── test_plotting.TestPlotAPIVisual.test_iris_comparative_scatter.png │ │ │ │ ├── test_plotting.TestPlotAPIVisual.test_iris_multiple_scatter.png │ │ │ │ ├── test_plotting.TestPlotAPIVisual.test_layer_opts.png │ │ │ │ ├── test_plotting.TestPlotAPIVisual.test_mercator_projection.png │ │ │ │ ├── test_plotting.TestPlotAPIVisual.test_mpl_kwargs.png │ │ │ │ ├── test_plotting.TestPlotAPIVisual.test_multiple_line.png │ │ │ │ ├── test_plotting.TestPlotAPIVisual.test_orographic_projection.png │ │ │ │ ├── test_plotting.TestPlotAPIVisual.test_polar_projection.png │ │ │ │ ├── test_plotting.TestPlotAPIVisual.test_scatter2d_over_bluemarble.png │ │ │ │ ├── test_plotting.TestPlotAPIVisual.test_scatter2d_over_bluemarble_coord_axis.png │ │ │ │ ├── test_plotting.TestPlotAPIVisual.test_scatter2d_over_bluemarble_explicit_axis.png │ │ │ │ ├── test_plotting.TestPlotAPIVisual.test_taylor_diagram_gridded.png │ │ │ │ ├── test_plotting.TestPlotVisual.test_aeronet_default_axes.png │ │ │ │ ├── test_plotting.TestPlotVisual.test_aerosol_cci_default_axes.png │ │ │ │ ├── test_plotting.TestPlotVisual.test_bluemarble_0_360.png │ │ │ │ ├── test_plotting.TestPlotVisual.test_bluemarble_minus_180_180.png │ │ │ │ ├── test_plotting.TestPlotVisual.test_coastline_color.png │ │ │ │ ├── test_plotting.TestPlotVisual.test_filled_contour_over_scatter2d.png │ │ │ │ ├── test_plotting.TestPlotVisual.test_filled_contour_over_scatter2d_with_cmin.png │ │ │ │ ├── test_plotting.TestPlotVisual.test_iris_comparative_scatter.png │ │ │ │ ├── test_plotting.TestPlotVisual.test_iris_contour.png │ │ │ │ ├── test_plotting.TestPlotVisual.test_iris_contour_over_heatmap.png │ │ │ │ ├── test_plotting.TestPlotVisual.test_iris_contour_over_heatmap_binary_cmap.png │ │ │ │ ├── test_plotting.TestPlotVisual.test_iris_contourf.png │ │ │ │ ├── test_plotting.TestPlotVisual.test_iris_heatmap.png │ │ │ │ ├── test_plotting.TestPlotVisual.test_iris_heatmap_force_minus_180_to_180.png │ │ │ │ ├── test_plotting.TestPlotVisual.test_iris_histogram.png │ │ │ │ ├── test_plotting.TestPlotVisual.test_iris_histogram2d.png │ │ │ │ ├── test_plotting.TestPlotVisual.test_iris_many_lines.png │ │ │ │ ├── test_plotting.TestPlotVisual.test_iris_one_line.png │ │ │ │ ├── test_plotting.TestPlotVisual.test_iris_one_line_with_step.png │ │ │ │ ├── test_plotting.TestPlotVisual.test_iris_scatter.png │ │ │ │ ├── test_plotting.TestPlotVisual.test_iris_scatter2d.png │ │ │ │ ├── test_plotting.TestPlotVisual.test_iris_scatter2d_overlay.png │ │ │ │ ├── test_plotting.TestPlotVisual.test_mercator_projection.png │ │ │ │ ├── test_plotting.TestPlotVisual.test_multiple_time_series_default_axes.png │ │ │ │ ├── test_plotting.TestPlotVisual.test_multiple_time_series_default_axes_files_with_named_xaxis.png │ │ │ │ ├── test_plotting.TestPlotVisual.test_multiple_time_series_incompatible_axes.png │ │ │ │ ├── test_plotting.TestPlotVisual.test_other_comparative_scatter.png │ │ │ │ ├── test_plotting.TestPlotVisual.test_other_contour.png │ │ │ │ ├── test_plotting.TestPlotVisual.test_other_contourf.png │ │ │ │ ├── test_plotting.TestPlotVisual.test_other_histogram.png │ │ │ │ ├── test_plotting.TestPlotVisual.test_other_histogram2d.png │ │ │ │ ├── test_plotting.TestPlotVisual.test_other_histogram2d_doesnt_plot_coastlines.png │ │ │ │ ├── test_plotting.TestPlotVisual.test_other_histogram_bin_width.png │ │ │ │ ├── test_plotting.TestPlotVisual.test_other_longitude_wrapping_0_360.png │ │ │ │ ├── test_plotting.TestPlotVisual.test_other_longitude_wrapping_0_360_forced_minus_180_to_180.png │ │ │ │ ├── test_plotting.TestPlotVisual.test_other_longitude_wrapping_minus_180_180.png │ │ │ │ ├── test_plotting.TestPlotVisual.test_other_longitude_wrapping_minus_180_180_forced_0_to_360.png │ │ │ │ ├── test_plotting.TestPlotVisual.test_other_longitude_wrapping_multiple_ranges_forced_0_to_360.png │ │ │ │ ├── test_plotting.TestPlotVisual.test_other_longitude_wrapping_multiple_ranges_forced_minus_180_to_180.png │ │ │ │ ├── test_plotting.TestPlotVisual.test_other_many_lines.png │ │ │ │ ├── test_plotting.TestPlotVisual.test_other_many_scatter_points.png │ │ │ │ ├── test_plotting.TestPlotVisual.test_other_many_scatter_points_given_color.png │ │ │ │ ├── test_plotting.TestPlotVisual.test_other_one_line.png │ │ │ │ ├── test_plotting.TestPlotVisual.test_other_scatter.png │ │ │ │ ├── test_plotting.TestPlotVisual.test_other_scatter2d.png │ │ │ │ ├── test_plotting.TestPlotVisual.test_other_taylor_diagram.png │ │ │ │ ├── test_plotting.TestPlotVisual.test_plotting_heatmap_of_aggregated_ungridded_data.png │ │ │ │ ├── test_plotting.TestPlotVisual.test_polar_projection.png │ │ │ │ ├── test_plotting.TestPlotVisual.test_scatter2d_over_contour.png │ │ │ │ ├── test_plotting.TestPlotVisual.test_setting_xrange_using_datetimes.png │ │ │ │ └── test_plotting.TestPlotVisual.test_transparent_contour_over_bluemarble.png │ │ └── test_plotting.py │ ├── runner.py │ ├── unit │ │ ├── __init__.py │ │ ├── aggregation │ │ │ ├── __init__.py │ │ │ ├── test_aggregation_kernels.py │ │ │ ├── test_gridded_aggregation.py │ │ │ └── test_ungridded_aggregation.py │ │ ├── colocate │ │ │ ├── __init__.py │ │ │ ├── test_GridCellBinIndex.py │ │ │ ├── test_colocate.py │ │ │ ├── test_constraint.py │ │ │ ├── test_dummy_col.py │ │ │ ├── test_general_gridded_col.py │ │ │ ├── test_general_ungridded_col.py │ │ │ ├── test_gridded_gridded_col.py │ │ │ ├── test_gridded_interpolation.py │ │ │ ├── test_gridded_ungridded_col.py │ │ │ └── test_kernel.py │ │ ├── eval │ │ │ ├── __init__.py │ │ │ └── test_calc.py │ │ ├── stats │ │ │ ├── __init__.py │ │ │ ├── test_stats_analyser.py │ │ │ └── test_stats_results.py │ │ ├── subset │ │ │ ├── __init__.py │ │ │ └── test_subset.py │ │ ├── test_api.py │ │ ├── test_coord.py │ │ ├── test_hyperpoint.py │ │ ├── test_io │ │ │ ├── __init__.py │ │ │ ├── test_NCAR_NetCDF_RAF_variable_name_decider.py │ │ │ ├── test_data_reader.py │ │ │ ├── test_gridded_data.py │ │ │ ├── test_hdf.py │ │ │ ├── test_hyperpoint_view.py │ │ │ └── test_ungridded_data.py │ │ ├── test_kdtree_col.py │ │ ├── test_overide_product.py │ │ ├── test_parse.py │ │ ├── test_parse_datetime.py │ │ ├── test_plot.py │ │ ├── test_time_util.py │ │ └── test_utils.py │ ├── util │ │ ├── __init__.py │ │ └── mock.py │ └── utils_for_testing.py ├── time_util.py └── utils.py ├── codecov.yml ├── conda_requirements.txt ├── doc ├── README ├── advanced_plugin_tutorial.rst ├── aggregation.rst ├── analysis_plugin_development.rst ├── cis_api.rst ├── cis_dependency.dot ├── collocation.rst ├── collocation_examples.rst ├── command_line.rst ├── conf.py ├── data_products.rst ├── easy_plugin_tutorial.rst ├── evaluation.rst ├── file_information.rst ├── gallery.rst ├── img │ ├── 2009-subset.png │ ├── 2010-subset.png │ ├── AOD550_on_MOD08_kdt_hsep_50km_full.png │ ├── AOD550_on_MOD08_kdt_hsep_50km_full_zoom.png │ ├── AOD550_on_MOD08_kdt_nn_full.png │ ├── AOD550n_3.png │ ├── Aerosol_CCI.png │ ├── Aerosol_CCI_4x4.png │ ├── Aerosol_CCI_col.png │ ├── Cloud_CCI.png │ ├── Cloud_CCI_col.png │ ├── CollocationDiagram.png │ ├── HorizontalLI.png │ ├── HorizontalNN.png │ ├── MOD08_on_AOD550_hsep_75km.png │ ├── MOD08_on_AOD550_kdt_hsep_100km_full.png │ ├── MOD08_on_AOD550_kdt_hsep_100km_var_full.png │ ├── MOD08_on_AOD550_nn_kdt.png │ ├── MOD08n_3.png │ ├── MODIS_L2.png │ ├── MODIS_L3.png │ ├── OriginalData.png │ ├── PressureCollocated.png │ ├── PressureCollocation.png │ ├── PressureOriginal.png │ ├── PressureSlice1.png │ ├── PressureSlice2.png │ ├── RF04.png │ ├── RF04_col.png │ ├── Screenshot%20of%20overlayed%20heatmap%20and%20scatter.png │ ├── Screenshot%20of%20overlayed%20line%20graphs.png │ ├── aerosol_cci.png │ ├── aggregation │ │ ├── MODIS-10.png │ │ ├── MODIS-6.png │ │ ├── MODIS-7.png │ │ ├── MODIS-8.png │ │ ├── MODIS-9.png │ │ ├── NCAR-RAF-1.png │ │ ├── NCAR-RAF-2.png │ │ ├── NCAR-RAF-3.png │ │ ├── NCAR-RAF-4.png │ │ ├── NCAR-RAF-5.png │ │ ├── gridded_collapse.png │ │ ├── lat-lon-coarser.png │ │ ├── lat-subset.png │ │ ├── max.png │ │ ├── months-days.png │ │ ├── stddev.png │ │ └── years.png │ ├── agoufou_18022013_all_three.gif │ ├── aircraft.png │ ├── caliop_l1b.png │ ├── cloudsat_RVOD.png │ ├── comparative_scatter_Aeronet.png │ ├── comparativehistogram3d.png │ ├── dep.png │ ├── eval │ │ ├── angstrom_exponent.png │ │ ├── echam_aggregated.png │ │ ├── echam_hadgem_difference.png │ │ ├── hadgem_aggregated.png │ │ ├── hadgem_collocated.png │ │ ├── modis_aggregated_aod.png │ │ ├── modis_cloud_fraction.png │ │ ├── modis_masked_optical_depth.png │ │ └── modis_optical_depth.png │ ├── line.png │ ├── model.png │ ├── overlay1.png │ ├── overlay2.png │ ├── overlay3.png │ ├── overlay4.png │ ├── overlay5.png │ ├── seviri-ctt.png │ ├── stats-aero440.png │ └── stats-aero500.png ├── index.rst ├── installation.rst ├── maintenance_and_development.rst ├── medium_plugin_tutorial.rst ├── overlay_examples.rst ├── plotting.rst ├── plugin │ ├── mycol.py │ └── myprod.py ├── plugin_development.rst ├── statistics.rst ├── subsetting.rst ├── whats_new.rst ├── whats_new_1.1.rst ├── whats_new_1.2.rst ├── whats_new_1.3.rst ├── whats_new_1.4.rst ├── whats_new_1.5.rst ├── whats_new_1.6.rst └── whats_new_1.7.rst └── setup.py /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | # Use the latest 2.1 version of CircleCI pipeline process engine. 2 | # See: https://circleci.com/docs/2.0/configuration-reference 3 | version: 2.1 4 | 5 | # Orbs are reusable packages of CircleCI configuration that you may share across projects, enabling you to create encapsulated, parameterized commands, jobs, and executors that can be used across multiple projects. 6 | # See: https://circleci.com/docs/2.0/orb-intro/ 7 | orbs: 8 | # The python orb contains a set of prepackaged CircleCI configuration you can use repeatedly in your configuration files 9 | # Orb commands and jobs help you with common scripting around a language/tool 10 | # so you dont have to copy and paste it everywhere. 11 | # See the orb documentation here: https://circleci.com/developer/orbs/orb/circleci/python 12 | python: circleci/python@1.5.0 13 | 14 | # Define a job to be invoked later in a workflow. 15 | # See: https://circleci.com/docs/2.0/configuration-reference/#jobs 16 | jobs: 17 | build-and-test: # This is the name of the job, feel free to change it to better match what you're trying to do! 18 | # These next lines defines a Docker executors: https://circleci.com/docs/2.0/executor-types/ 19 | # You can specify an image from Dockerhub or use one of the convenience images from CircleCI's Developer Hub 20 | # A list of available CircleCI Docker convenience images are available here: https://circleci.com/developer/images/image/cimg/python 21 | # The executor is the environment in which the steps below will be executed - below will use a python 3.10.2 container 22 | # Change the version below to your required version of python 23 | docker: 24 | - image: continuumio/miniconda3 25 | # Checkout the code as the first step. This is a dedicated CircleCI step. 26 | # The python orb's install-packages step will install the dependencies from a Pipfile via Pipenv by default. 27 | # Here we're making sure we use just use the system-wide pip. By default it uses the project root's requirements.txt. 28 | # Then run your tests! 29 | # CircleCI will report the results back to your VCS provider. 30 | steps: 31 | - checkout 32 | - run: conda config --add channels conda-forge 33 | - run: conda config --set always_yes true 34 | - run: conda config --set quiet true 35 | - run: 36 | name: Install test requirements 37 | command: conda install --file conda_requirements.txt 38 | - run: mkdir -p test_results/all_tests 39 | - run: 40 | name: Run tests 41 | # This assumes pytest is installed via the install-package step above 42 | command: | 43 | pytest cis/test/unit/ -v --cov=./cis --junitxml=./test_results/all_tests/results.xml -n 2 44 | - run: 45 | name: Upload coverage report 46 | command: | 47 | bash <(curl -s https://codecov.io/bash) -t "${CODECOV_TOKEN}" 48 | - store_test_results: 49 | path: ./test_results/all_tests 50 | 51 | # Invoke jobs via workflows 52 | # See: https://circleci.com/docs/2.0/configuration-reference/#workflows 53 | workflows: 54 | main: # This is the name of the workflow, feel free to change it to better match your workflow. 55 | # Inside the workflow, you define the jobs you want to run. 56 | jobs: 57 | - build-and-test 58 | -------------------------------------------------------------------------------- /.github/workflows/python-publish.yml: -------------------------------------------------------------------------------- 1 | # This workflow will upload a Python Package using Twine when a release is created 2 | # For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries 3 | 4 | name: Upload Python Package 5 | 6 | on: 7 | release: 8 | types: [created] 9 | 10 | jobs: 11 | deploy: 12 | 13 | runs-on: ubuntu-latest 14 | 15 | steps: 16 | - uses: actions/checkout@v2 17 | - name: Set up Python 18 | uses: actions/setup-python@v2 19 | with: 20 | python-version: '3.x' 21 | - name: Install dependencies 22 | run: | 23 | python -m pip install --upgrade pip 24 | pip install setuptools wheel twine 25 | - name: Build and publish 26 | env: 27 | TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} 28 | TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} 29 | run: | 30 | python setup.py sdist bdist_wheel 31 | twine upload dist/* 32 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .project 2 | .pydevproject 3 | /.idea 4 | *.log 5 | *.pyc 6 | build/ 7 | dist/ 8 | cis.egg-info/ 9 | .DS_Store 10 | *~ 11 | doc/_build 12 | cis/*.png 13 | 14 | cis/test/plot_tests/result_image_comparison -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | sudo: false 3 | 4 | #python: 5 | # - 2.7 6 | # - 3.6 7 | 8 | env: 9 | # - TEST_TARGET=unit PYTHON_TARGET=2.7 10 | - TEST_TARGET=unit PYTHON_TARGET=3.6 11 | - TEST_TARGET=unit PYTHON_TARGET=3.7 12 | # - TEST_TARGET=integration 13 | 14 | install: 15 | 16 | # Install miniconda 17 | # ----------------- 18 | - if [[ $PYTHON_TARGET == 2* ]]; then 19 | export CONDA_BASE=http://repo.continuum.io/miniconda/Miniconda2; 20 | else 21 | export CONDA_BASE=http://repo.continuum.io/miniconda/Miniconda3; 22 | fi 23 | - wget ${CONDA_BASE}-latest-Linux-x86_64.sh -O miniconda.sh; 24 | 25 | - bash miniconda.sh -b -p $HOME/miniconda 26 | - export PATH="$HOME/miniconda/bin:$PATH" 27 | 28 | # Create the basic testing environment 29 | # ------------------------------------ 30 | - conda config --set always_yes yes --set changeps1 no 31 | - conda config --set show_channel_urls True 32 | - conda update --quiet conda 33 | - ENV_NAME='test-environment' 34 | - conda create --quiet -n $ENV_NAME python=$PYTHON_TARGET 35 | - source activate $ENV_NAME 36 | 37 | # Customise the testing environment 38 | # --------------------------------- 39 | - conda install -c conda-forge --quiet --file conda_requirements.txt 40 | 41 | - PREFIX=$HOME/miniconda/envs/$ENV_NAME 42 | 43 | # Output debug info 44 | - conda list 45 | - conda info -a 46 | 47 | # Install 48 | - python setup.py --quiet install 49 | 50 | script: 51 | 52 | - if [[ $TEST_TARGET == 'unit' ]]; then 53 | python setup.py test; 54 | fi 55 | # Get test data, then run the integration tests 56 | - if [[ $TEST_TARGET == 'integration' ]] && [[ "$TRAVIS_OS_NAME" == "linux" ]]; then 57 | mkdir test_files; 58 | wget http://gws-access.ceda.ac.uk/public/cis/cis_repo_test_files.tar.gz; 59 | tar -xzvf cis_repo_test_files.tar.gz -C test_files; 60 | export CIS_DATA_HOME="$(pwd)/test_files"; 61 | python setup.py test -i -p 0 -d; 62 | fi 63 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include cis/logging.conf 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | CIS 2 | === 3 | 4 | [![Join the chat at https://gitter.im/cedadev/cis](https://badges.gitter.im/cedadev/cis.svg)](https://gitter.im/cedadev/cis?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) 5 | [![Build Status](https://circleci.com/gh/cedadev/cis/tree/master.svg?style=svg)](https://circleci.com/gh/cedadev/cis/tree/master) 6 | [![codecov](https://codecov.io/gh/cedadev/cis/branch/master/graph/badge.svg?token=8QTY6ZZN95)](https://codecov.io/gh/cedadev/cis) 7 | [![Documentation Status](https://readthedocs.org/projects/cis/badge/?version=latest)](https://readthedocs.org/projects/cis/?badge=latest) 8 | [![Downloads](https://anaconda.org/conda-forge/cis/badges/downloads.svg)](https://anaconda.org/conda-forge/cis/files) 9 | [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.4518867.svg)](https://doi.org/10.5281/zenodo.4518867) 10 | 11 | 12 | CIS is an open source Python library and command-line tool for the easy collocation, visualization, analysis, and comparison of a 13 | diverse set of gridded and ungridded datasets used across earth sciences. Visit our homepage at www.cistools.net. 14 | 15 | For issue tracking and improvement suggestions please see our JIRA project at: https://jira.ceh.ac.uk/projects/JASCIS/issues. 16 | 17 | Installation 18 | ------------ 19 | 20 | A pre-packaged version of CIS is available for installation using conda for Linux, Mac OSX and Windows. 21 | 22 | Once conda is installed, you can easily install CIS with the following command: 23 | 24 | conda install -c conda-forge cis 25 | 26 | If you don’t already have conda, you must first download and install it. 27 | Anaconda is a free conda package that includes Python and many common scientific and data analysis libraries, and is available here: http://continuum.io/downloads. 28 | Further documentation on using Anaconda and the features it provides can be found here: http://docs.continuum.io/anaconda/index.html 29 | 30 | More details for installing CIS from source, and other package sources, can be found in the get-started [documentation](http://cistools.net/get-started#installation). 31 | 32 | 33 | Contact 34 | ------- 35 | 36 | Philip.Kershaw@stfc.ac.uk, Philip.Stier@physics.ox.ac.uk or Duncan.Watson-Parris@physics.ox.ac.uk 37 | 38 | 39 | Copyright and licence 40 | --------------------- 41 | 42 | (C) University of Oxford 2013 43 | 44 | This file is part of the Community Intercomparison Suite (CIS). 45 | 46 | CIS is free software: you can redistribute it and/or modify it under 47 | the terms of the GNU Lesser General Public License as published by the 48 | Free Software Foundation, either version 3 of the License, or 49 | (at your option) any later version. 50 | 51 | CIS is distributed in the hope that it will be useful, 52 | but WITHOUT ANY WARRANTY; without even the implied warranty of 53 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 54 | GNU Lesser General Public License for more details. 55 | 56 | You should have received a copy of the GNU Lesser General Public License 57 | along with CIS. If not, see . 58 | 59 | We gratefully use a number of NASA Visible Earth 'Blue Marble' raster 60 | images in this repository. 61 | -------------------------------------------------------------------------------- /bin/cis: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # This file is part of the CIS package. 3 | # www.cistools.net 4 | 5 | from cis.cis_main import main 6 | main() 7 | -------------------------------------------------------------------------------- /bin/cis.lsf: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ "$1" = "-h" -o "$1" = "--help" ] 4 | then 5 | echo "cis.lsf: use to submit a cis job directly to lotus" 6 | echo " output is written to ~/cis.log" 7 | echo " e.g. 'cis.lsf version' runs cis on lotus with the argument 'version'," 8 | exit 9 | fi 10 | 11 | echo "Submitting job to LSF. Use bjobs to check if it has finished - output is in ~/cis.log" 12 | 13 | # Send the following lines to bsub 14 | cat <> ~/cis.log 18 | 19 | # run the cis command redirecting standard error to stanard out 20 | # prefix each line with the job id so they are easy to find 21 | # output to log 22 | cis $* 2>&1 | sed -e "s/^/\$LSB_JOBID: /" >> ~/cis.log 23 | 24 | # record finish of the job 25 | echo "\$LSB_JOBID: Job Finished" >> ~/cis.log 26 | EOF 27 | -------------------------------------------------------------------------------- /cis/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | CIS is an open source command-line tool and Python library for easy collocation, visualization, analysis, and 3 | comparison of diverse gridded and ungridded datasets used in the atmospheric sciences. 4 | 5 | .. note :: 6 | 7 | The CIS documentation has detailed usage information, including a :doc:`user guide <../index>` 8 | for new users. 9 | 10 | The :func:`read_data` function is a simple way to read a single gridded or ungridded data object (e.g. a NetCDF 11 | variable) from one or more files. CIS will determine the best way to interpret the datafile by comparing the file 12 | signature with the built-in data reading plugins and any user defined plugins. Specifying a particular ``product`` 13 | allows the user to override this automatic detection. 14 | 15 | The :func:`read_data_list` function is very similar to :func:`read_data` except that it allows the user to specify 16 | more than one variable name. This function returns a list of data objects, either all of which will be gridded, or all 17 | ungridded, but not a mix. For ungridded data lists it is assumed that all objects share the same coordinates. 18 | """ 19 | __author__ = "David Michel, Daniel Wallis, Duncan Watson-Parris, Richard Wilkinson, Ian Bush, Matt Kendall, John Holt" 20 | __version__ = "1.7.9" 21 | __status__ = "Stable" 22 | __website__ = "http://www.cistools.net/" 23 | 24 | __all__ = ['read_data', 'read_data_list', 'get_variables'] 25 | 26 | 27 | def read_data(filenames, variable, product=None): 28 | """ 29 | Read a specific variable from a list of files 30 | Files can be either gridded or ungridded but not a mix of both. 31 | First tries to read data as gridded, if that fails, tries as ungridded. 32 | 33 | :param filenames: The filenames of the files to read. This can be either a single filename as a string, a comma 34 | separated list, or a :class:`list` of string filenames. Filenames can include directories which will be expanded to 35 | include all files in that directory, or wildcards such as ``*`` or ``?``. 36 | :type filenames: string or list 37 | :param str variable: The variable to read from the files 38 | :param str product: The name of the data reading plugin to use to read the data (e.g. ``Cloud_CCI_L2``). 39 | :return: The specified data as either a :class:`GriddedData` or :class:`UngriddedData` object. 40 | """ 41 | data_list = read_data_list(filenames, variable, product) 42 | if len(data_list) > 1: 43 | raise ValueError("More than one {} variable found".format(variable)) 44 | return data_list[0] 45 | 46 | 47 | def read_data_list(filenames, variables, product=None, aliases=None): 48 | """ 49 | Read multiple data objects from a list of files. Files can be either gridded or ungridded but not a mix of both. 50 | 51 | :param filenames: The filenames of the files to read. This can be either a single filename as a string, a comma 52 | separated list, or a :class:`list` of string filenames. Filenames can include directories which will be expanded to 53 | include all files in that directory, or wildcards such as ``*`` or ``?``. 54 | :type filenames: string or list 55 | :param variables: One or more variables to read from the files 56 | :type variables: string or list 57 | :param str product: The name of the data reading plugin to use to read the data (e.g. ``Cloud_CCI_L2``). 58 | :param aliases: List of aliases to put on each variable's data object as an alternative means of identifying them. 59 | :type aliases: string or list 60 | :return: A list of the data read out (either a :class:`GriddedDataList` or :class:`UngriddedDataList` depending on 61 | the type of data contained in the files) 62 | """ 63 | from cis.data_io.data_reader import DataReader, expand_filelist 64 | try: 65 | file_set = expand_filelist(filenames) 66 | except ValueError as e: 67 | raise IOError(e) 68 | if len(file_set) == 0: 69 | raise IOError("No files found which match: {}".format(filenames)) 70 | return DataReader().read_data_list(file_set, variables, product, aliases) 71 | 72 | 73 | def get_variables(filenames, product=None, type=None): 74 | """ 75 | Get a list of variables names from a list of files. Files can be either gridded or ungridded but not a mix of both. 76 | 77 | :param filenames: The filenames of the files to read. This can be either a single filename as a string, a comma 78 | separated list, or a :class:`list` of string filenames. Filenames can include directories which will be expanded to 79 | include all files in that directory, or wildcards such as ``*`` or ``?``. 80 | :type filenames: string or list 81 | :param str product: The name of the data reading plugin to use to read the data (e.g. ``Cloud_CCI_L2``). 82 | :param str type: The type of HDF data to read, i.e. 'VD' or 'SD' 83 | :return: A list of the variables 84 | """ 85 | from cis.data_io.data_reader import expand_filelist 86 | from cis.data_io.products.AProduct import get_variables 87 | 88 | try: 89 | file_set = expand_filelist(filenames) 90 | except ValueError as e: 91 | raise IOError(e) 92 | if len(file_set) == 0: 93 | raise IOError("No files found which match: {}".format(filenames)) 94 | return get_variables(file_set, product=product, data_type=type) 95 | -------------------------------------------------------------------------------- /cis/aggregation/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/aggregation/__init__.py -------------------------------------------------------------------------------- /cis/aggregation/collapse_kernels.py: -------------------------------------------------------------------------------- 1 | """ 2 | Kernels used for the aggregation of GRIDDED data only. (Ungridded aggregation uses the standard collocation kernels.) 3 | """ 4 | import iris.analysis 5 | from numpy import ma, zeros 6 | 7 | 8 | class StddevKernel(iris.analysis.Aggregator): 9 | """ 10 | Custom standard deviation kernel (to allow calculation of standard deviation with appropriate metadata). 11 | """ 12 | 13 | def __init__(self): 14 | super(StddevKernel, self).__init__('standard_deviation', ma.std, ddof=1) 15 | 16 | def update_metadata(self, cube, coords, **kwargs): 17 | """ 18 | Update cube metadata after aggregation 19 | :param cube: 20 | :param coords: 21 | :param kwargs: 22 | :return: 23 | """ 24 | super(StddevKernel, self).update_metadata(cube, coords, **kwargs) 25 | cube.standard_name = None 26 | cube.long_name = 'Corrected sample standard deviation of {long_name}'.format(long_name=cube.long_name) 27 | cube.var_name = '{var_name}_std_dev'.format(var_name=cube.var_name) 28 | 29 | 30 | class CountKernel(iris.analysis.Aggregator): 31 | """ 32 | Custom counting kernel (to allow calculation of the number of points used in an aggregation cell, 33 | with appropriate metadata). 34 | """ 35 | 36 | def __init__(self): 37 | super(CountKernel, self).__init__('count', self.count_kernel_func) 38 | 39 | @staticmethod 40 | def count_kernel_func(data, axis, **kwargs): 41 | """ 42 | Count the number of (non-masked) points used in the aggregation for this cell. 43 | """ 44 | if not ma.isMaskedArray(data): 45 | data = ma.masked_array(data, zeros(data.shape)) 46 | return data.count(axis) 47 | 48 | def update_metadata(self, cube, coords, **kwargs): 49 | """ 50 | Update cube metadata after aggregation 51 | """ 52 | super(CountKernel, self).update_metadata(cube, coords, **kwargs) 53 | cube.standard_name = None 54 | cube.long_name = 'Number of points used to calculate the mean of {long_name}'.format(long_name=cube.long_name) 55 | cube.var_name = '{var_name}_num_points'.format(var_name=cube.var_name) 56 | cube.units = None 57 | 58 | 59 | class MultiKernel(object): 60 | """ 61 | Represents a set of kernels to be applied each in turn 62 | """ 63 | 64 | def __init__(self, cell_method, sub_kernels): 65 | """ 66 | Create a new MultiKernel 67 | :param cell_method: String name for the kernel 68 | :param sub_kernels: List of kernels to be applied each in turn 69 | :return: MultiKernel 70 | """ 71 | self.cell_method = cell_method 72 | self.sub_kernels = sub_kernels 73 | 74 | 75 | aggregation_kernels = {'sum': iris.analysis.SUM, 76 | 'median': iris.analysis.MEDIAN, 77 | 'gmean': iris.analysis.GMEAN, 78 | 'hmean': iris.analysis.HMEAN, 79 | 'peak': iris.analysis.PEAK, 80 | 'RMS': iris.analysis.RMS, 81 | 'mean': iris.analysis.MEAN, 82 | 'min': iris.analysis.MIN, 83 | 'max': iris.analysis.MAX, 84 | 'moments': MultiKernel('moments', [iris.analysis.MEAN, StddevKernel(), CountKernel()])} 85 | -------------------------------------------------------------------------------- /cis/collocation/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/collocation/__init__.py -------------------------------------------------------------------------------- /cis/collocation/col.py: -------------------------------------------------------------------------------- 1 | """ 2 | Top level collocation objects 3 | """ 4 | import logging 5 | from cis.collocation.col_implementations import moments 6 | import six 7 | 8 | 9 | def collocate(data, sample, collocator, constraint, kernel): 10 | """ 11 | Perform the collocation. 12 | 13 | :param CommonData or CommonDataList data: Data to collocate 14 | :param CommonData sample: Sampling to collocate onto 15 | :param cis.collocation.col_framework.Collocator collocator: The collocator object to use 16 | :param cis.collocation.col_framework.Constraint constraint: The constraint object 17 | :param cis.collocation.col_framework.Kernel kernel: The kernel to use 18 | :return CommonData: The collocated data 19 | :raises CoordinateNotFoundError: If the collocator was unable to compare the sample and data points 20 | """ 21 | from cis.exceptions import CoordinateNotFoundError 22 | from time import time 23 | from cis import __version__ 24 | 25 | logging.info("Collocator: " + str(collocator)) 26 | logging.info("Kernel: " + str(kernel)) 27 | 28 | logging.info("Collocating, this could take a while...") 29 | t1 = time() 30 | try: 31 | new_data = collocator.collocate(sample, data, constraint, kernel) 32 | except (TypeError, AttributeError) as e: 33 | raise CoordinateNotFoundError('Collocator was unable to compare data points, check the dimensions of each ' 34 | 'data set and the collocation methods chosen. \n' + str(e)) 35 | 36 | logging.info("Completed. Total time taken: " + str(time() - t1)) 37 | 38 | for d in new_data: 39 | history = "Collocated onto sampling from: " + str(getattr(sample, "filenames", "Unknown")) + " " + \ 40 | "\nusing CIS version " + __version__ + " " + \ 41 | "\nvariables: " + str(getattr(data, "var_name", "Unknown")) + " " + \ 42 | "\nwith files: " + str(getattr(data, "filenames", "Unknown")) + " " + \ 43 | "\nusing collocator: " + str(collocator) + " " + \ 44 | "\nkernel: " + str(kernel) 45 | d.add_history(history) 46 | return new_data 47 | 48 | 49 | def get_kernel(kernel, default=moments): 50 | """ 51 | Return a valid kernel instance from either an instance or a string, default is moments if no kernel is specified 52 | 53 | :param str or cis.collocation.col_framework.Kernel kernel: 54 | :param default: 55 | :return cis.collocation.col_framework.Kernel: 56 | """ 57 | from cis.collocation.col_framework import get_kernel, Kernel 58 | if not kernel: 59 | kernel = default() 60 | elif isinstance(kernel, six.string_types): 61 | kernel = get_kernel(kernel)() 62 | elif not isinstance(kernel, Kernel): 63 | raise ValueError("Invalid kernel argument, it must be either a string or a Kernel instance") 64 | return kernel 65 | -------------------------------------------------------------------------------- /cis/collocation/haversinedistancekdtreeindex.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from cis.collocation.kdtree import HaversineDistanceKDTree 4 | from cis.data_io.hyperpoint import HyperPoint 5 | 6 | 7 | def create_index(data, leafsize=10): 8 | """ 9 | Creates the k-D tree index. 10 | 11 | :param data: list of HyperPoints to index 12 | """ 13 | spatial_points = data[['latitude', 'longitude']] 14 | if hasattr(data, 'data'): 15 | mask = np.ma.getmask(data.data).ravel() 16 | else: 17 | mask = None 18 | return HaversineDistanceKDTree(spatial_points, mask=mask, leafsize=leafsize) 19 | 20 | 21 | class HaversineDistanceKDTreeIndex(object): 22 | """k-D tree index that can be used to query using distance along the Earth's surface. 23 | """ 24 | def __init__(self): 25 | self.index = None 26 | 27 | def index_data(self, points, data, coord_map, leafsize=10): 28 | """ 29 | Creates the k-D tree index. 30 | 31 | :param points: (not used) sample points 32 | :param data: list of HyperPoints to index 33 | :param coord_map: (not used) list of tuples relating index in HyperPoint 34 | to index in sample point coords and in coords to be output 35 | """ 36 | try: 37 | self.index = create_index(data, leafsize=leafsize) 38 | except KeyError: 39 | pass # Unable to create index 40 | 41 | def find_nearest_point(self, point): 42 | """Finds the indexed point nearest to a specified point. 43 | :param point: point for which the nearest point is required 44 | :return: index in data of closest point 45 | """ 46 | query_pt = point[['latitude', 'longitude']] 47 | (distances, indices) = self.index.query(query_pt) 48 | return indices 49 | 50 | def find_points_within_distance(self, point, distance): 51 | """Finds the points within a specified distance of a specified point. 52 | :param point: reference point 53 | :param distance: distance in kilometres 54 | :return: list indices in data of points 55 | """ 56 | query_pt = [[point.latitude, point.longitude]] 57 | return self.index.query_ball_point(query_pt, distance)[0] 58 | 59 | def find_points_within_distance_sample(self, sample, distance): 60 | """Finds the points within a specified distance of a specified point. 61 | :param sample: the sample points 62 | :param distance: distance in kilometres 63 | :return list of lists: 64 | For each element ``self.data[i]`` of this tree, ``results[i]`` is a 65 | list of the indices of its neighbors in ``other.data``. 66 | """ 67 | return create_index(sample).query_ball_tree(self.index, distance) 68 | -------------------------------------------------------------------------------- /cis/data_io/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/data_io/__init__.py -------------------------------------------------------------------------------- /cis/data_io/data_writer.py: -------------------------------------------------------------------------------- 1 | class DataWriter(object): 2 | """ 3 | High level class for writing data to a file 4 | """ 5 | 6 | def write_data(self, data, output_file): 7 | """ 8 | Write data to a file. 9 | 10 | :param CommonData data: Data to write 11 | :param str output_file: Output file name 12 | """ 13 | data.save_data(output_file) 14 | -------------------------------------------------------------------------------- /cis/data_io/products/HadGEM.py: -------------------------------------------------------------------------------- 1 | from cis.data_io.products import NetCDF_Gridded 2 | import logging 3 | 4 | 5 | class HadGEM_CONVSH(NetCDF_Gridded): 6 | """ 7 | HadGEM plugin for reading NetCDF files converted by CONVSH. It implements a callback to pass to iris when 8 | reading multiple files to allow correct merging 9 | """ 10 | 11 | def get_file_signature(self): 12 | return [r'[a-z]{6}[\._][pamd]{2}[0-9]{4,6}.*\.nc'] 13 | 14 | @staticmethod 15 | def load_multiple_files_callback(cube, field, filename): 16 | from iris.util import squeeze 17 | # We need to remove the history field when reading multiple files so that the cubes can be properly merged 18 | cube.attributes.pop('history') 19 | # cube.coord(name_or_coord='Hybrid height').attributes['formula_terms'] = 'a: lev b: b orog: orog' 20 | # We also need to remove the length one time dimension so that the cube can be merged correctly (iris preserves 21 | # the value as a scalar which then gets converted back into a full coordinate again on merge). 22 | return squeeze(cube) 23 | 24 | def _create_cube(self, filenames, variable): 25 | """Creates a cube for the specified variable. 26 | :param filenames: List of filenames to read coordinates from 27 | :param variable: Optional variable to read while we're reading the coordinates, can be a string or a 28 | VariableConstraint object 29 | :return: If variable was specified this will return an UngriddedData object, otherwise a CoordList 30 | """ 31 | import six 32 | from cis.exceptions import InvalidVariableError 33 | from cis.data_io.products.gridded_NetCDF import DisplayConstraint 34 | from cis.data_io.gridded_data import load_cube 35 | from iris.exceptions import CoordinateNotFoundError 36 | 37 | # Check if the files given actually exist. 38 | for filename in filenames: 39 | with open(filename) as f: 40 | pass 41 | 42 | variable_constraint = variable 43 | if isinstance(variable, six.string_types): 44 | # noinspection PyPep8 45 | variable_constraint = DisplayConstraint(cube_func=(lambda c: c.var_name == variable or 46 | c.standard_name == variable or 47 | c.long_name == variable), display=variable) 48 | if len(filenames) == 1: 49 | callback_function = self.load_single_file_callback 50 | else: 51 | callback_function = self.load_multiple_files_callback 52 | 53 | try: 54 | cube = load_cube(filenames, variable_constraint, callback=callback_function) 55 | except ValueError as e: 56 | if variable is None: 57 | message = "File contains more than one cube variable name must be specified" 58 | elif e.args[0] == "No cubes found": 59 | message = "Variable not found: {} \nTo see a list of variables run: cis info {}" \ 60 | .format(str(variable), filenames[0]) 61 | else: 62 | message = e.args[0] 63 | raise InvalidVariableError(message) 64 | 65 | try: 66 | hybrid_ht = cube.coord(name_or_coord='Hybrid height') 67 | hybrid_ht.attributes['formula'] = 'z(k,j,i) = a(k) + b(k)*orog(j,i)' 68 | hybrid_ht.convert_units('m') 69 | except CoordinateNotFoundError as e: 70 | pass 71 | 72 | try: 73 | cube.coord(long_name='t').standard_name = 'time' 74 | except CoordinateNotFoundError as e: 75 | pass 76 | 77 | self._add_available_aux_coords(cube, filenames) 78 | 79 | return cube 80 | 81 | def get_variable_names(self, filenames, data_type=None): 82 | # Don't do any checks on valid variables at the moment as iris can't parse the hybrid height dimension units... 83 | import iris 84 | from cis.utils import single_warnings_only 85 | # Filter the warnings so that they only appear once - otherwise you get lots of repeated warnings 86 | with single_warnings_only(): 87 | cubes = iris.load(filenames) 88 | 89 | return set(cube.name() for cube in cubes) 90 | 91 | 92 | class HadGEM_PP(NetCDF_Gridded): 93 | """ 94 | HadGEM plugin for reading native pp files 95 | """ 96 | 97 | def get_file_signature(self): 98 | return [r'.*\.pp'] 99 | 100 | @staticmethod 101 | def load_multiple_files_callback(cube, field, filename): 102 | # This method sets the var_name (used for outputting the cube to NetCDF) to the cube name. This can be quite 103 | # for some HadGEM variables but most commands allow the user to override this field on output. 104 | var_name = cube.name() 105 | try: 106 | cube.var_name = var_name 107 | except ValueError as e: 108 | logging.info("Unable to set var_name due to error: {}".format(e)) 109 | 110 | @staticmethod 111 | def load_single_file_callback(cube, field, filename): 112 | # This method sets the var_name (used for outputting the cube to NetCDF) to the cube name. This can be quite 113 | # for some HadGEM variables but most commands allow the user to override this field on output. 114 | var_name = cube.name() 115 | try: 116 | cube.var_name = var_name 117 | except ValueError as e: 118 | try: 119 | cube.var_name = var_name.replace(' ', '_') 120 | except ValueError as e: 121 | logging.info("Unable to set var_name due to error: {}".format(e)) 122 | -------------------------------------------------------------------------------- /cis/data_io/products/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Module with standard cis Data Products in it 3 | """ 4 | 5 | from .AProduct import AProduct 6 | from .products import Aeronet, ASCII_Hyperpoints, cis 7 | from .cloudsat import CloudSat 8 | from .gridded_NetCDF import NetCDF_Gridded 9 | from .NCAR_NetCDF_RAF import NCAR_NetCDF_RAF 10 | from .MODIS import MODIS_L2, MODIS_L3 11 | from .caliop import abstract_Caliop, Caliop_L1, Caliop_L2 12 | from .CCI import Aerosol_CCI_L2, Cloud_CCI_L2 13 | from .HadGEM import HadGEM_PP, HadGEM_CONVSH 14 | from .EarthCARE import EarthCARE_MSI_ATL 15 | 16 | # list of all data products 17 | __all__ = [ 18 | "EarthCARE_MSI_ATL", 19 | "AProduct", 20 | "NCAR_NetCDF_RAF", 21 | "NetCDF_Gridded", 22 | "MODIS_L2", 23 | "MODIS_L3", 24 | "Caliop_L1", 25 | "Caliop_L2", 26 | "CCI", 27 | "Cloud_CCI_L2", 28 | "Aerosol_CCI_L2", 29 | "Aeronet", 30 | "ASCII_Hyperpoints", 31 | "cis", 32 | "HadGEM_CONVSH", 33 | "HadGEM_PP" 34 | ] 35 | -------------------------------------------------------------------------------- /cis/exceptions.py: -------------------------------------------------------------------------------- 1 | """ 2 | Custom CIS exceptions 3 | """ 4 | 5 | 6 | class CISError(Exception): 7 | pass 8 | 9 | 10 | class InvalidPlotTypeError(CISError): 11 | pass 12 | 13 | 14 | class InvalidDimensionError(CISError): 15 | pass 16 | 17 | 18 | class InvalidVariableError(CISError): 19 | pass 20 | 21 | 22 | class InconsistentDimensionsError(CISError): 23 | pass 24 | 25 | 26 | class InvalidPlotFormatError(CISError): 27 | pass 28 | 29 | 30 | class InvalidFileExtensionError(CISError): 31 | pass 32 | 33 | 34 | class InvalidDataTypeError(CISError): 35 | pass 36 | 37 | 38 | class InvalidOperationError(CISError): 39 | pass 40 | 41 | 42 | class InvalidLineStyleError(CISError): 43 | pass 44 | 45 | 46 | class InvalidHistogramStyleError(CISError): 47 | pass 48 | 49 | 50 | class CoordinateNotFoundError(CISError): 51 | pass 52 | 53 | 54 | class DuplicateCoordinateError(CISError): 55 | pass 56 | 57 | 58 | class ClassNotFoundError(CISError): 59 | pass 60 | 61 | 62 | class InvalidCommandLineOptionError(CISError): 63 | pass 64 | 65 | 66 | class InvalidNumberOfDatagroupsSpecifiedError(CISError): 67 | pass 68 | 69 | 70 | class NotEnoughAxesSpecifiedError(CISError): 71 | pass 72 | 73 | 74 | class NoDataInSubsetError(CISError): 75 | pass 76 | 77 | 78 | class FileFormatError(CISError): 79 | """ 80 | Throw when there is an error determining the type of a file 81 | """ 82 | error_list = ['Unknown error'] 83 | 84 | def __init__(self, error_list, *args, **kwargs): 85 | super(FileFormatError, self).__init__(args, kwargs) 86 | self.error_list = error_list 87 | 88 | 89 | class UserPrintableException(CISError): 90 | """ 91 | This exception is thrown if the program has failed for a known reason. This message is printed without a stack trace 92 | """ 93 | 94 | def __init__(self, message): 95 | """ 96 | Constructor 97 | :param message: the message to show the user 98 | :return: nothing 99 | """ 100 | super(UserPrintableException, self).__init__() 101 | self.message = message 102 | 103 | def __str__(self): 104 | return self.message 105 | -------------------------------------------------------------------------------- /cis/info.py: -------------------------------------------------------------------------------- 1 | """ 2 | Routines for the info command 3 | """ 4 | 5 | 6 | def info(filenames, variables=None, product=None, type=None): 7 | """ 8 | Read all the variables from a file, or a specified selection of variables in more detail, and print to stdout. 9 | 10 | :param filenames: The filenames of the files to read 11 | :param variables: The user specified variables of interest 12 | :param product: The data reading plugin to use when reading the data 13 | :param type: String representing the type of HDF data to read, i.e. 'VD' or 'SD' 14 | """ 15 | from cis.data_io.products.AProduct import get_variables, get_data 16 | 17 | if variables is not None: 18 | for user_var in variables: 19 | print(str(get_data(filenames, user_var, product=product))) 20 | else: 21 | for variable in get_variables(filenames, product=product, data_type=type): 22 | print(variable) 23 | -------------------------------------------------------------------------------- /cis/logging.conf: -------------------------------------------------------------------------------- 1 | [loggers] 2 | keys=root 3 | 4 | [logger_root] 5 | handlers=screen,file 6 | level=NOTSET 7 | 8 | [formatters] 9 | keys=simple,complex 10 | 11 | 12 | [formatter_simple] 13 | format=%(asctime)s - %(levelname)s - %(message)s 14 | 15 | [formatter_complex] 16 | format=%(asctime)s - %(levelname)s - %(module)s : %(lineno)d - %(message)s 17 | 18 | [handlers] 19 | keys=file,screen 20 | 21 | [handler_file] 22 | class=handlers.WatchedFileHandler 23 | interval=midnight 24 | backupCount=5 25 | formatter=complex 26 | level=DEBUG 27 | args=(os.path.join(os.path.expanduser('~'), 'cis.log'),) 28 | 29 | [handler_screen] 30 | class=StreamHandler 31 | formatter=simple 32 | level=WARNING 33 | args=(sys.stdout,) 34 | -------------------------------------------------------------------------------- /cis/plotting/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/plotting/__init__.py -------------------------------------------------------------------------------- /cis/plotting/comparativescatter.py: -------------------------------------------------------------------------------- 1 | """ 2 | A scatter plot with one dataset plotted against another (as opposed to plotting data against a coordinate) 3 | """ 4 | import logging 5 | from cis.plotting.genericplot import APlot 6 | 7 | 8 | class ComparativeScatter(APlot): 9 | 10 | def __init__(self, packed_data, *mplargs, **mplkwargs): 11 | """ 12 | Note that the packed_data argument is ignored for this plot type - only xaxis and yaxis are plotted. 13 | """ 14 | super(ComparativeScatter, self).__init__(packed_data, *mplargs, **mplkwargs) 15 | 16 | logging.debug("Unpacking the data items") 17 | self.x, self.y = getattr(self.xaxis, 'points', None) or self.xaxis.data, packed_data.data 18 | 19 | self.label = packed_data.long_name if self.label is None else self.label 20 | 21 | self.xlabel = self.xaxis.name() 22 | self.ylabel = packed_data.name() 23 | 24 | def __call__(self, ax): 25 | if self.itemwidth is not None: 26 | self.mplkwargs["markersize"] = self.itemwidth 27 | 28 | if self.itemstyle is not None: 29 | self.mplkwargs["marker"] = self.itemstyle 30 | 31 | if self.edgecolor is not None: 32 | self.mplkwargs["edgecolors"] = self.edgecolor 33 | 34 | if self.color is not None: 35 | self.mplkwargs["c"] = self.color 36 | 37 | if self.label is not None: 38 | self.mplkwargs['label'] = self.label 39 | 40 | ax.plot(self.x.flatten(), self.y.flatten(), 'o', *self.mplargs, **self.mplkwargs) 41 | 42 | self._plot_xy_line(ax) 43 | 44 | ax.set_xlabel(self.xlabel) 45 | ax.set_ylabel(self.ylabel) 46 | 47 | def _plot_xy_line(self, ax): 48 | """ 49 | Plot an x-y dashed line as a guide for the eye 50 | """ 51 | import numpy as np 52 | from cis.utils import no_autoscale 53 | 54 | # Turn the scaling off so that we don't change the limits by plotting the line 55 | with no_autoscale(ax): 56 | lims = [np.min([ax.get_xlim(), ax.get_ylim()]), # min of both axes 57 | np.max([ax.get_xlim(), ax.get_ylim()])] # max of both axes 58 | 59 | # now plot both limits against each other for the x=y line 60 | ax.plot(lims, lims, color="black", linestyle="dashed") 61 | 62 | def is_map(self): 63 | return False 64 | -------------------------------------------------------------------------------- /cis/plotting/contourplot.py: -------------------------------------------------------------------------------- 1 | """ 2 | A contour plot, filled or not. 3 | """ 4 | from .genericplot import Generic2DPlot 5 | 6 | 7 | class ContourPlot(Generic2DPlot): 8 | 9 | def __init__(self, packed_data, contnlevels=7, vstep=None, 10 | contlevels=None, contlabel=False, contwidth=None, cont_label_kwargs=None, *args, **kwargs): 11 | """ 12 | 13 | :param packed_data: 14 | :param int contnlevels: The number of contour levels to plot (default is 7) 15 | :param float vstep: The step between contour levels 16 | :param list contlevels: A list of contour levels to plot 17 | :param bool contlabel: Plot contour labels (default is False) 18 | :param int contwidth: The thickness of the contour lines 19 | :param dict cont_label_kwargs: A dictionary of contour label keywork args (e.g. format) 20 | :param kwargs: Other keyword args to pass to plot 21 | """ 22 | super(ContourPlot, self).__init__(packed_data, *args, **kwargs) 23 | self.filled = False 24 | if contlevels: 25 | self.levels = contlevels 26 | elif self.vstep: 27 | self.levels = (kwargs.get('vmax', self.data.max()) - 28 | kwargs.get('vmin', self.data.min())) / vstep 29 | else: 30 | self.levels = contnlevels 31 | 32 | self.contlabel = contlabel 33 | self.contwidth = contwidth 34 | self.cont_label_kwargs = cont_label_kwargs if cont_label_kwargs is not None else {} 35 | 36 | def __call__(self, ax): 37 | from matplotlib import ticker 38 | 39 | # Set the options specific to a datagroup with the contour type 40 | mplkwargs = self.mplkwargs 41 | 42 | mplkwargs["colors"] = self.color 43 | 44 | mplkwargs["linewidths"] = self.contwidth 45 | 46 | if self.logv: 47 | mplkwargs['locator'] = ticker.LogLocator() 48 | 49 | contour_type = ax.contourf if self.filled else ax.contour 50 | 51 | self.mappable = contour_type(self.x, self.y, self.data, self.levels, **mplkwargs) 52 | 53 | if self.contlabel: 54 | ax.clabel(self.mappable, **self.cont_label_kwargs) 55 | 56 | super(ContourPlot, self).__call__(ax) 57 | 58 | 59 | class ContourfPlot(ContourPlot): 60 | 61 | def __init__(self, packed_data, *args, **kwargs): 62 | super(ContourfPlot, self).__init__(packed_data, *args, **kwargs) 63 | self.filled = True 64 | -------------------------------------------------------------------------------- /cis/plotting/heatmap.py: -------------------------------------------------------------------------------- 1 | """ 2 | A heatmap plot (pcolormesh) 3 | """ 4 | from cis.plotting.genericplot import Generic2DPlot 5 | from iris.coords import BOUND_MODE 6 | 7 | 8 | class Heatmap(Generic2DPlot): 9 | 10 | MODE = BOUND_MODE 11 | 12 | def __init__(self, packed_data, *args, **kwargs): 13 | # Do this here because if this is ungridded data, we won't be able to complete the super() call 14 | super(Heatmap, self).__init__(packed_data, *args, **kwargs) 15 | 16 | def __call__(self, ax): 17 | """ 18 | Plots a heatmap 19 | """ 20 | self.mappable = ax.pcolormesh(self.x, self.y, self.data, *self.mplargs, **self.mplkwargs) 21 | 22 | super(Heatmap, self).__call__(ax) 23 | -------------------------------------------------------------------------------- /cis/plotting/histogram.py: -------------------------------------------------------------------------------- 1 | """ 2 | A basic histogram plot 3 | """ 4 | from cis.plotting.genericplot import GenericPlot 5 | 6 | 7 | class Histogram(GenericPlot): 8 | valid_histogram_styles = ["bar", "step", "stepfilled"] 9 | 10 | def __init__(self, packed_data, xbins=10, *args, **kwargs): 11 | """ 12 | :param xbins: The number of bins to use on the xaxis 13 | """ 14 | super(Histogram, self).__init__(packed_data, *args, **kwargs) 15 | self.xbins = xbins 16 | # 17 | self.xlabel = packed_data.name() 18 | self.ylabel = "Frequency" 19 | 20 | def __call__(self, ax): 21 | from numpy.ma import MaskedArray 22 | 23 | self.mplkwargs["bins"] = self.xbins 24 | 25 | if self.itemstyle: 26 | if self.itemstyle in self.valid_histogram_styles: 27 | self.mplkwargs["histtype"] = self.itemstyle 28 | else: 29 | from cis.exceptions import InvalidHistogramStyleError 30 | raise InvalidHistogramStyleError( 31 | "'" + self.itemstyle + "' is not a valid histogram style, please use one of: " + str( 32 | self.valid_histogram_styles)) 33 | 34 | if self.color: 35 | self.mplkwargs["color"] = self.color 36 | 37 | if isinstance(self.data, MaskedArray): 38 | data = self.data.compressed() 39 | else: 40 | data = self.data.flatten() 41 | 42 | ax.hist(data, *self.mplargs, **self.mplkwargs) 43 | 44 | super(Histogram, self).__call__(ax) 45 | 46 | def unpack_data_items(self, packed_data_items): 47 | self.data = packed_data_items.data 48 | -------------------------------------------------------------------------------- /cis/plotting/histogram2d.py: -------------------------------------------------------------------------------- 1 | """ 2 | A '2D' Histogram plot - e.g. one that has data on both the x and y axis. This is generated using numpy.histogram2d and 3 | a call to pcolormesh. Although strictly a 2D plot it inherits from ComparativeScatter rather than Generic2D because 4 | the unpacking has more in common with that. 5 | """ 6 | from cis.plotting.comparativescatter import ComparativeScatter 7 | import numpy 8 | 9 | 10 | class Histogram2D(ComparativeScatter): 11 | 12 | def __init__(self, packed_data, logv=None, vstep=None, xbins=10, ybins=10, 13 | cbarscale=None, cbarorient=None, colourbar=True, cbarlabel=None, *args, **kwargs): 14 | """ 15 | Note that the packed_data argument is ignored for this plot type - only xaxis and yaxis are plotted. 16 | 17 | :param CommonData packed_data: IGNORED 18 | :param bool logv: Log the v (colour bar) axis? 19 | :param float vstep: The step to use for the v axis (colour bar) 20 | :param int xbins: The number of histogram bins to use on the xaxis 21 | :param int ybins: The number of histogram bins to use on the yaxis 22 | :param bool colourbar: Include colour bar? Default True 23 | :param float cbarscale: A scale factor for the colorbar 24 | :param string cbarorient: The colour bar orientation ('vertical' or 'horizontal') 25 | :param string cbarlabel: A label for the colour bar 26 | """ 27 | super(Histogram2D, self).__init__(packed_data, *args, **kwargs) 28 | 29 | self.logv = logv 30 | self.xbins = xbins 31 | self.ybins = ybins 32 | self.vstep = vstep 33 | self.cbarscale = cbarscale 34 | self.cbarorient = cbarorient 35 | self.colourbar = colourbar 36 | self.cbarlabel = cbarlabel or "Frequency" 37 | 38 | def __call__(self, ax): 39 | """ 40 | Plots a 2d histogram. 41 | """ 42 | from .plot import add_color_bar 43 | from cis.utils import apply_intersection_mask_to_two_arrays 44 | 45 | # Numpy histogram2D doesn't like masked arrays, so create the unmased intersection of the two datasets 46 | first_data_item, second_data_item = apply_intersection_mask_to_two_arrays(self.x, self.y) 47 | first_data_item = first_data_item.compressed() 48 | second_data_item = second_data_item.compressed() 49 | 50 | # Use Numpy histogram generator instead of hist2d to allow log scales to be properly plotted 51 | histogram2ddata, x, y = numpy.histogram2d(first_data_item, second_data_item, bins=[self.xbins, self.ybins]) 52 | histogram2ddata = numpy.ma.masked_equal(histogram2ddata, 0) 53 | self.map = ax.pcolor(x, y, histogram2ddata.T, *self.mplargs, **self.mplkwargs) 54 | 55 | self._plot_xy_line(ax) 56 | 57 | ax.set_xlabel(self.xlabel) 58 | ax.set_ylabel(self.ylabel) 59 | 60 | if self.colourbar: 61 | add_color_bar(ax, self.map, self.vstep, self.logv, self.cbarscale, self.cbarorient, self.cbarlabel) 62 | -------------------------------------------------------------------------------- /cis/plotting/lineplot.py: -------------------------------------------------------------------------------- 1 | """ 2 | A basic line plot 3 | """ 4 | from cis.plotting.genericplot import GenericPlot 5 | 6 | 7 | class LinePlot(GenericPlot): 8 | line_styles = ["solid", "dashed", "dashdot", "dotted"] 9 | 10 | def __call__(self, ax): 11 | """ 12 | Plots one line 13 | """ 14 | from cis.exceptions import InvalidDimensionError 15 | # Translate our argument names to mpl line kwargs 16 | self.mplkwargs["linewidth"] = self.itemwidth 17 | 18 | if self.itemstyle: 19 | if self.itemstyle in LinePlot.line_styles: 20 | self.mplkwargs["linestyle"] = self.itemstyle 21 | else: 22 | from cis.exceptions import InvalidLineStyleError 23 | raise InvalidLineStyleError( 24 | "'" + self.itemstyle + "' is not a valid line style, please use one of: " + str( 25 | self.line_styles)) 26 | 27 | if self.color: 28 | self.mplkwargs["color"] = self.color 29 | 30 | if self.x.shape[0] != self.data.shape[0]: 31 | raise InvalidDimensionError("The plot axes are incompatible, please check and specify at least one " 32 | "axis manually.") 33 | 34 | ax.plot(self.x, self.data, *self.mplargs, **self.mplkwargs) 35 | 36 | super(LinePlot, self).__call__(ax) 37 | -------------------------------------------------------------------------------- /cis/plotting/raster/world.topo.bathy.200407.3x1350x675.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/plotting/raster/world.topo.bathy.200407.3x1350x675.png -------------------------------------------------------------------------------- /cis/plotting/raster/world.topo.bathy.200407.3x2700x1350.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/plotting/raster/world.topo.bathy.200407.3x2700x1350.png -------------------------------------------------------------------------------- /cis/plotting/raster/world.topo.bathy.200407.3x5400x2700.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/plotting/raster/world.topo.bathy.200407.3x5400x2700.png -------------------------------------------------------------------------------- /cis/plotting/scatterplot.py: -------------------------------------------------------------------------------- 1 | """ 2 | Basic scatter plots, against either one or two coordinates. 3 | """ 4 | from cis.plotting.genericplot import GenericPlot, Generic2DPlot 5 | 6 | 7 | class ScatterPlot(GenericPlot): 8 | 9 | def __call__(self, ax): 10 | """ 11 | Plots one set of scatter points 12 | """ 13 | # Translate our argument names to mpl line kwargs 14 | if self.itemwidth is not None: 15 | self.mplkwargs["markersize"] = self.itemwidth 16 | 17 | if self.itemstyle is not None: 18 | self.mplkwargs["marker"] = self.itemstyle 19 | 20 | if self.edgecolor is not None: 21 | self.mplkwargs["edgecolors"] = self.edgecolor 22 | 23 | if self.color is not None: 24 | self.mplkwargs["c"] = self.color 25 | 26 | # By plotting the data as flat arrays with a line style of 'o' we get a scatter plot - with the added 27 | # benefit of not having to keep track of the layers ourselves. 28 | ax.plot(self.x.flatten(), self.data.flatten(), 'o', *self.mplargs, **self.mplkwargs) 29 | 30 | super(ScatterPlot, self).__call__(ax) 31 | 32 | 33 | class ScatterPlot2D(Generic2DPlot): 34 | 35 | def __call__(self, ax): 36 | """ 37 | Plots one set of scatter points with the colour determined by the data 38 | """ 39 | if self.itemwidth is not None: 40 | self.mplkwargs["s"] = self.itemwidth 41 | 42 | if self.itemstyle is not None: 43 | self.mplkwargs["marker"] = self.itemstyle 44 | 45 | if self.edgecolor is not None: 46 | self.mplkwargs["edgecolors"] = self.edgecolor 47 | else: 48 | # For 2D scatter plots set the edgecolors off by default 49 | self.mplkwargs["edgecolors"] = None 50 | 51 | self.mplkwargs["c"] = self.data 52 | 53 | self.mappable = ax.scatter(self.x, self.y, *self.mplargs, **self.mplkwargs) 54 | 55 | super(ScatterPlot2D, self).__call__(ax) 56 | -------------------------------------------------------------------------------- /cis/plugin.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | 4 | def get_all_subclasses(parent_class, mod): 5 | """ 6 | This will recursively find subclasses of parent_class in mod. 7 | The use of importlib allows mod to be of the form package.module 8 | Take extreme care when changing the function as it has been known to break! 9 | 10 | :param parent_class: The class to find subclasses of 11 | :param mod: The module to find subclasses in 12 | :return: A list of subclasses 13 | """ 14 | import importlib 15 | importlib.import_module(mod) 16 | subclasses = [] 17 | for subclass in parent_class.__subclasses__(): 18 | subclasses += get_all_subclasses(subclass, mod) 19 | subclasses += parent_class.__subclasses__() 20 | return subclasses 21 | 22 | 23 | def find_plugins(plugin_dir, parent_class_name, verbose): 24 | import logging 25 | import os 26 | import sys 27 | 28 | # if plugin_dir is None, there is no plugin to import, so return an empty list 29 | if plugin_dir is None: 30 | return [] 31 | 32 | if verbose: 33 | logging.info("Looking for plugins... ") 34 | 35 | plugin_files = [] 36 | try: 37 | for f in os.listdir(plugin_dir): 38 | 39 | if f.lower().endswith(('.pyc', '__init__.py')): 40 | continue 41 | 42 | if f.endswith(".py"): 43 | plugin_files.append(f[:-3]) 44 | except OSError: 45 | logging.warning("Unable to read plugin path: {}".format(plugin_dir)) 46 | 47 | if len(plugin_files) == 0: 48 | return [] 49 | 50 | sys.path.insert(0, plugin_dir) 51 | 52 | product_classes = [] 53 | for plugin in plugin_files: 54 | if verbose: 55 | logging.info("Importing plugin: " + str(plugin)) 56 | module = __import__(plugin) 57 | classes = [getattr(module, x) for x in dir(module) if isinstance(getattr(module, x), type)] 58 | product_classes = [cls for cls in classes if parent_class_name in str(cls.__bases__[0])] 59 | 60 | return product_classes 61 | 62 | 63 | def find_plugin_classes(parent_class, built_in_module, verbose=True): 64 | import os 65 | # find plugin classes, if any 66 | ENV_PATH = "CIS_PLUGIN_HOME" 67 | plugin_dir = os.environ.get(ENV_PATH, None) 68 | plugin_classes = find_plugins(plugin_dir, parent_class.__name__, verbose) 69 | 70 | # find built-in classes, i.e. subclasses of parent_class 71 | subclasses = get_all_subclasses(parent_class, built_in_module) 72 | all_classes = plugin_classes + subclasses 73 | 74 | for subclass in all_classes: 75 | if subclass.__name__.startswith('abstract') or subclass.__name__.startswith('Abstract'): 76 | all_classes.remove(subclass) 77 | 78 | all_classes = sorted(all_classes, key=lambda subclass: subclass.__name__) 79 | 80 | if verbose: 81 | logging.debug(parent_class.__name__ + " subclasses are: " + str(all_classes)) 82 | 83 | return all_classes 84 | -------------------------------------------------------------------------------- /cis/subsetting/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/subsetting/__init__.py -------------------------------------------------------------------------------- /cis/test/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/test/__init__.py -------------------------------------------------------------------------------- /cis/test/integration/__init__.py: -------------------------------------------------------------------------------- 1 | # Ensure we're using a headless matplotlib for testing. 2 | import matplotlib 3 | 4 | matplotlib.use("Agg") 5 | -------------------------------------------------------------------------------- /cis/test/integration/base_integration_test.py: -------------------------------------------------------------------------------- 1 | from netCDF4 import Dataset 2 | import os 3 | import unittest 4 | from hamcrest import assert_that, greater_than_or_equal_to, less_than_or_equal_to, is_ 5 | from functools import reduce 6 | 7 | 8 | class BaseIntegrationTest(unittest.TestCase): 9 | 10 | OUTPUT_FILENAME = "test_integration_out.nc" 11 | 12 | def setUp(self): 13 | # Set force overwrite in case working files are still present 14 | os.environ['CIS_FORCE_OVERWRITE'] = "True" 15 | self.clean_output() 16 | 17 | def tearDown(self): 18 | # Pop off the environemnt variable 19 | os.environ.pop('CIS_FORCE_OVERWRITE') 20 | self.clean_output() 21 | 22 | def clean_output(self): 23 | if hasattr(self, 'ds') and self.ds.isopen(): 24 | self.ds.close() 25 | if os.path.exists(self.OUTPUT_FILENAME): 26 | os.remove(self.OUTPUT_FILENAME) 27 | 28 | def check_output_contains_variables(self, output_path, var_names): 29 | self.ds = Dataset(output_path) 30 | for var in var_names: 31 | try: 32 | var = self.ds.variables[var] 33 | except KeyError: 34 | raise AssertionError("Variable %s not found in output file" % var) 35 | self.ds.close() 36 | 37 | def check_output_file_variable_attribute_contains_string(self, output_path, variable, attribute, string): 38 | self.ds = Dataset(output_path) 39 | try: 40 | var = self.ds.variables[variable] 41 | except KeyError: 42 | raise AssertionError("Variable %s not found in output file" % variable) 43 | try: 44 | att_string = getattr(var, attribute) 45 | except AttributeError: 46 | raise AssertionError("Attribute %s not found in variable" % attribute) 47 | assert string in att_string 48 | self.ds.close() 49 | 50 | def check_latlon_subsetting(self, lat_max, lat_min, lon_max, lon_min, lat_var='latitude', lon_var='longitude'): 51 | self.ds = Dataset(self.OUTPUT_FILENAME) 52 | lat = self.ds.variables[lat_var][:] 53 | lon = self.ds.variables[lon_var][:] 54 | assert_that(min(lon), greater_than_or_equal_to(lon_min)) 55 | assert_that(max(lon), less_than_or_equal_to(lon_max)) 56 | assert_that(min(lat), greater_than_or_equal_to(lat_min)) 57 | assert_that(max(lat), less_than_or_equal_to(lat_max)) 58 | self.ds.close() 59 | 60 | def check_alt_subsetting(self, alt_max, alt_min): 61 | self.ds = Dataset(self.OUTPUT_FILENAME) 62 | alt = self.ds.variables['altitude'][:] 63 | assert_that(min(alt), greater_than_or_equal_to(alt_min)) 64 | assert_that(max(alt), less_than_or_equal_to(alt_max)) 65 | self.ds.close() 66 | 67 | def check_pres_subsetting(self, pres_max, pres_min, pres_name='air_pressure'): 68 | import numpy as np 69 | self.ds = Dataset(self.OUTPUT_FILENAME) 70 | pres = self.ds.variables[pres_name][:] 71 | assert_that(np.min(pres), greater_than_or_equal_to(pres_min)) 72 | assert_that(np.max(pres), less_than_or_equal_to(pres_max)) 73 | self.ds.close() 74 | 75 | @staticmethod 76 | def _clean_sample_file_name(sample_file): 77 | import re 78 | return re.sub(r'([\\]):', r':', sample_file) 79 | 80 | def check_output_col_grid(self, sample_file, sample_var, output_file, output_vars, expected_shape=None): 81 | """ 82 | Check that the output grid matches the sample grid in shape. 83 | :param sample_file: 84 | :param sample_var: 85 | :param output_file: 86 | :param output_vars: 87 | :return: 88 | """ 89 | from cis import read_data 90 | from operator import mul 91 | if expected_shape is None: 92 | sample_shape = read_data(self._clean_sample_file_name(sample_file), sample_var).data.shape 93 | else: 94 | sample_shape = expected_shape 95 | self.ds = Dataset(self._clean_sample_file_name(output_file)) 96 | for output_var in output_vars: 97 | output_shape = self.ds.variables[output_var].shape 98 | # This copes with dims in different orders, length 1 values being taken out etc 99 | assert_that(reduce(mul, sample_shape), is_(reduce(mul, output_shape))) 100 | self.ds.close() 101 | 102 | def check_output_vars_are_different(self, output_file, output_vars): 103 | """ 104 | Check that the output variables are NOT exactly the same 105 | :param output_file: 106 | :param output_vars: 107 | :return: 108 | """ 109 | from itertools import combinations 110 | import numpy as np 111 | self.ds = Dataset(self._clean_sample_file_name(output_file)) 112 | # Loop over each possible pair of output var 113 | for a, b in combinations(output_vars, 2): 114 | a_data = self.ds.variables[a] 115 | b_data = self.ds.variables[b] 116 | assert not np.allclose(a_data, b_data) 117 | self.ds.close() -------------------------------------------------------------------------------- /cis/test/integration/cis-test_integration_out.nc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/test/integration/cis-test_integration_out.nc -------------------------------------------------------------------------------- /cis/test/integration/collocated_gassp.nc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/test/integration/collocated_gassp.nc -------------------------------------------------------------------------------- /cis/test/integration/test_io/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/test/integration/test_io/__init__.py -------------------------------------------------------------------------------- /cis/test/integration/test_io/test_aeronet.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from cis.test.integration_test_data import valid_aeronet_filename, valid_aeronet_variable 4 | from cis.data_io.aeronet import load_aeronet 5 | from nose.tools import assert_almost_equal, raises, assert_equal 6 | from cis.exceptions import InvalidVariableError 7 | 8 | 9 | class TestAeronet(unittest.TestCase): 10 | 11 | def test_aeronet_variable_name_parsing(self): 12 | from cf_units import Unit 13 | from cis import read_data_list 14 | 15 | aeronet_data = read_data_list(valid_aeronet_filename, ["AOT_440", "Water(cm)"]) 16 | 17 | assert_equal(aeronet_data[0].name(), "AOT_440") 18 | assert_equal(aeronet_data[0].units, Unit('1')) 19 | assert_equal(aeronet_data[1].name(), "Water(cm)") 20 | assert_equal(aeronet_data[1].var_name, "Water") 21 | assert_equal(aeronet_data[1].units, Unit('cm')) 22 | 23 | def test_aeronet_time_parsing(self): 24 | # 1.8s 25 | from datetime import datetime 26 | from cis.time_util import cis_standard_time_unit as ct 27 | 28 | aeronet_data = load_aeronet(valid_aeronet_filename, [valid_aeronet_variable]) 29 | 30 | assert_almost_equal(aeronet_data['datetime'][0], ct.date2num(datetime(2003, 9, 25, 6, 47, 9))) 31 | assert_almost_equal(aeronet_data['datetime'][5], ct.date2num(datetime(2003, 9, 25, 7, 10, 37))) 32 | assert_almost_equal(aeronet_data['datetime'][76], ct.date2num(datetime(2003, 9, 27, 13, 28, 2))) 33 | 34 | @raises(InvalidVariableError) 35 | def test_aeronet_missing_variable(self): 36 | aeronet_data = load_aeronet(valid_aeronet_filename, ['invalid_variable']) 37 | 38 | -------------------------------------------------------------------------------- /cis/test/integration/test_io/test_hdf.py: -------------------------------------------------------------------------------- 1 | from nose.tools import eq_, istest, raises 2 | 3 | from cis.data_io.hdf import _read_hdf4 4 | from cis.exceptions import InvalidVariableError 5 | from cis.test.integration_test_data import * 6 | 7 | 8 | @istest 9 | @skip_pyhdf 10 | @raises(IOError) 11 | def should_raise_io_error_with_non_hdf_file(): 12 | _read_hdf4(valid_cloud_cci_filename, valid_cloud_cci_variable) 13 | 14 | 15 | @istest 16 | @skip_pyhdf 17 | def test_read_hdf4(): 18 | filename = escape_colons(valid_hdf_sd_file) 19 | sds, vds = _read_hdf4(filename, ['Solution_Ocean', 'Path_Radiance_Land', 'Mean_Reflectance_Land']) 20 | 21 | # VD variable are listed in the VD part of the tuple, but not in the SD part 22 | eq_(True, 'Solution_Ocean' in vds) 23 | eq_(False, 'Solution_Ocean' in sds) 24 | 25 | # SD variable are listed in the SD part of the tuple, but not in the VD part 26 | eq_(True, 'Path_Radiance_Land' in sds) 27 | eq_(False, 'Path_Radiance_Land' in vds) 28 | eq_(True, 'Mean_Reflectance_Land' in sds) 29 | eq_(False, 'Mean_Reflectance_Land' in vds) 30 | 31 | 32 | @istest 33 | @skip_pyhdf 34 | @raises(InvalidVariableError) 35 | def test_that_cannot_read_unknown_variables(): 36 | filename = escape_colons(valid_hdf_sd_file) 37 | sds, vds = _read_hdf4(filename, ['athing', 'unechose', 'einding']) 38 | 39 | 40 | @istest 41 | @skip_pyhdf 42 | @raises(InvalidVariableError) 43 | def test_that_cannot_read_unknown_variables_and_valid_variables(): 44 | filename = escape_colons(valid_hdf_sd_file) 45 | sds, vds = _read_hdf4(filename, ['someBizarreVariableNobodyKnowsAbout', 'Solution_Ocean', 'Path_Radiance_Land', 46 | 'Mean_Reflectance_Land']) 47 | -------------------------------------------------------------------------------- /cis/test/integration/test_io/test_hdf_sd.py: -------------------------------------------------------------------------------- 1 | """ 2 | module to test the hdf4 utility function of hdf_sd.py 3 | """ 4 | from nose.tools import istest, eq_ 5 | 6 | from cis.test.integration_test_data import valid_hdf_sd_file, escape_colons, skip_pyhdf 7 | import cis.data_io.hdf_sd as hdf_sd 8 | 9 | 10 | @istest 11 | @skip_pyhdf 12 | def test_that_can_read_all_variables(): 13 | data_dict = hdf_sd.read(escape_colons(valid_hdf_sd_file)) 14 | eq_(len(data_dict), 67) 15 | 16 | 17 | @istest 18 | @skip_pyhdf 19 | def test_that_can_read_known_variables(): 20 | data_dict = hdf_sd.read(escape_colons(valid_hdf_sd_file), ['Latitude', 'Longitude']) 21 | eq_(len(data_dict), 2) 22 | 23 | 24 | @istest 25 | @skip_pyhdf 26 | def test_that_can_get_data(): 27 | data_dict = hdf_sd.read(escape_colons(valid_hdf_sd_file)) 28 | data = hdf_sd.get_data(data_dict['Latitude']) 29 | eq_(data.shape, (203, 135)) 30 | 31 | 32 | @istest 33 | @skip_pyhdf 34 | def test_that_can_get_metadata_for_known_variable(): 35 | data_dict = hdf_sd.read(escape_colons(valid_hdf_sd_file)) 36 | metadata = hdf_sd.get_metadata(data_dict['Latitude']) 37 | 38 | eq_(metadata._name, "Latitude") 39 | eq_(metadata.standard_name, "latitude") 40 | eq_(metadata.long_name, "Geodetic Latitude") 41 | eq_(metadata.shape, [203, 135]) 42 | eq_(metadata.units, "Degrees_north") 43 | eq_(metadata.factor, 1.0) 44 | eq_(metadata.offset, 0.0) 45 | eq_(metadata.missing_value, -999.0) 46 | 47 | attr = metadata.misc 48 | eq_(len(attr), 5) 49 | eq_(attr['Parameter_Type'], "MODIS Input") 50 | eq_(attr['valid_range'], [-90.0, 90.0]) 51 | -------------------------------------------------------------------------------- /cis/test/integration/test_io/test_hdf_vd.py: -------------------------------------------------------------------------------- 1 | """ 2 | module to test the hdf4 utility function of hdf_vd.py 3 | """ 4 | from nose.tools import istest, eq_ 5 | 6 | import cis.data_io.hdf_vd as hdf_vd 7 | from cis.test.integration_test_data import valid_hdf_vd_file, escape_colons, skip_pyhdf 8 | 9 | 10 | @istest 11 | @skip_pyhdf 12 | def test_that_can_read_all_variables(): 13 | dict = hdf_vd.get_hdf_VD_file_variables(escape_colons(valid_hdf_vd_file)) 14 | eq_(len(dict), 37) 15 | eq_(True, 'Longitude' in dict) 16 | 17 | 18 | @istest 19 | @skip_pyhdf 20 | def test_that_can_get_data(): 21 | vds_dict = hdf_vd.read(escape_colons(valid_hdf_vd_file), 'DEM_elevation') 22 | vds = vds_dict['DEM_elevation'] 23 | data = hdf_vd.get_data(vds) 24 | eq_(37081, len(data)) 25 | 26 | 27 | @istest 28 | @skip_pyhdf 29 | def test_that_can_get_variable_metadata(): 30 | vd = hdf_vd.read(escape_colons(valid_hdf_vd_file), 'DEM_elevation')['DEM_elevation'] 31 | metadata = hdf_vd.get_metadata(vd) 32 | eq_(metadata._name, "DEM_elevation") 33 | eq_(metadata.long_name, "Digital Elevation Map") 34 | eq_(metadata.shape, [37081]) 35 | eq_(metadata.units, "meters") 36 | eq_(metadata.factor, 1.0) 37 | eq_(metadata.offset, 0.0) 38 | eq_(metadata.missing_value, 9999) 39 | eq_(metadata.misc['valid_range'], [-9999, 8850]) 40 | 41 | 42 | @istest 43 | @skip_pyhdf 44 | def test_that_can_get_coord_metadata(): 45 | vd = hdf_vd.read(escape_colons(valid_hdf_vd_file), 'Longitude')['Longitude'] 46 | metadata = hdf_vd.get_metadata(vd) 47 | eq_(metadata._name, "Longitude") 48 | eq_(metadata.standard_name, "longitude") 49 | eq_(metadata.long_name, "Spacecraft Longitude") 50 | eq_(metadata.shape, [37081]) 51 | eq_(metadata.units, "degrees") 52 | eq_(metadata.factor, 1.0) 53 | eq_(metadata.offset, 0.0) 54 | eq_(metadata.missing_value, None) 55 | eq_(metadata.misc['valid_range'], [-180.0, 180.0]) 56 | -------------------------------------------------------------------------------- /cis/test/integration/test_io/test_products/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'duncan' 2 | -------------------------------------------------------------------------------- /cis/test/integration/test_io/test_products/test_Aproduct.py: -------------------------------------------------------------------------------- 1 | """ 2 | Module to test the abstract AProduct class and it's helper methods 3 | """ 4 | from unittest import TestCase 5 | 6 | from hamcrest import * 7 | from nose.tools import eq_, raises 8 | 9 | from cis.test.integration_test_data import * 10 | from cis.data_io.products.AProduct import get_data, __get_class as _get_class, get_coordinates 11 | from cis.exceptions import ClassNotFoundError 12 | 13 | 14 | class TestAProduct(TestCase): 15 | 16 | def test_that_get_data_accepts_valid_product(self): 17 | _get_class(valid_cloudsat_RVOD_file, product='CloudSat') 18 | 19 | def test_automatic_detection_of_product_for_existing_product(self): 20 | product_cls = _get_class(valid_cloudsat_RVOD_file) 21 | eq_(product_cls.__name__, 'CloudSat') 22 | 23 | product_cls = _get_class(valid_caliop_l2_filename) 24 | eq_(product_cls.__name__, 'Caliop_L2') 25 | 26 | @raises(ClassNotFoundError) 27 | def test_that_get_class_raises_ClassNotFoundError_for_non_existing_product(self): 28 | product_cls = _get_class('some_file_that_does_not_match_anything') 29 | 30 | @raises(ClassNotFoundError) 31 | def test_that_get_data_raises_ClassNotFoundError_for_missing_product(self): 32 | get_data(valid_cloudsat_RVOD_file, [valid_cloudsat_RVOD_sdata_variable], 33 | product='Product_Not_Yet_Implemented') 34 | 35 | @raises(ClassNotFoundError) 36 | def test_given_cls_which_implements_file_test_as_false_WHEN_call_get_data_for_product_THEN_cls_no_found_error(self): 37 | from cis.data_io.products.AProduct import AProduct 38 | 39 | class MyTestProduct(AProduct): 40 | # Ensure this doesn't get picked up as a genuine product (when running integration tests) 41 | priority = -1 42 | 43 | def create_data_object(self, filenames, variable): 44 | pass 45 | 46 | def create_coords(self, filenames): 47 | pass 48 | 49 | def get_file_signature(self): 50 | return [r'.*\.ending'] 51 | 52 | def get_file_type_error(self, filesname): 53 | return ["Not correct type"] 54 | 55 | get_coordinates(["file.ending"]) 56 | 57 | def test_given_class_which_implements_file_test_as_true_WHEN_call_get_data_for_product_THEN_test_is_checked(self): 58 | from cis.data_io.products.AProduct import AProduct 59 | global check 60 | check = False 61 | 62 | class MyTestProductTestFileTypeTrue(AProduct): 63 | # Ensure this doesn't get picked up as a genuine product (when running integration tests) 64 | priority = -1 65 | 66 | def create_data_object(self, filenames, variable): 67 | pass 68 | 69 | def create_coords(self, filenames): 70 | pass 71 | 72 | def get_file_signature(self): 73 | return [r'.*\.endingtrue'] 74 | 75 | def get_file_type_error(self, filesname): 76 | global check 77 | check = True 78 | return None 79 | 80 | get_coordinates(["file.endingtrue"]) 81 | assert_that(check, is_(True), "File type check was called") 82 | 83 | def test_that_get_product_full_name_returns_version_product_and_cis(self): 84 | from cis import __version__ 85 | from cis.data_io.products.AProduct import get_product_full_name 86 | from cis.data_io.products import CloudSat 87 | 88 | product_name = get_product_full_name([valid_cloudsat_RVOD_file]) 89 | 90 | assert_that(product_name, contains_string('CIS')) 91 | assert_that(product_name, contains_string(__version__)) 92 | assert_that(product_name, contains_string(CloudSat.__name__)) 93 | 94 | -------------------------------------------------------------------------------- /cis/test/integration/test_io/test_products/test_NetCDF_CF_Gridded.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from hamcrest import assert_that, is_, close_to 4 | 5 | from cis import read_data 6 | from cis.test.integration_test_data import * 7 | 8 | 9 | class TestNetCDF_CF_Gridded_ECHAM_Hybrid_Pressure_Geopotential(unittest.TestCase): 10 | 11 | data = None 12 | 13 | def read_data(self): 14 | # Only do this once 15 | if self.data is None: 16 | filename = valid_echamham_geopotential_filename 17 | var = valid_echamham_geopotential_variable 18 | self.data = read_data(filename, var) 19 | 20 | def test_GIVEN_hybrid_pressure_readable_by_iris_WHEN_read_THEN_pressure_coordinate_present(self): 21 | self.read_data() 22 | # The air pressure aux coord is a CF compliant aux coord which IRIS identifies 23 | pressure = self.data.coord('air_pressure') 24 | assert_that(pressure.shape, is_((248, 31, 96, 192))) 25 | 26 | def test_GIVEN_altitude_not_read_by_iris_WHEN_read_THEN_pressure_coordinate_present(self): 27 | self.read_data() 28 | # The altitude aux coord is CF compliant but not identified by IRIS so we manually create it 29 | altitude = self.data.coord('altitude') 30 | assert_that(altitude.shape, is_((248, 31, 96, 192))) 31 | # Check it has been converted to meters 32 | assert_that(str(altitude.units), is_('meter')) 33 | assert_that(altitude.points[0, 0, 0, 0], close_to(275081.0 / 9.80665, 0.1)) 34 | 35 | 36 | class TestNetCDF_CF_Gridded_ECHAM_Hybrid_Pressure_Geopotential_Height(unittest.TestCase): 37 | 38 | data = None 39 | 40 | def read_data(self): 41 | # Only need to do this once 42 | if self.data is None: 43 | filename = valid_echamham_geopotential_height_filename 44 | var = valid_echamham_geopotential_height_variable 45 | self.data = read_data(filename, var) 46 | 47 | def test_GIVEN_hybrid_pressure_readable_by_iris_WHEN_read_THEN_pressure_coordinate_present(self): 48 | self.read_data() 49 | # The air pressure aux coord is a CF compliant aux coord which IRIS identifies 50 | pressure = self.data.coord('air_pressure') 51 | assert_that(pressure.shape, is_((248, 31, 96, 192))) 52 | 53 | def test_GIVEN_altitude_not_read_by_iris_WHEN_read_THEN_pressure_coordinate_present(self): 54 | self.read_data() 55 | # The altitude aux coord is CF compliant but not identified by IRIS so we manually create it 56 | altitude = self.data.coord('altitude') 57 | assert_that(altitude.shape, is_((248, 31, 96, 192))) 58 | 59 | if __name__ == '__main__': 60 | unittest.main() 61 | -------------------------------------------------------------------------------- /cis/test/integration/test_io/test_products/test_ncar_raf.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from hamcrest import * 4 | 5 | from cis.data_io.products.NCAR_NetCDF_RAF import NCAR_NetCDF_RAF 6 | from cis.test.integration.test_io.test_products.test_data_products import ProductTests 7 | from cis.test.integration_test_data import cis_test_files 8 | 9 | 10 | class TestNCAR_NetCDF_RAF(ProductTests, unittest.TestCase): 11 | def setUp(self): 12 | self.setup(cis_test_files["NCAR_NetCDF_RAF"], NCAR_NetCDF_RAF) 13 | 14 | def test_can_concatenate_files_with_different_time_stamps(self): 15 | from cis import read_data 16 | import numpy as np 17 | from cis.test.integration_test_data import valid_GASSP_station_files_with_different_timestamps,\ 18 | valid_GASSP_station_var_with_different_timestamps 19 | var = valid_GASSP_station_var_with_different_timestamps 20 | filename = valid_GASSP_station_files_with_different_timestamps 21 | data = read_data(filename, var) 22 | time_coord = data.coord(axis='T') 23 | assert_that(np.min(time_coord.data), close_to(149107 + 54690.0/86400, 1e-5)) 24 | assert_that(np.max(time_coord.data), close_to(149110 + 81330.0/86400, 1e-5)) 25 | 26 | def test_can_concatenate_aircraft_files(self): 27 | from cis import read_data 28 | from cis.test.integration_test_data import valid_GASSP_aircraft_files_with_different_timestamps,\ 29 | valid_GASSP_aircraft_var_with_different_timestamps 30 | data = read_data(valid_GASSP_aircraft_files_with_different_timestamps, 31 | valid_GASSP_aircraft_var_with_different_timestamps) 32 | time_coord = data.coord(axis='T') 33 | assert_that(len(time_coord.data), equal_to(63609)) 34 | 35 | 36 | class TestNCAR_NetCDF_RAF_with_GASSP_aux_coord(ProductTests, unittest.TestCase): 37 | 38 | def setUp(self): 39 | self.setup(cis_test_files["GASSP_aux_coord"], NCAR_NetCDF_RAF) 40 | 41 | def test_variable_wildcarding(self): 42 | # We get all of the variables from the file like this - but this isn't the same as the set defined in the 43 | # test data because they are all the same shape. These aren't. 44 | self.vars = [u'AREADIST_DMA_OPC', u'VOLDIST_DMA_OPC', u'DYNAMIC_PRESSURE', u'NUMDIST_DMA_OPC', u'PRESSURE_ALTITUDE', 45 | u'LONGITUDE', u'RELATIVE_HUMIDITY', u'AIR_TEMPERATURE', u'AIR_PRESSURE', u'TIME', u'LATITUDE'] 46 | super(TestNCAR_NetCDF_RAF_with_GASSP_aux_coord, self).test_variable_wildcarding() 47 | 48 | 49 | class TestNCAR_NetCDF_RAF_with_GASSP_aeroplane(ProductTests, unittest.TestCase): 50 | 51 | def setUp(self): 52 | self.setup(cis_test_files["GASSP_aeroplane"], NCAR_NetCDF_RAF) 53 | 54 | 55 | class TestNCAR_NetCDF_RAF_with_GASSP_ship(ProductTests, unittest.TestCase): 56 | 57 | def setUp(self): 58 | self.setup(cis_test_files["GASSP_ship"], NCAR_NetCDF_RAF) 59 | 60 | 61 | class TestNCAR_NetCDF_RAF_with_GASSP_station(ProductTests, unittest.TestCase): 62 | 63 | def setUp(self): 64 | self.setup(cis_test_files["GASSP_station"], NCAR_NetCDF_RAF) 65 | 66 | 67 | class TestNCAR_NetCDF_RAF_get_file_type_error(unittest.TestCase): 68 | 69 | def test_WHEN_file_is_GASSP_THEN_no_errors(self): 70 | from cis.test.integration_test_data import valid_GASSP_station_filename 71 | product = NCAR_NetCDF_RAF() 72 | 73 | errors = product.get_file_type_error(valid_GASSP_station_filename) 74 | 75 | assert_that(errors, is_(None), "file should be GASSP") 76 | 77 | def test_WHEN_file_is_NCAR_RAF_THEN_no_errors(self): 78 | from cis.test.integration_test_data import valid_NCAR_NetCDF_RAF_filename 79 | product = NCAR_NetCDF_RAF() 80 | 81 | errors = product.get_file_type_error(valid_NCAR_NetCDF_RAF_filename) 82 | 83 | assert_that(errors, is_(None), "file should be GASSP") 84 | 85 | def test_WHEN_file_dose_not_exist_THEN_errors(self): 86 | from cis.test.integration_test_data import invalid_filename 87 | product = NCAR_NetCDF_RAF() 88 | 89 | errors = product.get_file_type_error(invalid_filename) 90 | 91 | assert_that(errors, is_(["File does not exist"]), "file should not exist") 92 | 93 | def test_WHEN_file_is_not_NCAR_RAF_OR_GASSP_THEN_errors(self): 94 | from cis.test.integration_test_data import valid_hadgem_filename 95 | product = NCAR_NetCDF_RAF() 96 | 97 | errors = product.get_file_type_error(valid_hadgem_filename) 98 | 99 | assert_that(len(errors), is_(1), "file should not be GASSP") 100 | 101 | def test_WHEN_file_is_not_netcdf_THEN_errors(self): 102 | from cis.test.integration_test_data import valid_aeronet_filename 103 | product = NCAR_NetCDF_RAF() 104 | 105 | errors = product.get_file_type_error(valid_aeronet_filename) 106 | 107 | assert_that(len(errors), is_(2), "file should not be netcdf") 108 | -------------------------------------------------------------------------------- /cis/test/integration/test_io/test_write_netcdf.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import os 3 | from netCDF4 import Dataset 4 | 5 | from cis.data_io.gridded_data import make_from_cube, GriddedDataList 6 | from cis.test.util.mock import make_dummy_2d_ungridded_data, make_mock_cube 7 | from cis.test.integration_test_data import valid_cis_col_file, valid_cis_col_variable 8 | from cis.data_io.write_netcdf import write 9 | 10 | tmp_file = "tmp_file.nc" 11 | 12 | 13 | class TestWriteNetcdf(unittest.TestCase): 14 | 15 | def tearDown(self): 16 | if hasattr(self, 'd') and self.d.isopen(): 17 | self.d.close() 18 | os.remove(tmp_file) 19 | 20 | def test_write_netcdf(self): 21 | data_object = make_dummy_2d_ungridded_data() 22 | write(data_object, tmp_file) 23 | 24 | def test_write_col_and_reload_1(self): 25 | # Copy a collocated file and try to reload it. This exposes a bug where 26 | # var.shape is set in the NetCDF metadata 27 | from cis.data_io.products import Aerosol_CCI_L2 28 | 29 | prod = Aerosol_CCI_L2() 30 | data_object = prod.create_data_object([valid_cis_col_file], valid_cis_col_variable) 31 | write(data_object, tmp_file) 32 | 33 | self.d = Dataset(tmp_file) 34 | 35 | v = self.d.variables['AOT_440'] 36 | 37 | # This will fail because var.shape is in the file 38 | print(v[:2]) 39 | 40 | def test_write_col_and_reload_2(self): 41 | # Copy a collocated file and try to reload it. This exposes a bug where 42 | # latitude and longitude aren't recognised on reload 43 | from cis.data_io.products import cis 44 | 45 | prod = cis() 46 | data_object = prod.create_data_object([valid_cis_col_file], valid_cis_col_variable) 47 | write(data_object, tmp_file) 48 | 49 | data_object2 = prod.create_data_object([tmp_file], valid_cis_col_variable) 50 | 51 | def test_ungridded_write_attributes(self): 52 | data = make_dummy_2d_ungridded_data() 53 | attrs = {'attr_name': 'attr_val', 54 | 'standard_name': 'std_val', 55 | 'long_name': 'lg_val', 56 | 'units': 'units'} 57 | data.add_attributes(attrs) 58 | write(data, tmp_file) 59 | self.d = Dataset(tmp_file) 60 | for key, val in attrs.items(): 61 | assert getattr(self.d.variables['rainfall_flux'], key) == val 62 | 63 | def test_gridded_write_attributes(self): 64 | data = make_from_cube(make_mock_cube()) 65 | data.var_name = 'rain' 66 | attrs = {'attr_name': 'attr_val', 67 | 'standard_name': 'convective_rainfall_amount', 68 | 'long_name': 'lg_val', 69 | 'units': 'units'} 70 | data.add_attributes(attrs) 71 | data.save_data(tmp_file) 72 | self.d = Dataset(tmp_file) 73 | for key, val in attrs.items(): 74 | assert getattr(self.d.variables['rain'], key) == val 75 | 76 | def test_ungridded_write_units(self): 77 | data = make_dummy_2d_ungridded_data() 78 | data.units = 'kg' 79 | write(data, tmp_file) 80 | self.d = Dataset(tmp_file) 81 | assert self.d.variables['rainfall_flux'].units == 'kg' 82 | 83 | def test_gridded_write_units(self): 84 | data = make_from_cube(make_mock_cube()) 85 | data.var_name = 'rain' 86 | data.units = 'ppm' 87 | data.save_data(tmp_file) 88 | self.d = Dataset(tmp_file) 89 | assert self.d.variables['rain'].units == 'ppm' 90 | 91 | def test_gridded_write_time_as_unlimited_dimension(self): 92 | data = make_from_cube(make_mock_cube(time_dim_length=7)) 93 | data.var_name = 'rain' 94 | data.save_data(tmp_file) 95 | self.d = Dataset(tmp_file) 96 | assert self.d.dimensions['time'].isunlimited() 97 | 98 | def test_gridded_write_no_time_has_no_unlimited_dimension(self): 99 | data = make_from_cube(make_mock_cube()) 100 | data.var_name = 'rain' 101 | data.save_data(tmp_file) 102 | self.d = Dataset(tmp_file) 103 | for d in self.d.dimensions.values(): 104 | assert not d.isunlimited() 105 | 106 | def test_gridded_list_write_time_as_unlimited_dimension(self): 107 | data = GriddedDataList([make_from_cube(make_mock_cube(time_dim_length=7))]) 108 | data[0].var_name = 'rain' 109 | data.save_data(tmp_file) 110 | self.d = Dataset(tmp_file) 111 | assert self.d.dimensions['time'].isunlimited() 112 | 113 | def test_gridded_list_write_no_time_has_no_unlimited_dimension(self): 114 | data = GriddedDataList([make_from_cube(make_mock_cube())]) 115 | data[0].var_name = 'rain' 116 | data.save_data(tmp_file) 117 | self.d = Dataset(tmp_file) 118 | for d in self.d.dimensions.values(): 119 | assert not d.isunlimited() 120 | 121 | # def test_can_write_hierarchical_group_variables(self): 122 | # from cis.test.integration_test_data import valid_nested_groups_file 123 | # from cis import read_data 124 | # from hamcrest import assert_that, is_ 125 | # var_name = 'group1/group2/var4' 126 | # data = read_data(valid_nested_groups_file, var_name, product='cis') 127 | # assert_that(data.data, is_([12321])) 128 | # data.save_data(tmp_file) 129 | # self.d = Dataset(tmp_file) 130 | # assert_that(self.d.variables[var_name][:], is_([12321])) 131 | -------------------------------------------------------------------------------- /cis/test/integration/test_read_api.py: -------------------------------------------------------------------------------- 1 | from nose.tools import istest, raises 2 | from cis.data_io.products.AProduct import ProductPluginException 3 | from cis.test.integration_test_data import cis_test_files, invalid_filename 4 | from cis import read_data, read_data_list 5 | 6 | 7 | @istest 8 | @raises(ProductPluginException) 9 | def test_read_data_raises_error_on_invalid_variable(): 10 | file = cis_test_files['Aerosol_CCI'].master_filename 11 | read_data(file, 'invalid_variable') 12 | 13 | 14 | @istest 15 | @raises(ValueError) 16 | def test_read_data_raises_error_on_more_than_one_variable_returned(): 17 | file = cis_test_files['Aerosol_CCI'].master_filename 18 | read_data(file, cis_test_files['Aerosol_CCI'].all_variable_names) 19 | 20 | 21 | @istest 22 | @raises(IOError) 23 | def test_read_data_raises_error_on_invalid_file(): 24 | read_data(invalid_filename, cis_test_files['Aerosol_CCI'].all_variable_names) 25 | 26 | 27 | @istest 28 | @raises(IOError) 29 | def test_read_data_raises_error_on_invalid_file_wildcard(): 30 | read_data(invalid_filename + '*', cis_test_files['Aerosol_CCI'].all_variable_names) 31 | 32 | 33 | @istest 34 | @raises(ProductPluginException) 35 | def test_read_data_list_raises_error_on_invalid_variable(): 36 | file = cis_test_files['Aerosol_CCI'].master_filename 37 | read_data_list(file, 'invalid_variable') 38 | 39 | 40 | @istest 41 | @raises(IOError) 42 | def test_read_data_list_raises_error_on_invalid_file(): 43 | read_data_list(invalid_filename, cis_test_files['Aerosol_CCI'].all_variable_names) 44 | 45 | 46 | @istest 47 | @raises(IOError) 48 | def test_read_data_list_raises_error_on_invalid_file_wildcard(): 49 | read_data_list(invalid_filename + '*', cis_test_files['Aerosol_CCI'].all_variable_names) 50 | -------------------------------------------------------------------------------- /cis/test/integration/test_stats.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from cis.parse import parse_args 4 | from cis.cis_main import stats_cmd, col_cmd 5 | from cis.test.integration_test_data import * 6 | from cis.test.integration.base_integration_test import BaseIntegrationTest 7 | 8 | 9 | class TestStats(BaseIntegrationTest): 10 | 11 | output_vars = [ 12 | "num_points", 13 | "dataset_mean_1", 14 | "dataset_mean_2", 15 | "dataset_stddev_1", 16 | "dataset_stddev_2", 17 | "abs_mean", 18 | "abs_stddev", 19 | "rel_mean", 20 | "rel_stddev", 21 | "spearman", 22 | "regression_gradient", 23 | "regression_intercept", 24 | "regression_r", 25 | "regression_stderr"] 26 | 27 | def test_Aeronet_wavelength_stats(self): 28 | # Takes 3s 29 | args = ['stats', '%s,%s:%s' % ('AOT_500', 'AOT_440', escape_colons(another_valid_aeronet_filename)), 30 | '-o', self.OUTPUT_FILENAME] 31 | arguments = parse_args(args) 32 | stats_cmd(arguments) 33 | self.check_output_contains_variables(self.OUTPUT_FILENAME, self.output_vars) 34 | 35 | def test_no_output_file(self): 36 | # Takes 3s 37 | args = ['stats', '%s,%s:%s' % ('AOT_500', 'AOT_440', escape_colons(another_valid_aeronet_filename))] 38 | arguments = parse_args(args) 39 | stats_cmd(arguments) 40 | 41 | def test_ECHAMHAM_wavelength_stats(self): 42 | # Takes 0.7s 43 | args = ['stats', "%s,%s:%s" % (valid_echamham_variable_1, valid_echamham_variable_2, escape_colons(valid_echamham_filename)), 44 | '-o', self.OUTPUT_FILENAME] 45 | arguments = parse_args(args) 46 | stats_cmd(arguments) 47 | self.check_output_contains_variables(self.OUTPUT_FILENAME, self.output_vars) 48 | 49 | def test_collocated_NetCDF_Gridded_onto_GASSP(self): 50 | # Takes 2s 51 | # First do a collocation of ECHAMHAM onto GASSP 52 | sample_file = valid_GASSP_aeroplane_filename 53 | sample_var = valid_GASSP_aeroplane_variable 54 | collocator_and_opts = 'nn,variable=%s' % sample_var 55 | arguments = ['col', '%s:%s' % (valid_echamham_variable_1, escape_colons(valid_echamham_filename)), 56 | escape_colons(sample_file) + ':collocator=' + collocator_and_opts, 57 | '-o', 'collocated_gassp'] 58 | main_arguments = parse_args(arguments) 59 | col_cmd(main_arguments) 60 | 61 | # Then do a statistics calculation using the collocated data: 62 | args = ['stats', "%s:%s" % (valid_echamham_variable_1, 'collocated_gassp.nc'), 63 | "%s:%s" % (valid_GASSP_aeroplane_variable, escape_colons(valid_GASSP_aeroplane_filename)), 64 | '-o', self.OUTPUT_FILENAME] 65 | arguments = parse_args(args) 66 | stats_cmd(arguments) 67 | self.check_output_contains_variables(self.OUTPUT_FILENAME, self.output_vars) 68 | os.remove('collocated_gassp.nc') 69 | 70 | @skip_pyhdf 71 | def test_CloudSat(self): 72 | # Takes 140s 73 | args = ['stats', "%s,%s:%s" % ("RVOD_liq_water_content", "RVOD_ice_water_content", 74 | escape_colons(valid_cloudsat_RVOD_file)), 75 | '-o', self.OUTPUT_FILENAME] 76 | arguments = parse_args(args) 77 | stats_cmd(arguments) 78 | self.check_output_contains_variables(self.OUTPUT_FILENAME, self.output_vars) 79 | 80 | 81 | if __name__ == '__main__': 82 | unittest.main() 83 | -------------------------------------------------------------------------------- /cis/test/integration/test_version.py: -------------------------------------------------------------------------------- 1 | """ 2 | Module to do integration tests of version. Does not check the version is correct just that the command runs 3 | created without errors. 4 | """ 5 | from unittest import TestCase 6 | 7 | from cis.cis_main import parse_and_run_arguments 8 | 9 | 10 | class TestVersionIntegration(TestCase): 11 | 12 | def test_should_do_scatter_plot_of_file_valid_aerosol_cci_file(self): 13 | 14 | arguments = ['version'] 15 | parse_and_run_arguments(arguments) 16 | -------------------------------------------------------------------------------- /cis/test/plot_tests/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Plotting integration tests. These tests use matplotlib to actually check the image output is 'similar' to some reference 3 | image. The tolerance is defined in the test_plotting script and based onthe default used in iris 1.8.0. 4 | """ -------------------------------------------------------------------------------- /cis/test/plot_tests/idiff.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # (C) British Crown Copyright 2010 - 2014, Met Office 3 | # 4 | # This file was heavily influenced by a similar file in the iris package. 5 | """ 6 | Provides "diff-like" comparison of images. 7 | 8 | Currently relies on matplotlib for image processing so limited to PNG format. 9 | 10 | """ 11 | 12 | from __future__ import (absolute_import, division, print_function) 13 | 14 | import os.path 15 | import shutil 16 | 17 | import matplotlib.pyplot as plt 18 | import matplotlib.image as mimg 19 | import matplotlib.widgets as mwidget 20 | 21 | 22 | def diff_viewer(expected_fname, result_fname, diff_fname): 23 | plt.figure(figsize=(16, 16)) 24 | plt.suptitle(os.path.basename(expected_fname)) 25 | ax = plt.subplot(221) 26 | ax.imshow(mimg.imread(expected_fname)) 27 | ax = plt.subplot(222, sharex=ax, sharey=ax) 28 | ax.imshow(mimg.imread(result_fname)) 29 | ax = plt.subplot(223, sharex=ax, sharey=ax) 30 | ax.imshow(mimg.imread(diff_fname)) 31 | 32 | def accept(event): 33 | # removes the expected result, and move the most recent result in 34 | print('ACCEPTED NEW FILE: %s' % (os.path.basename(expected_fname), )) 35 | os.remove(expected_fname) 36 | shutil.copy2(result_fname, expected_fname) 37 | os.remove(diff_fname) 38 | plt.close() 39 | 40 | def reject(event): 41 | print('REJECTED: %s' % (os.path.basename(expected_fname), )) 42 | plt.close() 43 | 44 | ax_accept = plt.axes([0.6, 0.35, 0.1, 0.075]) 45 | ax_reject = plt.axes([0.71, 0.35, 0.1, 0.075]) 46 | bnext = mwidget.Button(ax_accept, 'Accept change') 47 | bnext.on_clicked(accept) 48 | bprev = mwidget.Button(ax_reject, 'Reject') 49 | bprev.on_clicked(reject) 50 | 51 | plt.show() 52 | 53 | 54 | def step_over_diffs(): 55 | import cis.test.plot_tests 56 | image_dir = os.path.join(os.path.dirname(cis.test.plot_tests.__file__), 57 | 'reference', 'visual_tests') 58 | diff_dir = os.path.join(os.path.dirname(cis.test.plot_tests.__file__), 59 | 'result_image_comparison') 60 | 61 | for expected_fname in sorted(os.listdir(image_dir)): 62 | result_path = os.path.join(diff_dir, expected_fname) 63 | diff_path = result_path[:-4] + '-failed-diff.png' 64 | 65 | # if the test failed, there will be a diff file 66 | if os.path.exists(diff_path): 67 | expected_path = os.path.join(image_dir, expected_fname) 68 | diff_viewer(expected_path, result_path, diff_path) 69 | 70 | 71 | if __name__ == '__main__': 72 | step_over_diffs() 73 | -------------------------------------------------------------------------------- /cis/test/plot_tests/reference/kitten.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/test/plot_tests/reference/kitten.png -------------------------------------------------------------------------------- /cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotAPIVisual.test_can_specify_yaxis_altitude.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotAPIVisual.test_can_specify_yaxis_altitude.png -------------------------------------------------------------------------------- /cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotAPIVisual.test_contour_over_bluemarble.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotAPIVisual.test_contour_over_bluemarble.png -------------------------------------------------------------------------------- /cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotAPIVisual.test_explicit_comparative_scatter.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotAPIVisual.test_explicit_comparative_scatter.png -------------------------------------------------------------------------------- /cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotAPIVisual.test_heatmap.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotAPIVisual.test_heatmap.png -------------------------------------------------------------------------------- /cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotAPIVisual.test_histogram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotAPIVisual.test_histogram.png -------------------------------------------------------------------------------- /cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotAPIVisual.test_histogram_2d.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotAPIVisual.test_histogram_2d.png -------------------------------------------------------------------------------- /cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotAPIVisual.test_implicit_comparative_scatter.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotAPIVisual.test_implicit_comparative_scatter.png -------------------------------------------------------------------------------- /cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotAPIVisual.test_invalid_args.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotAPIVisual.test_invalid_args.png -------------------------------------------------------------------------------- /cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotAPIVisual.test_iris_comparative_scatter.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotAPIVisual.test_iris_comparative_scatter.png -------------------------------------------------------------------------------- /cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotAPIVisual.test_iris_multiple_scatter.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotAPIVisual.test_iris_multiple_scatter.png -------------------------------------------------------------------------------- /cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotAPIVisual.test_layer_opts.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotAPIVisual.test_layer_opts.png -------------------------------------------------------------------------------- /cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotAPIVisual.test_mercator_projection.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotAPIVisual.test_mercator_projection.png -------------------------------------------------------------------------------- /cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotAPIVisual.test_mpl_kwargs.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotAPIVisual.test_mpl_kwargs.png -------------------------------------------------------------------------------- /cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotAPIVisual.test_multiple_line.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotAPIVisual.test_multiple_line.png -------------------------------------------------------------------------------- /cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotAPIVisual.test_orographic_projection.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotAPIVisual.test_orographic_projection.png -------------------------------------------------------------------------------- /cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotAPIVisual.test_polar_projection.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotAPIVisual.test_polar_projection.png -------------------------------------------------------------------------------- /cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotAPIVisual.test_scatter2d_over_bluemarble.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotAPIVisual.test_scatter2d_over_bluemarble.png -------------------------------------------------------------------------------- /cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotAPIVisual.test_scatter2d_over_bluemarble_coord_axis.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotAPIVisual.test_scatter2d_over_bluemarble_coord_axis.png -------------------------------------------------------------------------------- /cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotAPIVisual.test_scatter2d_over_bluemarble_explicit_axis.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotAPIVisual.test_scatter2d_over_bluemarble_explicit_axis.png -------------------------------------------------------------------------------- /cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotAPIVisual.test_taylor_diagram_gridded.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotAPIVisual.test_taylor_diagram_gridded.png -------------------------------------------------------------------------------- /cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_aeronet_default_axes.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_aeronet_default_axes.png -------------------------------------------------------------------------------- /cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_aerosol_cci_default_axes.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_aerosol_cci_default_axes.png -------------------------------------------------------------------------------- /cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_bluemarble_0_360.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_bluemarble_0_360.png -------------------------------------------------------------------------------- /cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_bluemarble_minus_180_180.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_bluemarble_minus_180_180.png -------------------------------------------------------------------------------- /cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_coastline_color.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_coastline_color.png -------------------------------------------------------------------------------- /cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_filled_contour_over_scatter2d.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_filled_contour_over_scatter2d.png -------------------------------------------------------------------------------- /cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_filled_contour_over_scatter2d_with_cmin.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_filled_contour_over_scatter2d_with_cmin.png -------------------------------------------------------------------------------- /cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_iris_comparative_scatter.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_iris_comparative_scatter.png -------------------------------------------------------------------------------- /cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_iris_contour.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_iris_contour.png -------------------------------------------------------------------------------- /cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_iris_contour_over_heatmap.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_iris_contour_over_heatmap.png -------------------------------------------------------------------------------- /cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_iris_contour_over_heatmap_binary_cmap.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_iris_contour_over_heatmap_binary_cmap.png -------------------------------------------------------------------------------- /cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_iris_contourf.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_iris_contourf.png -------------------------------------------------------------------------------- /cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_iris_heatmap.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_iris_heatmap.png -------------------------------------------------------------------------------- /cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_iris_heatmap_force_minus_180_to_180.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_iris_heatmap_force_minus_180_to_180.png -------------------------------------------------------------------------------- /cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_iris_histogram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_iris_histogram.png -------------------------------------------------------------------------------- /cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_iris_histogram2d.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_iris_histogram2d.png -------------------------------------------------------------------------------- /cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_iris_many_lines.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_iris_many_lines.png -------------------------------------------------------------------------------- /cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_iris_one_line.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_iris_one_line.png -------------------------------------------------------------------------------- /cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_iris_one_line_with_step.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_iris_one_line_with_step.png -------------------------------------------------------------------------------- /cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_iris_scatter.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_iris_scatter.png -------------------------------------------------------------------------------- /cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_iris_scatter2d.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_iris_scatter2d.png -------------------------------------------------------------------------------- /cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_iris_scatter2d_overlay.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_iris_scatter2d_overlay.png -------------------------------------------------------------------------------- /cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_mercator_projection.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_mercator_projection.png -------------------------------------------------------------------------------- /cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_multiple_time_series_default_axes.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_multiple_time_series_default_axes.png -------------------------------------------------------------------------------- /cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_multiple_time_series_default_axes_files_with_named_xaxis.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_multiple_time_series_default_axes_files_with_named_xaxis.png -------------------------------------------------------------------------------- /cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_multiple_time_series_incompatible_axes.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_multiple_time_series_incompatible_axes.png -------------------------------------------------------------------------------- /cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_other_comparative_scatter.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_other_comparative_scatter.png -------------------------------------------------------------------------------- /cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_other_contour.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_other_contour.png -------------------------------------------------------------------------------- /cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_other_contourf.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_other_contourf.png -------------------------------------------------------------------------------- /cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_other_histogram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_other_histogram.png -------------------------------------------------------------------------------- /cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_other_histogram2d.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_other_histogram2d.png -------------------------------------------------------------------------------- /cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_other_histogram2d_doesnt_plot_coastlines.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_other_histogram2d_doesnt_plot_coastlines.png -------------------------------------------------------------------------------- /cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_other_histogram_bin_width.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_other_histogram_bin_width.png -------------------------------------------------------------------------------- /cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_other_longitude_wrapping_0_360.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_other_longitude_wrapping_0_360.png -------------------------------------------------------------------------------- /cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_other_longitude_wrapping_0_360_forced_minus_180_to_180.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_other_longitude_wrapping_0_360_forced_minus_180_to_180.png -------------------------------------------------------------------------------- /cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_other_longitude_wrapping_minus_180_180.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_other_longitude_wrapping_minus_180_180.png -------------------------------------------------------------------------------- /cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_other_longitude_wrapping_minus_180_180_forced_0_to_360.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_other_longitude_wrapping_minus_180_180_forced_0_to_360.png -------------------------------------------------------------------------------- /cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_other_longitude_wrapping_multiple_ranges_forced_0_to_360.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_other_longitude_wrapping_multiple_ranges_forced_0_to_360.png -------------------------------------------------------------------------------- /cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_other_longitude_wrapping_multiple_ranges_forced_minus_180_to_180.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_other_longitude_wrapping_multiple_ranges_forced_minus_180_to_180.png -------------------------------------------------------------------------------- /cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_other_many_lines.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_other_many_lines.png -------------------------------------------------------------------------------- /cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_other_many_scatter_points.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_other_many_scatter_points.png -------------------------------------------------------------------------------- /cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_other_many_scatter_points_given_color.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_other_many_scatter_points_given_color.png -------------------------------------------------------------------------------- /cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_other_one_line.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_other_one_line.png -------------------------------------------------------------------------------- /cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_other_scatter.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_other_scatter.png -------------------------------------------------------------------------------- /cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_other_scatter2d.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_other_scatter2d.png -------------------------------------------------------------------------------- /cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_other_taylor_diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_other_taylor_diagram.png -------------------------------------------------------------------------------- /cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_plotting_heatmap_of_aggregated_ungridded_data.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_plotting_heatmap_of_aggregated_ungridded_data.png -------------------------------------------------------------------------------- /cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_polar_projection.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_polar_projection.png -------------------------------------------------------------------------------- /cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_scatter2d_over_contour.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_scatter2d_over_contour.png -------------------------------------------------------------------------------- /cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_setting_xrange_using_datetimes.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_setting_xrange_using_datetimes.png -------------------------------------------------------------------------------- /cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_transparent_contour_over_bluemarble.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/test/plot_tests/reference/visual_tests/test_plotting.TestPlotVisual.test_transparent_contour_over_bluemarble.png -------------------------------------------------------------------------------- /cis/test/runner.py: -------------------------------------------------------------------------------- 1 | from setuptools.command.test import test as TestCommand 2 | import multiprocessing 3 | 4 | 5 | def run(test_set='cis/test/unit', n_processors=1, stop=False, debug=False): 6 | import nose 7 | import logging 8 | import sys 9 | 10 | if debug: 11 | logging.basicConfig(level=logging.DEBUG, 12 | format="%(asctime)s - %(levelname)s - %(module)s : %(lineno)d - %(message)s", 13 | stream=sys.stdout) 14 | 15 | args = ['', test_set, '--processes=%s' % n_processors, '--verbosity=2', '--process-timeout=60'] 16 | 17 | if stop: 18 | args.append('--stop') 19 | 20 | nose.run_exit(argv=args) 21 | 22 | 23 | class nose_test(TestCommand): 24 | """ 25 | Command to run unit tests 26 | """ 27 | description = "Run CIS tests. By default this will run all of the unit tests. Optionally the integration tests can"\ 28 | " be run instead." 29 | user_options = [('integration-tests', 'i', 'Run the integration tests.'), 30 | ('stop', 'x', 'Stop running tests after the first error or failure.'), 31 | ('num-processors=', 'p', 'The number of processors used for running the tests.'), 32 | ('debug', 'd', 'Send debug level logging information to stdout.')] 33 | 34 | def initialize_options(self): 35 | TestCommand.initialize_options(self) 36 | self.integration_tests = False 37 | self.stop = False 38 | self.num_processors = None 39 | self.debug = False 40 | 41 | def finalize_options(self): 42 | TestCommand.finalize_options(self) 43 | self.test_args = [] 44 | self.test_suite = True 45 | if self.integration_tests: 46 | self.test_set = 'cis/test/integration' 47 | else: 48 | self.test_set = 'cis/test/unit' 49 | 50 | if self.num_processors is None: 51 | self.num_processors = multiprocessing.cpu_count() - 1 52 | else: 53 | self.num_processors = int(self.num_processors) 54 | 55 | def run_tests(self): 56 | run(self.test_set, self.num_processors, self.stop, self.debug) 57 | 58 | # nose.run_exit(argv=['nosetests',os.path.join(os.path.dirname(__file__), 'unit')]) 59 | 60 | -------------------------------------------------------------------------------- /cis/test/unit/__init__.py: -------------------------------------------------------------------------------- 1 | # Ensure we're using a headless matplotlib for testing. 2 | import matplotlib 3 | 4 | matplotlib.use("Agg") 5 | -------------------------------------------------------------------------------- /cis/test/unit/aggregation/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/test/unit/aggregation/__init__.py -------------------------------------------------------------------------------- /cis/test/unit/aggregation/test_aggregation_kernels.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from cis.collocation.col_framework import get_kernel 4 | from cis.test.util import mock 5 | from cis.aggregation.collapse_kernels import aggregation_kernels, CountKernel 6 | from cis.test.utils_for_testing import * 7 | from cis.data_io.gridded_data import make_from_cube 8 | 9 | 10 | class TestMomentsKernel(unittest.TestCase): 11 | 12 | def test_GIVEN_gridded_data_WHEN_full_collapse_THEN_calculations_correct(self): 13 | cube = make_from_cube(mock.make_mock_cube()) 14 | kernel = aggregation_kernels['moments'] 15 | result = cube.collapsed(['y'], how=kernel) 16 | 17 | expected_means = numpy.array([7, 8, 9]) 18 | expected_std_dev = numpy.array(3 * [numpy.sqrt(22.5)]) 19 | expected_no = numpy.array([5, 5, 5]) 20 | assert_that(len(result), is_(3)) 21 | assert_that(numpy.allclose(result[0].data, expected_means)) 22 | assert_that(numpy.allclose(result[1].data, expected_std_dev)) 23 | assert_that(numpy.array_equal(result[2].data, expected_no)) 24 | 25 | def test_GIVEN_gridded_data_WHEN_full_collapse_THEN_metadata_correct(self): 26 | cube = make_from_cube(mock.make_mock_cube()) 27 | cube.standard_name = 'age_of_sea_ice' # Use a CF compliant name 28 | cube.long_name = 'Age of sea ice' 29 | cube.var_name = 'age_ice' 30 | cube.units = 'years' 31 | kernel = aggregation_kernels['moments'] 32 | result = cube.collapsed(['y'], how=kernel) 33 | 34 | mean, stddev, num = result 35 | assert_that(mean.standard_name, is_('age_of_sea_ice')) 36 | assert_that(stddev.standard_name, is_(None)) 37 | assert_that(num.standard_name, is_(None)) 38 | assert_that(mean.long_name, is_('Age of sea ice')) 39 | assert_that(stddev.long_name, is_('Corrected sample standard deviation of Age of sea ice')) 40 | assert_that(num.long_name, is_('Number of points used to calculate the mean of Age of sea ice')) 41 | assert_that(mean.var_name, is_('age_ice')) 42 | assert_that(stddev.var_name, is_('age_ice_std_dev')) 43 | assert_that(num.var_name, is_('age_ice_num_points')) 44 | assert_that(mean.units, is_('years')) 45 | assert_that(stddev.units, is_('years')) 46 | assert_that(num.units, is_(None)) 47 | 48 | def test_GIVEN_grid_contains_single_points_WHEN_collapse_THEN_stddev_undefined(self): 49 | cube = make_from_cube(mock.make_mock_cube(2, 2)) 50 | cube.data = numpy.ma.masked_invalid([[float('Nan'), 1], [float('Nan'), float('Nan')]]) 51 | kernel = aggregation_kernels['moments'] 52 | result = cube.collapsed(['y'], how=kernel) 53 | 54 | assert_that(result[1].data.mask.all()) 55 | 56 | def test_GIVEN_ungridded_data_WHEN_collapse_THEN_calculations_correct(self): 57 | grid = {'y': slice(-12.5, 12.5, 12.5)} 58 | data = mock.make_regular_2d_ungridded_data() 59 | kernel_class = get_kernel('moments') 60 | kernel = kernel_class() 61 | result = data.aggregate(how=kernel, **grid) 62 | 63 | expected_means = numpy.array([3.5, 11]) 64 | expected_std_dev = numpy.array([numpy.sqrt(3.5), numpy.sqrt(7.5)]) 65 | expected_no = numpy.array([6, 9]) 66 | assert_that(len(result), is_(3)) 67 | assert_arrays_almost_equal(result[0].data.flatten(), expected_means) 68 | assert_arrays_almost_equal(result[1].data.flatten(), expected_std_dev) 69 | assert_that(numpy.array_equal(result[2].data.flatten(), expected_no)) 70 | 71 | def test_GIVEN_ungridded_data_WHEN_collapse_THEN_metadata_correct(self): 72 | grid = {'y': slice(-10, 10, 10)} 73 | data = mock.make_regular_2d_ungridded_data() 74 | kernel_class = get_kernel('moments') 75 | kernel = kernel_class() 76 | result = data.aggregate(how=kernel, **grid) 77 | 78 | mean, stddev, num = result 79 | assert_that(mean.standard_name, is_('rainfall_rate')) 80 | assert_that(stddev.standard_name, is_(None)) 81 | assert_that(num.standard_name, is_(None)) 82 | assert_that(mean.long_name, is_('TOTAL RAINFALL RATE: LS+CONV KG/M2/S')) 83 | assert_that(stddev.long_name, 84 | is_('Corrected sample standard deviation of TOTAL RAINFALL RATE: LS+CONV KG/M2/S')) 85 | assert_that(num.long_name, is_('Number of points used to calculate the mean of ' 86 | 'TOTAL RAINFALL RATE: LS+CONV KG/M2/S')) 87 | assert_that(mean.var_name, is_('rain')) 88 | assert_that(stddev.var_name, is_('rain_std_dev')) 89 | assert_that(num.var_name, is_('rain_num_points')) 90 | assert_that(mean.units, is_('kg m-2 s-1')) 91 | assert_that(stddev.units, is_('kg m-2 s-1')) 92 | assert_that(num.units, is_(None)) 93 | 94 | 95 | class TestCountKernel(unittest.TestCase): 96 | 97 | def test_GIVEN_missing_data_WHEN_count_THEN_calculation_correct(self): 98 | cube = mock.make_5x3_lon_lat_2d_cube_with_missing_data() 99 | kernel = CountKernel() 100 | data = kernel.count_kernel_func(cube.data, 0) 101 | assert_that(numpy.array_equal(data, numpy.array([4, 4, 4]))) 102 | 103 | if __name__ == '__main__': 104 | unittest.main() 105 | -------------------------------------------------------------------------------- /cis/test/unit/colocate/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/test/unit/colocate/__init__.py -------------------------------------------------------------------------------- /cis/test/unit/colocate/test_GridCellBinIndex.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from hamcrest import * 4 | 5 | from cis.collocation import data_index 6 | from cis.data_io.hyperpoint import HyperPoint 7 | from cis.collocation.col_implementations import make_coord_map, BinnedCubeCellOnlyConstraint 8 | from cis.test.util.mock import * 9 | from cis.time_util import convert_datetime_to_std_time 10 | 11 | 12 | class TestGridCellBinIndex(unittest.TestCase): 13 | 14 | def test_GIVEN_single_point_in_cube_WHEN_iterate_THEN_return_point_in_middle(self): 15 | 16 | sample_cube = make_square_5x3_2d_cube_with_time(offset=0, time_offset=0) 17 | data_point = make_dummy_ungridded_data_single_point(0.5, 0.5, 1.2, time=datetime.datetime(1984, 8, 28, 0, 0)) 18 | coord_map = make_coord_map(sample_cube, data_point) 19 | coords = sample_cube.coords() 20 | for (hpi, ci, shi) in coord_map: 21 | coord = coords[ci] 22 | if coord.ndim > 1: 23 | raise NotImplementedError("Co-location of data onto a cube with a coordinate of dimension greater" 24 | " than one is not supported (coordinate %s)", coord.name()) 25 | # Ensure that bounds exist. 26 | if not coord.has_bounds(): 27 | coord.guess_bounds() 28 | 29 | constraint = BinnedCubeCellOnlyConstraint() 30 | data_index.create_indexes(constraint, coords, data_point.get_non_masked_points(), coord_map) 31 | iterator = constraint.get_iterator(False, coord_map, coords, data_point.get_non_masked_points(), None, 32 | sample_cube, None) 33 | 34 | final_points_index = [(out_index, hp, points) for out_index, hp, points in iterator] 35 | assert_that(len(final_points_index), is_(1), "There is one mapping from sample_cube to the final grid") 36 | assert_that(final_points_index[0][0], is_((2, 1, 1)), "The points should map to index") 37 | assert_that(final_points_index[0][1], is_(HyperPoint(lat=0, lon=0, t=datetime.datetime(1984, 8, 28))), 38 | "The points should map to index") 39 | assert_that(final_points_index[0][2].latitudes, is_([0.5]), "The points should map to index") 40 | assert_that(final_points_index[0][2].longitudes, is_([0.5]), "The points should map to index") 41 | assert_that(final_points_index[0][2].times, 42 | is_([convert_datetime_to_std_time(datetime.datetime(1984, 8, 28, 0, 0))]), 43 | "The points should map to index") 44 | assert_that(final_points_index[0][2].vals, is_([1.2]), "The points should map to index") 45 | 46 | def test_GIVEN_single_masked_point_in_cube_WHEN_iterate_THEN_return_no_points(self): 47 | 48 | sample_cube = make_square_5x3_2d_cube_with_time(offset=0, time_offset=0) 49 | data_point = make_dummy_ungridded_data_single_point(0.5, 0.5, 1.2, time=datetime.datetime(1984, 8, 28, 0, 0), 50 | mask=True) 51 | coord_map = make_coord_map(sample_cube, data_point) 52 | coords = sample_cube.coords() 53 | for (hpi, ci, shi) in coord_map: 54 | coord = coords[ci] 55 | if coord.ndim > 1: 56 | raise NotImplementedError("Co-location of data onto a cube with a coordinate of dimension greater" 57 | " than one is not supported (coordinate %s)", coord.name()) 58 | # Ensure that bounds exist. 59 | if not coord.has_bounds(): 60 | coord.guess_bounds() 61 | 62 | constraint = BinnedCubeCellOnlyConstraint() 63 | data_index.create_indexes(constraint, coords, data_point.get_non_masked_points(), coord_map) 64 | iterator = constraint.get_iterator(False, coord_map, coords, data_point.get_non_masked_points(), None, 65 | sample_cube, None) 66 | 67 | final_points_index = [(out_index, hp, points) for out_index, hp, points in iterator] 68 | assert_that(len(final_points_index), is_(0), "Masked points should not be iterated over") 69 | -------------------------------------------------------------------------------- /cis/test/unit/colocate/test_colocate.py: -------------------------------------------------------------------------------- 1 | """ 2 | Tests the cis.col.Collocate class 3 | """ 4 | import unittest 5 | from nose.tools import assert_raises 6 | 7 | 8 | class TestCollocate(unittest.TestCase): 9 | def test_get_kernel(self): 10 | from cis.collocation.col_implementations import moments, nn_pressure, mean 11 | from cis.collocation.col import get_kernel 12 | 13 | assert isinstance(get_kernel('moments'), moments) 14 | assert isinstance(get_kernel(nn_pressure()), nn_pressure) 15 | assert isinstance(get_kernel('mean', default=moments), mean) 16 | assert isinstance(get_kernel('', default=moments), moments) 17 | 18 | with assert_raises(ValueError): 19 | get_kernel('foo') 20 | 21 | with assert_raises(ValueError): 22 | get_kernel('foo', default=moments) -------------------------------------------------------------------------------- /cis/test/unit/colocate/test_dummy_col.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | import numpy 4 | 5 | from cis.data_io.ungridded_data import UngriddedDataList 6 | import cis.test.util.mock as mock 7 | from cis.collocation.col_implementations import DummyCollocator 8 | 9 | 10 | class TestDummyCollocator(unittest.TestCase): 11 | 12 | def test_single_data(self): 13 | sample = mock.make_regular_2d_ungridded_data() 14 | data = mock.make_regular_2d_ungridded_data(data_offset=10) 15 | col = DummyCollocator() 16 | con = None 17 | kernel = None 18 | 19 | output = col.collocate(sample, data, con, kernel) 20 | 21 | assert len(output) == 1 22 | assert numpy.array_equal(output[0].data, data.data) 23 | 24 | def test_list_of_data(self): 25 | sample = mock.make_regular_2d_ungridded_data() 26 | data = UngriddedDataList([mock.make_regular_2d_ungridded_data(data_offset=5), 27 | mock.make_regular_2d_ungridded_data(data_offset=10)]) 28 | col = DummyCollocator() 29 | con = None 30 | kernel = None 31 | 32 | output = col.collocate(sample, data, con, kernel) 33 | 34 | assert len(output) == 2 35 | assert numpy.array_equal(output[0].data, data[0].data) 36 | assert numpy.array_equal(output[1].data, data[1].data) 37 | 38 | if __name__ == '__main__': 39 | unittest.main() 40 | -------------------------------------------------------------------------------- /cis/test/unit/eval/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/test/unit/eval/__init__.py -------------------------------------------------------------------------------- /cis/test/unit/stats/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/test/unit/stats/__init__.py -------------------------------------------------------------------------------- /cis/test/unit/subset/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/test/unit/subset/__init__.py -------------------------------------------------------------------------------- /cis/test/unit/test_coord.py: -------------------------------------------------------------------------------- 1 | from hamcrest import * 2 | from nose.tools import istest, raises 3 | import numpy 4 | from cis.data_io.Coord import Coord, CoordList 5 | from cis.data_io.ungridded_data import Metadata 6 | from cis.exceptions import DuplicateCoordinateError 7 | 8 | 9 | def create_dummy_coordinates_list(): 10 | coord1 = Coord(numpy.array([5, 4]), Metadata(standard_name='grid_latitude'), axis='Y') 11 | coord2 = Coord(numpy.array([5, 4]), Metadata(standard_name='grid_longitude'), axis='X') 12 | return CoordList([coord1, coord2]) 13 | 14 | 15 | @istest 16 | def can_create_a_valid_list_of_coordinates(): 17 | list = create_dummy_coordinates_list() 18 | assert (len(list) == 2) 19 | assert (list[0].standard_name == 'grid_latitude') 20 | assert (list[1].standard_name == 'grid_longitude') 21 | assert (list[0].axis == 'Y') 22 | assert (list[1].axis == 'X') 23 | 24 | 25 | @istest 26 | def can_append_to_list_of_coordinates(): 27 | list = create_dummy_coordinates_list() 28 | list.append(Coord(numpy.array([5, 4]), Metadata(standard_name='altitude'), axis='Z')) 29 | assert (len(list) == 3) 30 | assert (list[2].standard_name == 'altitude') 31 | assert (list[2].axis == 'Z') 32 | 33 | 34 | @istest 35 | @raises(DuplicateCoordinateError) 36 | def append_a_duplicate_to_a_list_of_coordinates_fails(): 37 | list = create_dummy_coordinates_list() 38 | list.append(Coord(numpy.array([5, 4]), Metadata(standard_name='grid_longitude'), axis='X')) 39 | 40 | 41 | @istest 42 | def can_find_a_coord_from_a_list_of_coordinates(): 43 | list = create_dummy_coordinates_list() 44 | coord = list.get_coord(standard_name='grid_longitude') 45 | assert (coord.standard_name == 'grid_longitude') 46 | assert (coord.axis == 'X') 47 | 48 | 49 | @istest 50 | def can_find_many_coords_from_a_list_of_coordinates(): 51 | list = create_dummy_coordinates_list() 52 | list.append(Coord(numpy.array([5, 4]), Metadata(name='testZ'), axis='Z')) 53 | list.append(Coord(numpy.array([5, 4]), Metadata(name='testZ', standard_name='air_pressure'))) 54 | assert (len(list) == 4) 55 | coords = list.get_coords(var_name='testZ') 56 | assert (len(coords) == 2) 57 | assert (coords[0].axis == 'Z') 58 | assert (coords[1].axis == '') 59 | assert (coords[1].name() == 'air_pressure') 60 | 61 | @istest 62 | def can_convert_time_without_since_in_units(): 63 | times = numpy.array([0, 1]) 64 | units = "Days from the file reference point 1601-01-01" 65 | time_stamp_info = "1601-01-01" 66 | coord = Coord(times, Metadata(units=units)) 67 | 68 | coord.convert_to_std_time(time_stamp_info) 69 | 70 | assert_that(coord.data_flattened[0], is_(366.0), "time point") 71 | 72 | 73 | @istest 74 | def can_convert_time_with_since_in_units(): 75 | times = numpy.array([0, 1]) 76 | units = "days since 1601-01-01" 77 | coord = Coord(times, Metadata(units=units)) 78 | 79 | coord.convert_to_std_time() 80 | 81 | assert_that(coord.data_flattened[0], is_(366.0), "time point") 82 | 83 | 84 | @istest 85 | @raises(ValueError) 86 | def can_not_convert_time_without_since_in_units_but_no_timestamp(): 87 | times = numpy.array([0, 1]) 88 | units = "Days from the file reference point 1601-01-01" 89 | coord = Coord(times, Metadata(units=units)) 90 | 91 | coord.convert_to_std_time() 92 | 93 | 94 | @istest 95 | @raises(ValueError) 96 | def can_not_convert_time_without_since_in_units_with_no_units(): 97 | times = numpy.array([0, 1]) 98 | units = "" 99 | time_stamp_info = "1601-01-01" 100 | coord = Coord(times, Metadata(units=units)) 101 | 102 | coord.convert_to_std_time(time_stamp_info) 103 | -------------------------------------------------------------------------------- /cis/test/unit/test_io/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/test/unit/test_io/__init__.py -------------------------------------------------------------------------------- /cis/test/unit/test_io/test_hdf.py: -------------------------------------------------------------------------------- 1 | # Based on examples here: http://erikzaadi.com/2012/07/03/mocking-python-imports/ 2 | """ 3 | Tests for checking correct behaviour when Python HDF is not installed. ImportError should be raised when HDF is used 4 | not when CIS is first started. 5 | """ 6 | from nose.tools import raises 7 | from mock import MagicMock, patch 8 | 9 | 10 | class HDFNotInstalledTests(object): 11 | 12 | def setup(self): 13 | """ 14 | We have to patch the sys.modules dictionary to use our mocked versions of the pyhdf library to test it not 15 | being installed. This creates the appropriate mocks and imports them into the namespace. 16 | """ 17 | 18 | self.pyhdf_mock = MagicMock() 19 | self.pyhdf_mock.SD = None 20 | self.pyhdf_mock.HDF = None 21 | modules = { 22 | 'pyhdf': self.pyhdf_mock, 23 | 'pyhdf.SD': self.pyhdf_mock.SD, 24 | 'pyhdf.HDF': self.pyhdf_mock.HDF 25 | } 26 | 27 | self.module_patcher = patch.dict('sys.modules', modules) 28 | self.module_patcher.start() 29 | 30 | from cis.data_io import hdf, hdf_vd, hdf_sd 31 | self.hdf_vd = hdf_vd 32 | self.hdf_sd = hdf_sd 33 | self.hdf = hdf 34 | 35 | def teardown(self): 36 | """ 37 | Remove the patches 38 | """ 39 | self.module_patcher.stop() 40 | 41 | @raises(ImportError) 42 | def test_no_pyhdf_raises_not_installed_in_VD_variables(self): 43 | _ = self.hdf_vd.get_hdf_VD_file_variables('some_file') 44 | 45 | @raises(ImportError) 46 | def test_no_pyhdf_raises_not_installed_in_VD_read(self): 47 | _ = self.hdf_vd.read('some_file', 'some_variable') 48 | 49 | @raises(ImportError) 50 | def test_no_pyhdf_raises_not_installed_in_SD_variables(self): 51 | _ = self.hdf_sd.get_hdf_SD_file_variables('some_file') 52 | 53 | @raises(ImportError) 54 | def test_no_pyhdf_raises_not_installed_in_SD_read(self): 55 | _ = self.hdf_sd.read('some_file', 'some_variable') 56 | 57 | @raises(ImportError) 58 | def test_no_pyhdf_raises_not_installed_in_HDF_get_metadata(self): 59 | _ = self.hdf.get_hdf4_file_metadata('some_file') 60 | -------------------------------------------------------------------------------- /cis/test/unit/test_overide_product.py: -------------------------------------------------------------------------------- 1 | from nose.tools import istest, eq_, raises 2 | from cis.data_io.products.caliop import Caliop_L2 3 | from cis.exceptions import ClassNotFoundError 4 | from cis.data_io.products.AProduct import __get_class 5 | from cis.parse import parse_args 6 | 7 | # Note that the below is only used as a filename to test the product matching routines - there is no need for the actual 8 | # file to be present 9 | example_caliop_l2_filename = "CAL_LID_L2_05kmAPro-Prov-V3-01.2009-12-31T23-36-08ZN.hdf" 10 | 11 | 12 | @istest 13 | def can_overide_default_product(): 14 | from cis.data_io.products.gridded_NetCDF import NetCDF_Gridded 15 | eq_(__get_class(example_caliop_l2_filename), Caliop_L2) 16 | eq_(__get_class(example_caliop_l2_filename, "NetCDF_Gridded"), NetCDF_Gridded) 17 | 18 | 19 | @istest 20 | def should_raise_error_with_unknown_product_specified(): 21 | try: 22 | parse_args(["plot", "var:" + example_caliop_l2_filename + "::::unknownproduct"]) 23 | assert False 24 | except SystemExit as e: 25 | if e.code != 2: 26 | raise e 27 | 28 | 29 | @istest 30 | @raises(ClassNotFoundError) 31 | def files_which_match_some_of_regexp_but_not_the_end_shouldnt_match(): 32 | """ 33 | Because we're using re.match the regular expression has to match the start - but not necassarily the end. This 34 | test ensures that we have taken care of that as often the extension is the most important part. 35 | """ 36 | import cis.plugin as plugin 37 | from cis.data_io.products.AProduct import AProduct 38 | # We have to patch all of the plugin classes because get_file_type_error gets called and this file doesn't exist 39 | # we are only testing the wildcard matching logic. 40 | product_classes = plugin.find_plugin_classes(AProduct, 'cis.data_io.products') 41 | for p in product_classes: 42 | p.get_file_type_error = lambda self, f: None 43 | _ = __get_class(example_caliop_l2_filename+".ext") 44 | -------------------------------------------------------------------------------- /cis/test/unit/test_parse_datetime.py: -------------------------------------------------------------------------------- 1 | """Tests for parse_datetime module 2 | """ 3 | from nose.tools import istest, raises, assert_almost_equal, eq_ 4 | from cis.parse_datetime import _parse_partial_datetime, parse_as_number_or_datetime, \ 5 | parse_datetimestr_delta_to_float_days, parse_datetimestr_to_std_time 6 | from cis.time_util import PartialDateTime 7 | 8 | 9 | # Tests for parse_datetime 10 | @istest 11 | def parse_datetime_can_parse_year(): 12 | dt = _parse_partial_datetime('2010') 13 | assert (dt == PartialDateTime(2010)) 14 | 15 | 16 | @istest 17 | def parse_datetime_can_parse_year_month(): 18 | dt = _parse_partial_datetime('2010-07') 19 | print(dt) 20 | assert (dt == PartialDateTime(2010, 7)) 21 | 22 | 23 | @istest 24 | def parse_datetime_can_parse_date(): 25 | dt = _parse_partial_datetime('2010-07-01') 26 | assert (dt == PartialDateTime(2010, 7, 1)) 27 | 28 | 29 | @istest 30 | def parse_datetime_can_parse_date_hour(): 31 | dt = _parse_partial_datetime('2010-07-01T13') 32 | assert (dt == PartialDateTime(2010, 7, 1, 13)) 33 | 34 | 35 | @istest 36 | def parse_datetime_can_parse_date_hour_min(): 37 | dt = _parse_partial_datetime('2010-07-01T13:27') 38 | assert (dt == PartialDateTime(2010, 7, 1, 13, 27)) 39 | 40 | 41 | @istest 42 | def parse_datetime_can_parse_date_hour_min_sec(): 43 | dt = _parse_partial_datetime('2010-07-01T13:27:43') 44 | assert (dt == PartialDateTime(2010, 7, 1, 13, 27, 43)) 45 | 46 | 47 | @istest 48 | def parse_datetime_can_parse_date_hour_min_sec_no_leading_zeros(): 49 | dt = _parse_partial_datetime('2010-3-4T5:6:7') 50 | assert (dt == PartialDateTime(2010, 3, 4, 5, 6, 7)) 51 | 52 | 53 | @istest 54 | def parse_datetime_can_parse_date_time_with_space_separator(): 55 | dt = _parse_partial_datetime('2010-07-01 13:27:43') 56 | assert (dt == PartialDateTime(2010, 7, 1, 13, 27, 43)) 57 | 58 | 59 | @istest 60 | def parse_datetime_can_parse_date_time_with_colon_separator(): 61 | dt = _parse_partial_datetime('2010-07-01:13:27:43') 62 | assert (dt == PartialDateTime(2010, 7, 1, 13, 27, 43)) 63 | 64 | 65 | # parse_datetime: Parse errors 66 | @istest 67 | @raises(ValueError) 68 | def parse_datetime_raises_error_if_invalid_character_in_year(): 69 | dt = _parse_partial_datetime('2X10') 70 | 71 | 72 | @istest 73 | @raises(ValueError) 74 | def parse_datetime_raises_error_if_time_but_incomplete_date(): 75 | dt = _parse_partial_datetime('2010-10T12:00') 76 | 77 | 78 | @istest 79 | @raises(ValueError) 80 | def parse_datetime_raises_error_if_too_many_date_components(): 81 | dt = _parse_partial_datetime('2010-10-05-06') 82 | 83 | 84 | @istest 85 | @raises(ValueError) 86 | def parse_datetime_raises_error_if_too_many_time_components(): 87 | dt = _parse_partial_datetime('2010-10-05T12:01:02:03') 88 | 89 | 90 | # parse_datetime: Strings that parse correctly but correspond to invalid date/times 91 | @istest 92 | @raises(ValueError) 93 | def parse_datetime_raises_error_if_invalid_month(): 94 | dt = _parse_partial_datetime('2010-13') 95 | 96 | 97 | @istest 98 | @raises(ValueError) 99 | def parse_datetime_raises_error_if_invalid_day(): 100 | dt = _parse_partial_datetime('2010-06-31') 101 | 102 | 103 | # Tests for parse_as_number_or_datetime 104 | @istest 105 | def parse_as_number_or_datetime_can_parse_date_as_datetime(): 106 | from datetime import datetime 107 | dt = parse_as_number_or_datetime('2010-07-01') 108 | assert (dt == datetime(2010, 7, 1)) 109 | 110 | 111 | @istest 112 | def parse_as_number_or_datetime_can_parse_integer(): 113 | dt = parse_as_number_or_datetime('2010') 114 | assert (dt == 2010) 115 | 116 | 117 | @istest 118 | def parse_as_number_or_datetime_can_parse_float(): 119 | dt = parse_as_number_or_datetime('12.345') 120 | assert (dt == 12.345) 121 | 122 | 123 | @istest 124 | def test_that_can_parse_time_deltas(): 125 | delta = parse_datetimestr_delta_to_float_days("P2y15m3dT5M10H3S") 126 | assert_almost_equal(1183.420173611111, delta) 127 | 128 | 129 | @istest 130 | @raises(ValueError) 131 | def test_that_raise_an_error_when_datetimestr_delta_is_invalid(): 132 | parse_datetimestr_delta_to_float_days("some wierd string") 133 | 134 | @istest 135 | def test_that_can_parse_datetimestr_to_obj(): 136 | from cis.time_util import convert_datetime_to_std_time 137 | import datetime as dt 138 | # when not specifying the hours, minutes or seconds, 0 is used 139 | eq_(parse_datetimestr_to_std_time("2010-02-05 02:15:45"), 140 | convert_datetime_to_std_time(dt.datetime(2010, 2, 5, 2, 15, 45))) 141 | eq_(parse_datetimestr_to_std_time("2010-02-05 02:15"), 142 | convert_datetime_to_std_time(dt.datetime(2010, 2, 5, 2, 15, 0))) 143 | eq_(parse_datetimestr_to_std_time("2010-02-05 02"), 144 | convert_datetime_to_std_time(dt.datetime(2010, 2, 5, 2, 0, 0))) 145 | eq_(parse_datetimestr_to_std_time("2010-02-05"), 146 | convert_datetime_to_std_time(dt.datetime(2010, 2, 5, 0, 0, 0))) 147 | 148 | # GOTCHA: when not specifying an element of a date (i.e. the year, month or day), the current date is used 149 | now = dt.datetime.now() 150 | eq_(parse_datetimestr_to_std_time("2010-02-05"), 151 | convert_datetime_to_std_time(dt.datetime(2010, 2, 5))) 152 | eq_(parse_datetimestr_to_std_time("2010-12"), 153 | convert_datetime_to_std_time(dt.datetime(2010, 12, now.day))) 154 | eq_(parse_datetimestr_to_std_time("2010-"), 155 | convert_datetime_to_std_time(dt.datetime(2010, now.month, now.day))) 156 | 157 | 158 | if __name__ == '__main__': 159 | import nose 160 | 161 | nose.runmodule() 162 | -------------------------------------------------------------------------------- /cis/test/util/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/cis/test/util/__init__.py -------------------------------------------------------------------------------- /cis/test/utils_for_testing.py: -------------------------------------------------------------------------------- 1 | from hamcrest import assert_that, is_ 2 | import numpy 3 | 4 | 5 | def assert_arrays_equal(result, expected): 6 | assert_that(numpy.array_equal(result, expected), is_(True), 7 | "arrays not the same. Expected\n {}\n was\n {}".format(expected, result)) 8 | 9 | 10 | def assert_arrays_almost_equal(result, expected, tol=1.0e-15): 11 | assert_that(numpy.allclose(result, expected, atol=tol) 12 | , is_(True), 13 | "arrays not almost the same. Expected\n {}\n was\n {}".format(expected, result)) 14 | 15 | 16 | def compare_masked_arrays(a1, a2): 17 | """ 18 | Compare two masked arrays: 19 | - Masks should be the same 20 | - Unmasked data should be same 21 | - Shape should be same 22 | - Numeric values that are 'masked out' don't matter 23 | """ 24 | flat_1 = a1.compressed() 25 | flat_2 = a2.compressed() 26 | assert_that(numpy.allclose(flat_1, flat_2), 'Masked arrays have different values') 27 | assert_that(numpy.array_equal(a1.mask, a2.mask), 'Masked arrays have different masks') 28 | assert_that(a1.shape, is_(a2.shape), 'Masked arrays have different shapes') 29 | -------------------------------------------------------------------------------- /codecov.yml: -------------------------------------------------------------------------------- 1 | ignore: 2 | - "cis/test" # Ignore tests (which includes lots of integration tests) 3 | - "cis/plotting" # Ignore plotting which isn't covered by unit tests 4 | 5 | -------------------------------------------------------------------------------- /conda_requirements.txt: -------------------------------------------------------------------------------- 1 | # This file may be used to create an environment using: 2 | # $ conda create --name --file 3 | 4 | # Python dependencies 5 | cf-units>=3.0.0 6 | matplotlib>=3.1.0 7 | numpy>=1.10 8 | iris>=2.0.0 9 | scipy 10 | pandas 11 | six 12 | 13 | # IO Dependencies 14 | hdf4 15 | hdf5 16 | pyhdf>=0.9.0 17 | libnetcdf 18 | netcdf4 19 | zlib 20 | 21 | # jpeg=8d=0 22 | 23 | # Testing dependencies 24 | nose 25 | mock 26 | psutil 27 | pyhamcrest 28 | pytest-cov 29 | pytest-xdist 30 | -------------------------------------------------------------------------------- /doc/README: -------------------------------------------------------------------------------- 1 | Documentation specific to how CIS works should go here in `Sphinx format `. 2 | 3 | The configured theme requires the theme "sphinx_rtd_theme" which can be imported as follows:: 4 | 5 | $ pip install sphinx_rtd_theme 6 | 7 | -------------------------------------------------------------------------------- /doc/cis_dependency.dot: -------------------------------------------------------------------------------- 1 | digraph cis { 2 | cis -> iris; 3 | cis -> nose; 4 | cis -> netcdf4; 5 | cis -> pyhdf; 6 | cis -> psutil; 7 | cis -> numpy; 8 | cis -> matplotlib; 9 | cis -> pyhamcrest; 10 | cis -> mock; 11 | cis -> scipy; 12 | 13 | iris -> netcdf4; 14 | iris -> nose; 15 | iris -> numpy; 16 | iris -> scipy; 17 | iris -> udunits2; 18 | iris -> PyKE; 19 | iris -> matplotlib; 20 | iris -> cartopy; 21 | iris -> PythonImagingLibrary; 22 | iris -> shapely; 23 | 24 | cartopy -> cython; 25 | cartopy -> "proj.4"; 26 | cartopy -> geos; 27 | cartopy -> shapely; 28 | cartopy -> pyshp; 29 | cartopy -> nose; 30 | cartopy -> scipy; 31 | 32 | } 33 | 34 | -------------------------------------------------------------------------------- /doc/command_line.rst: -------------------------------------------------------------------------------- 1 | ====================== 2 | Using the command line 3 | ====================== 4 | 5 | Run the following command to print help and check that it runs: ``cis --help`` 6 | 7 | The following should be displayed:: 8 | 9 | usage: cis [-h] [-v | -q] [--force-overwrite] 10 | {plot,info,col,aggregate,subset,eval,stats,version} ... 11 | 12 | positional arguments: 13 | {plot,info,col,aggregate,subset,eval,stats,version} 14 | plot Create plots 15 | info Get information about a file 16 | col Perform collocation 17 | aggregate Perform aggregation 18 | subset Perform subsetting 19 | eval Evaluate a numeric expression 20 | stats Perform statistical comparison of two datasets 21 | version Display the CIS version number 22 | 23 | optional arguments: 24 | -h, --help Show this help message and exit 25 | -v, --verbose Increase the level of logging information output to 26 | screen to include 'Info' statements 27 | -vv All log messages will be output to the screen including 'Debug' statements 28 | -q, --quiet Suppress all output to the screen, only 'Error' 29 | messages will be displayed (which are always fatal). 30 | --force-overwrite Do not prompt when an output file already exists - 31 | always overwrite. This can also be set by setting the 32 | 'CIS_FORCE_OVERWRITE' environment variable to 'TRUE' 33 | 34 | There are 8 commands the program can execute: 35 | 36 | * ``plot`` which is used to plot the data 37 | * ``info`` which prints information about a given input file 38 | * ``col`` which is used to perform collocation on data 39 | * ``aggregate`` which is used to perform aggregation along coordinates in the data 40 | * ``subset`` which is used to perform subsetting of the data 41 | * ``eval`` which is used to evaluate a numeric expression on data 42 | * ``stats`` which is used to perform a statistical comparison of two datasets 43 | * ``version`` which is used to display the version number of CIS 44 | 45 | 46 | If an error occurs while running any of these commands, you may wish to increase the level of output using the verbose 47 | option, or check the log file 'cis.log'; the default location for this is the current user's home directory. 48 | 49 | LSF Batch Job Submission 50 | ------------------------ 51 | 52 | CIS jobs may be submitted to an LSF type batch submission system (e.g. the JASMIN environment) by using the 53 | command ``cis.lsf`` instead of cis. In this case the job will be sent to the batch system and any output will be written 54 | to the log file. -------------------------------------------------------------------------------- /doc/file_information.rst: -------------------------------------------------------------------------------- 1 | ======================== 2 | Getting file information 3 | ======================== 4 | 5 | The info command provides a visual summary of the data within any of the data files CIS supports. 6 | 7 | To get this summary, run a command of the format:: 8 | 9 | $ cis info [--type ["VD" | "SD"]] 10 | 11 | where: 12 | 13 | ```` 14 | is a :ref:`CIS datagroup ` specifying the variables and files to read and is of the format 15 | ``[...:][:product=]`` where: 16 | 17 | * ``variable`` is an optional variable or list of variables to use. 18 | * ``filenames`` is a mandatory file or list of files to read from. 19 | * ``product`` is an optional CIS data product to use (see :ref:`Data Products `): 20 | 21 | Note that the product can only be specified if a variable is specified. See :ref:`datagroups` for a more detailed explanation 22 | of datagroups. 23 | 24 | ``--type`` allows the user to list only ``SD`` or ``VD`` variables from an HDF file, the default is ``All`` 25 | 26 | 27 | Running without a variable (``$ cis info ``) will print a list of the variables available in those files 28 | such as:: 29 | 30 | Trop 31 | latitude 32 | longitude_1 33 | surface 34 | unspecified_1 35 | level6 36 | ht 37 | msl 38 | latitude_1 39 | 40 | To get more specific information about one or more variables in those files, simply pass those as well:: 41 | 42 | $ cis info var1,var2: 43 | 44 | where ``$var1`` and ``$var2`` are the names of the variables to get the information for. 45 | 46 | Here is an example output:: 47 | 48 | Ungridded data: SO4 / (ug m-3) 49 | Shape = (6478,) 50 | Total number of points = 6478 51 | Number of non-masked points = 6478 52 | Long name = Sulphate 53 | Standard name = SO4 54 | Units = ug m-3 55 | Missing value = -9999 56 | Range = (-0.57346399999999997, 7.0020300000000004) 57 | History = 58 | Coordinates: 59 | time 60 | Long name = Starting time 61 | Standard name = time 62 | Units = days since 1600-01-01 00:00:00 63 | Calendar = gregorian 64 | Missing value = -9999 65 | Range = ('2008-07-10 02:04:35', '2008-07-20 09:50:33') 66 | History = 67 | latitude 68 | Long name = Latitude 69 | Standard name = latitude 70 | Units = N degree 71 | Missing value = -9999 72 | Range = (4.0211802, 7.14886) 73 | History = 74 | longitude 75 | Long name = Longitude 76 | Standard name = longitude 77 | Units = E degree 78 | Missing value = -9999 79 | Range = (114.439, 119.733) 80 | History = 81 | altitude 82 | Long name = Altitude 83 | Standard name = altitude 84 | Units = m 85 | Missing value = -9999 86 | Range = (51.164299, 6532.6401) 87 | History = 88 | 89 | -------------------------------------------------------------------------------- /doc/gallery.rst: -------------------------------------------------------------------------------- 1 | ======= 2 | Gallery 3 | ======= 4 | 5 | A collection of example CIS plots along with the commands used to generate them. 6 | 7 | .. figure:: img/model.png 8 | :width: 400px 9 | 10 | Model output data 11 | 12 | .. figure:: img/line.png 13 | :width: 400px 14 | 15 | Aggregated model data 16 | 17 | % cis plot precip:xenida_zonal.nc --itemwidth=2 --xaxis latitude --xlabel "Latitude (degrees)" --yaxis precip --ylabel "Precipitation (\$kg/m^2/s\$)" --title "Zonal mean of total precipitation rate" -o line.png 18 | 19 | .. figure:: img/MODIS_L2.png 20 | :width: 400px 21 | 22 | MODIS Level 2 23 | 24 | .. figure:: img/MODIS_L3.png 25 | :width: 400px 26 | 27 | MODIS Level 3 28 | 29 | .. figure:: img/seviri-ctt.png 30 | :width: 400px 31 | 32 | Seviri Cloud top temperature 33 | 34 | .. figure:: img/agoufou_18022013_all_three.gif 35 | :width: 400px 36 | 37 | Aeronet time series 38 | 39 | .. figure:: img/comparative_scatter_Aeronet.png 40 | :width: 400px 41 | 42 | Aeronet comparative scatter 43 | 44 | % cis plot 440-870Angstrom:../cis_repo_test_files/920801_091128_Agoufou_small.lev20 AOT_440:../cis_repo_test_files/920801_091128_Agoufou_small.lev20 --xlabel "440-870nm Angstrom Exponent" --ylabel "AOT at 440nm" --title "" --type comparativescatter -o comparative_scatter_Aeronet.png 45 | 46 | .. figure:: img/comparativehistogram3d.png 47 | :width: 400px 48 | 49 | Aeronet comparatice histogram 50 | 51 | % cis plot 440-870Angstrom:920801_091128_Agoufou_small.lev20 AOT_440:../cis_repo_test_files/920801_091128_Agoufou_small.lev20 --xlabel "440-870nm Angstrom Exponent" --ylabel "AOT at 440nm" --title "" --type histogram3d -o comparativehistogram3d 52 | 53 | .. figure:: img/aerosol_cci.png 54 | :width: 400px 55 | 56 | Aerosol CCI 57 | 58 | .. figure:: img/Cloud_CCI.png 59 | :width: 400px 60 | 61 | Cloud CCI 62 | 63 | %cis plot cwp:20080620072500-ESACCI-L2_CLOUD-CLD_PRODUCTS-MODIS-AQUA-fv1.0.nc 64 | -o Cloud_CCI --xmin 75 --xmax 110 --xstep 5 65 | 66 | .. figure:: img/cloudsat_RVOD.png 67 | :width: 400px 68 | 69 | CloudSat Liquid water content 70 | 71 | .. figure:: img/caliop_l1b.png 72 | :width: 400px 73 | 74 | CALIOP Level 1b 75 | 76 | .. figure:: img/aircraft.png 77 | :width: 400px 78 | 79 | NCAR-RAF ambient temperature 80 | 81 | % cis plot ATX:RF04.20090114.192600_035100.PNI.nc --xaxis latitude --xlabel 82 | "Latitude (degrees north)" --yaxis altitude --ylabel "Altitude (\$m\$)" --cbarlabel "\$^{\circ}C\$" -o aircraft.png -------------------------------------------------------------------------------- /doc/img/2009-subset.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/doc/img/2009-subset.png -------------------------------------------------------------------------------- /doc/img/2010-subset.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/doc/img/2010-subset.png -------------------------------------------------------------------------------- /doc/img/AOD550_on_MOD08_kdt_hsep_50km_full.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/doc/img/AOD550_on_MOD08_kdt_hsep_50km_full.png -------------------------------------------------------------------------------- /doc/img/AOD550_on_MOD08_kdt_hsep_50km_full_zoom.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/doc/img/AOD550_on_MOD08_kdt_hsep_50km_full_zoom.png -------------------------------------------------------------------------------- /doc/img/AOD550_on_MOD08_kdt_nn_full.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/doc/img/AOD550_on_MOD08_kdt_nn_full.png -------------------------------------------------------------------------------- /doc/img/AOD550n_3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/doc/img/AOD550n_3.png -------------------------------------------------------------------------------- /doc/img/Aerosol_CCI.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/doc/img/Aerosol_CCI.png -------------------------------------------------------------------------------- /doc/img/Aerosol_CCI_4x4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/doc/img/Aerosol_CCI_4x4.png -------------------------------------------------------------------------------- /doc/img/Aerosol_CCI_col.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/doc/img/Aerosol_CCI_col.png -------------------------------------------------------------------------------- /doc/img/Cloud_CCI.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/doc/img/Cloud_CCI.png -------------------------------------------------------------------------------- /doc/img/Cloud_CCI_col.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/doc/img/Cloud_CCI_col.png -------------------------------------------------------------------------------- /doc/img/CollocationDiagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/doc/img/CollocationDiagram.png -------------------------------------------------------------------------------- /doc/img/HorizontalLI.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/doc/img/HorizontalLI.png -------------------------------------------------------------------------------- /doc/img/HorizontalNN.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/doc/img/HorizontalNN.png -------------------------------------------------------------------------------- /doc/img/MOD08_on_AOD550_hsep_75km.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/doc/img/MOD08_on_AOD550_hsep_75km.png -------------------------------------------------------------------------------- /doc/img/MOD08_on_AOD550_kdt_hsep_100km_full.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/doc/img/MOD08_on_AOD550_kdt_hsep_100km_full.png -------------------------------------------------------------------------------- /doc/img/MOD08_on_AOD550_kdt_hsep_100km_var_full.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/doc/img/MOD08_on_AOD550_kdt_hsep_100km_var_full.png -------------------------------------------------------------------------------- /doc/img/MOD08_on_AOD550_nn_kdt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/doc/img/MOD08_on_AOD550_nn_kdt.png -------------------------------------------------------------------------------- /doc/img/MOD08n_3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/doc/img/MOD08n_3.png -------------------------------------------------------------------------------- /doc/img/MODIS_L2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/doc/img/MODIS_L2.png -------------------------------------------------------------------------------- /doc/img/MODIS_L3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/doc/img/MODIS_L3.png -------------------------------------------------------------------------------- /doc/img/OriginalData.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/doc/img/OriginalData.png -------------------------------------------------------------------------------- /doc/img/PressureCollocated.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/doc/img/PressureCollocated.png -------------------------------------------------------------------------------- /doc/img/PressureCollocation.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/doc/img/PressureCollocation.png -------------------------------------------------------------------------------- /doc/img/PressureOriginal.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/doc/img/PressureOriginal.png -------------------------------------------------------------------------------- /doc/img/PressureSlice1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/doc/img/PressureSlice1.png -------------------------------------------------------------------------------- /doc/img/PressureSlice2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/doc/img/PressureSlice2.png -------------------------------------------------------------------------------- /doc/img/RF04.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/doc/img/RF04.png -------------------------------------------------------------------------------- /doc/img/RF04_col.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/doc/img/RF04_col.png -------------------------------------------------------------------------------- /doc/img/Screenshot%20of%20overlayed%20heatmap%20and%20scatter.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/doc/img/Screenshot%20of%20overlayed%20heatmap%20and%20scatter.png -------------------------------------------------------------------------------- /doc/img/Screenshot%20of%20overlayed%20line%20graphs.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/doc/img/Screenshot%20of%20overlayed%20line%20graphs.png -------------------------------------------------------------------------------- /doc/img/aerosol_cci.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/doc/img/aerosol_cci.png -------------------------------------------------------------------------------- /doc/img/aggregation/MODIS-10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/doc/img/aggregation/MODIS-10.png -------------------------------------------------------------------------------- /doc/img/aggregation/MODIS-6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/doc/img/aggregation/MODIS-6.png -------------------------------------------------------------------------------- /doc/img/aggregation/MODIS-7.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/doc/img/aggregation/MODIS-7.png -------------------------------------------------------------------------------- /doc/img/aggregation/MODIS-8.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/doc/img/aggregation/MODIS-8.png -------------------------------------------------------------------------------- /doc/img/aggregation/MODIS-9.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/doc/img/aggregation/MODIS-9.png -------------------------------------------------------------------------------- /doc/img/aggregation/NCAR-RAF-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/doc/img/aggregation/NCAR-RAF-1.png -------------------------------------------------------------------------------- /doc/img/aggregation/NCAR-RAF-2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/doc/img/aggregation/NCAR-RAF-2.png -------------------------------------------------------------------------------- /doc/img/aggregation/NCAR-RAF-3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/doc/img/aggregation/NCAR-RAF-3.png -------------------------------------------------------------------------------- /doc/img/aggregation/NCAR-RAF-4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/doc/img/aggregation/NCAR-RAF-4.png -------------------------------------------------------------------------------- /doc/img/aggregation/NCAR-RAF-5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/doc/img/aggregation/NCAR-RAF-5.png -------------------------------------------------------------------------------- /doc/img/aggregation/gridded_collapse.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/doc/img/aggregation/gridded_collapse.png -------------------------------------------------------------------------------- /doc/img/aggregation/lat-lon-coarser.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/doc/img/aggregation/lat-lon-coarser.png -------------------------------------------------------------------------------- /doc/img/aggregation/lat-subset.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/doc/img/aggregation/lat-subset.png -------------------------------------------------------------------------------- /doc/img/aggregation/max.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/doc/img/aggregation/max.png -------------------------------------------------------------------------------- /doc/img/aggregation/months-days.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/doc/img/aggregation/months-days.png -------------------------------------------------------------------------------- /doc/img/aggregation/stddev.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/doc/img/aggregation/stddev.png -------------------------------------------------------------------------------- /doc/img/aggregation/years.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/doc/img/aggregation/years.png -------------------------------------------------------------------------------- /doc/img/agoufou_18022013_all_three.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/doc/img/agoufou_18022013_all_three.gif -------------------------------------------------------------------------------- /doc/img/aircraft.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/doc/img/aircraft.png -------------------------------------------------------------------------------- /doc/img/caliop_l1b.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/doc/img/caliop_l1b.png -------------------------------------------------------------------------------- /doc/img/cloudsat_RVOD.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/doc/img/cloudsat_RVOD.png -------------------------------------------------------------------------------- /doc/img/comparative_scatter_Aeronet.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/doc/img/comparative_scatter_Aeronet.png -------------------------------------------------------------------------------- /doc/img/comparativehistogram3d.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/doc/img/comparativehistogram3d.png -------------------------------------------------------------------------------- /doc/img/dep.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/doc/img/dep.png -------------------------------------------------------------------------------- /doc/img/eval/angstrom_exponent.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/doc/img/eval/angstrom_exponent.png -------------------------------------------------------------------------------- /doc/img/eval/echam_aggregated.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/doc/img/eval/echam_aggregated.png -------------------------------------------------------------------------------- /doc/img/eval/echam_hadgem_difference.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/doc/img/eval/echam_hadgem_difference.png -------------------------------------------------------------------------------- /doc/img/eval/hadgem_aggregated.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/doc/img/eval/hadgem_aggregated.png -------------------------------------------------------------------------------- /doc/img/eval/hadgem_collocated.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/doc/img/eval/hadgem_collocated.png -------------------------------------------------------------------------------- /doc/img/eval/modis_aggregated_aod.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/doc/img/eval/modis_aggregated_aod.png -------------------------------------------------------------------------------- /doc/img/eval/modis_cloud_fraction.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/doc/img/eval/modis_cloud_fraction.png -------------------------------------------------------------------------------- /doc/img/eval/modis_masked_optical_depth.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/doc/img/eval/modis_masked_optical_depth.png -------------------------------------------------------------------------------- /doc/img/eval/modis_optical_depth.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/doc/img/eval/modis_optical_depth.png -------------------------------------------------------------------------------- /doc/img/line.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/doc/img/line.png -------------------------------------------------------------------------------- /doc/img/model.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/doc/img/model.png -------------------------------------------------------------------------------- /doc/img/overlay1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/doc/img/overlay1.png -------------------------------------------------------------------------------- /doc/img/overlay2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/doc/img/overlay2.png -------------------------------------------------------------------------------- /doc/img/overlay3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/doc/img/overlay3.png -------------------------------------------------------------------------------- /doc/img/overlay4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/doc/img/overlay4.png -------------------------------------------------------------------------------- /doc/img/overlay5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/doc/img/overlay5.png -------------------------------------------------------------------------------- /doc/img/seviri-ctt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/doc/img/seviri-ctt.png -------------------------------------------------------------------------------- /doc/img/stats-aero440.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/doc/img/stats-aero440.png -------------------------------------------------------------------------------- /doc/img/stats-aero500.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cedadev/cis/c207b4e3b47f770da404991eedeeb55422817772/doc/img/stats-aero500.png -------------------------------------------------------------------------------- /doc/index.rst: -------------------------------------------------------------------------------- 1 | .. Community Intercomparison Suite documentation master file, created by 2 | sphinx-quickstart on Tue Mar 25 10:10:43 2014. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Welcome to the Community Intercomparison Suite's documentation! 7 | =============================================================== 8 | 9 | Contents: 10 | 11 | .. toctree:: 12 | :maxdepth: 2 13 | :numbered: 14 | 15 | installation 16 | whats_new 17 | data_products 18 | command_line 19 | cis_api 20 | file_information 21 | subsetting 22 | aggregation 23 | collocation 24 | collocation_examples 25 | plotting 26 | gallery 27 | evaluation 28 | statistics 29 | overlay_examples 30 | plugin_development 31 | analysis_plugin_development 32 | maintenance_and_development 33 | 34 | Indices and tables 35 | ================== 36 | 37 | * :ref:`genindex` 38 | * :ref:`modindex` 39 | * :ref:`search` 40 | 41 | -------------------------------------------------------------------------------- /doc/installation.rst: -------------------------------------------------------------------------------- 1 | 2 | ============== 3 | Installing CIS 4 | ============== 5 | 6 | A pre-packaged version of CIS is available for installation using conda for 64-bit Linux, Mac OSX and Windows. 7 | 8 | Once conda is installed, you can easily install CIS with the following command:: 9 | 10 | $ conda install -c conda-forge cis 11 | 12 | 13 | If you don't already have conda, you must first download and install it. Anaconda is a free conda package that includes Python and many common scientific and data analysis libraries, and is available `here `_. Further documentation on using Anaconda and the features it provides can be found at http://docs.continuum.io/anaconda/index.html. 14 | 15 | To check that CIS is installed correctly, simply type ``cis version`` to display the version number, for example:: 16 | 17 | $ cis version 18 | Using CIS version: V1R4M0 (Stable) 19 | 20 | In order to upgrade CIS to the latest version use:: 21 | 22 | $ conda update -c conda-forge cis 23 | 24 | Dependencies 25 | ============ 26 | 27 | If you choose to install the dependencies yourself, use the following command to check the required dependencies are present:: 28 | 29 | $ python setup.py checkdep 30 | 31 | -------------------------------------------------------------------------------- /doc/overlay_examples.rst: -------------------------------------------------------------------------------- 1 | ===================== 2 | Overlay Plot Examples 3 | ===================== 4 | 5 | First subset some gridded data that will be used for the examples:: 6 | 7 | cis subset od550aer:aerocom.HadGEM3-A-GLOMAP.A2.CTRL.monthly.od550aer.2006.nc t=[2006-10-13] -o HadGEM_od550aer-subset 8 | 9 | cis subset rsutcs:aerocom.HadGEM3-A-GLOMAP.A2.CTRL.monthly.rsutcs.2006.nc t=[2006-10-13] -o HadGEM_rsutcs-subset 10 | 11 | 12 | Contour over heatmap 13 | ==================== 14 | 15 | :: 16 | 17 | cis plot od550aer:HadGEM_od550aer-subset.nc:type=heatmap rsutcs:HadGEM_rsutcs-subset.nc:type=contour,color=white,contlevels=[1,10,25,50,175] --width 20 --height 15 --cbarscale 0.5 -o overlay1.png 18 | 19 | 20 | 21 | .. image:: img/overlay1.png 22 | :width: 900px 23 | 24 | :: 25 | 26 | cis plot od550aer:HadGEM_od550aer-subset.nc:type=heatmap,cmap=binary rsutcs:HadGEM_rsutcs-subset.nc:type=contour,cmap=jet,contlevels=[1,10,25,50,175] --xmin -180 --xmax 180 --width 20 --height 15 --cbarscale 0.5 -o overlay2.png 27 | 28 | 29 | .. image:: img/overlay2.png 30 | :width: 900px 31 | 32 | Filled contour with transparency on NASA Blue Marble 33 | ==================================================== 34 | 35 | :: 36 | 37 | cis plot od550aer:HadGEM_od550aer-subset.nc:cmap=Reds,type=contourf,transparency=0.5,cmin=0.15 --xmin -180 --xmax 180 --width 20 --height 15 --cbarscale 0.5 --nasabluemarble 38 | 39 | 40 | .. image:: img/overlay3.png 41 | :width: 900px 42 | 43 | Scatter plus Filled Contour 44 | =========================== 45 | 46 | :: 47 | 48 | cis subset rsutcs:HadGEM_rsutcs-subset.nc x=[-180,-90],y=[0,90] -o HadGEM_rsutcs-subset2 49 | 50 | cis plot GGALT:RF04.20090114.192600_035100.PNI.nc:type=scatter rsutcs:HadGEM_rsutcs-subset2.nc:type=contourf,contlevels=[0,10,20,30,40,50,100],transparency=0.7,contlabel=true,contfontsize=18 --width 20 --height 15 --xaxis longitude --yaxis latitude --xmin -180 --xmax -90 --ymin 0 --ymax 90 --itemwidth 20 -o overlay4.png 51 | 52 | 53 | .. image:: img/overlay4.png 54 | :width: 600px 55 | 56 | :: 57 | 58 | cis plot GGALT:RF04.20090114.192600_035100.PNI.nc:type=scatter rsutcs:HadGEM_rsutcs-subset2.nc:type=contourf,contlevels=[40,50,100],transparency=0.3,contlabel=true,contfontsize=18,cmap=Reds --width 20 --height 15 --xaxis longitude --yaxis latitude --xmin -180 --xmax -90 --ymin 0 --ymax 90 --itemwidth 20 --nasabluemarble -o overlay5.png 59 | 60 | 61 | .. image:: img/overlay5.png 62 | :width: 600px 63 | 64 | File Locations 65 | ============== 66 | 67 | The gridded data files can be found at:: 68 | 69 | /group_workspaces/jasmin/cis/AeroCom/A2/HadGEM3-A-GLOMAP.A2.CTRL/renamed 70 | 71 | and the ungridded:: 72 | 73 | /group_workspaces/jasmin/cis/jasmin_cis_repo_test_files 74 | 75 | -------------------------------------------------------------------------------- /doc/plugin/mycol.py: -------------------------------------------------------------------------------- 1 | from cis.collocation.col_framework import Collocator, Constraint, Kernel 2 | from cis.data_io.ungridded_data import LazyData 3 | 4 | 5 | class MyCollocator(Collocator): 6 | 7 | def collocate(self, points, data, constraint, kernel): 8 | values = [] 9 | for point in points: 10 | con_points = constraint.constrain_points(point, data) 11 | try: 12 | values.append(kernel.get_value(point, con_points)) 13 | except ValueError: 14 | values.append(constraint.fill_value) 15 | new_data = LazyData(values, data.metadata) 16 | new_data.missing_value = constraint.fill_value 17 | return new_data 18 | 19 | 20 | class MyConstraint(Constraint): 21 | 22 | def constrain_points(self, ref_point, data): 23 | con_points = [] 24 | for point in data: 25 | if point.value > self.val_check: 26 | con_points.append(point) 27 | return con_points 28 | 29 | 30 | class MyKernel(Kernel): 31 | 32 | def get_value(self, point, data): 33 | nearest_point = point.furthest_point_from() 34 | for data_point in data: 35 | if point.compdist(nearest_point, data_point): 36 | nearest_point = data_point 37 | return nearest_point.val 38 | -------------------------------------------------------------------------------- /doc/plugin/myprod.py: -------------------------------------------------------------------------------- 1 | from cis.data_io.products.AProduct import AProduct 2 | from cis.data_io.Coord import Coord, CoordList 3 | from cis.data_io.ungridded_data import UngriddedData 4 | from cis.data_io.ungridded_data import Metadata 5 | 6 | import logging 7 | 8 | 9 | class MyProd(AProduct): 10 | def get_file_signature(self): 11 | return [r'.*something*'] 12 | 13 | def create_coords(self, filenames): 14 | 15 | logging.info("gathering coordinates") 16 | for filename in filenames: 17 | data1 = [] 18 | data2 = [] 19 | data3 = [] 20 | 21 | logging.info("gathering coordinates metadata") 22 | metadata1 = Metadata() 23 | metadata2 = Metadata() 24 | metadata3 = Metadata() 25 | 26 | coord1 = Coord(data1, metadata1, 'X') # this coordinate will be used as the 'X' axis when plotting 27 | coord2 = Coord(data2, metadata2, 'Y') # this coordinate will be used as the 'Y' axis when plotting 28 | coord3 = Coord(data3, metadata3) 29 | 30 | return CoordList([coord1, coord2, coord3]) 31 | 32 | def create_data_object(self, filenames, variable): 33 | 34 | logging.info("gathering data for variable " + str(variable)) 35 | for filename in filenames: 36 | data = [] 37 | 38 | logging.info("gatherings metadata for variable " + str(variable)) 39 | metadata = Metadata() 40 | 41 | coords = self.create_coords(filenames) 42 | return UngriddedData(data, metadata, coords) 43 | -------------------------------------------------------------------------------- /doc/statistics.rst: -------------------------------------------------------------------------------- 1 | .. _statistics: 2 | .. |nbsp| unicode:: 0xA0 3 | 4 | ********** 5 | Statistics 6 | ********** 7 | 8 | The Community Intercomparison Suite allows you to perform statistical analysis on two variables using the 'stats' 9 | command. For example, you might wish to examine the correlation between a model data variable and actual measurements. 10 | The 'stats' command will calculate: 11 | 12 | #. Number of data points used in the analysis. 13 | #. The mean and standard deviation of each dataset (separately). 14 | #. The mean and standard deviation of the absolute difference (var2 - var1). 15 | #. The mean and standard deviation of the relative difference ((var2 - var1) / var1). 16 | #. The `Linear Pearson `_ correlation coefficient. 17 | #. The `Spearman Rank `_ correlation coefficient. 18 | #. The coefficients of linear regression (i.e. var2 = a var1 + b ), r-value, and standard error of the estimate. 19 | 20 | These values will be displayed on screen and can optionally be save as NetCDF output. 21 | 22 | .. note:: 23 | Both variables used in a statistical analysis **must** be of the same shape in order to be compatible, i.e. the 24 | same number of points in each dimension, and of the same type (ungridded or gridded). This means that, for example, 25 | operations between different data products are unlikely to work correctly - performing a collocation or aggregation 26 | onto a common grid would be a good pre-processing step. 27 | 28 | .. note:: 29 | Only points which have non-missing values for both variables will be included in the analysis. The number of points 30 | this includes is part of the output of the stats command. 31 | 32 | .. warning:: 33 | Unlike :ref:`aggregation `, ``stats`` does **not** currently use latitude weighting to account for the 34 | relative areas of different grid cells. 35 | 36 | The statistics syntax looks like this:: 37 | 38 | $ cis stats ... [-o ] 39 | 40 | where: 41 | 42 | ```` 43 | is a :ref:`CIS datagroup ` specifying the variables and files to read and is of the format 44 | ``...:[:product=]`` where: 45 | 46 | * ```` is a mandatory variable or list of variables to use. 47 | * ```` is a mandatory file or list of files to read from. 48 | * ```` is an optional CIS data product to use (see :ref:`Data Products `): 49 | 50 | One or more datagroups should be given, but the total number of variables declared in all datagroups must be exactly 51 | two. See :ref:`datagroups` for a more detailed explanation of datagroups. 52 | 53 | 54 | ```` 55 | is an optional argument specifying a file to output to. This will be automatically given a ``.nc`` extension if not 56 | present. This must not be the same file path as any of the input files. If not provided, then the output will not be 57 | saved to a file and will only be displayed on screen. 58 | 59 | 60 | Statistics Example 61 | ================== 62 | 63 | In this example, we perform a statistical comparison of Aeronet aerosol optical thickness at two wavelengths. 64 | The data we are using is shown in the following CIS plot commands 65 | and can be found at ``/group_workspaces/jasmin/cis/data``:: 66 | 67 | $ cis plot AOT_500:aeronet/AOT/LEV20/ALL_POINTS/920801_121229_Yonsei_University.lev20 --title "Aerosol optical thickness 550nm" 68 | $ cis plot AOT_440:aeronet/AOT/LEV20/ALL_POINTS/920801_121229_Yonsei_University.lev20 --title "Aerosol optical thickness 440nm" 69 | 70 | .. image:: img/stats-aero500.png 71 | :width: 450px 72 | 73 | .. image:: img/stats-aero440.png 74 | :width: 450px 75 | 76 | 77 | We then perform a statistical comparison of these variables using:: 78 | 79 | $ cis stats AOT_500,AOT_440:aeronet/AOT/LEV20/ALL_POINTS/920801_121229_Yonsei_University.lev20 80 | 81 | Which gives the following output:: 82 | 83 | =================================================================== 84 | RESULTS OF STATISTICAL COMPARISON: 85 | ------------------------------------------------------------------- 86 | Compared all points which have non-missing values in both variables 87 | =================================================================== 88 | Number of points: 10727 89 | Mean value of dataset 1: 0.427751965508 90 | Mean value of dataset 2: 0.501316673814 91 | Standard deviation for dataset 1: 0.307680514916 92 | Standard deviation for dataset 2: 0.346274598431 93 | Mean of absolute difference: 0.0735647083061 94 | Standard deviation of absolute difference: 0.0455684788406 95 | Mean of relative difference: 0.188097066086 96 | Standard deviation of relative difference: 0.0528621773819 97 | Spearman's rank coefficient: 0.998289763952 98 | Linear regression gradient: 1.12233533743 99 | Linear regression intercept: 0.0212355272705 100 | Linear regression r-value: 0.997245296339 101 | Linear regression standard error: 0.0256834603945 102 | -------------------------------------------------------------------------------- /doc/whats_new.rst: -------------------------------------------------------------------------------- 1 | 2 | ================= 3 | What's new in CIS 4 | ================= 5 | 6 | .. toctree:: 7 | whats_new_1.5 8 | whats_new_1.4 9 | whats_new_1.3 10 | whats_new_1.2 11 | whats_new_1.1 12 | -------------------------------------------------------------------------------- /doc/whats_new_1.1.rst: -------------------------------------------------------------------------------- 1 | 2 | ===================== 3 | What's new in CIS 1.1 4 | ===================== 5 | 6 | This page documents the new features added, and bugs fixed in CIS since version 1.0. For more detail see all changes here: https://github.com/cedadev/cis/compare/1.0.0...1.1.0 7 | 8 | CIS 1.1 features 9 | ================ 10 | 11 | * JASMIN-CIS is now called CIS, and the packages, modules and documentation have been renamed accordingly. 12 | * Conda packages are now available to allow much easier installation of CIS, and across more platforms: Linux, OSX and Windows. 13 | * PyHDF is now an optional dependency. This makes the installation of CIS on e.g. Windows much easier when HDF reading is not required. 14 | 15 | Bugs fixed 16 | ========== 17 | 18 | * JASCIS-243 - Error when reading multiple GASSP aircraft files 19 | * JASCIS-139 - Updated ungridded aggregation to rename any variables which clash with coordinate variables, as this breaks during the output otherwise. 20 | * Compatibility fixes for Numpy versions >1.8 and Python-NetCDF versions >1.1. 21 | * Fix Caliop pressure units which were stored as hPA, but need to be hPa to conform to CF. 22 | * The integration test data has been moved completely out of the repository - making the download quicker and less bloated. It's location can be specified by setting the CIS_DATA_HOME environment variable. 23 | * A test runner has been created to allow easy running of the unit and integration test. 24 | 25 | 26 | What's new in CIS 1.1.1 27 | ======================= 28 | 29 | This section documents changes in CIS since version 1.1, these were primarily bug fixes and documentation updates. See all changes here: https://github.com/cedadev/cis/compare/1.1.0...1.1.1 30 | 31 | Bugs fixed 32 | ========== 33 | 34 | * JASCIS-181 - Updated eval documentation 35 | * JASCIS-239 - Documented the requirement of PyHamCrest for running tests 36 | * JASCIS-249 - CIS will now accept variables and filenames (such as Windows paths) which include a colon as long as they are escaped with a backslash. E.g. ``cis plot my_var:C\:\my_file.nc``. 37 | * Occasionally HDF will exit when reading an invalid HDF file without throwing any exceptions. To protect against this the HDF reader will now insist on an .hdf extension for any files it reads. -------------------------------------------------------------------------------- /doc/whats_new_1.2.rst: -------------------------------------------------------------------------------- 1 | 2 | ===================== 3 | What's new in CIS 1.2 4 | ===================== 5 | 6 | This page documents the new features added, and bugs fixed in CIS since version 1.1. See all changes here: https://github.com/cedadev/cis/compare/1.1.0...1.2.0 7 | 8 | 9 | CIS 1.2 features 10 | ================ 11 | 12 | * All new ``cis info`` command provides much more detailed information about ungridded data variables and enables multiple variables to be output at a time. 13 | * Updated a number of routines to take advantage of Iris 1.8 features. In particular gridded-gridded collocation using the nearest neighbour kernel should be significantly faster. Iris 1.8 is now the minimum version required for CIS. 14 | * Gridded-ungridded collocation now supports collocation from cubes with hybrid height or hybrid pressure coordinates for both nearest neighbour and linear interpolation kernels. 15 | * Built-in support for reading multiple HadGEM .pp files directly. 16 | * All new API and plugin development documentation, including a number of tutorials 17 | 18 | Bugs fixed 19 | ========== 20 | 21 | * JASCIS-253 - Any ungridded points which contain a NaN in any of its coordinate values will now be ignored by CIS 22 | * JASCIS-250 - Multiple HadGEM files can now be read correctly through the new data plugins. 23 | * JASCIS-197 - Gridded-gridded collocation now respects scalar coordinates 24 | * JASCIS-199 - Aggregation now correctly uses the bounds supplied by the user, even when collapsing to length one coordinates. 25 | * Speed improvement to the ungridded-gridded collocation using linear interpolation 26 | * Several bug fixes for reading multiple GASSP ship files 27 | * Renamed and restructured the collocation modules for consistency 28 | * Many documentation spelling and formatting updates 29 | * Many code formatting updates for PEP8 compliance 30 | 31 | CIS 1.2.1 features 32 | ================== 33 | * Updated CCI plugin to support Aerosol CCI v3 files. 34 | -------------------------------------------------------------------------------- /doc/whats_new_1.3.rst: -------------------------------------------------------------------------------- 1 | 2 | ===================== 3 | What's new in CIS 1.3 4 | ===================== 5 | 6 | This page documents the new features added, and bugs fixed in CIS since version 1.2. See all changes here: https://github.com/cedadev/cis/compare/1.2.1...1.3.0 7 | 8 | 9 | CIS 1.3 features 10 | ================ 11 | 12 | * Some significant optimisations have been made in reading Caliop, CCI and Aeronet datasets, there have also been speed 13 | improvements for ungridded data subsetting 14 | * New Pandas interface allows the easy creation of DataFrames through the 'as_data_frame' method on Gridded or Ungridded 15 | data. Pandas is an extensive python library providing many powerful data analysis algorithms and routines. 16 | * Compatibility updates for newer versions of Numpy and SciPy. The minimum require version of SciPy is now 0.16.0 17 | * Swapped out Basemap plotting routines for Cartopy. This removed a dependancy (as Cartopy was already required by 18 | Iris), and has given us more flexibility for plotting different projections in the future 19 | * Plots now automatically try to use the most appropriate resolution background images for plots over coastlines NASA 20 | blue marble images. 21 | * 'scatter_overlay' plots have been completely removed (they have been deprecated for the last two versions), the same 22 | functionality can be achieved through the more generic 'overlay' plots. 23 | * Update to the UngriddedData.coord() and .coords() API to match the changes in IRIS >=1.8. This allows users to also 24 | search for coordinates by supplying a :class:`Coord` instance to compare against. Currently this only compares 25 | standard names, but this may be extended in the future. 26 | 27 | Bugs fixed 28 | ========== 29 | 30 | * JASCIS-279 - This release removes the basemap dependency and means we can use a much newer version of GEOS which 31 | doesn't clash with the SciTools version 32 | * JASCIS-267 - Fixed ASCII file reading to be compatible with Numpy 1.9 33 | * JASCIS-259 - Fixed Stats unit tests to reflect updates in SciPy (>0.15.0) linear regression routines for masked arrays 34 | * JASCIS-211 - Subsetting now accepts variable names (rather than axes shorthands) more consistently, the docs have 35 | been updated to make the dangers of relying on axes shorthands clear and an error is now thrown if a specific subset 36 | coordinate is not found. 37 | * JASCIS-275 - The ungridded subsetting is now done array-wise rather than element wise giving large performance 38 | improvements 39 | 40 | CIS 1.3.1 fixes 41 | =============== 42 | * JASCIS-231 & JASCIS-209 - CIS now better determines the yaxis when the user specifies the xaxis as 'time' so that overlaying multiple time series is easy 43 | * JASCIS-283 - An issue with setting xmin or xmax using datetimes 44 | * A minor fix to the AerosolCCI product 45 | -------------------------------------------------------------------------------- /doc/whats_new_1.4.rst: -------------------------------------------------------------------------------- 1 | 2 | ===================== 3 | What's new in CIS 1.4 4 | ===================== 5 | 6 | This page documents the new features added, and bugs fixed in CIS since version 1.3.1. See all changes here: https://github.com/cedadev/cis/compare/1.3.1...1.4.0 7 | 8 | 9 | CIS 1.4 features 10 | ================ 11 | * An all new Python interface for subsetting any data read by CIS. Just call the :meth:`subset` method on any CIS GriddedData 12 | or UngriddedData object to access the same functionality as through the command line - without reading or writing to 13 | disk. See :doc:`CIS API` for more details. 14 | 15 | * CIS now includes full support for Python => 3.4, as well as Python 2.7 16 | * New verbose and quiet flags allow for control over how much CIS commands output to the screen. The default verbosity 17 | has also changed so that by default only warnings and errors will be output to the screen. The full debug output 18 | remains for the cis.log file. 19 | * Significant optimizations have been made in gridded -> ungridded collocation which should now be considerably faster. 20 | Also, when collocating multiple gridded source datasets the interpolation indices are now cached internally leading 21 | to further time savings. 22 | * Any ``valid_range`` attributes in supported NetCDF or HDF files (including MODIS, CALIOP and CloudSat) files are now 23 | automatically respected by CIS. All data values outside of the valid range are masked. Data from NetCDF files with 24 | ``valid_min`` or ``valid_max`` attributes is also masked appropriately. 25 | * CloudSat ``missing`` and ``missop`` attributes are now read and combined to mask out values which don't conform to the 26 | inequality defined. 27 | * [JASCIS-342] The extrapolation modes are now consistent across both gridded->gridded and gridded->ungridded collocation 28 | modes. The default is no extrapolation (gridded->gridded would previously extrapolate). This can still be overridden 29 | by the user. 30 | * [JASCIS-128] If the output file already exists the user is now prompted to overwrite it. This prompt can be disabled 31 | by using the --force-overwrite argument, or setting the ``CIS_FORCE_OVERWRITE`` environment variable to 'TRUE'. 32 | 33 | Incompatible changes 34 | ==================== 35 | * To accommodate the new verbose flags (-v) the info command now takes a single datagroup argument, and optional 36 | variable names, as reflected in the updated documentation. 37 | * CIS no longer prepends ungridded output files with 'cis-'. Instead CIS creates a global attribute in the output file 38 | called source which contains 'CIS'. This is checked in the updated CIS plugin when reading any NetCDF file. 39 | 40 | .. note:: 41 | While this is much neater going forward and will hopefully save a lot of head scratching it will mean CIS is unable 42 | to read old files produced by CIS automatically. All commands can be forced to use the CIS product by including the 43 | product=cis keyword argument. Alternatively you can update the data file manually using the following command: 44 | ``ncatted -O -a source,global,a,c,"CIS" in.nc`` 45 | 46 | Bugs fixed 47 | ========== 48 | 49 | * [JASCIS-34] MODIS L3 data is now correctly treated as gridded data. 50 | * [JASCIS-345] Product regular expression matching now matches the whole string rather than just the start. 51 | * [JASCIS-360] Collocation now correctly applies the 'missing_data_for_missing_sample' logic for all collocations. 52 | * [JASCIS-361] Fixed the CloudSat scale and offset transformation so that they are now applied correctly. 53 | * [JASCIS-281] Fixed a caching error when aggregating multiple ungridded datasets which could lead to incorrect values 54 | * CIS no longer crashes when the CIS_PLUGIN_HOME path cannot be found -------------------------------------------------------------------------------- /doc/whats_new_1.5.rst: -------------------------------------------------------------------------------- 1 | 2 | ===================== 3 | What's new in CIS 1.5 4 | ===================== 5 | 6 | This page documents the new features added, and bugs fixed in CIS since version 1.4.0. See all changes here: https://github.com/cedadev/cis/compare/1.4.0...1.5.0 7 | 8 | 9 | CIS 1.5 features 10 | ================ 11 | * The biggest change is that CIS can now be used as a Python library, all of the command line tools are now easily 12 | available through Python. This allows commands to be run sequentially in memory, slicing of gridded or ungridded 13 | datasets and easy integration with other Python packages such as Iris and Pandas. 14 | * Taylor diagrams - CIS is now able to plot Taylor diagrams which are an excellent way of quantitatively comparing two 15 | or more (collocated) datasets 16 | * All map plots are now able to be plotted in any of the available Cartopy projections, see 17 | http://scitools.org.uk/cartopy/docs/latest/crs/projections.html for a full list. 18 | 19 | 20 | Incompatible changes 21 | ==================== 22 | * Since aggregation of gridded datasets has quite a different set of options as compared to the aggregation of 23 | ungridded datasets, the ``aggregate`` command has been deprecated for gridded datasets. It is still supported through 24 | command line for the time being, but will be removed in future releases. Please use the ``collapse`` command instead. 25 | 26 | Bugs fixed 27 | ========== 28 | 29 | * [JASCIS-268] The plotting routines have been re-architected to allow easier testing and extension. 30 | * [JASCIS-357] Added deprecation for the aggregation of gridded datasets 31 | * [JASCIS-329] Metadata objects now attempt to use cf_units for all units, but will fall back to strings if needed. In 32 | future releases we may insist on plugins providing standard units. 33 | 34 | CIS 1.5.1 fixes 35 | =============== 36 | * Minor fix in interpreting units when reading some NetCDF data in Python 2 37 | * Fixed an issue where line and scatter plots weren't respecting the yaxis keyword 38 | 39 | CIS 1.5.2 fixes 40 | =============== 41 | * Gridded and ungridded datasets can now be subset to an arbitrary lat/lon (shapely) shape. 42 | * Slicing and copying Coords now preserves the axis 43 | * Fixed an issue where subsetting gridded data over multiple coordinates sometimes resulted in an error 44 | * CIS will now catch errors when writing out metadata values which might have special types and can't be safely 45 | cast (e.g. VALID_RANGE). 46 | * Minor fix for log scale color bars 47 | * Minor fix for parsing the command aliases 48 | * Minor fix for creating data lists from iterators 49 | 50 | CIS 1.5.3 fixes 51 | =============== 52 | * Fixed a (potentially serious) bug in unit parsing which would convert any string to lowercase. 53 | * [JASCIS-367] Make the name() method more consistent between gridded and ungridded data 54 | * Minor fix when reading variables from PP files with spaces in the name 55 | 56 | CIS 1.5.4 fixes 57 | =============== 58 | * Minor fix for the info command on Windows -------------------------------------------------------------------------------- /doc/whats_new_1.6.rst: -------------------------------------------------------------------------------- 1 | 2 | ===================== 3 | What's new in CIS 1.6 4 | ===================== 5 | 6 | This page documents the new features added, and bugs fixed in CIS since version 1.5.0. See all changes here: https://github.com/cedadev/cis/compare/1.5.4...1.6.0 7 | 8 | 9 | CIS 1.6 features 10 | ================ 11 | * Implemented ungridded - ungridded collocation performance improvements (#9) 12 | * Improved reading of NCAR-RAF style NetCDF files 13 | * Performance improvement when reading many (100s of) NetCDF files 14 | * Improved support for reading CALIOP L2 data 15 | 16 | Incompatible changes 17 | ==================== 18 | * 19 | 20 | Bugs fixed 21 | ========== 22 | 23 | * In previous versions of CIS cartographic weights were calculated for all collapse operations regardless of the 24 | dimensions being collapsed. This may have given unexpected values in some cases. CIS now only calculates weights 25 | when collapsing over latitude. 26 | * Plot colourbars are now associated with the plot axes rather than the figure, which makes it easier to make 27 | multi-axes plots with CIS. 28 | * Fixed an issue where `UngriddedData.copy()` didn't copy metadata 29 | * [JASCIS-373] get_variable_names now gets passed the product argument from read_data_list 30 | * Fix #11 by relying on shapely exception which moved 31 | * [JASCIS-375] Ensure the mask is retained when expanding coordinates from 1d to 2d 32 | * The Cloudsat reader no longer expands coordinate arrays for 1-d datasets (such as ice-water path) 33 | -------------------------------------------------------------------------------- /doc/whats_new_1.7.rst: -------------------------------------------------------------------------------- 1 | 2 | ===================== 3 | What's new in CIS 1.7 4 | ===================== 5 | 6 | This page documents the new features added, and bugs fixed in CIS since version 1.6.0. See all changes here: 7 | https://github.com/cedadev/cis/compare/1.6.0...1.7.0 8 | 9 | CIS 1.7 features 10 | ================ 11 | * Extended the definition of the MODIS_L3 plugin to include monthly files. 12 | * Added plugins Aerosol_CCI_L3 and Cloud_CCI_L3 to read Level 3 CCI products. 13 | * Extended the Aeronet plugin to read files from: 14 | * The Version 3 Direct Sun Algorithm, 15 | * The Maritime Aerosol Network, 16 | * All versions of the Spectral Decomposition Algorithm, 17 | * The All_Sites_Times_*.dat files now distributed under the "Download All Sites" link. 18 | 19 | Incompatible changes 20 | ==================== 21 | * Renamed the plugins Aerosol_CCI and Cloud_CCI to Aerosol_CCI_L2 and Cloud_CCI_L2 to be consistent with MODIS. 22 | * Updated to Iris 2.0.0 and pyHDF 0.9.0 (removing previous workarounds). 23 | * Gridded / gridded collocation of time coordinates is no longer supported since iris no longer allows the 24 | determination of whether a point lies within a bounded region for datetime-like objects 25 | 26 | Bugs fixed 27 | ========== 28 | * In a PP file, if a var_name contains spaces the plugin will now attempt to replace them with underscores. 29 | 30 | CIS 1.7.1 fixes 31 | =============== 32 | * Fixed an issue where interpolation would unmask masked source arrays 33 | * Support for Pandas 0.24 34 | 35 | CIS 1.7.2 fixes 36 | =============== 37 | * We no-longer officially support Python 2.7. We won't yet explicitly remove Python 2.7 features but we no longer 38 | test against Python 2 and won't fix issues relating to it. 39 | * Support for Pandas 1.1 40 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import os 2 | import os.path 3 | 4 | from setuptools import setup, find_packages, Command 5 | from pkg_resources import require, DistributionNotFound, VersionConflict 6 | from cis.test.runner import nose_test 7 | from cis import __version__, __website__ 8 | 9 | 10 | root_path = os.path.dirname(__file__) 11 | 12 | # If we're building docs on readthedocs we don't have any dependencies as they're all mocked out 13 | on_rtd = os.environ.get('READTHEDOCS', None) == 'True' 14 | if on_rtd: 15 | dependencies = [] 16 | optional_dependencies = {} 17 | test_dependencies = [] 18 | 19 | else: 20 | dependencies = ["matplotlib>=2.0", 21 | "netcdf4>=1.0", 22 | "numpy", 23 | "scipy>=0.15.0", 24 | "scitools-iris>=1.8.0", 25 | "psutil>=2.0.0", 26 | "six"] 27 | 28 | optional_dependencies = {"HDF": ["pyhdf>=0.9.0"], "Pandas": ["pandas"]} 29 | 30 | test_dependencies = ["pyhamcrest", "mock", "nose"] 31 | 32 | 33 | class check_dep(Command): 34 | """ 35 | Command to check that the required dependencies are installed on the system 36 | """ 37 | description = "Checks that the required dependencies are installed on the system" 38 | user_options = [] 39 | 40 | def initialize_options(self): 41 | pass 42 | 43 | def finalize_options(self): 44 | pass 45 | 46 | def run(self): 47 | 48 | for dep in dependencies: 49 | try: 50 | require(dep) 51 | print(dep + " ...[ok]") 52 | except (DistributionNotFound, VersionConflict): 53 | print(dep + "... MISSING!") 54 | 55 | # Extract long-description from README 56 | README = open(os.path.join(root_path, 'README.md'), 'rb').read().decode('utf-8') 57 | 58 | setup( 59 | name='cis', 60 | version=__version__, 61 | description='Community Intercomparison Suite', 62 | long_description=README, 63 | maintainer='Philip Kershaw', 64 | maintainer_email='Philip.Kershaw@stfc.ac.uk', 65 | url=__website__, 66 | classifiers=[ 67 | 'License :: OSI Approved :: GNU Lesser General Public License v3 or later (LGPLv3+)', 68 | 'Topic :: Scientific/Engineering :: Atmospheric Science', 69 | 'Topic :: Scientific/Engineering :: Visualization', 70 | 'Environment :: Console', 71 | ], 72 | packages=find_packages(), 73 | package_data={'': ['logging.conf', 'plotting/raster/*.png']}, 74 | scripts=['bin/cis', 'bin/cis.lsf'], 75 | cmdclass={"checkdep": check_dep, 76 | "test": nose_test}, 77 | install_requires=dependencies, 78 | extras_require=optional_dependencies, 79 | tests_require=test_dependencies 80 | ) 81 | --------------------------------------------------------------------------------