├── .dockerignore ├── .gitignore ├── .project ├── .pydevproject ├── CITATION.cff ├── Dockerfile ├── LICENSE ├── README.md ├── bin ├── aseg2nc ├── aseg2nc.bat ├── csw_find ├── csw_find.bat ├── nc2aseg ├── nc2aseg.bat ├── rechunk └── rechunk.bat ├── build_all_images.sh ├── docker ├── Dockerfile_0.1.0_gdal3.5.3_py3.9_alpine3.15_normal ├── Dockerfile_0.1.0_gdal3.6.4_py3.10_alpine3.16_normal ├── Dockerfile_0.1.0_gdal3.7.0_mcpy3.10_ubuntu22.04_full ├── Dockerfile_0.1.0_gdal3.7.0_py3.10_alpine3.16_normal └── Dockerfile_0.1.0_gdal3.7.0_py3.10_ubuntu22.04.2_full ├── environment.yml ├── examples ├── 10_gravity_point_discovery_and_access_demo.ipynb ├── 1_CSW_data_discovery.ipynb ├── 2_geophys_netcdf_grid_utils_demo.ipynb ├── 3_geophys_netcdf_line_utils_demo.ipynb ├── 4_geophys_data_fusion_demo.ipynb ├── 5_aem_test.ipynb ├── 5a_aem_test-EFTF.ipynb ├── 6_aem_test_multiline.ipynb ├── 7_aem_test_3d_voxel.ipynb ├── 7_aem_test_3d_voxel_netCDF_output.ipynb ├── 8_line_dataset_discovery_and_use_demo.ipynb ├── 9_gravity_point_dataset.ipynb ├── 9a_line_dataset_plot.ipynb ├── 9b_line_dataset_plot_line_elevations.ipynb ├── CSW Request.xml ├── CSW Response.xml ├── NCSS_Demo.ipynb ├── aem_netcdf_test.ipynb ├── dynamic_kmls │ └── app.py ├── dynamic_point_gridding_utils.py ├── fetch_points_demo.ipynb ├── geophys_point_fetcher.py ├── geophysics_line_nc_metadata.csv ├── get_nearest_points_demo.ipynb ├── get_netcdf_util_demo.py ├── gravity_point_metadata.csv ├── line_data_demo.py ├── nearest_geophys_point_finder.py ├── netcdf_grid_utils_shape_test.ipynb ├── new_9_gravity_point_dataset.ipynb ├── plot_point_dataset.py ├── point_gravity_access.ipynb └── tile_gridder.py ├── geophys_utils ├── __init__.py ├── _array2file.py ├── _array_pieces.py ├── _blrb.py ├── _concave_hull.py ├── _crs_utils.py ├── _csw_utils.py ├── _data_stats.py ├── _datetime_utils.py ├── _dem_utils.py ├── _gdal_grid_utils.py ├── _get_netcdf_util.py ├── _netcdf_grid_utils.py ├── _netcdf_line_utils.py ├── _netcdf_point_utils.py ├── _netcdf_utils.py ├── _polygon_utils.py ├── _transect_utils.py ├── _vincenty.py ├── csw_find.py ├── csw_utils_settings.yml ├── dataset_metadata_cache │ ├── __init__.py │ ├── _dataset_metadata_cache.py │ ├── _postgres_dataset_metadata_cache.py │ ├── _sqlite_dataset_metadata_cache.py │ ├── csw2dataset_metadata_cache.py │ ├── data │ │ └── dataset_metadata_cache.sqlite │ ├── dataset_metadata_cache_settings.yml │ ├── dataset_search_demo.py │ ├── netcdf2dataset_metadata_cache.py │ ├── postgres_dataset_metadata_cache_ddl.sql │ └── sqlite_dataset_metadata_cache_ddl.sql ├── nc2aseg.py ├── netcdf_converter │ ├── __init__.py │ ├── _to_netcdf_converter.py │ ├── _to_netcdf_converter_national.py │ ├── aseg_gdf2netcdf_converter.py │ ├── aseg_gdf_settings.yml │ ├── aseg_gdf_utils.py │ ├── csv2netcdf_converter.py │ ├── grav2netcdf_converter.py │ ├── grav2netcdf_converter_national.py │ ├── grav2netcdf_converter_national_global_attribute_setter.py │ ├── grav2netcdf_converter_national_remove_points_outside_bbox.py │ ├── grav2netcdf_converter_national_settings.yml │ ├── grav2netcdf_converter_national_sql_strings.yml │ ├── grav2netcdf_converter_settings.yml │ ├── grav2netcdf_converter_settings_ga_metadata │ ├── grav2netcdf_converter_sql_strings.yml │ ├── netcdf2aseg_gdf_converter.py │ └── test_netcdf_converter.py ├── rechunk.py └── test │ ├── __init__.py │ ├── __main__.py │ ├── test_array_pieces.py │ ├── test_crs_utils.py │ ├── test_data_stats.py │ ├── test_grid.nc │ ├── test_netcdf_grid_utils.py │ ├── test_netcdf_line_utils.py │ └── test_netcdf_point_utils.py ├── pyproject.toml ├── requirements.txt └── setup.py /.dockerignore: -------------------------------------------------------------------------------- 1 | .* 2 | !.git* 3 | !.dockerignore 4 | build 5 | dist 6 | *.egg-info 7 | Temp 8 | temp -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | dataset_metadata_cache.sqlite 6 | 7 | # C extensions 8 | *.so 9 | 10 | # Distribution / packaging 11 | .Python 12 | env/ 13 | build/ 14 | develop-eggs/ 15 | dist/ 16 | downloads/ 17 | eggs/ 18 | .eggs/ 19 | lib/ 20 | lib64/ 21 | parts/ 22 | sdist/ 23 | var/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *,cover 47 | 48 | # Translations 49 | *.mo 50 | *.pot 51 | 52 | # Django stuff: 53 | *.log 54 | 55 | # Sphinx documentation 56 | docs/_build/ 57 | 58 | # PyBuilder 59 | target/ 60 | 61 | # pycharm project files 62 | .idea/ 63 | 64 | # iPython checkpoints 65 | *-checkpoint.ipynb 66 | 67 | # Temporary files 68 | /Temp/ 69 | 70 | # Tiling tests 71 | examples/dataset_lists 72 | examples/point_lists 73 | examples/tiles 74 | 75 | -------------------------------------------------------------------------------- /.project: -------------------------------------------------------------------------------- 1 | 2 | 3 | geophys_utils 4 | 5 | 6 | 7 | 8 | 9 | org.python.pydev.PyDevBuilder 10 | 11 | 12 | 13 | 14 | 15 | org.python.pydev.pythonNature 16 | 17 | 18 | -------------------------------------------------------------------------------- /.pydevproject: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | /${PROJECT_DIR_NAME} 5 | 6 | python 2.7 7 | Default 8 | 9 | -------------------------------------------------------------------------------- /CITATION.cff: -------------------------------------------------------------------------------- 1 | cff-version: 1.2.0 2 | message: "If you use this software, please cite it as below." 3 | authors: 4 | - family-names: Ip 5 | given-names: Alex 6 | orcid: https://orcid.org/0000-0001-8937-8904 7 | title: "Geophysics Utilities for netCDF encodings" 8 | version: 0.1.0 9 | date-released: 2016-11-16 -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # This Dockerfile exists only for Binder deployment 2 | # It uses a base image created using repo2docker in BinderHub to speed up the build process 3 | FROM alexip/conda-geophys_utils_base:0.1.0_gdal3.7.0_mcpy3.10_r2d 4 | 5 | USER root 6 | 7 | # Take a complete copy of the project directory into /geophys_utils (could do a git pull) 8 | COPY . /home/jovyan/geophys_utils 9 | 10 | # install geophys_utils and set up scripts 11 | RUN chown -R jovyan:jovyan /home/jovyan/geophys_utils && \ 12 | chmod -R u+rwX /home/jovyan/geophys_utils && \ 13 | /srv/conda/envs/notebook/bin/pip install -e /home/jovyan/geophys_utils && \ 14 | for script in $(ls /home/jovyan/geophys_utils/bin/* | grep -v '\.bat$'); \ 15 | do chmod 755 "${script}"; \ 16 | ln -s "/home/jovyan/geophys_utils/bin/${script}" /usr/local/sbin/$(basename "${script}"); \ 17 | done 18 | 19 | USER jovyan -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # geophys\_utils - Utilities for Accessing netCDF Encoded Geophysics Data 2 | The geophys_utils Python library is a collection of utilities for discovering, accessing and using geophysics data via 3 | web services or from netCDF files. This library is maintained by Geoscience Australia with ongoing contributions from 4 | the original Author, Alex Ip. 5 | 6 | An overview of the netCDF encodings for grid, point and line geophysical data can be viewed at 7 | 8 | 9 | Any feedback, suggestions or contributions will be gratefully accepted. 10 | 11 | # Binder deployment 12 | Click this button to launch an example notebook in the ARDC Binder service: 13 | 14 | [![Binder](https://mybinder.org/badge_logo.svg)](https://binderhub.rc.nectar.org.au/v2/gh/alex-ip/geophys_utils/0.1.0_dev?labpath=geophys_utils%2Fexamples%2F10_gravity_point_discovery_and_access_demo.ipynb) 15 | 16 | Note that the BinderHub deployment uses a prebuilt base image providing a Conda environment with GDAL, netCDF4 and 17 | other dependencies to speed up the overall build process. The Dockerfile in the root of this repository is used to 18 | create the notebook container for geophys_utils in BinderHub. 19 | 20 | # Docker deployment 21 | There is a docker directory in the project root which contains Dockerfiles which will allow you to create fresh Docker 22 | images from the current source code using the command 23 | ```docker build --progress=plain -t : -f docker/Dockerfile_ .``` 24 | in the project root directory. Alternatively, you can run the ```build_all_images.sh``` script to build all the images 25 | defined. 26 | Note that, depending on the OS, Python, and GDAL version, some builds can initially take longer due to the extensive 27 | dependencies for large packages such as NetCDF and SciPy. After the initial builds, the layers should be cached for 28 | faste subsequent builds. 29 | 30 | Alternatively, you can pull a pre-made Ubuntu image with the latest versions using the following commands: 31 | ```bash 32 | # Pull down the latest stable Ubuntu image from DockerHub 33 | docker pull alexip/geophys_utils:latest 34 | # Run unit tests 35 | docker run --rm --name geophys_utils alexip/geophys_utils:latest python -m geophys_utils.test 36 | # Start an interactive shell in the container 37 | docker run -it --name geophys_utils alexip/geophys_utils:latest /bin/bash 38 | ``` 39 | 40 | ## License 41 | The content of this repository is licensed for use under the 42 | [Apache 2.0 License](http://www.apache.org/licenses/LICENSE-2.0). 43 | See the [license deed](https://github.com/GeoscienceAustralia/geophys_utils/blob/master/LICENSE) for full details. 44 | 45 | ## Contacts 46 | **Geoscience Australia** 47 | *Publisher* 48 | 49 | [clientservices@ga.gov.au](mailto://clientservices@ga.gov.au) 50 | 51 | **Alex Ip** 52 | *Principal Author* 53 | [alex@trentham.net.au](mailto://alex@trentham.net.au) 54 | 55 | 56 | **Andrew Turner** 57 | *Contributing Author* 58 | 59 | -------------------------------------------------------------------------------- /bin/aseg2nc: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #=============================================================================== 3 | # Copyright 2017 Geoscience Australia 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | #=============================================================================== 17 | # Shell script to invoke aseg_gdf2netcdf_converter Python script in BASH 18 | # Written by Alex Ip 27/6/2018 19 | # Example invocation: aseg2nc 20 | 21 | python -m geophys_utils.netcdf_converter.aseg_gdf2netcdf_converter "$@" 22 | -------------------------------------------------------------------------------- /bin/aseg2nc.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | ::=============================================================================== 3 | :: Copyright 2017 Geoscience Australia 4 | :: 5 | :: Licensed under the Apache License, Version 2.0 (the "License"); 6 | :: you may not use this file except in compliance with the License. 7 | :: You may obtain a copy of the License at 8 | :: 9 | :: http://www.apache.org/licenses/LICENSE-2.0 10 | :: 11 | :: Unless required by applicable law or agreed to in writing, software 12 | :: distributed under the License is distributed on an "AS IS" BASIS, 13 | :: WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | :: See the License for the specific language governing permissions and 15 | :: limitations under the License. 16 | ::=============================================================================== 17 | :: Batch script to invoke aseg_gdf2netcdf_converter Python script in MS-Windows 18 | :: Written by Alex Ip 27/6/2018 19 | :: Example invocation: Example invocation: aseg2nc 20 | 21 | python -m geophys_utils.netcdf_converter.aseg_gdf2netcdf_converter %* 22 | -------------------------------------------------------------------------------- /bin/csw_find: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #=============================================================================== 3 | # Copyright 2017 Geoscience Australia 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | #=============================================================================== 17 | # Shell script to invoke _csw_utils Python script in BASH 18 | # Written by Alex Ip & Andrew Turner 27/2/2017 19 | # Example invocation: csw_find -k "NCI, AU, grid, potassium" -b 148.996,-35.48,149.399,-35.124 20 | 21 | python -m geophys_utils.csw_find "$@" 22 | -------------------------------------------------------------------------------- /bin/csw_find.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | ::=============================================================================== 3 | :: Copyright 2017 Geoscience Australia 4 | :: 5 | :: Licensed under the Apache License, Version 2.0 (the "License"); 6 | :: you may not use this file except in compliance with the License. 7 | :: You may obtain a copy of the License at 8 | :: 9 | :: http://www.apache.org/licenses/LICENSE-2.0 10 | :: 11 | :: Unless required by applicable law or agreed to in writing, software 12 | :: distributed under the License is distributed on an "AS IS" BASIS, 13 | :: WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | :: See the License for the specific language governing permissions and 15 | :: limitations under the License. 16 | ::=============================================================================== 17 | :: Batch file to invoke _csw_utils Python script in MS-Windows 18 | :: Written by Written by Alex Ip & Andrew Turner 27/2/2017 19 | :: Example invocation: csw_find -k "NCI, AU, grid, potassium" -b 148.996,-35.48,149.399,-35.124 20 | 21 | python -m geophys_utils.csw_find %* 22 | -------------------------------------------------------------------------------- /bin/nc2aseg: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #=============================================================================== 3 | # Copyright 2017 Geoscience Australia 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | #=============================================================================== 17 | # Shell script to invoke netcdf2aseg_gdf_converter Python script in BASH 18 | # Written by Alex Ip 27/6/2018 19 | # Example invocation: aseg2nc 20 | 21 | python -m geophys_utils.nc2aseg "$@" 22 | -------------------------------------------------------------------------------- /bin/nc2aseg.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | ::=============================================================================== 3 | :: Copyright 2017 Geoscience Australia 4 | :: 5 | :: Licensed under the Apache License, Version 2.0 (the "License"); 6 | :: you may not use this file except in compliance with the License. 7 | :: You may obtain a copy of the License at 8 | :: 9 | :: http://www.apache.org/licenses/LICENSE-2.0 10 | :: 11 | :: Unless required by applicable law or agreed to in writing, software 12 | :: distributed under the License is distributed on an "AS IS" BASIS, 13 | :: WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | :: See the License for the specific language governing permissions and 15 | :: limitations under the License. 16 | ::=============================================================================== 17 | :: Batch script to invoke netcdf2aseg_gdf_converter Python script in MS-Windows 18 | :: Written by Alex Ip 27/6/2018 19 | :: Example invocation: aseg2nc 20 | 21 | python -m geophys_utils.nc2aseg %* 22 | -------------------------------------------------------------------------------- /bin/rechunk: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #=============================================================================== 3 | # Copyright 2017 Geoscience Australia 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | #=============================================================================== 17 | # Shell script to invoke _netcdf_utils Python script to re-chunk netCDF file in BASH 18 | # Written by Alex Ip 2/3/2017 19 | # Example invocation: rechunk --chunkspec=latitude/256,longitude/256 infile.nc outfile.nc 20 | 21 | python3 -m geophys_utils.rechunk "$@" 22 | -------------------------------------------------------------------------------- /bin/rechunk.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | ::=============================================================================== 3 | :: Copyright 2017 Geoscience Australia 4 | :: 5 | :: Licensed under the Apache License, Version 2.0 (the "License"); 6 | :: you may not use this file except in compliance with the License. 7 | :: You may obtain a copy of the License at 8 | :: 9 | :: http://www.apache.org/licenses/LICENSE-2.0 10 | :: 11 | :: Unless required by applicable law or agreed to in writing, software 12 | :: distributed under the License is distributed on an "AS IS" BASIS, 13 | :: WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | :: See the License for the specific language governing permissions and 15 | :: limitations under the License. 16 | ::=============================================================================== 17 | :: Batch file to invoke _netcdf_utils Python script to re-chunk netCDF file in MS-Windows 18 | :: Written by Written by Alex Ip 2/3/2017 19 | :: Example invocation: rechunk --chunkspec=latitude/256,longitude/256 infile.nc outfile.nc 20 | 21 | python -m geophys_utils.rechunk %* 22 | -------------------------------------------------------------------------------- /build_all_images.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # build_all_images.sh - A script to build all Docker images defined in the Dockerfiles in the docker subdirectory 3 | # Run this script from the project root (i.e. the parent directory of the "docker" subdirectory) 4 | for dockerfile in $(ls docker/Dockerfile_*) 5 | do 6 | tag="$(echo "${dockerfile}" | cut -d_ -f2-)" 7 | image="$(basename "$(pwd)"):$tag" 8 | docker build --progress=plain -t "${image}" -f "${dockerfile}" . 9 | done 10 | -------------------------------------------------------------------------------- /docker/Dockerfile_0.1.0_gdal3.5.3_py3.9_alpine3.15_normal: -------------------------------------------------------------------------------- 1 | # alexip/geophys_utils:0.1.0 contains geophys_utils v0.1.0 and all dependencies 2 | # GDAL Docker image sourced from https://github.com/OSGeo/gdal/ 3 | 4 | FROM ghcr.io/osgeo/gdal:alpine-normal-3.5.3 AS build 5 | 6 | ENV PYTHONUNBUFFERED=1 7 | 8 | # Install system dependencies for geophys_utils 0.1.0 / GDAL 3.5.3 / Python 3.9 9 | RUN apk update && apk upgrade && \ 10 | apk add --update --no-cache \ 11 | 'build-base' \ 12 | 'gcc' \ 13 | 'g++' \ 14 | 'gfortran' \ 15 | 'make' \ 16 | 'cmake' \ 17 | 'musl-dev' \ 18 | 'python3-dev' \ 19 | 'git' \ 20 | 'openblas-dev' \ 21 | 'lapack-dev' \ 22 | 'py3-scipy' \ 23 | 'py3-pip' \ 24 | 'python3-dev' \ 25 | 'hdf5' \ 26 | 'hdf5-dev' \ 27 | 'netcdf' \ 28 | 'netcdf-dev' \ 29 | 'netcdf-utils' \ 30 | 'geos' \ 31 | 'geos-dev' 32 | 33 | RUN python -m venv /opt/venv 34 | ENV PATH="/opt/venv/bin:$PATH" 35 | 36 | # Install dependencies 37 | # need to force scikit-image==0.19.3 due to broken scipy==1.9.1 dependency 38 | COPY requirements.txt . 39 | RUN pip install --no-cache-dir --upgrade pip && \ 40 | pip install --no-cache-dir --upgrade setuptools wheel && \ 41 | pip install --no-cache-dir --upgrade gdal==3.5.3 scikit-image==0.19.3 && \ 42 | pip install --no-cache-dir -r requirements.txt 43 | 44 | 45 | # Create runtime image 46 | FROM ghcr.io/osgeo/gdal:alpine-normal-3.5.3 AS runtime 47 | 48 | RUN apk update && apk upgrade --no-cache && \ 49 | apk add --update --no-cache \ 50 | 'hdf5' \ 51 | 'netcdf' \ 52 | 'netcdf-utils' \ 53 | 'geos' \ 54 | 'py3-scipy' \ 55 | 'py3-pip' 56 | 57 | COPY --from=build /opt/venv /opt/venv 58 | 59 | # Take a complete copy of the project directory into /geophys_utils (could do a git pull) 60 | RUN mkdir /geophys_utils 61 | WORKDIR /geophys_utils 62 | COPY . /geophys_utils 63 | 64 | ENV PATH="/opt/venv/bin:$PATH" \ 65 | PYTHONPATH="/geophys_utils:$PYTHONPATH" \ 66 | PYTHONUNBUFFERED=1 67 | 68 | # Install geophys_utils package from current source 69 | RUN pip install --no-cache-dir -e . && \ 70 | echo $(scripts=$(find ./bin -type f -not -path '*.bat'); for script in $scripts; do cat $script | sed s/\\/bin\\/bash/\\/bin\\/sh/g > /opt/venv/bin/$(basename $script); chmod 755 /opt/venv/bin/$(basename $script); done) 71 | -------------------------------------------------------------------------------- /docker/Dockerfile_0.1.0_gdal3.6.4_py3.10_alpine3.16_normal: -------------------------------------------------------------------------------- 1 | # alexip/geophys_utils:0.1.0 contains geophys_utils v0.1.0 and all dependencies 2 | # GDAL Docker image sourced from https://github.com/OSGeo/gdal/ 3 | 4 | FROM ghcr.io/osgeo/gdal:alpine-normal-3.6.4 AS build 5 | 6 | ENV PYTHONUNBUFFERED=1 7 | 8 | # Install system dependencies for geophys_utils 0.1.0 / GDAL 3.6.4 / Python 3.9 9 | RUN apk update && apk upgrade && \ 10 | apk add --update --no-cache \ 11 | 'build-base' \ 12 | 'gcc' \ 13 | 'g++' \ 14 | 'gfortran' \ 15 | 'make' \ 16 | 'cmake' \ 17 | 'musl-dev' \ 18 | 'python3-dev' \ 19 | 'git' \ 20 | 'openblas-dev' \ 21 | 'lapack-dev' \ 22 | 'py3-scipy' \ 23 | 'py3-pip' \ 24 | 'python3-dev' \ 25 | 'hdf5' \ 26 | 'hdf5-dev' \ 27 | 'netcdf' \ 28 | 'netcdf-dev' \ 29 | 'netcdf-utils' \ 30 | 'geos' \ 31 | 'geos-dev' 32 | 33 | RUN python -m venv /opt/venv 34 | ENV PATH="/opt/venv/bin:$PATH" 35 | 36 | # Install dependencies 37 | COPY requirements.txt . 38 | RUN pip install --no-cache-dir --upgrade pip && \ 39 | pip install --no-cache-dir --upgrade setuptools wheel && \ 40 | pip install --no-cache-dir --upgrade gdal==3.6.4 && \ 41 | pip install --no-cache-dir -r requirements.txt 42 | 43 | 44 | # Create runtime image 45 | FROM ghcr.io/osgeo/gdal:alpine-normal-3.6.4 AS runtime 46 | 47 | RUN apk update && apk upgrade --no-cache && \ 48 | apk add --update --no-cache \ 49 | 'hdf5' \ 50 | 'netcdf' \ 51 | 'netcdf-utils' \ 52 | 'geos' \ 53 | 'py3-scipy' \ 54 | 'py3-pip' 55 | 56 | COPY --from=build /opt/venv /opt/venv 57 | 58 | # Take a complete copy of the project directory into /geophys_utils (could do a git pull) 59 | RUN mkdir /geophys_utils 60 | WORKDIR /geophys_utils 61 | COPY . /geophys_utils 62 | 63 | ENV PATH="/opt/venv/bin:$PATH" \ 64 | PYTHONPATH="/geophys_utils:$PYTHONPATH" \ 65 | PYTHONUNBUFFERED=1 66 | 67 | # Install geophys_utils package from current source 68 | RUN pip install --no-cache-dir -e . && \ 69 | echo $(scripts=$(find ./bin -type f -not -path '*.bat'); for script in $scripts; do cat $script | sed s/\\/bin\\/bash/\\/bin\\/sh/g > /opt/venv/bin/$(basename $script); chmod 755 /opt/venv/bin/$(basename $script); done) 70 | -------------------------------------------------------------------------------- /docker/Dockerfile_0.1.0_gdal3.7.0_mcpy3.10_ubuntu22.04_full: -------------------------------------------------------------------------------- 1 | FROM ubuntu:22.04 2 | 3 | ENV PYTHONUNBUFFERED=1 4 | 5 | # N.B: We are working as root for the container instead of ec2-user for the EC2 instance 6 | WORKDIR /geophys_utils 7 | 8 | RUN apt update -y && \ 9 | apt upgrade -y && \ 10 | apt install -y \ 11 | wget && \ 12 | echo "export PATH=\"$PATH:/usr/local/bin\"" | tee /etc/profile.d/setup_path.sh && \ 13 | echo "export LD_LIBRARY_PATH=\"/usr/local/lib:$LD_LIBRARY_PATH\"" | tee -a /etc/profile.d/setup_path.sh && \ 14 | echo "export PROJ_LIB=\"/miniconda/share/proj:$PROJ_LIB\"" | tee -a /etc/profile.d/setup_path.sh && \ 15 | . /etc/profile.d/setup_path.sh 16 | 17 | # Download & install miniconda 18 | RUN wget https://repo.anaconda.com/miniconda/Miniconda3-py310_23.3.1-0-Linux-x86_64.sh && \ 19 | chmod 755 Miniconda3-py310_23.3.1-0-Linux-x86_64.sh 20 | 21 | RUN ./Miniconda3-py310_23.3.1-0-Linux-x86_64.sh -b -p /miniconda && \ 22 | eval "$(/miniconda/bin/conda shell.bash hook)" && \ 23 | /miniconda/bin/conda init && \ 24 | rm Miniconda3-py310_23.3.1-0-Linux-x86_64.sh 25 | 26 | # Install python 3.10 27 | COPY environment.yml . 28 | RUN export PATH="/miniconda/bin:/miniconda/condabin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/bin:$PATH" && \ 29 | export LD_LIBRARY_PATH="/usr/local/lib:$LD_LIBRARY_PATH" && \ 30 | export PROJ_LIB="/miniconda/share/proj:$PROJ_LIB" && \ 31 | conda update -y --name base -c defaults conda && \ 32 | conda update -y --name base --all && \ 33 | conda env update --name base --file environment.yml 34 | 35 | # Take a complete copy of the project directory into /geophys_utils (could do a git pull) 36 | COPY . /geophys_utils 37 | 38 | # install geophys_utils and set up scripts 39 | RUN /miniconda/bin/pip install -e /geophys_utils && \ 40 | for script in $(ls bin/* | grep -v '\.bat$'); \ 41 | do chmod 755 "${script}"; \ 42 | ln -s "$(pwd)/${script}" /usr/local/sbin/$(basename "${script}"); \ 43 | done 44 | -------------------------------------------------------------------------------- /docker/Dockerfile_0.1.0_gdal3.7.0_py3.10_alpine3.16_normal: -------------------------------------------------------------------------------- 1 | # alexip/geophys_utils:0.1.0 contains geophys_utils v0.1.0 and all dependencies 2 | # GDAL Docker image sourced from https://github.com/OSGeo/gdal/ 3 | 4 | FROM ghcr.io/osgeo/gdal:alpine-normal-3.7.0 AS build 5 | 6 | ENV PYTHONUNBUFFERED=1 7 | 8 | # System dependencies for geophys_utils 0.1.0 / GDAL 3.5.3 / Python 3.9 9 | RUN apk update && apk upgrade && \ 10 | apk add --update --no-cache \ 11 | 'build-base' \ 12 | 'gcc' \ 13 | 'g++' \ 14 | 'gfortran' \ 15 | 'make' \ 16 | 'cmake' \ 17 | 'musl-dev' \ 18 | 'python3-dev' \ 19 | 'git' \ 20 | 'openblas-dev' \ 21 | 'lapack-dev' \ 22 | 'py3-scipy' \ 23 | 'py3-pip' \ 24 | 'python3-dev' \ 25 | 'hdf5' \ 26 | 'hdf5-dev' \ 27 | 'netcdf' \ 28 | 'netcdf-dev' \ 29 | 'netcdf-utils' \ 30 | 'geos' \ 31 | 'geos-dev' 32 | 33 | RUN python -m venv /opt/venv 34 | ENV PATH="/opt/venv/bin:$PATH" 35 | 36 | # Install dependencies 37 | COPY requirements.txt . 38 | RUN pip install --no-cache-dir --upgrade pip && \ 39 | pip install --no-cache-dir --upgrade setuptools wheel && \ 40 | pip install --no-cache-dir -r requirements.txt 41 | 42 | # Create runtime image 43 | FROM ghcr.io/osgeo/gdal:alpine-normal-3.7.0 AS runtime 44 | 45 | RUN apk update && apk upgrade --no-cache && \ 46 | apk add --update --no-cache \ 47 | 'hdf5' \ 48 | 'netcdf' \ 49 | 'netcdf-utils' \ 50 | 'geos' \ 51 | 'py3-scipy' \ 52 | 'py3-pip' 53 | 54 | COPY --from=build /opt/venv /opt/venv 55 | 56 | # Take a complete copy of the project directory into /geophys_utils (could do a git pull) 57 | RUN mkdir /geophys_utils 58 | WORKDIR /geophys_utils 59 | COPY . /geophys_utils 60 | 61 | ENV PATH="/opt/venv/bin:$PATH" \ 62 | PYTHONPATH="/geophys_utils:$PYTHONPATH" \ 63 | PYTHONUNBUFFERED=1 64 | 65 | # Install geophys_utils package from current source 66 | RUN pip install --no-cache-dir -e . && \ 67 | echo $(scripts=$(find ./bin -type f -not -path '*.bat'); for script in $scripts; do cat $script | sed s/\\/bin\\/bash/\\/bin\\/sh/g > /opt/venv/bin/$(basename $script); chmod 755 /opt/venv/bin/$(basename $script); done) 68 | -------------------------------------------------------------------------------- /docker/Dockerfile_0.1.0_gdal3.7.0_py3.10_ubuntu22.04.2_full: -------------------------------------------------------------------------------- 1 | # alexip/geophys_utils:0.1.0 contains geophys_utils v0.1.0 and all dependencies 2 | # GDAL Docker image sourced from https://github.com/OSGeo/gdal/ 3 | 4 | FROM ghcr.io/osgeo/gdal:ubuntu-full-3.7.0 AS build 5 | 6 | ENV PYTHONUNBUFFERED=1 7 | 8 | # System dependencies for geophys_utils 0.1.0 / GDAL 3.7.0 Python 3.10 9 | RUN apt update -y && \ 10 | apt upgrade -y && \ 11 | apt install -y \ 12 | libgeos-dev \ 13 | python3-pip \ 14 | python3.10-venv 15 | 16 | RUN python -m venv /opt/venv 17 | ENV PATH="/opt/venv/bin:$PATH" 18 | 19 | # Install dependencies 20 | COPY requirements.txt . 21 | RUN pip install --no-cache-dir --upgrade pip && \ 22 | pip install --no-cache-dir --upgrade setuptools wheel && \ 23 | pip install --no-cache-dir -r requirements.txt 24 | 25 | 26 | # Create runtime image 27 | FROM ghcr.io/osgeo/gdal:ubuntu-full-3.7.0 AS runtime 28 | 29 | # Upgrade OS, fix certificates issue, install system requirements 30 | RUN apt update -y && \ 31 | apt upgrade -y && \ 32 | mkdir -p /etc/pki/tls/certs/ && ln -s /etc/ssl/certs/ca-certificates.crt /etc/pki/tls/certs/ca-bundle.crt && \ 33 | apt install -y python3.10-venv && \ 34 | apt clean 35 | 36 | COPY --from=build /opt/venv /opt/venv 37 | 38 | # Take a complete copy of the project directory into /geophys_utils (could do a git pull) 39 | RUN mkdir /geophys_utils 40 | WORKDIR /geophys_utils 41 | COPY . /geophys_utils 42 | 43 | ENV PATH="/opt/venv/bin:$PATH" \ 44 | PYTHONPATH="/geophys_utils:$PYTHONPATH" \ 45 | PYTHONUNBUFFERED=1 46 | 47 | # Fix certificates issue, upgrade install utilities and install geophys_utils package from disk 48 | RUN pip install --no-cache-dir -e . && \ 49 | echo $(scripts=$(find ./bin -type f -not -path '*.bat'); for script in $scripts; do cp $script /opt/venv/bin/$(basename $script); chmod 755 /opt/venv/bin/$(basename $script); done) 50 | -------------------------------------------------------------------------------- /environment.yml: -------------------------------------------------------------------------------- 1 | name: geophys_utils 2 | channels: 3 | - conda-forge 4 | dependencies: 5 | - python=3.10 6 | - pip 7 | - numpy=1.24.3 8 | - gdal=3.6.2 9 | - affine=2.3.0 10 | - cartopy=0.18.0 11 | - matplotlib=3.7.1 12 | - numexpr=2.8.4 13 | - owslib=0.24.1 14 | - pyyaml=6.0 15 | - requests=2.29.0 16 | - scikit-image=0.19.3 17 | - scipy=1.10.1 18 | - shapely=2.0.1 19 | - unidecode=1.3.6 20 | - pip: 21 | - netCDF4==1.6.4 22 | - zipstream-new==1.1.8 -------------------------------------------------------------------------------- /examples/1_CSW_data_discovery.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "metadata": { 3 | "anaconda-cloud": {}, 4 | "kernelspec": { 5 | "display_name": "Python 3 (ipykernel)", 6 | "language": "python", 7 | "name": "python3" 8 | }, 9 | "language_info": { 10 | "codemirror_mode": { 11 | "name": "ipython", 12 | "version": 3 13 | }, 14 | "file_extension": ".py", 15 | "mimetype": "text/x-python", 16 | "name": "python", 17 | "nbconvert_exporter": "python", 18 | "pygments_lexer": "ipython3", 19 | "version": "3.10.12" 20 | } 21 | }, 22 | "nbformat_minor": 4, 23 | "nbformat": 4, 24 | "cells": [ 25 | { 26 | "cell_type": "markdown", 27 | "source": "# 1. Catalogue Service for the Web (CSW) Data Discovery Demonstration\nFinds all national gravity grids from GA Catalogue", 28 | "metadata": {} 29 | }, 30 | { 31 | "cell_type": "code", 32 | "source": "# %matplotlib inline", 33 | "metadata": { 34 | "trusted": true 35 | }, 36 | "execution_count": 1, 37 | "outputs": [] 38 | }, 39 | { 40 | "cell_type": "code", 41 | "source": "from geophys_utils import CSWUtils\nfrom geophys_utils import DataStats # https://github.com/alex-ip/geophys2netcdf/tree/develop\nimport os\nimport re\nfrom netCDF4 import Dataset\nfrom pprint import pprint", 42 | "metadata": { 43 | "trusted": true 44 | }, 45 | "execution_count": 2, 46 | "outputs": [] 47 | }, 48 | { 49 | "cell_type": "code", 50 | "source": "# Setup proxy as required\nGA_STAFF_WIFI = False\n\nif GA_STAFF_WIFI:\n os.environ['http_proxy'] = 'http://proxy.inno.lan:3128'\n os.environ['https_proxy'] = 'http://proxy.inno.lan:3128'", 51 | "metadata": { 52 | "trusted": true 53 | }, 54 | "execution_count": 3, 55 | "outputs": [] 56 | }, 57 | { 58 | "cell_type": "code", 59 | "source": "csw_url = 'https://ecat.ga.gov.au/geonetwork/srv/eng/csw' # GA's externally-facing eCat\n#csw_url = 'https://internal.ecat.ga.gov.au/geonetwork/srv/eng/csw' # GA's internally-facing eCat\n#csw_url = 'http://geonetworkrr2.nci.org.au/geonetwork/srv/eng/csw' # NCI GeoNetwork", 60 | "metadata": { 61 | "trusted": true 62 | }, 63 | "execution_count": 4, 64 | "outputs": [] 65 | }, 66 | { 67 | "cell_type": "code", 68 | "source": "# Define search parameters\nkeywords = \"complete, gravity, grid\" # Comma-separated list of keywords for GA catalogue to bring up national gravity grids\nallwords = \"NCI, complete, gravity, grid\"\n#bounds = [110, -45, 160, -5] # Bounding box slightly larger than national coverage\n#bounds = [115, -40, 150, -10] # Bounding box slightly smaller than national coverage\nbounds = [148.996,-35.48,149.399,-35.124] # Bounding box way smaller than national coverage\n#bounds = [110, -40, 160, -10] # Bounding box slightly wider than national coverage\n#bounds = [115, -45, 150, -5] # Bounding box slightly taller than national coverage\n#bounds = [0, 0, 1, 1] # Invalid bounding box somewhere South of England\ntitlewords = \"onshore, gravity, grid, Australia, 2016\"\nnot_title = '%image%'", 69 | "metadata": { 70 | "trusted": true 71 | }, 72 | "execution_count": 5, 73 | "outputs": [] 74 | }, 75 | { 76 | "cell_type": "code", 77 | "source": "# Find all datasets of interest.\n#create a csw_utils object and populate the parameters with search parameters\ncswu = CSWUtils(csw_url)\nrecord_generator = cswu.query_csw(keyword_list=keywords,\n #anytext_list=allwords,\n #titleword_list=titlewords,\n bounding_box=bounds,\n #start_datetime=start_date,\n #stop_datetime=end_date,\n #max_total_records=2000\n )\n\n# Access datasets and print some info\nfor distribution in cswu.get_netcdf_urls(record_generator):\n dataset = Dataset(distribution['url'])\n data_variable = [variable for variable in dataset.variables.values() if hasattr(variable, 'grid_mapping')][0]\n dataset_extent = [round(ordinate, 6) for ordinate in [dataset.geospatial_lon_min,\n dataset.geospatial_lat_min,\n dataset.geospatial_lon_max,\n dataset.geospatial_lat_max\n ]]\n print(f'{distribution[\"title\"]}')\n print(f'\\tNetCDF {distribution[\"url\"]}')\n print(f'\\tUUID {distribution[\"uuid\"]}')\n print(f'\\textent {dataset_extent}')\n print(f'\\tgrid shape {data_variable.shape}')\n print(f'\\tkeywords {distribution[\"keywords\"]}')\n", 78 | "metadata": { 79 | "trusted": true 80 | }, 81 | "execution_count": 6, 82 | "outputs": [ 83 | { 84 | "name": "stdout", 85 | "text": "National Gravity Compilation 2019 includes airborne - CSCBA 0.5VD grid\n\tNetCDF https://dapds00.nci.org.au/thredds/dodsC/iv65/Geoscience_Australia_Geophysics_Reference_Data_Collection/national_geophysical_compilations/Gravmap2019/Gravmap2019-grid-grv_cscba_05vd-IncludesAirborne.nc\n\tUUID 86db5fff-e32e-41d8-b9c2-a6f4d431529e\n\textent [107.997917, -48.002083, 164.002083, -7.997917]\n\tgrid shape (9601, 13441)\n\tkeywords 05vd, Australia, B series, Bouguer, EARTH SCIENCES, Earth sciences, GADDS2.0, NCI, None, Published_External, complete, geophysical survey, geophysics, geoscientificInformation, gradiometry. Airborne, grav, gravity, grid, grid, ground digital data, marine, raster, satellite, spherical cap, survey 202008\nNational Gravity Compilation 2019 includes airborne tilt grid\n\tNetCDF https://dapds00.nci.org.au/thredds/dodsC/iv65/Geoscience_Australia_Geophysics_Reference_Data_Collection/national_geophysical_compilations/Gravmap2019/Gravmap2019-grid-grv_cscba_tilt-IncludesAirborne.nc\n\tUUID 4b91ba6f-1553-4b1e-9297-4a48d1263d65\n\textent [107.997917, -48.002083, 164.002083, -7.997917]\n\tgrid shape (9601, 13441)\n\tkeywords Australia, B series, Bouguer, EARTH SCIENCES, Earth sciences, GADDS2.0, NCI, None, Published_External, airborne, complete, digital data, geophysical survey, geophysics, geoscientificInformation, gradiometry, grav, gravity, grid, grid, marine, raster, round, satellite, spherical cap, survey 202008, tilt\nNational Gravity Compilation 2019 - CSCBA 0.5VD grid\n\tNetCDF https://dapds00.nci.org.au/thredds/dodsC/iv65/Geoscience_Australia_Geophysics_Reference_Data_Collection/national_geophysical_compilations/Gravmap2019/Gravmap2019-grid-grv_cscba_05vd.nc\n\tUUID feff54f5-bd85-4647-973f-dbc1dfece415\n\textent [107.997917, -48.002083, 164.002083, -7.997917]\n\tgrid shape (9601, 13441)\n\tkeywords 05vd, A series, Australia, Bouguer, EARTH SCIENCES, Earth sciences, GADDS2.0, NCI, None, Published_External, complete, geophysical survey, geophysics, geoscientificInformation, grav, gravity, grid, grid, ground digital data, half vertical derivative, marine, national, raster, satellite, spherical cap, survey 202008\nNational Gravity Compilation 2019 tilt grid\n\tNetCDF https://dapds00.nci.org.au/thredds/dodsC/iv65/Geoscience_Australia_Geophysics_Reference_Data_Collection/national_geophysical_compilations/Gravmap2019/Gravmap2019-grid-grv_cscba_tilt.nc\n\tUUID d15bdefa-924b-4b87-8b82-0adc5b15e20e\n\textent [107.997917, -48.002083, 164.002083, -7.997917]\n\tgrid shape (9601, 13441)\n\tkeywords A series, Australia, Bouguer, EARTH SCIENCES, Earth sciences, GADDS2.0, NCI, None, Published_External, complete, digital data, geophysical survey, geophysics, geoscientificInformation, grav, gravity, grid, grid, ground, marine, national, raster, satellite, spherical cap, survey 202008, tilt\nNational Gravity Compilation 2019 includes airborne CSCBA 1VD image\n\tNetCDF https://dapds00.nci.org.au/thredds/dodsC/iv65/Geoscience_Australia_Geophysics_Reference_Data_Collection/national_geophysical_compilations/Gravmap2019/Gravmap2019-image-gravity-grv_cscba_1vd-IncludesAirborne.nc\n\tUUID 8b86e595-140f-41d2-ac79-d03affebafa5\n\textent [107.997917, -48.002083, 164.002083, -7.997917]\n\tgrid shape (9601, 13441)\n\tkeywords 1vd, Australia, B series, Bouguer, EARTH SCIENCES, Earth sciences, GADDS2.0, NCI, None, Published_External, airborne, complete, digital data, geophysical survey, geophysics, geoscientificInformation, gradiometry, grav, gravity, grid, ground, image, image, marine, raster, satellite, spherical cap, survey 202008\nNational Gravity Compilation 2019 includes airborne - CSCBA 0.5VD image\n\tNetCDF https://dapds00.nci.org.au/thredds/dodsC/iv65/Geoscience_Australia_Geophysics_Reference_Data_Collection/national_geophysical_compilations/Gravmap2019/Gravmap2019-image-gravity-grv_cscba_05vd-IncludesAirborne.nc\n\tUUID bfca1271-b5fe-4cda-a8b2-d9f5c1b559eb\n\textent [107.997917, -48.002083, 164.002083, -7.997917]\n\tgrid shape (9601, 13441)\n\tkeywords 05vd, Airborne, Australia, B series, Bouguer, EARTH SCIENCES, Earth sciences, GADDS2.0, NCI, None, Published_External, complete, geophysical survey, geophysics, geoscientificInformation, gradiometry, grav, gravity, grid, ground digital data, image, image, marine, raster, satellite, spherical cap, survey 202008\nNational Gravity Compilation 2019 includes airborne tilt image\n\tNetCDF https://dapds00.nci.org.au/thredds/dodsC/iv65/Geoscience_Australia_Geophysics_Reference_Data_Collection/national_geophysical_compilations/Gravmap2019/Gravmap2019-image-gravity-grv_cscba_tilt-IncludesAirborne.nc\n\tUUID bf89b910-25c2-4eba-9343-182d7d134599\n\textent [107.997917, -48.002083, 164.002083, -7.997917]\n\tgrid shape (9601, 13441)\n\tkeywords Australia, B series, Bouguer, EARTH SCIENCES, Earth sciences, GADDS2.0, NCI, None, Published_External, airborne, complete, digital data, geophysical survey, geophysics, geoscientificInformation, gradiometry, grav, gravity, grid, image, image, marine, raster, round, satellite, spherical cap, survey 202008, tilt\n", 86 | "output_type": "stream" 87 | } 88 | ] 89 | }, 90 | { 91 | "cell_type": "code", 92 | "source": "", 93 | "metadata": { 94 | "trusted": true 95 | }, 96 | "execution_count": null, 97 | "outputs": [] 98 | } 99 | ] 100 | } 101 | -------------------------------------------------------------------------------- /examples/CSW Request.xml: -------------------------------------------------------------------------------- 1 | 7 | 8 | full 9 | 10 | 11 | 13 | anyText 14 | dbcc0c59-81e6-4eed-e044-00144fdd4fa6 15 | 16 | 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /examples/CSW Response.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | dbcc0c59-81e6-4eed-e044-00144fdd4fa6 7 | 2013-05-03T00:00:00 8 | Isostatic Residual Gravity Anomaly Grid of Onshore Australia - 2011 9 | dataset 10 | National dataset 11 | gravity 12 | grid 13 | NCI 14 | Gravimetrics 15 | Isostatic anomaly 16 | Geophysical National Coverage 17 | Gravity Digital Data 18 | geoscientificInformation 19 | html 20 | NetCDF 21 | NetCDF 22 | Zipped ERS 23 | html 24 | WCS 25 | WMS 26 | html 27 | html 28 | NetCDF 29 | Geoscience Australia 30 | Gravity data measure small changes in gravity due to changes in the density of rocks beneath the Earth surface. This grid represents isostatic residual gravity anomalies over onshore Australia made from onshore gravity measurements collected on geophysical surveys conducted by Commonwealth, State and Northern Territory governments, and the private sector. The grid cell size is approximately 800 metres. As the effect of the long wavelength anomalies are removed, the isostatic residual anomalies reveal better than most gravity maps, the density distributions within the upper crust. 31 | Gravity data measure small changes in gravity due to changes in the density of rocks beneath the Earth surface. This grid represents isostatic residual gravity anomalies over onshore Australia made from onshore gravity measurements collected on geophysical surveys conducted by Commonwealth, State and Northern Territory governments, and the private sector. The grid cell size is approximately 800 metres. As the effect of the long wavelength anomalies are removed, the isostatic residual anomalies reveal better than most gravity maps, the density distributions within the upper crust. 32 | license 33 | license 34 | The gravity data set of onshore Australia contains more than 1.57 million reliable onshore stations gathered during more than 1800 surveys and held in the Australian National Gravity Database (ANGD). Continental Australia has a basic station spacing coverage of 11 km, with South Australia, Tasmania and part of New South Wales covered at a spacing of 7 km. Victoria has station coverage of approximately 1.5 km. Some areas of scientific or economic interest have been infilled with station spacings between 2 km and 4 km by recent Commonwealth, State and Territory Government initiatives. Other areas of detailed coverage have been surveyed by private companies for exploration purposes. Only open file data as held in the ANGD at March 2011 were used in the creation of the grid. 35 | The data values contained in the grid are Isostatic Residual Gravity anomalies over Continental Australia. A depth to mantle model and subsequent isostatic corrections were produced using a modified version of the USGS program AIRYROOT (Simpson et al., 1983) provided by Intrepid Geophysics. Geoscience Australia's 2009 Bathymetry and Topography Grid (Whiteway, 2009) was used to calculate the depth to crustal bottom following the Airy-Heiskanen crustal-root model. The isostatic corrections were then applied to the complete Bouguer anomalies (Tracey and Nakamura, 2010) to produce the Isostatic Residual Gravity Anomaly Grid of Australia. The gravity anomalies are based on the Australian Absolute Gravity Datum 2007 and 1994 Geodetic Datum of Australia (Tracey et al., 2008). 36 | A crustal density of 2.67 tonnes per cubic meter was used for the calculation, with an assumed density contrast between the crust and mantle of 0.4 tonnes per cubic meter. A depth to mantle at sea level of 37 km was used in the calculation. This was derived from the average Australian depth to the Mohorovi¿i¿ discontinuity (Moho) at sea level using data from seismic studies around Australia (Goncharov et al., 2007). 37 | The original grid was converted from ERMapper (.ers) format to netCDF4_classic format using GDAL1.11.1. The main purpose of this conversion is to enable access to the data by relevant open source tools and software. The netCDF grid was created on 2016-03-29. 38 | References 39 | Goncharov, A., Deighton, I., Tischer, M. and Collins, C., 2007. Crustal thickness in Australia: where, how and what for?: ASEG Extended Abstracts, vol. 2007, pp. 1-4, ASEG2007 19th Geophysical Conference. 40 | Simpson, R.W., Jachens, R.C. and Blakely, R.J., 1983. AIRYROOT: A FORTRAN program for calculating the gravitational attraction of an Airy isostatic root out to 166.7 km: US Geological Survey Open File Report 83-883. 41 | Tracey, R., Bacchin, M., and Wynne, P., 2008. AAGD07: A new absolute gravity datum for Australian gravity and new standards for the Australian National Gravity Database: ASEG Extended Abstracts, vol. 2007, No.1, pp. 1-3, ASEG2007 19th Geophysical Conference. 42 | Tracey, R. and Nakamura, A., 2010. Complete Bouguer Anomalies for the Australian National Gravity Database: ASEG Extended Abstracts, vol. 2010, pp. 1-3, ASEG2010 21st Geophysical Conference. 43 | Whiteway, T.G., 2009. Australian Bathymetry and Topography Grid: Geoscience Australia Record 2009/21, 46pp. 44 | c6b58f54-102c-19e9-e044-00144fdd4fa6 45 | html 46 | NetCDF 47 | NetCDF 48 | Zipped ERS 49 | html 50 | WCS 51 | WMS 52 | html 53 | html 54 | NetCDF 55 | 56 | 156.741882 -44.204934 57 | 109.100234 -9.354948 58 | 59 | 60 | 61 | -------------------------------------------------------------------------------- /examples/dynamic_kmls/app.py: -------------------------------------------------------------------------------- 1 | from flask import Flask 2 | from flask_restful import Api 3 | from flask import request 4 | import simplekml 5 | import time 6 | from shapely.geometry import Polygon 7 | from shapely import wkt 8 | from dynamic_kmls import netcdf2kml 9 | from geophys_utils.dataset_metadata_cache import get_dataset_metadata_cache 10 | import logging 11 | 12 | DATABASE_ENGINE = 'SQLite' 13 | #DATABASE_ENGINE = 'Postgres' 14 | 15 | # Define maximum bounding box width for point display. Uses survey convex-hull polygons for anything larger. 16 | MAX_BOX_WIDTH_FOR_POINTS = 1.5 17 | 18 | app = Flask(__name__) 19 | api = Api(app) 20 | 21 | # Setup logging handlers if required 22 | logger = logging.getLogger(__name__) # Get logger 23 | logger.setLevel(logging.DEBUG) # Initial logging level for this module 24 | 25 | 26 | @app.route('/',methods=['GET']) 27 | def do_everything(bounding_box): 28 | 29 | t0 = time.time() # retrieve coordinates from query 30 | 31 | query_string = request.args 32 | bbox = query_string['BBOX'] 33 | bbox_list = bbox.split(',') 34 | west = float(bbox_list[0]) 35 | south = float(bbox_list[1]) 36 | east = float(bbox_list[2]) 37 | north = float(bbox_list[3]) 38 | 39 | bbox_polygon = Polygon(((west, south), 40 | (east, south), 41 | (east, north), 42 | (west, north), 43 | (west, south) 44 | )) 45 | 46 | t1 = time.time() 47 | logger.debug("Retrieve bbox values from get request...") 48 | logger.debug("Time: " + str(t1-t0)) 49 | 50 | # Get the point_data_tuple surveys from the database that are within the bbox 51 | sdmc = get_dataset_metadata_cache(db_engine=DATABASE_ENGINE, debug=False) 52 | point_data_tuple_list = sdmc.search_dataset_distributions( 53 | keyword_list=['AUS', 'ground digital data', 'gravity', 'geophysical survey', 'points'], 54 | protocol='opendap', 55 | ll_ur_coords=[[west, south], [east, north]] 56 | ) 57 | 58 | logger.debug([[west, south], [east, north]]) 59 | t2 = time.time() 60 | logger.debug("Retrieve point_data_tuple strings from database...") 61 | logger.debug("Time: " + str(t2-t1)) 62 | 63 | kml = simplekml.Kml() 64 | 65 | # ---------------------------------------------------------------------------------------------------------------- 66 | # High zoom: show points rather than polygons. 67 | if east - west < MAX_BOX_WIDTH_FOR_POINTS: 68 | 69 | if len(point_data_tuple_list) > 0: 70 | # set point style 71 | point_style = simplekml.Style() 72 | point_style.iconstyle.icon.href = "http://maps.google.com/mapfiles/kml/paddle/grn-blank.png" 73 | point_style.iconstyle.scale = 0.7 74 | point_style.labelstyle.scale = 0 # removes the label 75 | 76 | netcdf_file_folder = kml.newfolder(name="Ground Gravity Survey Observations") 77 | 78 | for point_data_tuple in point_data_tuple_list: 79 | logger.debug("Building NETCDF: " + str(point_data_tuple[2])) 80 | netcdf2kml_obj = netcdf2kml.NetCDF2kmlConverter(point_data_tuple) 81 | t3 = time.time() 82 | logger.debug("set style and create netcdf2kmlconverter instance of point_data_tuple file ...") 83 | logger.debug("Time: " + str(t3 - t2)) 84 | 85 | #logger.debug("Number of points in file: " + str(netcdf2kml_obj.npu.point_count)) 86 | 87 | if netcdf2kml_obj.npu.point_count > 0: 88 | ta = time.time() 89 | netcdf2kml_obj.build_points(netcdf_file_folder, bbox_list, point_style) 90 | tb = time.time() 91 | logger.debug("do the things time: " + str(tb-ta)) 92 | logger.debug("Build the point ...") 93 | dataset_points_region = netcdf2kml_obj.build_region(100, -1, 200, 800) 94 | netcdf_file_folder.region = dataset_points_region 95 | netcdf2kml_obj.netcdf_dataset.close() # file must be closed after use to avoid errors when accessed again. 96 | del netcdf2kml_obj # Delete netcdf2kml_obj to removenetcdf2kml_obj.npu cache file 97 | t4 = time.time() 98 | 99 | return str(netcdf_file_folder) 100 | 101 | else: 102 | logger.debug("No surveys in view") 103 | 104 | # ---------------------------------------------------------------------------------------------------------------- 105 | # Low zoom: show polygons and not points. 106 | else: 107 | t_polygon_1 = time.time() 108 | 109 | # set polygon style 110 | polygon_style = simplekml.Style() 111 | polygon_style.polystyle.color = 'B30000ff' # Transparent red 112 | #polygon_style.polystyle.color = 'ff4545' 113 | polygon_style.polystyle.outline = 1 114 | 115 | polygon_style_background = simplekml.Style() 116 | polygon_style_background.polystyle.color = '7FFFFFFF' # Transparent white 117 | polygon_style_background.polystyle.outline = 1 118 | 119 | if len(point_data_tuple_list) > 0: 120 | netcdf_file_folder = kml.newfolder(name="Ground Gravity Survey Extents") 121 | for point_data_tuple in point_data_tuple_list: 122 | logger.debug("point_data_tuple: " + str(point_data_tuple)) 123 | netcdf2kml_obj = netcdf2kml.NetCDF2kmlConverter(point_data_tuple) 124 | t_polygon_2 = time.time() 125 | logger.debug("set style and create netcdf2kmlconverter instance from point_data_tuple for polygon ...") 126 | logger.debug("Time: " + str(t_polygon_2 - t_polygon_1)) 127 | 128 | try: 129 | survey_polygon = wkt.loads(point_data_tuple[3]) 130 | except Exception as e: 131 | #print(e) 132 | continue # Skip this polygon 133 | 134 | if survey_polygon.intersects(bbox_polygon): 135 | #if survey_polygon.within(bbox_polygon): 136 | #if not survey_polygon.contains(bbox_polygon): 137 | #if survey_polygon.centroid.within(bbox_polygon): 138 | #if not survey_polygon.contains(bbox_polygon) and survey_polygon.centroid.within(bbox_polygon): 139 | 140 | polygon_folder = netcdf2kml_obj.build_polygon(netcdf_file_folder, polygon_style) 141 | else: 142 | polygon_folder = netcdf2kml_obj.build_polygon(netcdf_file_folder, polygon_style, False) 143 | 144 | dataset_polygon_region = netcdf2kml_obj.build_region(-1, -1, 200, 800) 145 | polygon_folder.region = dataset_polygon_region # insert built polygon region into polygon folder 146 | 147 | #else: # for surveys with 1 or 2 points. Can't make a polygon. Still save the points? 148 | # logger.debug("not enough points") 149 | 150 | # neww = kml.save("test_polygon.kml") 151 | return str(netcdf_file_folder) 152 | 153 | else: 154 | empty_folder = kml.newfolder(name="no points in view") 155 | return str(empty_folder) 156 | 157 | app.run(debug=True) 158 | 159 | -------------------------------------------------------------------------------- /examples/get_netcdf_util_demo.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on 13 Dec 2019 3 | 4 | @author: Alex Ip 5 | ''' 6 | import logging 7 | import re 8 | import sys 9 | 10 | from geophys_utils import get_netcdf_util 11 | 12 | logger = logging.getLogger(__name__) 13 | 14 | 15 | def main(): 16 | ''' 17 | Quick and dirty test routine. Takes the name of a file containing a list of NCI OPeNDAP endpoints 18 | and prints each endpoint address along with the class name of the NetCDFUtils subclass 19 | ''' 20 | assert sys.argv, 'Please supply filename containing a list of NCI OPeNDAP endpoints' 21 | nc_list_file_path = sys.argv[1] 22 | 23 | with open(nc_list_file_path, 'r') as nc_list_file: 24 | nc_list = [nc_file_path.strip() 25 | for nc_file_path in nc_list_file 26 | if re.match('^http.*/dodsC/.*\.nc$', nc_file_path) 27 | ] 28 | nc_list_file.close() 29 | 30 | nc_list.sort() 31 | # logger.debug(nc_list) 32 | 33 | for nc in nc_list: 34 | ncu = get_netcdf_util(nc) 35 | print(nc, type(ncu).__name__) 36 | if ncu: # If we have a recognised netCDF file (i.e. grid, point or line) 37 | 38 | # TODO: Do some stuff here 39 | 40 | ncu.close() 41 | 42 | 43 | if __name__ == '__main__': 44 | # Setup logging handler if required 45 | if not logger.handlers: 46 | # Set handler for root root_logger to standard output 47 | console_handler = logging.StreamHandler(sys.stdout) 48 | # console_handler.setLevel(logging.INFO) 49 | console_handler.setLevel(logging.DEBUG) 50 | console_formatter = logging.Formatter('%(message)s') 51 | console_handler.setFormatter(console_formatter) 52 | logger.addHandler(console_handler) 53 | 54 | main() 55 | -------------------------------------------------------------------------------- /examples/plot_point_dataset.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # =============================================================================== 4 | # Copyright 2017 Geoscience Australia 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # =============================================================================== 18 | ''' 19 | Created on 18Jun.,2018 20 | 21 | @author: Andrew Turner & Alex Ip 22 | ''' 23 | import sys 24 | from math import log10, floor, pow 25 | 26 | import cartopy.crs as ccrs 27 | import cartopy.io.img_tiles as cimgt 28 | import matplotlib.pyplot as plt 29 | import netCDF4 30 | import numpy as np 31 | 32 | from geophys_utils import NetCDFPointUtils 33 | from geophys_utils import get_spatial_ref_from_wkt 34 | 35 | 36 | def plot_point_dataset(netcdf_point_utils, 37 | variable_to_map, 38 | utm_bbox=None, 39 | plot_title=None, 40 | colour_scheme='binary', 41 | point_size=10, 42 | point_step=1 43 | ): 44 | ''' 45 | Function to plot data points on a map 46 | @param netcdf_point_utils: NetCDFPointUtils object wrapping a netCDF dataset 47 | @param variable_to_map: String specifying variable name for colour map 48 | @param utm_bbox: UTM Bounding box of form [xmin, ymin, xmax, ymax] or None for all points. Default=None 49 | @param plot_title: String to prefix before dataset title. Default=None for dataset title or dataset basename 50 | @param colour_scheme: String specifying colour scheme for data points. Default='binary' 51 | @param point_size: Point size for data points. Default=10 52 | @param point_step: Point step between plotted points - used to skip points in dense datasets. Default=1 53 | ''' 54 | 55 | def rescale_array(input_np_array, new_range_min=0, new_range_max=1): 56 | old_min = input_np_array.min() 57 | old_range = input_np_array.max() - old_min 58 | new_range = new_range_max - new_range_min 59 | 60 | scaled_np_array = ((input_np_array - old_min) / old_range * new_range) + new_range_min 61 | 62 | return scaled_np_array 63 | 64 | if plot_title is None: 65 | if hasattr(netcdf_point_utils.netcdf_dataset, 'title'): 66 | plot_title = netcdf_point_utils.netcdf_dataset.title 67 | else: 68 | plot_title = netcdf_point_utils.netcdf_dataset.filepath() 69 | 70 | utm_wkt, utm_coords = netcdf_point_utils.utm_coords(netcdf_point_utils.xycoords) 71 | 72 | utm_zone = get_spatial_ref_from_wkt(utm_wkt).GetUTMZone() # -ve for Southern Hemisphere 73 | southern_hemisphere = (utm_zone < 0) 74 | utm_zone = abs(utm_zone) 75 | projection = ccrs.UTM(zone=utm_zone, southern_hemisphere=southern_hemisphere) 76 | print('utm_zone = {}'.format(utm_zone)) 77 | 78 | variable = netcdf_point_utils.netcdf_dataset.variables[variable_to_map] 79 | 80 | # Set geographic range of plot 81 | if utm_bbox is None: 82 | utm_bbox = [ 83 | np.min(utm_coords[:, 0]), 84 | np.min(utm_coords[:, 1]), 85 | np.max(utm_coords[:, 0]), 86 | np.max(utm_coords[:, 1]) 87 | ] 88 | spatial_mask = np.ones(shape=variable.shape, dtype=bool) 89 | else: 90 | spatial_mask = np.logical_and(np.logical_and((utm_bbox[0] <= utm_coords[:, 0]), 91 | (utm_coords[:, 0] <= utm_bbox[2])), 92 | np.logical_and((utm_bbox[1] <= utm_coords[:, 1]), 93 | (utm_coords[:, 1] <= utm_bbox[3])) 94 | ) 95 | utm_coords = utm_coords[spatial_mask] 96 | 97 | print('{} points in UTM bounding box: {}'.format(np.count_nonzero(spatial_mask), utm_bbox)) 98 | 99 | colour_array = rescale_array(variable[spatial_mask], 0, 1) 100 | 101 | fig = plt.figure(figsize=(30, 30)) 102 | 103 | ax = fig.add_subplot(1, 1, 1, projection=projection) 104 | 105 | ax.set_title(plot_title) 106 | 107 | # map_image = cimgt.OSM() # https://www.openstreetmap.org/about 108 | # map_image = cimgt.StamenTerrain() # http://maps.stamen.com/ 109 | map_image = cimgt.GoogleTiles(style='satellite') 110 | 111 | ax.add_image(map_image, 10) 112 | 113 | # Compute and set regular tick spacing 114 | range_x = utm_bbox[2] - utm_bbox[0] 115 | range_y = utm_bbox[3] - utm_bbox[1] 116 | x_increment = pow(10.0, floor(log10(range_x))) / 2 117 | y_increment = pow(10.0, floor(log10(range_y))) / 2 118 | x_ticks = np.arange((utm_bbox[0] // x_increment + 1) * x_increment, utm_bbox[2], x_increment) 119 | y_ticks = np.arange((utm_bbox[1] // y_increment + 1) * y_increment, utm_bbox[3], y_increment) 120 | plt.xticks(x_ticks, rotation=45) 121 | plt.yticks(y_ticks) 122 | 123 | # set the x and y axis labels 124 | plt.xlabel("Eastings (m)", rotation=0, labelpad=20) 125 | plt.ylabel("Northings (m)", rotation=90, labelpad=20) 126 | 127 | # See link for possible colourmap schemes: https://matplotlib.org/examples/color/colormaps_reference.html 128 | cm = plt.cm.get_cmap(colour_scheme) 129 | 130 | # build a scatter plot of the specified data, define marker, spatial reference system, and the chosen colour map type 131 | sc = ax.scatter(utm_coords[::point_step, 0], 132 | utm_coords[::point_step, 1], 133 | marker='o', 134 | c=colour_array[::point_step], 135 | s=point_size, 136 | alpha=0.9, 137 | transform=projection, 138 | cmap=cm 139 | ) 140 | 141 | # set the colour bar ticks and labels 142 | try: # not all variables have units. These will fail on the try and produce the map without tick labels. 143 | cb = plt.colorbar(sc, ticks=[0, 1]) 144 | cb.ax.set_yticklabels( 145 | [str(np.min(variable[spatial_mask])), str(np.max(variable[spatial_mask]))]) # vertically oriented colorbar 146 | cb.set_label("{} {}".format(variable.long_name, variable.units)) 147 | except: 148 | pass 149 | print("show") 150 | plt.show() 151 | 152 | 153 | def main(): 154 | ''' 155 | main function for quick and dirty testing 156 | ''' 157 | nc_path = sys.argv[1] 158 | variable_to_plot = sys.argv[2] 159 | 160 | netcdf_dataset = netCDF4.Dataset(nc_path) 161 | npu = NetCDFPointUtils(netcdf_dataset) 162 | 163 | print('1D Point variables:\n\t{}'.format('\n\t'.join([key for key, value in netcdf_dataset.variables.items() 164 | if value.dimensions == ('point',)]))) 165 | # Plot spatial subset 166 | plot_point_dataset(npu, 167 | variable_to_plot, 168 | # utm_bbox=[660000,7080000,680000,7330000], 169 | colour_scheme='gist_heat', 170 | point_size=30, 171 | point_step=100 172 | ) 173 | 174 | 175 | if __name__ == '__main__': 176 | main() 177 | -------------------------------------------------------------------------------- /geophys_utils/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # =============================================================================== 4 | # Copyright 2017 Geoscience Australia 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # =============================================================================== 18 | ''' 19 | Created on 16Nov.,2016 20 | 21 | @author: Alex Ip 22 | ''' 23 | import logging 24 | 25 | logger = logging.getLogger(__name__) 26 | logger.setLevel(logging.INFO) # Initial logging level for this module 27 | 28 | # =============================================================================== 29 | # if not logger.handlers: 30 | # # Set handler for root logger to standard output 31 | # console_handler = logging.StreamHandler(sys.stdout) 32 | # #console_handler.setLevel(logging.INFO) 33 | # console_handler.setLevel(logging.DEBUG) 34 | # console_formatter = logging.Formatter('%(message)s') 35 | # console_handler.setFormatter(console_formatter) 36 | # logger.addHandler(console_handler) 37 | # =============================================================================== 38 | 39 | from geophys_utils._netcdf_utils import NetCDFUtils 40 | from geophys_utils._netcdf_grid_utils import NetCDFGridUtils 41 | from geophys_utils._netcdf_point_utils import NetCDFPointUtils 42 | from geophys_utils._netcdf_line_utils import NetCDFLineUtils 43 | from geophys_utils._csw_utils import CSWUtils 44 | from geophys_utils._array_pieces import array_pieces 45 | from geophys_utils._data_stats import DataStats 46 | from geophys_utils._polygon_utils import get_grid_edge_points, get_netcdf_edge_points, points2convex_hull, \ 47 | points2alpha_shape, netcdf2convex_hull 48 | from geophys_utils._crs_utils import get_spatial_ref_from_wkt, get_wkt_from_spatial_ref, get_coordinate_transformation, \ 49 | get_utm_wkt, transform_coords, get_reprojected_bounds 50 | from geophys_utils._gdal_grid_utils import get_gdal_wcs_dataset, get_gdal_grid_values 51 | from geophys_utils._transect_utils import line_length, point_along_line, utm_coords, coords2distance, sample_transect 52 | from geophys_utils._dem_utils import DEMUtils 53 | from geophys_utils._array2file import array2file 54 | from geophys_utils._datetime_utils import date_string2datetime 55 | from geophys_utils._get_netcdf_util import get_netcdf_util 56 | -------------------------------------------------------------------------------- /geophys_utils/_array2file.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on 20 Oct. 2017 3 | 4 | @author: Alex Ip 5 | ''' 6 | from osgeo import gdal 7 | 8 | 9 | def array2file(data_arrays, 10 | projection, 11 | geotransform, 12 | file_path, 13 | file_format='GTiff', 14 | dtype=gdal.GDT_Float32): 15 | ''' 16 | Function to output array to any(?) GDAL supported raster format and return GDAL dataset 17 | Formats defined in http://www.gdal.org/formats_list.html 18 | ''' 19 | data_array_shape = data_arrays[0].shape 20 | assert [data_array.shape for data_array in data_arrays].count(data_array_shape) == len( 21 | data_arrays), 'Data arrays are of different shape' 22 | 23 | driver = gdal.GetDriverByName(file_format) 24 | gdal_dataset = driver.Create(file_path, 25 | data_array_shape[1], data_array_shape[0], # Array must be ordered yx 26 | len(data_arrays), 27 | dtype) 28 | gdal_dataset.SetGeoTransform(geotransform) 29 | gdal_dataset.SetProjection(projection) 30 | 31 | # Write arrays to file 32 | for band_index in range(len(data_arrays)): 33 | raster_band = gdal_dataset.GetRasterBand(band_index + 1) 34 | raster_band.WriteArray(data_arrays[band_index]) 35 | raster_band.FlushCache() 36 | 37 | gdal_dataset.FlushCache() 38 | return gdal_dataset 39 | -------------------------------------------------------------------------------- /geophys_utils/_array_pieces.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # =============================================================================== 4 | # Copyright 2017 Geoscience Australia 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # =============================================================================== 18 | ''' 19 | process_array.py 20 | Created on 1 Sep,2016 21 | 22 | @author: Alex Ip 23 | ''' 24 | import itertools 25 | import math 26 | import sys 27 | from functools import reduce 28 | 29 | import netCDF4 30 | 31 | 32 | def array_pieces(ndarray, max_bytes=None, overlap=0): 33 | ''' 34 | Generator to return a series of numpy arrays less than max_bytes in size and the offset within the complete data from a NetCDF variable 35 | Parameters: 36 | ndarray: Numpy array or NetCDF array variable 37 | overlap: number of pixels to add to each edge 38 | max_bytes: Maximum number of bytes to retrieve. Defaults to 500,000,000 for NCI's OPeNDAP 39 | 40 | Yields: 41 | piece_array: array subset less than max_bytes in size 42 | array_offset: start indices of subset in whole array 43 | ''' 44 | max_bytes = max_bytes or 500000000 # Defaults to 500MB for NCI's OPeNDAP 45 | 46 | array_shape = ndarray.shape 47 | array_dimensions = len(array_shape) 48 | 49 | # Determine overall array size in bytes 50 | array_bytes = ndarray.dtype.itemsize * \ 51 | reduce(lambda x, y: x * y, array_shape) 52 | 53 | if array_bytes > max_bytes: # Multiple pieces required 54 | # Determine number of divisions in each axis required to keep pieces 55 | # under max_bytes in size 56 | axis_divisions = int(math.ceil( 57 | math.pow(math.ceil(array_bytes / float(max_bytes)), 1.0 / array_dimensions))) 58 | 59 | # Determine chunk size for pieces or default to natural divisions if no 60 | # chunking set 61 | try: 62 | chunking = ndarray.chunking() or (1, 1) 63 | except: # Numpy arrays don't have chunking 64 | chunking = (1, 1) 65 | 66 | # Disregard chunking if it's too big to be useful 67 | chunking = [chunking[index] if chunking[index] < array_shape[index] // axis_divisions else 1 68 | for index in range(array_dimensions)] 69 | 70 | # Determine piece shape rounded down to chunk sizes 71 | piece_shape = [array_shape[index] // axis_divisions // chunking[index] 72 | * chunking[index] for index in range(array_dimensions)] 73 | 74 | # Determine total number of pieces in each axis 75 | axis_pieces = [int(math.ceil(float(array_shape[index]) // piece_shape[index])) 76 | for index in range(array_dimensions)] 77 | 78 | # Iterate over every piece of array 79 | for piece_indices in itertools.product(*[range(axis_pieces[dimension_index]) 80 | for dimension_index in range(array_dimensions)]): 81 | # Compute base start indices with no overlap 82 | start_indices = [piece_indices[dimension_index] * piece_shape[dimension_index] 83 | for dimension_index in range(array_dimensions)] 84 | 85 | # Compute end indices plus overlap from start indices 86 | end_indices = [min(start_indices[dimension_index] + piece_shape[dimension_index] + overlap, 87 | array_shape[dimension_index]) 88 | for dimension_index in range(array_dimensions)] 89 | 90 | # Subtract overlap from base start indices 91 | start_indices = [max(0, start_indices[dimension_index] - overlap) 92 | for dimension_index in range(array_dimensions)] 93 | 94 | array_slices = tuple([slice(start_indices[dimension_index], 95 | end_indices[dimension_index]) 96 | for dimension_index in range(array_dimensions)]) 97 | 98 | piece_array = ndarray[array_slices] 99 | yield piece_array, tuple(start_indices) 100 | 101 | else: # Only one piece required 102 | yield ndarray[...], (0, 0) 103 | 104 | 105 | def main(): 106 | ''' 107 | Main function for testing 108 | ''' 109 | netcdf_path = sys.argv[1] 110 | netcdf_dataset = netCDF4.Dataset(netcdf_path) 111 | 112 | # Find variable with "grid_mapping" attribute - assumed to be 2D data 113 | # variable 114 | try: 115 | data_variable = [variable for variable in netcdf_dataset.variables.values( 116 | ) if hasattr(variable, 'grid_mapping')][0] 117 | except: 118 | raise Exception( 119 | 'Unable to determine data variable (must have "grid_mapping" attribute') 120 | 121 | piece_count = 0 122 | for piece_array, array_offset in array_pieces(data_variable, overlap=0): 123 | piece_count += 1 124 | piece_bytes = data_variable.dtype.itemsize * \ 125 | reduce(lambda x, y: x * y, piece_array.shape) 126 | print('piece_array.shape = %s, array_offset = %s, piece_bytes = %d'.format(piece_array.shape, array_offset, 127 | piece_bytes)) 128 | 129 | print('piece_count = %s'.format(piece_count)) 130 | 131 | 132 | if __name__ == '__main__': 133 | main() 134 | -------------------------------------------------------------------------------- /geophys_utils/_blrb.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # =============================================================================== 4 | # Copyright 2017 Geoscience Australia 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # =============================================================================== 18 | """ 19 | Author: Roger Edberg (roger.edberg@ga.gov.au) 20 | Functions for BiLinear Recursive Bisection (BLRB). 21 | 22 | All shape references here follow the numpy convention (nrows, ncols), which 23 | makes some of the code harder to follow. 24 | """ 25 | 26 | import logging 27 | import numpy 28 | 29 | logger = logging.getLogger('root.' + __name__) 30 | 31 | DEFAULT_ORIGIN = (0, 0) 32 | DEFAULT_SHAPE = (8, 8) 33 | 34 | 35 | def bilinear(shape, fUL, fUR, fLR, fLL, dtype=numpy.float64): 36 | """ 37 | Bilinear interpolation of four scalar values. 38 | 39 | :param shape: 40 | Shape of interpolated grid (nrows, ncols). 41 | 42 | :param fUL: 43 | Data value at upper-left (NW) corner. 44 | 45 | :param fUR: 46 | Data value at upper-right (NE) corner. 47 | 48 | :param fLR: 49 | Data value at lower-right (SE) corner. 50 | 51 | :param fLL: 52 | Data value at lower-left (SW) corner. 53 | 54 | :param dtype: 55 | Data type (numpy I presume?). 56 | 57 | :return: 58 | Array of data values interpolated between corners. 59 | """ 60 | 61 | s, t = [a.astype(dtype) for a in numpy.ogrid[0:shape[0], 0:shape[1]]] 62 | 63 | s /= (shape[0] - 1.0) 64 | t /= (shape[1] - 1.0) 65 | 66 | return s * (t * fLR + (1.0 - t) * fLL) + (1.0 - s) * (t * fUR + (1.0 - t) * fUL) 67 | 68 | 69 | def indices(origin=DEFAULT_ORIGIN, shape=DEFAULT_SHAPE): 70 | """ 71 | Generate corner indices for a grid block. 72 | 73 | :param origin: 74 | Block origin (2-tuple). 75 | 76 | :param shape: 77 | Block shape (2-tuple: nrows, ncols). 78 | 79 | :return: 80 | Corner indices: (xmin, xmax, ymin, ymax). 81 | """ 82 | return (origin[0], origin[0] + shape[0] - 1, 83 | origin[1], origin[1] + shape[1] - 1) 84 | 85 | 86 | def subdivide(origin=DEFAULT_ORIGIN, shape=DEFAULT_SHAPE): 87 | """ 88 | Generate indices for grid sub-blocks. 89 | 90 | :param origin: 91 | Block origin (2-tuple). 92 | 93 | :param shape: 94 | Block shape (nrows, ncols). 95 | 96 | :return: 97 | Dictionary containing sub-block corner indices: 98 | { 'UL': , 99 | 'UR': , 100 | 'LL': , 101 | 'LR': } 102 | """ 103 | i0, ie, j0, je = indices(origin, shape) 104 | ic = origin[0] + shape[0] / 2 105 | jc = origin[1] + shape[1] / 2 106 | 107 | return { 108 | 'UL': [(i0, j0), (i0, jc), (ic, j0), (ic, jc)], 109 | 'LL': [(ic, j0), (ic, jc), (ie, j0), (ie, jc)], 110 | 'UR': [(i0, jc), (i0, je), (ic, jc), (ic, je)], 111 | 'LR': [(ic, jc), (ic, je), (ie, jc), (ie, je)], 112 | } 113 | 114 | 115 | def interpolate_block(origin=DEFAULT_ORIGIN, shape=DEFAULT_SHAPE, eval_func=None, grid=None): 116 | """ 117 | Interpolate a grid block. 118 | 119 | :param origin: 120 | Block origin (2-tuple). 121 | 122 | :param shape: 123 | Block shape (nrows, ncols). 124 | 125 | :param eval_func: 126 | Evaluator function. 127 | :type eval_func: 128 | callable; accepts grid indices i, j and returns a scalar value. 129 | 130 | :param grid: 131 | Grid array. 132 | :type grid: 133 | :py:class:`numpy.array`. 134 | 135 | :return: 136 | Interpolated block array if grid argument is None. If grid argument 137 | is supplied its elements are modified in place and this function 138 | does not return a value. 139 | """ 140 | i0, i1, j0, j1 = indices(origin, shape) 141 | 142 | fUL = eval_func(i0, j0) 143 | fLL = eval_func(i1, j0) 144 | fUR = eval_func(i0, j1) 145 | fLR = eval_func(i1, j1) 146 | 147 | if grid is None: 148 | return bilinear(shape, fUL, fUR, fLR, fLL) 149 | 150 | grid[i0:i1 + 1, j0:j1 + 1] = bilinear(shape, fUL, fUR, fLR, fLL) 151 | 152 | 153 | def interpolate_grid(depth=0, origin=DEFAULT_ORIGIN, shape=DEFAULT_SHAPE, eval_func=None, grid=None): 154 | """ 155 | Interpolate a data grid. 156 | 157 | :param depth: 158 | Recursive bisection depth. 159 | :type depth: 160 | :py:class:`int` 161 | 162 | :param origin: 163 | Block origin, 164 | :type origin: 165 | :py:class:`tuple` of length 2. 166 | 167 | :param shape: 168 | Block shape. 169 | :type shape: 170 | :py:class:`tuple` of length 2 ``(nrows, ncols)``. 171 | 172 | :param eval_func: 173 | Evaluator function. 174 | :type eval_func: 175 | callable; accepts grid indices i, j and returns a scalar value. 176 | 177 | :param grid: 178 | Grid array. 179 | :type grid: 180 | :py:class:`numpy.array`. 181 | 182 | :todo: 183 | Move arguments ``eval_func`` and ``grid`` to positions 1 and 2, and remove 184 | defaults (and the check that they are not ``None`` at the top of the function 185 | body). 186 | """ 187 | assert eval_func is not None 188 | assert grid is not None 189 | 190 | if depth == 0: 191 | interpolate_block(origin, shape, eval_func, grid) 192 | else: 193 | blocks = subdivide(origin, shape) 194 | for (kUL, _kUR, _kLL, kLR) in blocks.itervalues(): 195 | block_shape = (kLR[0] - kUL[0] + 1, kLR[1] - kUL[1] + 1) 196 | interpolate_grid(depth - 1, kUL, block_shape, eval_func, grid) 197 | -------------------------------------------------------------------------------- /geophys_utils/_data_stats.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # =============================================================================== 4 | # Copyright 2017 Geoscience Australia 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # =============================================================================== 18 | ''' 19 | Created on 15Aug.,2016 20 | 21 | @author: Alex 22 | ''' 23 | import os 24 | import sys 25 | 26 | import netCDF4 27 | import numpy as np 28 | 29 | from geophys_utils._array_pieces import array_pieces 30 | 31 | 32 | class DataStats(object): 33 | ''' 34 | DataStats class definition. Obtains statistics for gridded data 35 | ''' 36 | key_list = ['nc_path', 'data_type', 'nodata_value', 'x_size', 'y_size', 'min', 37 | 'max', 'mean'] # , 'median', 'std_dev', 'percentile_1', 'percentile_99'] 38 | 39 | def __init__(self, netcdf_path=None, netcdf_dataset=None, 40 | max_bytes=500000000): 41 | ''' 42 | DataStats Constructor 43 | Parameter: 44 | netcdf_path - string representing path to NetCDF file or URL for an OPeNDAP endpoint 45 | max_bytes - maximum number of bytes to pull into memory 46 | ''' 47 | assert netcdf_dataset or netcdf_path, 'Either netcdf_dataset or netcdf_path must be defined' 48 | assert not ( 49 | netcdf_dataset and netcdf_path), 'netcdf_dataset and netcdf_path cannot both be defined' 50 | 51 | netcdf_path = os.path.abspath(netcdf_path) if netcdf_path else None 52 | netcdf_dataset = netcdf_dataset or netCDF4.Dataset(netcdf_path, 'r') 53 | 54 | # Find variable with "grid_mapping" attribute - assumed to be 2D data 55 | # variable 56 | try: 57 | self.data_variable = [variable for variable in netcdf_dataset.variables.values( 58 | ) if hasattr(variable, 'grid_mapping')][0] 59 | except: 60 | raise Exception( 61 | 'Unable to determine data variable (must have "grid_mapping" attribute') 62 | 63 | self._data_stats = {} 64 | self._data_stats['nc_path'] = netcdf_path or netcdf_dataset.filepath() 65 | self._data_stats['data_type'] = str(self.data_variable.dtype) 66 | self._data_stats['nodata_value'] = self.data_variable._FillValue 67 | 68 | shape = self.data_variable.shape 69 | # Array is ordered YX 70 | self._data_stats['x_size'] = shape[1] 71 | self._data_stats['y_size'] = shape[0] 72 | 73 | length_read = 0 74 | weighted_mean = 0.0 75 | 76 | for piece_array, _piece_offsets in array_pieces( 77 | self.data_variable, max_bytes=max_bytes): 78 | 79 | if isinstance(piece_array, np.ma.core.MaskedArray): 80 | piece_array = piece_array.data 81 | 82 | # Discard all no-data elements 83 | piece_array = np.array( 84 | piece_array[piece_array != self.data_variable._FillValue]) 85 | 86 | piece_size = len(piece_array) 87 | 88 | if piece_size: 89 | try: 90 | self._data_stats['min'] = min( 91 | self._data_stats['min'], np.nanmin(piece_array)) 92 | except: 93 | self._data_stats['min'] = np.nanmin(piece_array) 94 | 95 | try: 96 | self._data_stats['max'] = max( 97 | self._data_stats['max'], np.nanmax(piece_array)) 98 | except: 99 | self._data_stats['max'] = np.nanmax(piece_array) 100 | 101 | weighted_mean += np.nanmean(piece_array) * piece_size 102 | length_read += piece_size 103 | # ================================================================== 104 | # else: 105 | # print 'Empty array' 106 | # ================================================================== 107 | 108 | self._data_stats['mean'] = weighted_mean / length_read if length_read else None 109 | 110 | # =================================================================== 111 | # #TODO: Implement something clever for these 112 | # self._data_stats['median'] = np.NaN 113 | # self._data_stats['std_dev'] = np.NaN 114 | # self._data_stats['percentile_1'] = np.NaN 115 | # self._data_stats['percentile_99'] = np.NaN 116 | # =================================================================== 117 | 118 | # TODO: Do something nicer than this to get at the values, A property 119 | # might be good. 120 | def value(self, key): 121 | return self._data_stats[key] 122 | 123 | 124 | def main(): 125 | print(','.join(DataStats.key_list)) 126 | for netcdf_path in sys.argv[1:]: 127 | datastats = DataStats(netcdf_path) 128 | print(','.join([str(datastats.value(key)) for key in DataStats.key_list])) 129 | 130 | 131 | if __name__ == '__main__': 132 | main() 133 | -------------------------------------------------------------------------------- /geophys_utils/_datetime_utils.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on 26 Oct. 2018 3 | 4 | @author: alex 5 | ''' 6 | from datetime import datetime 7 | 8 | 9 | def date_string2datetime(date_string): 10 | ''' 11 | Helper function to convert date string in one of several possible formats to a datetime object 12 | @param date_string: date string in one of several possible formats 13 | 14 | @return datetime object 15 | ''' 16 | # ? 17 | DATE_FORMAT_LIST = ['%Y%m{}', '%Y-%m-{}', '{}/%m/%y', '{}/%m/%Y'] 18 | # If there is a date_string (start date or end date from argparse) then for each format type 19 | # in the DATE FORMAT LIST try to use the datetime.strptime method. 20 | # datetime.strptime() returns a datetime variable from the input parsed into the correct format. 21 | # OR does it check if it is the correct format? 22 | datetime_result = None 23 | 24 | if date_string: 25 | for format_string in DATE_FORMAT_LIST: 26 | try: 27 | datetime_result = datetime.strptime(date_string, format_string) 28 | break 29 | except ValueError: 30 | pass 31 | 32 | # if successful return the input as a datetime class object or None. 33 | return datetime_result 34 | -------------------------------------------------------------------------------- /geophys_utils/_gdal_grid_utils.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # =============================================================================== 4 | # Copyright 2017 Geoscience Australia 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # =============================================================================== 18 | ''' 19 | Created on 8Dec.,2016 20 | 21 | @author: Alex Ip 22 | ''' 23 | import os 24 | import re 25 | import tempfile 26 | 27 | import numpy as np 28 | from osgeo import gdal 29 | from owslib.wcs import WebCoverageService 30 | 31 | from geophys_utils._crs_utils import transform_coords 32 | 33 | 34 | def get_gdal_wcs_dataset(wcs_url): 35 | '''\ 36 | Helper function to return a GDAL dataset for a WCS endpoint 37 | ''' 38 | clean_url = re.match('http[^?]+', wcs_url).group(0) 39 | temp_xml_path = os.path.join(tempfile.gettempdir(), re.sub('\W', '_', clean_url) + '.xml') 40 | 41 | wcs = WebCoverageService(wcs_url, version='1.0.0') 42 | variable_name = list(wcs.contents.keys())[0] # Only work with first variable 43 | 44 | xml_string = ''' 45 | %s? 46 | %s 47 | ''' % (clean_url, variable_name) 48 | 49 | temp_xml_file = open(temp_xml_path, 'w') 50 | temp_xml_file.write(xml_string) 51 | temp_xml_file.close() 52 | 53 | return gdal.Open(temp_xml_path) 54 | 55 | 56 | def get_gdal_grid_values(gdal_dataset, sample_points, from_crs, band_no=1): 57 | ''' 58 | Function to return values at a series of points from a GDAL dataset 59 | ''' 60 | geotransform = gdal_dataset.GetGeoTransform() 61 | to_crs = gdal_dataset.GetProjection() 62 | gdal_band = gdal_dataset.GetRasterBand(band_no) 63 | 64 | native_sample_points = transform_coords(sample_points, from_crs, to_crs) 65 | 66 | # TODO: Make this faster 67 | values = [] 68 | for point in native_sample_points: 69 | indices = (int((point[0] - geotransform[0]) / geotransform[1] + 0.5), 70 | int((point[1] - geotransform[3]) / geotransform[5] + 0.5)) 71 | value = gdal_band.ReadAsArray(xoff=indices[0], yoff=indices[1], win_xsize=1, win_ysize=1)[0, 0] 72 | # print point, indices, value 73 | values.append(value) 74 | 75 | return np.array(values) 76 | -------------------------------------------------------------------------------- /geophys_utils/_get_netcdf_util.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on 12 Dec 2019 3 | 4 | @author: Alex Ip 5 | ''' 6 | import logging 7 | 8 | import netCDF4 9 | 10 | from geophys_utils._netcdf_grid_utils import NetCDFGridUtils 11 | from geophys_utils._netcdf_line_utils import NetCDFLineUtils 12 | from geophys_utils._netcdf_point_utils import NetCDFPointUtils 13 | 14 | logger = logging.getLogger(__name__) 15 | 16 | 17 | def get_netcdf_util(netcdf_dataset, open_mode='r', debug=False): 18 | ''' 19 | Function to take a netCDF4 Dataset object, a path to a netCDF file, or an OPeNDAP endpoint 20 | and return a NetCDFUtils subclass object (i.e. NetCDFPointUtils, NetCDFLineUtils, or NetCDFGridUtils) 21 | ''' 22 | if type(netcdf_dataset) == str: # String provided as path to netCDF file 23 | try: 24 | try: 25 | _netcdf_dataset = netCDF4.Dataset(netcdf_dataset, open_mode) 26 | except OSError: 27 | if not _netcdf_dataset.startswith('http'): 28 | raise 29 | _netcdf_dataset = netCDF4.Dataset(netcdf_dataset + '#fillmismatch', open_mode) 30 | netcdf_dataset = _netcdf_dataset 31 | except Exception as e: 32 | logger.error('Unable to open {}: {}'.format(netcdf_dataset, e)) 33 | return 34 | 35 | elif type(netcdf_dataset) != netCDF4.Dataset: # NetCDF4.Dataset object provided 36 | raise TypeError('Invalid netcdf_dataset type') 37 | 38 | # Dataset has line and line_index variables => must be a line dataset 39 | if set(['line', 'line_index']) <= set(netcdf_dataset.variables.keys()): 40 | return NetCDFLineUtils(netcdf_dataset, debug=debug) 41 | 42 | # Dataset has 2D (or higher dimensionality) variable with grid_mapping attribute and indexing variables 43 | elif len([variable 44 | for variable in netcdf_dataset.variables.values() 45 | if hasattr(variable, 'grid_mapping') # "crs" or similar variable specified 46 | and variable.grid_mapping in netcdf_dataset.variables.keys() # "crs" or similar variable exists 47 | and len(variable.dimensions) >= 2 # Data variable is a grid 48 | and set(variable.dimensions) <= set(netcdf_dataset.variables.keys()) # Indexing variables exist 49 | ]) > 0: 50 | return NetCDFGridUtils(netcdf_dataset, debug=debug) 51 | 52 | # TODO: Make sure that there are no other tests we could apply here for point datasets 53 | else: 54 | return NetCDFPointUtils(netcdf_dataset, debug=debug) 55 | -------------------------------------------------------------------------------- /geophys_utils/_transect_utils.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # =============================================================================== 4 | # Copyright 2017 Geoscience Australia 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # =============================================================================== 18 | ''' 19 | Created on 23Nov.,2016 20 | 21 | @author: Alex Ip 22 | ''' 23 | import math 24 | 25 | import numpy as np 26 | 27 | from ._crs_utils import get_utm_wkt, transform_coords 28 | 29 | 30 | def line_length(line): 31 | ''' 32 | Function to return length of line 33 | @param line: iterable containing two two-ordinate iterables, e.g. 2 x 2 array or 2-tuple of 2-tuples 34 | 35 | @return length: Distance between start & end points in native units 36 | ''' 37 | return math.sqrt(math.pow(line[1][0] - line[0][0], 2.0) + 38 | math.pow(line[1][1] - line[0][1], 2.0)) 39 | 40 | 41 | def point_along_line(line, distance): 42 | ''' 43 | Function to return a point the specified distance along the line 44 | @param line: iterable containing two two-ordinate iterables, e.g. 2 x 2 array or 2-tuple of 2-tuples 45 | @param distance: Distance along line new point should be 46 | 47 | @return point: Coordinates of point along line or None if distance > line length 48 | ''' 49 | length = line_length(line) 50 | proportion = distance / length 51 | 52 | if proportion < 0 or proportion > 1: 53 | return None 54 | 55 | return tuple([line[0][dim_index] + proportion * 56 | (line[1][dim_index] - line[0][dim_index]) for dim_index in range(2)]) 57 | 58 | 59 | def utm_coords(coordinate_array, wkt): 60 | ''' 61 | Function to convert coordinates to the appropriate UTM CRS 62 | @param coordinate_array: Array of shape (n, 2) or iterable containing coordinate pairs 63 | 64 | @return wkt: WKT for UTM CRS 65 | @return coordinate_array: Array of shape (n, 2) containing UTM coordinate pairs 66 | ''' 67 | native_centre_coords = (np.nanmean(coordinate_array[:, 0]), np.nanmean(coordinate_array[:, 1])) 68 | utm_wkt = get_utm_wkt(native_centre_coords, wkt) 69 | return utm_wkt, np.array(transform_coords(coordinate_array, wkt, utm_wkt)) 70 | 71 | 72 | def coords2distance(coordinate_array): 73 | ''' 74 | Function to calculate cumulative distance in metres from native (lon/lat) coordinates 75 | @param coordinate_array: Array of shape (n, 2) or iterable containing coordinate pairs 76 | 77 | @return distance_array: Array of shape (n) containing cumulative distances from first coord 78 | ''' 79 | coord_count = coordinate_array.shape[0] 80 | distance_array = np.zeros((coord_count,), coordinate_array.dtype) 81 | cumulative_distance = 0.0 82 | distance_array[0] = cumulative_distance 83 | last_point = coordinate_array[0] 84 | 85 | for coord_index in range(1, coord_count): 86 | point = coordinate_array[coord_index] 87 | distance = math.sqrt(math.pow(point[0] - last_point[0], 2.0) + math.pow(point[1] - last_point[1], 2.0)) 88 | distance = line_length((point, last_point)) 89 | cumulative_distance += distance 90 | distance_array[coord_index] = cumulative_distance 91 | last_point = point 92 | 93 | return distance_array 94 | 95 | 96 | def sample_transect(transect_vertices, wkt, sample_metres): 97 | ''' 98 | Function to return a list of sample points sample_metres apart along lines between transect vertices 99 | @param transect_vertices: list or array of transect vertex coordinates 100 | @param wkt: coordinate reference system for transect_vertices 101 | @param sample_metres: distance between sample points in metres 102 | ''' 103 | transect_vertex_array = np.array(transect_vertices) 104 | # print 'transect_vertex_array = %s' % transect_vertex_array 105 | nominal_utm_wkt, utm_transect_vertices = utm_coords(transect_vertex_array, wkt) 106 | # print 'nominal_utm_wkt = %s' % nominal_utm_wkt 107 | # print 'utm_transect_vertices = %s' % utm_transect_vertices 108 | 109 | sample_points = [] 110 | residual = 0 111 | for vertex_index in range(len(utm_transect_vertices) - 1): 112 | utm_line = (utm_transect_vertices[ 113 | vertex_index], utm_transect_vertices[vertex_index + 1]) 114 | # print 'utm_line = %s' % (utm_line,) 115 | utm_line_length = line_length(utm_line) 116 | # print 'utm_line_length = %s' % utm_line_length 117 | 118 | # Skip lines of infinite length 119 | if utm_line_length == float('inf'): 120 | continue 121 | 122 | sample_count = int((utm_line_length + residual) // sample_metres) 123 | # print 'sample_count = %s' % sample_count 124 | if not sample_count: 125 | residual += utm_line_length 126 | continue 127 | 128 | if residual: # Use un-sampled distance from last line 129 | start_point = point_along_line( 130 | utm_line, sample_metres - residual) 131 | else: 132 | start_point = utm_line[0] # Start at beginning 133 | # print 'start_point = %s' % (start_point,) 134 | 135 | # Calculate new residual 136 | residual = (utm_line_length + residual) % sample_metres 137 | # print 'residual = %s' % residual 138 | 139 | end_point = point_along_line(utm_line, utm_line_length - residual) 140 | # print 'end_point = %s' % (end_point,) 141 | 142 | try: 143 | sample_point_array = np.stack([np.linspace(start_point[dim_index], end_point[ 144 | dim_index], sample_count + 1) for dim_index in range(2)]).transpose() 145 | # print 'sample_point_array.shape = %s' % 146 | # (sample_point_array.shape,) 147 | except Exception as e: 148 | print('Line sampling failed: {}'.format(e)) 149 | residual = 0 150 | continue 151 | 152 | sample_points += list(sample_point_array) 153 | 154 | # Don't double up end point with next start point 155 | if (not residual) and (vertex_index < 156 | len(utm_transect_vertices) - 1): 157 | sample_points.pop() 158 | 159 | return transform_coords( 160 | sample_points, nominal_utm_wkt, wkt), sample_metres 161 | -------------------------------------------------------------------------------- /geophys_utils/csw_find.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # =============================================================================== 4 | # Copyright 2017 Geoscience Australia 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # =============================================================================== 18 | ''' 19 | Created on 19 Oct. 2018 20 | 21 | @author: Alex Ip - Geoscience Australia 22 | ''' 23 | import argparse 24 | import logging 25 | import re 26 | import sys 27 | 28 | from unidecode import unidecode 29 | 30 | from geophys_utils import CSWUtils, date_string2datetime 31 | 32 | root_logger = logging.getLogger() 33 | root_logger.setLevel(logging.INFO) # Initial logging level for this module 34 | 35 | 36 | def main(): 37 | ''' 38 | Main function to take command line parameters, perform CSW query and print required output 39 | ''' 40 | 41 | def quote_delimitedtext(text, delimiter, quote_char='"'): 42 | ''' 43 | Helper function to quote text containing delimiters or whitespace 44 | ''' 45 | try: 46 | if delimiter in text or quote_char in text or re.search('\s', text): 47 | if delimiter == ',': # Use double quote to escape quote character for CSV 48 | return quote_char + text.replace(quote_char, quote_char + quote_char) + quote_char 49 | else: # Use backslash to escape quote character for tab or space delimited text 50 | return quote_char + text.replace(quote_char, '\\' + quote_char) + quote_char 51 | else: 52 | return text 53 | except UnicodeEncodeError: 54 | root_logger.error(text) 55 | raise 56 | 57 | # Define command line arguments 58 | parser = argparse.ArgumentParser() 59 | 60 | parser.add_argument("-i", "--identifiers", 61 | help="comma separated list of possible metadata identifiers (UUID) for search", type=str) 62 | parser.add_argument("-j", "--alt_identifiers", 63 | help="comma separated list of possible metadata alternate identifiers (eCat ID) for search", 64 | type=str) 65 | parser.add_argument("-k", "--keywords", help="comma-separated list of required keywords for search", type=str) 66 | parser.add_argument("-t", "--titlewords", help="comma-separated list of required titlewords for search", type=str) 67 | parser.add_argument("-a", "--anytext", help="comma-separated list of required text snippets for search", type=str) 68 | parser.add_argument("-b", "--bounds", 69 | help='comma-separated ,,, ordinates of bounding box for search. N.B: A leading "-" sign on this list should NOT be preceded by a space', 70 | type=str) 71 | parser.add_argument("-c", "--crs", 72 | help='coordinate reference system for bounding box coordinates for search. Default determined by settings file.', 73 | type=str) 74 | parser.add_argument("-s", "--start_date", help="start date for search", type=str) 75 | parser.add_argument("-e", "--end_date", help="end date for search", type=str) 76 | # Parameters to define output 77 | parser.add_argument("-p", "--protocols", 78 | help='comma-separated list of possible distribution protocols for output. Default determined by settings file, "*" = wildcard, "None" = no distributions.', 79 | type=str) 80 | parser.add_argument("-f", "--fields", 81 | help='comma-separated list of fields for output. Default determined by settings file, "*" = wildcard.', 82 | type=str) 83 | parser.add_argument("-d", "--delimiter", help='field delimiter for output. Defaults to "\t"', type=str) 84 | parser.add_argument("-u", "--urls", 85 | help="CSW URL(s) to query (comma separated list). Default determined by settings file.", 86 | type=str) 87 | parser.add_argument("-m", "--max_results", help="Maximum number of records to return", type=int) 88 | parser.add_argument('-r', '--header_row', action='store_const', const=True, 89 | help='show header row. Default determined by settings file') 90 | parser.add_argument('-l', '--get_layers', action='store_const', const=True, 91 | help='get WMS/WCS layer names. Default determined by settings file') 92 | parser.add_argument('--debug', action='store_const', const=True, default=False, 93 | help='output debug information. Default is no debug info') 94 | parser.add_argument("-y", "--types", help="comma-separated list of possible record types for search", type=str) 95 | 96 | args = parser.parse_args() 97 | 98 | # CONVERTING INPUT TO CORRECT DATA TYPES AND FORMATS 99 | 100 | # Convert string to list of floats 101 | # if the user inputs an argument for bounds, then convert this string to a list of floats 102 | # whereas each list object is split at the commas. 103 | # If the user does not call the bounds argument, then don't use it. 104 | if args.bounds: 105 | bounds = [float(ordinate) for ordinate in args.bounds.split(',')] 106 | else: 107 | bounds = None 108 | 109 | # Convert string to datetime 110 | start_date = date_string2datetime(args.start_date) 111 | 112 | # Convert string to datetime 113 | end_date = date_string2datetime(args.end_date) 114 | 115 | # creatse a CSW object and populates the parameters with argparse inputs 116 | 117 | url_list = ([url.strip() for url in args.urls.split(',')] if args.urls else None) 118 | 119 | cswu = CSWUtils(url_list, 120 | debug=args.debug) 121 | 122 | # If there is a protocol list, then create a list of protocols that are split at the comma, use defaults if there isn't 123 | # Replace "None" with empty string 124 | protocol_list = (([protocol.strip().lower().replace('none', '') for protocol in args.protocols.split(',')] 125 | if args.protocols is not None else None) 126 | or cswu.settings['OUTPUT_DEFAULTS']['DEFAULT_PROTOCOLS']) 127 | 128 | # Allow wildcard - protocol_list=None means show all protocols 129 | if '*' in protocol_list: 130 | protocol_list = None 131 | 132 | # formatting the output for fields 133 | field_list = ([field.strip().lower() for field in args.fields.split(',')] if args.fields else None) or \ 134 | cswu.settings['OUTPUT_DEFAULTS']['DEFAULT_FIELDS'] 135 | # Allow wildcard - field_list=None means show all fields 136 | if '*' in field_list: 137 | field_list = None 138 | 139 | # Set default delimiter to tab character 140 | delimiter = args.delimiter or cswu.settings['OUTPUT_DEFAULTS']['DEFAULT_DELIMITER'] 141 | 142 | record_generator = cswu.query_csw(identifier_list=args.identifiers, 143 | alt_identifier_list=args.alt_identifiers, 144 | keyword_list=args.keywords, 145 | anytext_list=args.anytext, 146 | titleword_list=args.titlewords, 147 | bounding_box=bounds, 148 | start_datetime=start_date, 149 | stop_datetime=end_date, 150 | record_type_list=args.types, 151 | max_total_records=args.max_results, 152 | get_layers=args.get_layers 153 | ) 154 | 155 | header_row_required = (cswu.settings['OUTPUT_DEFAULTS']['DEFAULT_SHOW_HEADER_ROW'] 156 | if args.header_row is None 157 | else args.header_row) 158 | 159 | # Print results 160 | header_printed = False 161 | distribution_count = 0 162 | for distribution in cswu.get_distributions(protocol_list, record_generator): 163 | # for distribution in cswu.get_netcdf_urls(record_generator): 164 | distribution_count += 1 165 | 166 | # Print header if required 167 | if header_row_required and not header_printed: 168 | print(delimiter.join([field 169 | for field in (field_list or sorted(distribution.keys())) 170 | ] 171 | ) 172 | ) 173 | header_printed = True; 174 | 175 | # Decode and quote fields if required 176 | print(delimiter.join([quote_delimitedtext(unidecode(distribution.get(field) or ''), delimiter) 177 | for field in (field_list or sorted(distribution.keys())) 178 | ] 179 | ) 180 | ) 181 | 182 | root_logger.debug('{} results found.'.format(distribution_count)) 183 | 184 | 185 | if __name__ == '__main__': 186 | if not root_logger.handlers: 187 | # Set handler for root root_logger to standard output 188 | console_handler = logging.StreamHandler(sys.stdout) 189 | # console_handler.setLevel(logging.INFO) 190 | console_handler.setLevel(logging.DEBUG) 191 | console_formatter = logging.Formatter('%(message)s') 192 | console_handler.setFormatter(console_formatter) 193 | root_logger.addHandler(console_handler) 194 | 195 | main() 196 | -------------------------------------------------------------------------------- /geophys_utils/csw_utils_settings.yml: -------------------------------------------------------------------------------- 1 | #=============================================================================== 2 | # Copyright 2017 Geoscience Australia 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | #=============================================================================== 16 | # YAML settings file for csw_utils 17 | --- 18 | DEFAULT_CSW_URLS: # List of CSW URLs to query 19 | - "https://ecat.ga.gov.au/geonetwork/srv/eng/csw" # GA's externally-facing eCat 20 | # - "https://internal.ecat.ga.gov.au/geonetwork/srv/eng/csw" # GA's internal eCat 21 | # - "https://geonetwork.nci.org.au/geonetwork/srv/eng/csw" # NCI's GeoNetwork 22 | # - "http://localhost:8080/geonetwork/srv/eng/csw" # Local GeoNetwork 23 | ENVIRONMENT_VARIABLES: # Environment variables defined as key-value pairs 24 | HTTP_PROXY: "" 25 | HTTPS_PROXY: "" 26 | NO_PROXY: "" 27 | # HTTP_PROXY: "http://proxy.ga.gov.au:8080" 28 | # HTTPS_PROXY: "https://proxy.ga.gov.au:8080" 29 | # HTTP_PROXY: "http://user:password@proxy.ga.gov.au:8080" 30 | # HTTPS_PROXY: "https://user:password@proxy.ga.gov.au:8080" 31 | # NO_PROXY: "intranet.ga.gov.au" 32 | # HTTP_PROXY: "http://sun-web-intdev.ga.gov.au:2710" 33 | # HTTPS_PROXY: "https://sun-web-intdev.ga.gov.au:2710" 34 | 35 | DEFAULT_TIMEOUT: 30 # Timeout in seconds 36 | DEFAULT_CRS: "CRS84" # Unprojected WGS84 with lon-lat ordering. See https://gis.stackexchange.com/questions/124050/how-do-i-specify-the-lon-lat-ordering-in-csw-bounding-box-request 37 | DEFAULT_MAXQUERYRECORDS: 100 # Retrieve only this many datasets per CSW query per server 38 | DEFAULT_MAXTOTALRECORDS: 2000 # Maximum total number of records to retrieve per server 39 | DEFAULT_GET_LAYERS: False # Boolean flag indicating whether WMS & WCS layer names should be discovered (potentially slow) 40 | DEFAULT_RECORD_TYPES: # List of record types to return (empty means return all) 41 | # - "dataset" 42 | 43 | OUTPUT_DEFAULTS: # Parameters governing the defaults for printed output 44 | DEFAULT_PROTOCOLS: # List of protocols to display by default. * = wildcard 45 | - "file" 46 | 47 | DEFAULT_FIELDS: # List of fields to display by default. * = wildcard 48 | - "protocol" 49 | - "url" 50 | - "title" 51 | 52 | DEFAULT_DELIMITER: "\t" # Delimiter character used to separate fields 53 | DEFAULT_SHOW_HEADER_ROW: False # Boolean flag indicating whether to show header row containing field labels 54 | -------------------------------------------------------------------------------- /geophys_utils/dataset_metadata_cache/__init__.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on 20 Jul. 2018 3 | 4 | @author: Alex 5 | ''' 6 | from ._dataset_metadata_cache import settings, DatasetMetadataCache, Dataset, Distribution 7 | from ._postgres_dataset_metadata_cache import PostgresDatasetMetadataCache 8 | from ._sqlite_dataset_metadata_cache import SQLiteDatasetMetadataCache 9 | 10 | 11 | def get_dataset_metadata_cache(db_engine='SQLite', *args, **kwargs): 12 | ''' 13 | Class factory function to return subclass of DatasetMetadataCache for specified db_engine 14 | ''' 15 | if db_engine == 'SQLite': 16 | return SQLiteDatasetMetadataCache(*args, **kwargs) 17 | elif db_engine == 'Postgres': 18 | return PostgresDatasetMetadataCache(*args, **kwargs) 19 | else: 20 | raise BaseException('Unhandled db_engine "{}"'.format(db_engine)) 21 | -------------------------------------------------------------------------------- /geophys_utils/dataset_metadata_cache/_dataset_metadata_cache.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on 19 Jul. 2018 3 | 4 | @author: Alex Ip 5 | ''' 6 | 7 | import abc 8 | import logging 9 | import os 10 | 11 | import yaml 12 | 13 | logger = logging.getLogger(__name__) 14 | logger.setLevel(logging.INFO) # Initial logging level for this module 15 | 16 | settings = yaml.safe_load( 17 | open(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'dataset_metadata_cache_settings.yml'))) 18 | 19 | 20 | class Distribution(object): 21 | ''' 22 | Distribution class definition 23 | ''' 24 | 25 | def __init__(self, 26 | url, 27 | protocol 28 | ): 29 | ''' 30 | Distribution class Constructor 31 | ''' 32 | self.url = url 33 | self.protocol = protocol 34 | 35 | 36 | class Dataset(object): 37 | ''' 38 | Dataset class definition 39 | ''' 40 | 41 | def __init__(self, 42 | dataset_title, 43 | ga_survey_id, 44 | longitude_min, 45 | longitude_max, 46 | latitude_min, 47 | latitude_max, 48 | convex_hull_polygon, 49 | keyword_list, 50 | distribution_list, 51 | point_count, 52 | metadata_uuid=None, 53 | start_date=None, 54 | end_date=None 55 | ): 56 | ''' 57 | Dataset class Constructor 58 | ''' 59 | self.dataset_title = dataset_title 60 | self.ga_survey_id = ga_survey_id 61 | self.longitude_min = longitude_min 62 | self.longitude_max = longitude_max 63 | self.latitude_min = latitude_min 64 | self.latitude_max = latitude_max 65 | self.convex_hull_polygon = convex_hull_polygon 66 | self.metadata_uuid = metadata_uuid 67 | self.keyword_list = keyword_list 68 | self.distribution_list = distribution_list 69 | self.point_count = point_count 70 | self.start_date = start_date 71 | self.end_date = end_date 72 | 73 | 74 | class DatasetMetadataCache(object): 75 | ''' 76 | DatasetMetadataCache class definition 77 | ''' 78 | # Tuple containing field names for results of search_dataset_distributions function 79 | dataset_distribution_search_fields = ('ga_survey_id', 80 | 'dataset_title', 81 | 'distribution_url', 82 | 'convex_hull_polygon', 83 | 'longitude_min', 84 | 'longitude_max', 85 | 'latitude_min', 86 | 'latitude_max', 87 | 'point_count', 88 | 'start_date', 89 | 'end_date', 90 | 'metadata_uuid' 91 | ) 92 | 93 | _db_engine = None 94 | 95 | @abc.abstractmethod 96 | def __init__(self, debug): 97 | ''' 98 | DatasetMetadataCache class Constructor 99 | @parameter debug: Boolean flag indicating whether debug output is required 100 | ''' 101 | # Initialise and set debug property 102 | self._debug = None 103 | self.debug = debug 104 | 105 | @abc.abstractmethod 106 | def add_dataset(self, dataset): 107 | ''' 108 | Function to insert or update dataset record 109 | ''' 110 | return 111 | 112 | @abc.abstractmethod 113 | def add_survey(self, 114 | ga_survey_id, 115 | survey_name=None 116 | ): 117 | ''' 118 | Function to insert survey if necessary 119 | ''' 120 | return 121 | 122 | @abc.abstractmethod 123 | def add_keywords(self, 124 | dataset_id, 125 | keyword_list): 126 | ''' 127 | Function to insert new keywords 128 | ''' 129 | return 130 | 131 | @abc.abstractmethod 132 | def add_distributions(self, 133 | dataset_id, 134 | distribution_list): 135 | ''' 136 | Function to insert new distributions 137 | ''' 138 | return 139 | 140 | @abc.abstractmethod 141 | def search_dataset_distributions(self, 142 | keyword_list, 143 | protocol, 144 | ll_ur_coords=None 145 | ): 146 | ''' 147 | Function to return list of dicts containing metadata for all datasets with specified keywords and bounding box 148 | Note that keywords are searched exclusively, i.e. using "and", not "or" 149 | ''' 150 | return 151 | 152 | @property 153 | def db_engine(self): 154 | return type(self)._db_engine 155 | 156 | @property 157 | def debug(self): 158 | return self._debug 159 | 160 | @debug.setter 161 | def debug(self, debug_value): 162 | if self._debug != debug_value or self._debug is None: 163 | self._debug = debug_value 164 | 165 | if self._debug: 166 | logger.setLevel(logging.DEBUG) 167 | logging.getLogger(self.__module__).setLevel(logging.DEBUG) 168 | else: 169 | logger.setLevel(logging.INFO) 170 | logging.getLogger(self.__module__).setLevel(logging.INFO) 171 | 172 | logger.debug('Logger {} set to level {}'.format(logger.name, logger.level)) 173 | logging.getLogger(self.__module__).debug('Logger {} set to level {}'.format(self.__module__, logger.level)) 174 | -------------------------------------------------------------------------------- /geophys_utils/dataset_metadata_cache/data/dataset_metadata_cache.sqlite: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GeoscienceAustralia/geophys_utils/ecd8da91310eceb3aa6bf4f342b37ec275fb457a/geophys_utils/dataset_metadata_cache/data/dataset_metadata_cache.sqlite -------------------------------------------------------------------------------- /geophys_utils/dataset_metadata_cache/dataset_metadata_cache_settings.yml: -------------------------------------------------------------------------------- 1 | --- 2 | DEBUG: True 3 | DB_ENGINE: "Postgres" 4 | LOGFILE: "dataset_metadata_cache.log" 5 | LOGLEVEL: "WARNING" 6 | SQLITE_DB_PATH: "data/dataset_metadata_cache.sqlite" 7 | POSTGRES_SERVER: "localhost" 8 | POSTGRES_PORT: 5432 9 | POSTGRES_DBNAME: "dataset_metadata_cache" 10 | POSTGRES_USER: "db_user" 11 | POSTGRES_PASSWORD: "db_password" 12 | LOCAL_TIMEZONE: "AEDT" -------------------------------------------------------------------------------- /geophys_utils/dataset_metadata_cache/dataset_search_demo.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on 20 Jul. 2018 3 | 4 | @author: Alex 5 | ''' 6 | from geophys_utils.dataset_metadata_cache import get_dataset_metadata_cache 7 | 8 | DEBUG = False 9 | 10 | DATABASE_ENGINE = 'SQLite' 11 | 12 | 13 | # DATABASE_ENGINE = 'Postgres' 14 | 15 | def main(): 16 | dmc = get_dataset_metadata_cache(db_engine=DATABASE_ENGINE, debug=DEBUG) 17 | 18 | metadata_list = dmc.search_dataset_distributions( 19 | keyword_list=['AUS', 'ground digital data', 'gravity', 'geophysical survey', 'points'], 20 | protocol='opendap', 21 | # ll_ur_coords=[[-179.9, -90.0], [180.0, 90.0]] 22 | ll_ur_coords=[[138.193588256836, -30.5767288208008], [138.480285644531, -30.1188278198242]] 23 | ) 24 | 25 | print('Search results:') 26 | for metadata in metadata_list: 27 | print(metadata) # You would do your own thing here. 28 | 29 | print('{} datasets found.'.format(len(metadata_list))) 30 | 31 | 32 | if __name__ == '__main__': 33 | main() 34 | -------------------------------------------------------------------------------- /geophys_utils/dataset_metadata_cache/sqlite_dataset_metadata_cache_ddl.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE dataset ( 2 | dataset_id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL UNIQUE, 3 | dataset_title character varying(128) NOT NULL, 4 | survey_id INTEGER, 5 | longitude_min double precision NOT NULL CHECK((-180 < longitude_min) AND (longitude_min <= 180)), 6 | longitude_max double precision NOT NULL CHECK((-180 < longitude_max) AND (longitude_max <= 180)), 7 | latitude_min double precision NOT NULL CHECK((-90 <= latitude_min) AND (latitude_min <= 90)), 8 | latitude_max double precision NOT NULL CHECK((-90 <= latitude_max) AND (latitude_max <= 90)), 9 | convex_hull_polygon text, 10 | metadata_uuid character(36) NOT NULL UNIQUE, 11 | point_count INTEGER, 12 | FOREIGN KEY (survey_id) REFERENCES survey(survey_id) ON UPDATE CASCADE 13 | ); 14 | 15 | CREATE TABLE dataset_keyword ( 16 | dataset_id INTEGER NOT NULL, 17 | keyword_id INTEGER NOT NULL, 18 | FOREIGN KEY (dataset_id) REFERENCES dataset(dataset_id) ON UPDATE CASCADE, 19 | FOREIGN KEY (keyword_id) REFERENCES keyword(keyword_id) ON UPDATE CASCADE, 20 | UNIQUE (dataset_id, keyword_id) 21 | ); 22 | 23 | CREATE TABLE distribution ( 24 | distribution_id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL UNIQUE, 25 | dataset_id INTEGER NOT NULL, 26 | distribution_url character varying(256) NOT NULL, -- Should be globally unique 27 | protocol_id INTEGER NOT NULL, 28 | FOREIGN KEY (dataset_id) REFERENCES dataset(dataset_id) ON UPDATE CASCADE, 29 | FOREIGN KEY (protocol_id) REFERENCES protocol(protocol_id) ON UPDATE CASCADE 30 | UNIQUE (dataset_id, distribution_url) 31 | ); 32 | 33 | CREATE TABLE keyword ( 34 | keyword_id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL UNIQUE, 35 | keyword_value character varying(64) NOT NULL UNIQUE, 36 | keyword_url character varying(256) 37 | ); 38 | 39 | CREATE TABLE protocol ( 40 | protocol_id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL UNIQUE, 41 | protocol_value character varying(32) NOT NULL UNIQUE 42 | ); 43 | 44 | CREATE TABLE survey ( 45 | survey_id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL UNIQUE, 46 | ga_survey_id character varying(16) NOT NULL UNIQUE, 47 | survey_name character varying(128), 48 | start_date date, 49 | end_date date 50 | ); 51 | 52 | CREATE INDEX fki_dataset_keyword_dataset_id ON dataset_keyword (dataset_id); 53 | 54 | CREATE INDEX fki_dataset_keyword_keyword_id ON dataset_keyword (keyword_id); 55 | 56 | CREATE INDEX fki_dataset_survey_id ON dataset (survey_id); 57 | 58 | CREATE INDEX fki_distribution_dataset_id ON distribution (dataset_id); 59 | 60 | CREATE INDEX fki_distribution_protocol_id ON distribution (protocol_id); 61 | 62 | 63 | 64 | CREATE INDEX dataset_longitude_min_idx ON dataset (longitude_min); 65 | CREATE INDEX dataset_longitude_max_idx ON dataset (longitude_max); 66 | CREATE INDEX dataset_latitude_min_idx ON dataset (latitude_min); 67 | CREATE INDEX dataset_latitude_max_idx ON dataset (latitude_max); 68 | 69 | -------------------------------------------------------------------------------- /geophys_utils/netcdf_converter/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Created on 13 Jun. 2018 3 | 4 | @author: Alex Ip 5 | """ 6 | from geophys_utils.netcdf_converter._to_netcdf_converter import ( 7 | NetCDFVariable, 8 | ToNetCDFConverter, 9 | ) 10 | from geophys_utils.netcdf_converter._to_netcdf_converter_national import ( 11 | NetCDFVariableNational, 12 | ToNetCDFConverterNational, 13 | ) 14 | -------------------------------------------------------------------------------- /geophys_utils/netcdf_converter/aseg_gdf_settings.yml: -------------------------------------------------------------------------------- 1 | # Default settings file for ASEGGDF2NetCDFConverter class 2 | 3 | # Field definition overrides for definitions read from .dfn file 4 | # keyed by lower-case ASEG-GDF field name. 5 | # N.B: overriding short_name will effectively rename field for subsequent use 6 | field_definitions: 7 | # 1D Variables 8 | doi: { short_name: depth_of_investigation, units: m } 9 | dataresidual: { long_name: Data residual, short_name: data_residual } 10 | residual2: { long_name: Data residual 2, short_name: data_residual2 } 11 | residual3: { long_name: Data residual 3, short_name: data_residual3 } 12 | stddev_TxHeightInverted: { long_name: Uncertainty of the inverted height 13 | of the TXRX frame, short_name: stddev_tx_height_inverted, units: m } 14 | totalresidual: { long_name: Total Residual, short_name: total_residual } 15 | txheightinverted: { long_name: Inverted height of the TXRX frame, short_name: tx_height_inverted, 16 | units: m } 17 | txheightmeasured: { long_name: Measured height of the TXRX frame, short_name: tx_height_measured, 18 | units: m } 19 | easting: { long_name: UTM Easting (x) ordinate, short_name: easting, units: m } 20 | elevation: { long_name: Ground elevation relative to sea-level, short_name: elevation, 21 | units: m } 22 | lat: { long_name: Latitude, short_name: latitude, units: degrees North } 23 | lon: { long_name: Longitude, short_name: longitude, units: degrees East } 24 | gda94lat: { long_name: Latitude, short_name: latitude, units: degrees North } 25 | gda94lon: { long_name: Longitude, short_name: longitude, units: degrees East } 26 | gda94long: { long_name: Longitude, short_name: longitude, units: degrees East } 27 | gda94llg: { long_name: Longitude, short_name: longitude, units: degrees East } 28 | latgda94: { long_name: Latitude, short_name: latitude, units: degrees North } 29 | longda94: { long_name: Longitude, short_name: longitude, units: degrees East } 30 | longgda94: { long_name: Longitude, short_name: longitude, units: degrees East } 31 | gnss_latitude_gda94: { short_name: latitude, units: degrees North } 32 | gnss_longitude_gda94: { short_name: longitude, units: degrees East } 33 | line: { long_name: Line number, short_name: line } 34 | flight: { long_name: Flight number, short_name: flight } 35 | flt: { long_name: Flight number, short_name: flight } 36 | nlayers: { long_name: layers, short_name: layer } 37 | ndatlm: { long_name: low moment gates, short_name: low_moment_gate } 38 | ndathm: { long_name: high moment gates, short_name: high_moment_gate } 39 | northing: { long_name: UTM Northing (y) ordinate, short_name: northing, units: m } 40 | sequence: { long_name: Inversion sequence number, short_name: sequence } 41 | # 2D Variables - must have dimensions attribute defined 42 | # dimensions attribute must either be a single dimension name or list of part dimension names 43 | depth_top: { long_name: Depth to the top of the layer, short_name: layer_top_depth, units: m, dimensions: layer } 44 | layer_top_depth: { long_name: Depth to the top of the layer, short_name: layer_top_depth, units: m, dimensions: layer } 45 | resistivity: { long_name: Layer resistivity, short_name: resistivity, 46 | units: ohm.m, dimensions: layer } 47 | conductivity: { long_name: Layer conductivity, short_name: conductivity, 48 | units: S/m, dimensions: layer } 49 | resistivity_uncertainty: { long_name: Uncertainty of the layer resistivity, 50 | short_name: resistivity_uncertainty, dimensions: layer } 51 | data_values: { long_name: Data values measured at each gate, short_name: data_values, units: V/(A.turns.m4), 52 | dimensions: [ low_moment_gate, high_moment_gate ] } 53 | data_response: { long_name: Data response, short_name: data_response, dimensions: [ low_moment_gate, high_moment_gate ] } 54 | data_relative_uncertainty: { long_name: Relative uncertainty of data value, short_name: data_relative_uncertainty, 55 | dimensions: [ low_moment_gate, high_moment_gate ] } 56 | thickness: { dimensions: layer } 57 | emsystem_1_xs: { dimensions: secondary_field_window } 58 | emsystem_1_zs: { dimensions: secondary_field_window } 59 | emx_nonhprg: { dimensions: emx_window } 60 | emx_hprg: { dimensions: emx_window } 61 | emz_nonhprg: { dimensions: emx_window } 62 | emz_hprg: { dimensions: emx_window } 63 | cond_xz: { dimensions: layer } 64 | cond_xz_depth: { dimensions: layer } 65 | 66 | # Known custom .dfn attributes to write into variable attributes with mappings to new names 67 | # N.B: Do NOT include standard attributes like "NAME", "UNIT" or "UNITS" 68 | variable_attributes: 69 | DATUM: datum_name 70 | PROJECTION: projection_name 71 | 72 | # String containing comma-separated list of keywords 73 | keywords: geophysics, airborne, AEM, conductivity 74 | 75 | # List of possible dimension field names definining dimension sizes. 76 | # Uses overridden short_name values (if set) 77 | # Not used for output - should be defined in .dfn file instead 78 | dimension_fields: [ layer, low_moment_gate, high_moment_gate ] 79 | 80 | # List of possible lookup field names to indicate which fields need to be turned into lookup arrays 81 | # Uses overridden short_name values (if set) 82 | lookup_fields: [ line, flight ] 83 | 84 | # List of possible ASEG-GDF fields to be ignored (i.e. not output to netCDF) 85 | # Uses overridden short_name values (if set) 86 | ignored_fields: [ ] # e.g. Proj_CGG, Proj_Client 87 | 88 | # NetCDF variable name to ASEG-GDF field name mappings for ASEG-GDF output 89 | aseg_field_mapping: 90 | longitude: LONGITUD 91 | latitude: LATITUDE 92 | easting: EASTING 93 | northing: NORTHING 94 | 95 | # WKT for unprojected CRS if only UTM provided 96 | default_crs_wkt: GDA94 -------------------------------------------------------------------------------- /geophys_utils/netcdf_converter/csv2netcdf_converter.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # =============================================================================== 4 | # Copyright 2017 Geoscience Australia 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # =============================================================================== 18 | ''' 19 | CSV2NetCDFConverter concrete class for converting data to netCDF 20 | 21 | Created on 28Mar.2018 22 | 23 | @author: Alex Ip 24 | ''' 25 | from collections import OrderedDict 26 | 27 | import numpy as np 28 | 29 | from geophys_utils.netcdf_converter import ToNetCDFConverter, NetCDFVariable 30 | 31 | 32 | class CSV2NetCDFConverter(ToNetCDFConverter): 33 | ''' 34 | CSV2NetCDFConverter concrete class for converting CSV data to netCDF 35 | ''' 36 | 37 | def __init__(self, nc_out_path, csv_path, netcdf_format='NETCDF4_CLASSIC'): 38 | ''' 39 | Concrete constructor for subclass CSV2NetCDFConverter 40 | Needs to initialise object with everything that is required for the other Concrete methods 41 | N.B: Make sure the base class constructor is called from the subclass constructor 42 | ''' 43 | ToNetCDFConverter.__init__(self, nc_out_path, netcdf_format) 44 | 45 | self.csv_path = csv_path 46 | 47 | def get_global_attributes(self): 48 | ''' 49 | Concrete method to return dict of global attribute : pairs 50 | ''' 51 | return {'title': 'test dataset'} 52 | 53 | def get_dimensions(self): 54 | ''' 55 | Concrete method to return OrderedDict of : pairs 56 | ''' 57 | dimensions = OrderedDict() 58 | 59 | # Example lat/lon dimensions 60 | dimensions['lon'] = 509 61 | dimensions['lat'] = 639 62 | 63 | return dimensions 64 | 65 | def variable_generator(self): 66 | ''' 67 | Concrete generator to yield NetCDFVariable objects 68 | ''' 69 | # Example of latitude dimension variable creation 70 | yield self.build_dimension_variable(dimension_name='lat', 71 | min_value=-22.9247209891964, 72 | max_value=-20.5641209891964, 73 | long_name='latitude', 74 | units='degrees north', 75 | standard_name='latitude', 76 | descending=True # Invert Y axis 77 | ) 78 | 79 | # Example of longitude dimension variable creation 80 | yield self.build_dimension_variable(dimension_name='lon', 81 | min_value=121.122089060582, 82 | max_value=123.001689060582, 83 | long_name='longitude', 84 | units='degrees east', 85 | standard_name='longitude', 86 | descending=False 87 | ) 88 | 89 | # Example of crs variable creation for GDA94 90 | yield self.build_crs_variable('''\ 91 | GEOGCS["GDA94", 92 | DATUM["Geocentric_Datum_of_Australia_1994", 93 | SPHEROID["GRS 1980",6378137,298.257222101, 94 | AUTHORITY["EPSG","7019"]], 95 | TOWGS84[0,0,0,0,0,0,0], 96 | AUTHORITY["EPSG","6283"]], 97 | PRIMEM["Greenwich",0, 98 | AUTHORITY["EPSG","8901"]], 99 | UNIT["degree",0.0174532925199433, 100 | AUTHORITY["EPSG","9122"]], 101 | AUTHORITY["EPSG","4283"]] 102 | ''' 103 | ) 104 | 105 | yield NetCDFVariable(short_name='test_data', 106 | data=np.random.random((self.nc_output_dataset.dimensions['lat'].size, 107 | self.nc_output_dataset.dimensions['lon'].size)), 108 | dimensions=['lat', 'lon'], 109 | fill_value=0.0, 110 | attributes={'units': 'random crap', 111 | 'long_name': 'random numbers between 0 and 1' 112 | }, 113 | dtype=np.float32 114 | ) 115 | 116 | return 117 | 118 | 119 | def main(): 120 | nc_out_path = 'C:\\Temp\\test.nc' 121 | c2n = CSV2NetCDFConverter(nc_out_path) 122 | c2n.convert2netcdf() 123 | print('Finished writing netCDF file {}'.format(nc_out_path)) 124 | 125 | 126 | if __name__ == '__main__': 127 | main() 128 | -------------------------------------------------------------------------------- /geophys_utils/netcdf_converter/grav2netcdf_converter_national_global_attribute_setter.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import cx_Oracle 3 | from netCDF4 import Dataset 4 | import yaml 5 | import numpy as np 6 | from datetime import datetime 7 | from geophys_utils import points2convex_hull 8 | import logging 9 | 10 | logger = logging.getLogger(__name__) 11 | logger.setLevel(logging.DEBUG) 12 | 13 | assert len(sys.argv) == 5, '....' 14 | nc_path = sys.argv[1] 15 | oracle_db = sys.argv[2] 16 | u_id = sys.argv[3] 17 | pw = sys.argv[4] 18 | con = cx_Oracle.connect(u_id, pw, oracle_db) 19 | cursor = con.cursor() 20 | 21 | ds = Dataset("{}".format(nc_path), mode='a') 22 | 23 | yaml_sql_settings = yaml.safe_load(open('grav2netcdf_converter_national_sql_strings.yml')) 24 | sql_strings_dict = yaml_sql_settings['sql_strings_dict'] 25 | 26 | start_date_sql = sql_strings_dict['get_national_survey_metadata_order_by_startdate'] 27 | start_date_query_result = cursor.execute(start_date_sql) 28 | field_names = [field_desc[0] for field_desc in start_date_query_result.description] 29 | start_date_zipped_data = dict(zip(field_names, start_date_query_result.fetchone())) 30 | earliest_start_date = start_date_zipped_data['STARTDATE'] 31 | 32 | end_date_sql = sql_strings_dict['get_national_survey_metadata_order_by_enddate_desc'] 33 | end_date_query_result = cursor.execute(end_date_sql) 34 | field_names = [field_desc[0] for field_desc in end_date_query_result.description] 35 | end_date_zipped_data = dict(zip(field_names, end_date_query_result.fetchone())) 36 | latest_end_date = end_date_zipped_data['ENDDATE'] 37 | 38 | print(earliest_start_date) 39 | print(latest_end_date) 40 | 41 | metadata_dict = { 42 | 'title': "Australian National Ground Gravity Compilation 2023", 43 | 'Conventions': "CF-1.6,ACDD-1.3", 44 | 'keywords': "points, gravity, geophysical survey, Earth sciences, geophysics, geoscientific Information", 45 | 'geospatial_lon_min': np.min(ds.variables['longitude']), 46 | 'geospatial_lon_max': np.max(ds.variables['longitude']), 47 | 'geospatial_lon_units': "degrees_east", 48 | 'geospatial_lon_resolution': "point", 49 | 'geospatial_lat_min': np.min(ds.variables['latitude']), 50 | 'geospatial_lat_max': np.max(ds.variables['latitude']), 51 | 'geospatial_lat_units': "degrees_north", 52 | 'geospatial_lat_resolution': "point", 53 | 'history': "Pulled from point gravity database at Geoscience Australia", 54 | 'summary': "Compilation of ground gravity surveys acquired in Australia. Data acquired from State and " 55 | "National Geological Surveys, Academics, and private companies. Data has gone through Quality " 56 | "Control processes. Station spacing ranges from 11km to less than 1km. The accuracy of the data " 57 | "varies generally with date of acquisition, later data using high precision gravity meters and " 58 | "GPS for better accuracies.", 59 | 'location_accuracy_min': np.min(ds.variables['locacc']), 60 | 'location_accuracy_max': np.max(ds.variables['locacc']), 61 | 'location_accuracy_units': "m", 62 | 'elevation_accuracy_min': np.min(ds.variables['gndelevacc']), 63 | 'elevation_accuracy_max': np.max(ds.variables['gndelevacc']), 64 | 'elevation_accuracy_units': "m", 65 | 'gravity_accuracy_min': np.min(ds.variables['gravacc']), 66 | 'gravity_accuracy_max': np.max(ds.variables['gravacc']), 67 | 'gravity_accuracy_units': "um/s^2", 68 | 'time_coverage_start': str(earliest_start_date), 69 | 'time_coverage_end': str(latest_end_date), 70 | 'time_coverage_duration': str(latest_end_date - earliest_start_date), 71 | 'date_created': datetime.now().isoformat(), 72 | 'institution': 'Geoscience Australia', 73 | 'source': 'ground observation', 74 | 'cdm_data_type': 'Point' 75 | } 76 | 77 | try: 78 | # Compute convex hull and add GML representation to metadata 79 | coordinates = np.array(list(zip(ds.variables['longitude'][:], 80 | ds.variables['latitude'][:] 81 | ) 82 | ) 83 | ) 84 | if len(coordinates) >= 3: 85 | convex_hull = points2convex_hull(coordinates) 86 | metadata_dict['geospatial_bounds'] = 'POLYGON((' + ', '.join([' '.join( 87 | ['%.4f' % ordinate for ordinate in coordinates]) for coordinates in convex_hull]) + '))' 88 | elif len(coordinates) == 2: # Two points - make bounding box 89 | bounding_box = [[min(coordinates[:, 0]), min(coordinates[:, 1])], 90 | [max(coordinates[:, 0]), min(coordinates[:, 1])], 91 | [max(coordinates[:, 0]), max(coordinates[:, 1])], 92 | [min(coordinates[:, 0]), max(coordinates[:, 1])], 93 | [min(coordinates[:, 0]), min(coordinates[:, 1])] 94 | ] 95 | metadata_dict['geospatial_bounds'] = 'POLYGON((' + ', '.join([' '.join( 96 | ['%.4f' % ordinate for ordinate in coordinates]) for coordinates in bounding_box]) + '))' 97 | elif len(coordinates) == 1: # Single point 98 | # TODO: Check whether this is allowable under ACDD 99 | metadata_dict['geospatial_bounds'] = 'POINT((' + ' '.join( 100 | ['%.4f' % ordinate for ordinate in coordinates[0]]) + '))' 101 | except: 102 | logger.warning('Unable to write global attribute "geospatial_bounds"') 103 | 104 | print(metadata_dict) 105 | 106 | for attribute_name, attribute_value in iter(metadata_dict.items()): 107 | setattr(ds, attribute_name, attribute_value or '') 108 | 109 | cursor.close() 110 | ds.close() 111 | -------------------------------------------------------------------------------- /geophys_utils/netcdf_converter/grav2netcdf_converter_national_remove_points_outside_bbox.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from netCDF4 import Dataset 3 | import numpy as np 4 | 5 | assert len(sys.argv) == 3, '...' 6 | nc_in_path = sys.argv[1] 7 | nc_out_path = sys.argv[2] 8 | 9 | # Input file 10 | ds_in = Dataset('{}'.format(nc_in_path)) 11 | 12 | # Output file 13 | ds_out = Dataset('{}'.format(nc_out_path), mode='w', format='NETCDF4') 14 | 15 | # Copy dimensions 16 | for dim_name, dim in ds_in.dimensions.items(): 17 | ds_out.createDimension(dim_name, len(dim) if not dim.isunlimited() else None) 18 | 19 | # Get the indices of the points that lie outside the national bounding box 20 | lon = ds_in.variables['longitude'][:] 21 | lat = ds_in.variables['latitude'][:] 22 | lon_indices_outside_box = np.where((lon < 110.8) | (lon > 156))[0] 23 | lat_indices_outside_box = np.where((lat < -45) | (lat > -9))[0] 24 | indices_outside_box = np.concatenate((lon_indices_outside_box, lat_indices_outside_box)) 25 | 26 | base_variable_parameters = {'complevel': 4, 'endian': 'little', 'fletcher32': True, 'shuffle': True, 'zlib': True} 27 | 28 | for var_name, var in ds_in.variables.items(): 29 | 30 | if var_name == 'crs': 31 | variable_parameters = base_variable_parameters 32 | else: 33 | variable_parameters = base_variable_parameters 34 | variable_parameters['chunksizes'] = var.chunking() 35 | variable_parameters['fill_value'] = var._FillValue 36 | 37 | out_var = ds_out.createVariable(var_name, var.datatype, var.dimensions, **variable_parameters) 38 | 39 | if 'point' in var.dimensions: 40 | out_var[:] = np.delete(var[:], indices_outside_box) 41 | else: 42 | out_var[:] = var[:] 43 | 44 | out_var.setncatts({attr: var.getncattr(attr) for attr in var.ncattrs() if attr != '_FillValue'}) 45 | 46 | ds_out.close() 47 | 48 | -------------------------------------------------------------------------------- /geophys_utils/netcdf_converter/grav2netcdf_converter_national_settings.yml: -------------------------------------------------------------------------------- 1 | field_names: 2 | Obsno: {short_name: Obsno, long_name: Observation Number, database_field_name: obsno, dtype: int32, fill_value: -9999.9} 3 | Stationno: {short_name: Stationno, long_name: Station Number, database_field_name: STATIONNO, dtype: int64, fill_value: -999999999.9} 4 | Stattype: {short_name: Stattype, long_name: Station Type, database_field_name: STATIONTYPE, dtype: int8, lookup_table: STATIONTYPES, convert_keys_and_data_to_int8: True, fill_value: -99} 5 | 6 | Lat: {standard_name: latitude, long_name: Latitude, database_field_name: dlat, dtype: float64, units: degrees_north, fill_value: -99999.9, axis: Y} 7 | Long: {standard_name: longitude, long_name: Longitude, database_field_name: dlong, dtype: float64, units: degrees_east, fill_value: -99999.9, axis: X} 8 | Locmethod: {short_name: Locmethod, long_name: Location Method, database_field_name: LOCMETH, dtype: int8, lookup_table: LOCMETHODS, convert_keys_and_data_to_int8: True, fill_value: -99} 9 | Locacc: {short_name: Locacc, long_name: Location Accuracy, database_field_name: LOCACC, dtype: float32, units: m, fill_value: -99999.9} 10 | Locaccmethod: {short_name: Locaccmethod, long_name: Location Accuracy Method, database_field_name: LOCACCMETHOD, dtype: int8, lookup_table: ACCURACYMETHOD, fill_value: -99} 11 | 12 | ##TODO Spherical Cap Bouguer Anomaly (SCBA) include acryonm? SphericalCapBouguerAnomaly as ncname? 13 | Freeair: {short_name: Freeair, long_name: Spherical Cap Bouguer Anomaly, database_field_name: 'gravity.GRAV_FA07(o1.dlat, o1.ellipsoidhgt, o1.grav, o1.ellipsoidmeterhgt)', dtype: float32, units: um/s^2, fill_value: -99999.9} 14 | Bouguer: {short_name: Bouguer, long_name: Ellipsoidal Free, database_field_name: 'gravity.grav_BA07(o1.dlat, o1.ellipsoidhgt, o1.grav, o1.ellipsoidmeterhgt, o1.gndelevtype, o1.gndelev, o1.nvalue)', dtype: float32, units: um/s^2, fill_value: -99999.9} 15 | 16 | Grav: {short_name: Grav, long_name: Gravity, database_field_name: GRAV, dtype: float64, units: um/s^2, fill_value: -99999.9} 17 | Gravmeth: {short_name: Gravmeth, long_name: Gravity Method, database_field_name: GRAVMETH, dtype: int8, lookup_table: GRAVMETHODS, fill_value: -99} 18 | Gravacc: {short_name: Gravacc, long_name: Gravity Accuracy, database_field_name: GRAVACC, dtype: float32, units: um/s^2, datum: GRAVDATUM, convert_key_to_lookup_value_for_datum: True, convert_keys_and_data_to_int8: True, fill_value: -99} 19 | Gravdatum: {short_name: Gravdatum, long_name: Gravity Datum, database_field_name: GRAVDATUM, dtype: int8, lookup_table: GRAVDATUMS, fill_value: -99} 20 | Gravaccmeth: {short_name: Gravaccmeth, long_name: Gravity Accuracy Method, database_field_name: GRAVACCMETHOD, dtype: int8, lookup_table: ACCURACYMETHOD, fill_value: -99} 21 | 22 | Gndelev: {short_name: Gndelev, long_name: Ground Elevation, database_field_name: GNDELEV, dtype: float32, units: m, fill_value: -99999.9} 23 | Gndelevacc: {short_name: Gndelevacc, long_name: Ground Level Accuracy, database_field_name: GNDELEVACC, dtype: float32, units: m, fill_value: -99} 24 | Gndelevtype: {short_name: Gndelevtype, long_name: Ground Level Type, database_field_name: GNDELEVTYPE, dtype: int8, lookup_table: GNDELEVTYPES, convert_keys_and_data_to_int8: True, fill_value: -99} 25 | Gndelevdatum: {short_name: Gndelevdatum, long_name: Ground Level Datum, database_field_name: GNDELEVDATUM, dtype: int8, lookup_table: GNDELEVDATUMS, fill_value: -99} 26 | Gndelevmeth: {short_name: Gndelevmeth, long_name: Ground Level Method, database_field_name: GNDELEVMETH, dtype: int8, lookup_table: GNDELEVMETHODS, convert_keys_and_data_to_int8: True, fill_value: -99} 27 | Gndelevaccmethod: {short_name: Gndelevaccmethod, long_name: Ground Level Accuracy Method, database_field_name: GNDELEVACCMETHOD, dtype: int8, lookup_table: ACCURACYMETHOD, fill_value: -99} 28 | 29 | Insthgt: {short_name: Insthgt, long_name: Instrument Height, database_field_name: METERHGT, dtype: float32, units: m, fill_value: -99999.9} 30 | Insthgterr: {short_name: Insthgterr, long_name: Instrument Height Error, database_field_name: METERHGTERR, dtype: float32, units: m, fill_value: -99} 31 | Insthgtmeth: {short_name: Insthgtmeth, long_name: Instrument Height Method, database_field_name: METERHGTMETH, dtype: int8, lookup_table: GNDELEVMETHODS, convert_keys_and_data_to_int8: True, fill_value: -99} 32 | Insthgterrmeth: {short_name: Insthgterrmeth, long_name: Instrument Height Error Method, database_field_name: METERHGTERRMETHOD, dtype: int8, lookup_table: ACCURACYMETHOD, fill_value: -99} 33 | 34 | Ellipsoidinsthgt: {short_name: Ellipsoidinsthgt, long_name: Ellipsoid Instrument Height, database_field_name: ELLIPSOIDMETERHGT, dtype: float32, units: m, fill_value: -99999.9} 35 | Ellipsoidinsthgterr: {short_name: Ellipsoidinsthgterr, long_name: Ellipsoid Instrument Height Error, database_field_name: ELLIPSOIDMETERHGTERR, dtype: float32, unit: m, fill_value: -99999.9} 36 | Ellipsoidinsthgterrmethod: {short_name: Ellipsoidinsthgterrmethod, long_name: Ellipsoid Instrument Height Accuracy Method, database_field_name: ELLIPSOIDMETERHGTERRMETHOD, dtype: int8, lookup_table: ACCURACYMETHOD, fill_value: -99} 37 | 38 | Ellipsoidhgt: {short_name: Ellipsoidhgt, long_name: Ellipsoid Height, database_field_name: ELLIPSOIDHGT, dtype: float32, units: m, fill_value: -99999.9} 39 | Ellipsoidhgtacc: {short_name: Ellipsoidhgtacc, long_name: Ellipsoid Height Accuracy, database_field_name: ELLIPSOIDHGTACC, dtype: float32, units: m, fill_value: -99999.9} 40 | Ellipsoidhgtmeth: {short_name: Ellipsoidhgtmeth, long_name: Ellipsoid Height Method, database_field_name: ELLIPSOIDHGTMETH, dtype: int8, lookup_table: GNDELEVMETHODS, convert_keys_and_data_to_int8: True, fill_value: -99} 41 | Ellipsoidhgtaccmethod: {short_name: Ellipsoidhgtaccmethod, long_name: Ellipsoid Height Accuracy Method, database_field_name: ELLIPSOIDHGTACCMETHOD, dtype: int8, lookup_table: ACCURACYMETHOD, fill_value: -99} 42 | Ellipsoiddatum: {short_name: Ellipsoiddatum, long_name: Ellipsoid Datum, database_field_name: ELLIPSOIDHGTDATUM, dtype: int8, lookup_table: ELLIPSOIDHGTDATUM, convert_keys_and_data_to_int8: True, fill_value: -99} 43 | 44 | Tc: {short_name: Tc, long_name: Terrain Correction, database_field_name: TC, dtype: float32, units: um/s^2, dem: TCDEM, fill_value: -99999.9} 45 | Tcdensity: {short_name: Tcdensity, long_name: TC Density, database_field_name: TCDENSITY, dtype: float32, units: um/s^2, dem: TCDEM, fill_value: -99999.9} 46 | Tcerr: {short_name: Tcerr, long_name: TC Error, database_field_name: TCERR, dtype: float32, units: um/s^2, dem: TCDEM, fill_value: -99999.9} 47 | Tcmeth: {short_name: Tcmeth, long_name: TC Method, database_field_name: TCMETH, dtype: int8, lookup_table: TCMETHODS, convert_keys_and_data_to_int8: True, fill_value: -99} 48 | 49 | Gridflag: {short_name: Gridflag, long_name: Grid Flag, database_field_name: GRIDFLAG, dtype: int8, lookup_table: GRIDFLAGS, fill_value: -99} 50 | Reliab: {short_name: Reliab, long_name: Estimation of Station Reliability, database_field_name: reliab, dtype: int8, lookup_table: RELIABILITY, fill_value: -99} 51 | ## 52 | # 53 | # 54 | # 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | #TODO it would be nice to have some logic that checks if a value such as gndelevdatum or gravmeth are consistent for all points in a survey and create a variable level attribute, otherwise create the lookup table. However, whats more importance, consistency in structure or file size? 66 | 67 | 68 | 69 | # GRAVMETH 70 | 71 | # GRAVACCMETHOD 72 | 73 | # LOCMETH 74 | 75 | # LOCACCMETHOD 76 | 77 | # GNDELEVMETH 78 | 79 | # GNDELEVACCMETHOD 80 | 81 | # METERHGTMETH 82 | 83 | # METERHGTERRMETHOD? 84 | # 85 | # TCMETH 86 | # no tc error method 87 | 88 | # ELLIPSOIDHGTMETH 89 | 90 | # ELLIPSOIDHGTACCMETHOD 91 | # ELLIPSOIDMETERHGTMETH 92 | 93 | # ELLIPSOIDMETERHGTERRMETHOD 94 | 95 | # NVALUEMETH 96 | # NVALUEACCMETHOD 97 | # NONGATCERRMETHOD 98 | # NONGATCMETH 99 | -------------------------------------------------------------------------------- /geophys_utils/netcdf_converter/grav2netcdf_converter_national_sql_strings.yml: -------------------------------------------------------------------------------- 1 | sql_strings_dict: 2 | sql_get_surveyids: 3 | > 4 | select Surveyid from gravity.GRAVSURVEYS gs 5 | where exists (select o1.Surveyid from gravity.OBSERVATIONS o1 6 | left join gravity.OBSERVATIONS o2 7 | on 8 | o1.surveyid = o2.surveyid 9 | and (o1.entrydate > o2.entrydate OR(o1.entrydate = o2.entrydate and o1.obsno > o2.obsno)) 10 | and o1.geodetic_datum = o2.geodetic_datum 11 | and o1.dlat = o2.dlat 12 | and o1.dlong = o2.dlong 13 | and o1.access_code = o2.access_code 14 | and o1.status = o2.status 15 | where 16 | o1.surveyid = gs.surveyid 17 | and o1.status = 'A' 18 | and o1.access_code = 'O' 19 | and o1.grav is not null 20 | and o1.gndelev is not null 21 | and o1.meterhgt is not null 22 | and o1.nvalue is not null 23 | and o1.ellipsoidhgt is not null 24 | and o1.ellipsoidmeterhgt is not null 25 | and o1.eno in (select eno from a.surveys where countryid is null or countryid = 'AUS') 26 | and o2.obsno is null) 27 | -- and gs.surveyid = '201780' -- *** Use this for testing only *** 28 | order by gs.SURVEYID 29 | sql_get_count_surveyids: 30 | > 31 | select count(Surveyid) from gravity.GRAVSURVEYS gs 32 | where exists (select o1.Surveyid from gravity.OBSERVATIONS o1 33 | left join gravity.OBSERVATIONS o2 34 | on 35 | o1.surveyid = o2.surveyid 36 | and (o1.entrydate > o2.entrydate OR(o1.entrydate = o2.entrydate and o1.obsno > o2.obsno)) 37 | and o1.geodetic_datum = o2.geodetic_datum 38 | and o1.dlat = o2.dlat 39 | and o1.dlong = o2.dlong 40 | and o1.access_code = o2.access_code 41 | and o1.status = o2.status 42 | where 43 | o1.surveyid = gs.surveyid 44 | and o1.status = 'A' 45 | and o1.access_code = 'O' 46 | and o1.grav is not null 47 | and o1.gndelev is not null 48 | and o1.meterhgt is not null 49 | and o1.nvalue is not null 50 | and o1.ellipsoidhgt is not null 51 | and o1.ellipsoidmeterhgt is not null 52 | and o1.eno in (select eno from a.surveys where countryid is null or countryid = 'AUS') 53 | and o2.obsno is null) 54 | -- and gs.surveyid = '201780' -- *** Use this for testing only *** 55 | order by gs.SURVEYID 56 | get_survey_metadata: 57 | > 58 | select * from gravity.GRAVSURVEYS gs 59 | inner join a.surveys using(eno) 60 | where 61 | gs.surveyid = {0} 62 | and exists 63 | (select o1.* from gravity.OBSERVATIONS o1 64 | left join gravity.OBSERVATIONS o2 65 | on 66 | o1.surveyid = o2.surveyid 67 | and (o1.entrydate > o2.entrydate OR(o1.entrydate = o2.entrydate and o1.obsno > o2.obsno)) 68 | and o1.geodetic_datum = o2.geodetic_datum 69 | and o1.dlat = o2.dlat 70 | and o1.dlong = o2.dlong 71 | and o1.access_code = o2.access_code 72 | and o1.status = o2.status 73 | where 74 | o1.surveyid = {0} 75 | and o1.status = 'A' 76 | and o1.access_code = 'O' 77 | and o1.dlat is not null 78 | and o1.dlong is not null 79 | and o1.grav is not null 80 | and o1.gndelev is not null 81 | and o1.meterhgt is not null 82 | and o1.nvalue is not null 83 | and o1.ellipsoidhgt is not null 84 | and o1.ellipsoidmeterhgt is not null 85 | and o1.eno in (select eno from a.surveys where countryid is null or countryid = 'AUS') 86 | and o2.obsno is null) 87 | get_national_survey_metadata: 88 | > 89 | select a.SURVEYNAME, a.SURVEYID, gs.LAYOUT, a.SURVEYTYPE, a.OWNER, a.OPERATOR, a.CONTRACTOR, a.PROCESSOR, a.ON_OFF, 90 | a.SPACEMIN, a.SPACEMAX, gs.STATIONS, a.STARTDATE, a.ENDDATE 91 | from gravity.GRAVSURVEYS gs 92 | inner join a.surveys a using(eno) 93 | where exists 94 | (select o1.* from gravity.OBSERVATIONS o1 95 | left join gravity.OBSERVATIONS o2 96 | on 97 | o1.surveyid = o2.surveyid 98 | and (o1.entrydate > o2.entrydate OR(o1.entrydate = o2.entrydate and o1.obsno > o2.obsno)) 99 | and o1.geodetic_datum = o2.geodetic_datum 100 | and o1.dlat = o2.dlat 101 | and o1.dlong = o2.dlong 102 | and o1.access_code = o2.access_code 103 | and o1.status = o2.status 104 | where 105 | o1.surveyid = gs.surveyid 106 | and o1.status = 'A' 107 | and o1.access_code = 'O' 108 | and o1.dlat is not null 109 | and o1.dlong is not null 110 | and o1.grav is not null 111 | and o1.gndelev is not null 112 | and o1.meterhgt is not null 113 | and o1.nvalue is not null 114 | and o1.ellipsoidhgt is not null 115 | and o1.ellipsoidmeterhgt is not null 116 | and o1.eno in (select eno from a.surveys where countryid is null or countryid = 'AUS') 117 | and o2.obsno is null) 118 | order by gs.SURVEYID 119 | get_national_survey_metadata_order_by_startdate: 120 | > 121 | select a.STARTDATE 122 | from gravity.GRAVSURVEYS gs 123 | inner join a.surveys a using(eno) 124 | where exists 125 | (select o1.* from gravity.OBSERVATIONS o1 126 | left join gravity.OBSERVATIONS o2 127 | on 128 | o1.surveyid = o2.surveyid 129 | and (o1.entrydate > o2.entrydate OR(o1.entrydate = o2.entrydate and o1.obsno > o2.obsno)) 130 | and o1.geodetic_datum = o2.geodetic_datum 131 | and o1.dlat = o2.dlat 132 | and o1.dlong = o2.dlong 133 | and o1.access_code = o2.access_code 134 | and o1.status = o2.status 135 | where 136 | o1.surveyid = gs.surveyid 137 | and o1.status = 'A' 138 | and o1.access_code = 'O' 139 | and o1.dlat is not null 140 | and o1.dlong is not null 141 | and o1.grav is not null 142 | and o1.gndelev is not null 143 | and o1.meterhgt is not null 144 | and o1.nvalue is not null 145 | and o1.ellipsoidhgt is not null 146 | and o1.ellipsoidmeterhgt is not null 147 | and o1.eno in (select eno from a.surveys where countryid is null or countryid = 'AUS') 148 | and o2.obsno is null) 149 | order by a.STARTDATE 150 | get_national_survey_metadata_order_by_enddate_desc: 151 | > 152 | select a.ENDDATE 153 | from gravity.GRAVSURVEYS gs 154 | inner join a.surveys a using(eno) 155 | where exists 156 | (select o1.* from gravity.OBSERVATIONS o1 157 | left join gravity.OBSERVATIONS o2 158 | on 159 | o1.surveyid = o2.surveyid 160 | and (o1.entrydate > o2.entrydate OR(o1.entrydate = o2.entrydate and o1.obsno > o2.obsno)) 161 | and o1.geodetic_datum = o2.geodetic_datum 162 | and o1.dlat = o2.dlat 163 | and o1.dlong = o2.dlong 164 | and o1.access_code = o2.access_code 165 | and o1.status = o2.status 166 | where 167 | o1.surveyid = gs.surveyid 168 | and o1.status = 'A' 169 | and o1.access_code = 'O' 170 | and o1.dlat is not null 171 | and o1.dlong is not null 172 | and o1.grav is not null 173 | and o1.gndelev is not null 174 | and o1.meterhgt is not null 175 | and o1.nvalue is not null 176 | and o1.ellipsoidhgt is not null 177 | and o1.ellipsoidmeterhgt is not null 178 | and o1.eno in (select eno from a.surveys where countryid is null or countryid = 'AUS') 179 | and o2.obsno is null) 180 | order by a.ENDDATE DESC 181 | get_data: 182 | > 183 | select NVL({0}, {1}) from gravity.OBSERVATIONS o1 184 | left join gravity.OBSERVATIONS o2 185 | on 186 | o1.surveyid = o2.surveyid 187 | and (o1.entrydate > o2.entrydate OR(o1.entrydate = o2.entrydate and o1.obsno > o2.obsno)) 188 | and o1.geodetic_datum = o2.geodetic_datum 189 | and o1.dlat = o2.dlat 190 | and o1.dlong = o2.dlong 191 | and o1.access_code = o2.access_code 192 | and o1.status = o2.status 193 | where 194 | o1.surveyid = {2} 195 | and o1.status = 'A' 196 | and o1.access_code = 'O' 197 | and o1.grav is not null 198 | and o1.gndelev is not null 199 | and o1.meterhgt is not null 200 | and o1.nvalue is not null 201 | and o1.ellipsoidhgt is not null 202 | and o1.ellipsoidmeterhgt is not null 203 | and o1.eno in (select eno from a.surveys where countryid is null or countryid = 'AUS') 204 | and o2.obsno is null 205 | order by o1.obsno 206 | get_field_description: 207 | > 208 | SELECT COMMENTS 209 | FROM ALL_COL_COMMENTS 210 | WHERE TABLE_NAME = 'OBSERVATIONS' 211 | AND COLUMN_NAME = '{}' 212 | get_dimensions: 213 | > 214 | select count(*) from gravity.OBSERVATIONS o1 215 | left join gravity.OBSERVATIONS o2 216 | on 217 | o1.surveyid = o2.surveyid 218 | and (o1.entrydate > o2.entrydate OR(o1.entrydate = o2.entrydate and o1.obsno > o2.obsno)) 219 | and o1.geodetic_datum = o2.geodetic_datum 220 | and o1.dlat = o2.dlat 221 | and o1.dlong = o2.dlong 222 | and o1.access_code = o2.access_code 223 | and o1.status = o2.status 224 | where 225 | o1.surveyid = {} 226 | and o1.status = 'A' 227 | and o1.access_code = 'O' 228 | and o1.grav is not null 229 | and o1.gndelev is not null 230 | and o1.meterhgt is not null 231 | and o1.nvalue is not null 232 | and o1.ellipsoidhgt is not null 233 | and o1.ellipsoidmeterhgt is not null 234 | and o1.eno in (select eno from a.surveys where countryid is null or countryid = 'AUS') 235 | and o2.obsno is null 236 | check_all_values_in_survey_consistent: 237 | > 238 | SELECT DISTINCT {0} 239 | FROM gravity.OBSERVATIONS o1 240 | WHERE o1.surveyid = {1} 241 | get_ellipsoidhgt_datums_lookup: 242 | > 243 | SELECT DISTINCT ELLIPSOIDHGTDATUM 244 | FROM gravity.OBSERVATIONS o1 -------------------------------------------------------------------------------- /geophys_utils/netcdf_converter/grav2netcdf_converter_settings.yml: -------------------------------------------------------------------------------- 1 | field_names: 2 | Obsno: { short_name: Obsno, long_name: Observation Number, database_field_name: obsno, dtype: int32, fill_value: -9999.9 } 3 | Stationno: { short_name: Stationno, long_name: Station Number, database_field_name: STATIONNO, dtype: int64, fill_value: -999999999.9 } 4 | Stattype: { short_name: Stattype, long_name: Station Type, database_field_name: STATIONTYPE, dtype: int8, lookup_table: STATIONTYPES, convert_keys_and_data_to_int8: True, fill_value: -99 } 5 | 6 | Lat: { standard_name: latitude, long_name: Latitude, database_field_name: dlat, dtype: float64, units: degrees_north, fill_value: -99999.9, axis: Y } 7 | Long: { standard_name: longitude, long_name: Longitude, database_field_name: dlong, dtype: float64, units: degrees_east, fill_value: -99999.9, axis: X } 8 | Locmethod: { short_name: Locmethod, long_name: Location Method, database_field_name: LOCMETH, dtype: int8, lookup_table: LOCMETHODS, convert_keys_and_data_to_int8: True, fill_value: -99 } 9 | Locacc: { short_name: Locacc, long_name: Location Accuracy, database_field_name: LOCACC, dtype: float32, units: m, fill_value: -99999.9 } 10 | Locaccmethod: { short_name: Locaccmethod, long_name: Location Accuracy Method, database_field_name: LOCACCMETHOD, dtype: int8, lookup_table: ACCURACYMETHOD, fill_value: -99 } 11 | 12 | ##TODO Spherical Cap Bouguer Anomaly (SCBA) include acryonm? SphericalCapBouguerAnomaly as ncname? 13 | Freeair: { short_name: Freeair, long_name: Spherical Cap Bouguer Anomaly, database_field_name: 'gravity.GRAV_FA07(o1.dlat, o1.ellipsoidhgt, o1.grav, o1.ellipsoidmeterhgt)', dtype: float32, units: um/s^2, fill_value: -99999.9 } 14 | Bouguer: { short_name: Bouguer, long_name: Ellipsoidal Free, database_field_name: 'gravity.grav_BA07(o1.dlat, o1.ellipsoidhgt, o1.grav, o1.ellipsoidmeterhgt, o1.gndelevtype, o1.gndelev, o1.nvalue)', dtype: float32, units: um/s^2, fill_value: -99999.9 } 15 | 16 | Grav: { short_name: Grav, long_name: Gravity, database_field_name: GRAV, dtype: float64, units: um/s^2, fill_value: -99999.9 } 17 | Gravmeth: { short_name: Gravmeth, long_name: Gravity Method, database_field_name: GRAVMETH, dtype: int8, lookup_table: GRAVMETHODS, fill_value: -99 } 18 | Gravacc: { short_name: Gravacc, long_name: Gravity Accuracy, database_field_name: GRAVACC, dtype: float32, units: um/s^2, datum: GRAVDATUM, convert_key_to_lookup_value_for_datum: True, convert_keys_and_data_to_int8: True, fill_value: -99 } 19 | Gravdatum: { short_name: Gravdatum, long_name: Gravity Datum, database_field_name: GRAVDATUM, dtype: int8, lookup_table: GRAVDATUMS, fill_value: -99 } 20 | Gravaccmeth: { short_name: Gravaccmeth, long_name: Gravity Accuracy Method, database_field_name: GRAVACCMETHOD, dtype: int8, lookup_table: ACCURACYMETHOD, fill_value: -99 } 21 | 22 | Gndelev: { short_name: Gndelev, long_name: Ground Elevation, database_field_name: GNDELEV, dtype: float32, units: m, fill_value: -99999.9 } 23 | Gndelevacc: { short_name: Gndelevacc, long_name: Ground Level Accuracy, database_field_name: GNDELEVACC, dtype: float32, units: m, fill_value: -99 } 24 | Gndelevtype: { short_name: Gndelevtype, long_name: Ground Level Type, database_field_name: GNDELEVTYPE, dtype: int8, lookup_table: GNDELEVTYPES, convert_keys_and_data_to_int8: True, fill_value: -99 } 25 | Gndelevdatum: { short_name: Gndelevdatum, long_name: Ground Level Datum, database_field_name: GNDELEVDATUM, dtype: int8, lookup_table: GNDELEVDATUMS, fill_value: -99 } 26 | Gndelevmeth: { short_name: Gndelevmeth, long_name: Ground Level Method, database_field_name: GNDELEVMETH, dtype: int8, lookup_table: GNDELEVMETHODS, convert_keys_and_data_to_int8: True, fill_value: -99 } 27 | Gndelevaccmethod: { short_name: Gndelevaccmethod, long_name: Ground Level Accuracy Method, database_field_name: GNDELEVACCMETHOD, dtype: int8, lookup_table: ACCURACYMETHOD, fill_value: -99 } 28 | 29 | Insthgt: { short_name: Insthgt, long_name: Instrument Height, database_field_name: METERHGT, dtype: float32, units: m, fill_value: -99999.9 } 30 | Insthgterr: { short_name: Insthgterr, long_name: Instrument Height Error, database_field_name: METERHGTERR, dtype: float32, units: m, fill_value: -99 } 31 | Insthgtmeth: { short_name: Insthgtmeth, long_name: Instrument Height Method, database_field_name: METERHGTMETH, dtype: int8, lookup_table: GNDELEVMETHODS, convert_keys_and_data_to_int8: True, fill_value: -99 } 32 | Insthgterrmeth: { short_name: Insthgterrmeth, long_name: Instrument Height Error Method, database_field_name: METERHGTERRMETHOD, dtype: int8, lookup_table: ACCURACYMETHOD, fill_value: -99 } 33 | 34 | Ellipsoidinsthgt: { short_name: Ellipsoidinsthgt, long_name: Ellipsoid Instrument Height, database_field_name: ELLIPSOIDMETERHGT, dtype: float32, units: m, fill_value: -99999.9 } 35 | Ellipsoidinsthgterr: { short_name: Ellipsoidinsthgterr, long_name: Ellipsoid Instrument Height Error, database_field_name: ELLIPSOIDMETERHGTERR, dtype: float32, unit: m, fill_value: -99999.9 } 36 | Ellipsoidinsthgterrmethod: { short_name: Ellipsoidinsthgterrmethod, long_name: Ellipsoid Instrument Height Accuracy Method, database_field_name: ELLIPSOIDMETERHGTERRMETHOD, dtype: int8, lookup_table: ACCURACYMETHOD, fill_value: -99 } 37 | 38 | Ellipsoidhgt: { short_name: Ellipsoidhgt, long_name: Ellipsoid Height, database_field_name: ELLIPSOIDHGT, dtype: float32, units: m, fill_value: -99999.9 } 39 | Ellipsoidhgtacc: { short_name: Ellipsoidhgtacc, long_name: Ellipsoid Height Accuracy, database_field_name: ELLIPSOIDHGTACC, dtype: float32, units: m, fill_value: -99999.9 } 40 | Ellipsoidhgtmeth: { short_name: Ellipsoidhgtmeth, long_name: Ellipsoid Height Method, database_field_name: ELLIPSOIDHGTMETH, dtype: int8, lookup_table: GNDELEVMETHODS, convert_keys_and_data_to_int8: True, fill_value: -99 } 41 | Ellipsoidhgtaccmethod: { short_name: Ellipsoidhgtaccmethod, long_name: Ellipsoid Height Accuracy Method, database_field_name: ELLIPSOIDHGTACCMETHOD, dtype: int8, lookup_table: ACCURACYMETHOD, fill_value: -99 } 42 | Ellipsoiddatum: { short_name: Ellipsoiddatum, long_name: Ellipsoid Datum, database_field_name: ELLIPSOIDHGTDATUM, dtype: int8, lookup_table: ELLIPSOIDHGTDATUM, convert_keys_and_data_to_int8: True, fill_value: -99 } 43 | 44 | Tc: { short_name: Tc, long_name: Terrain Correction, database_field_name: TC, dtype: float32, units: um/s^2, dem: TCDEM, fill_value: -99999.9 } 45 | Tcdensity: { short_name: Tcdensity, long_name: TC Density, database_field_name: TCDENSITY, dtype: float32, units: um/s^2, dem: TCDEM, fill_value: -99999.9 } 46 | Tcerr: { short_name: Tcerr, long_name: TC Error, database_field_name: TCERR, dtype: float32, units: um/s^2, dem: TCDEM, fill_value: -99999.9 } 47 | Tcmeth: { short_name: Tcmeth, long_name: TC Method, database_field_name: TCMETH, dtype: int8, lookup_table: TCMETHODS, convert_keys_and_data_to_int8: True, fill_value: -99 } 48 | 49 | Gridflag: { short_name: Gridflag, long_name: Grid Flag, database_field_name: GRIDFLAG, dtype: int8, lookup_table: GRIDFLAGS, fill_value: -99 } 50 | Reliab: { short_name: Reliab, long_name: Estimation of Station Reliability, database_field_name: reliab, dtype: int8, lookup_table: RELIABILITY, fill_value: -99 } 51 | ## 52 | # 53 | # 54 | # 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | #TODO it would be nice to have some logic that checks if a value such as gndelevdatum or gravmeth are consistent for all points in a survey and create a variable level attribute, otherwise create the lookup table. However, whats more importance, consistency in structure or file size? 66 | 67 | 68 | 69 | # GRAVMETH 70 | 71 | # GRAVACCMETHOD 72 | 73 | # LOCMETH 74 | 75 | # LOCACCMETHOD 76 | 77 | # GNDELEVMETH 78 | 79 | # GNDELEVACCMETHOD 80 | 81 | # METERHGTMETH 82 | 83 | # METERHGTERRMETHOD? 84 | # 85 | # TCMETH 86 | # no tc error method 87 | 88 | # ELLIPSOIDHGTMETH 89 | 90 | # ELLIPSOIDHGTACCMETHOD 91 | # ELLIPSOIDMETERHGTMETH 92 | 93 | # ELLIPSOIDMETERHGTERRMETHOD 94 | 95 | # NVALUEMETH 96 | # NVALUEACCMETHOD 97 | # NONGATCERRMETHOD 98 | # NONGATCMETH 99 | -------------------------------------------------------------------------------- /geophys_utils/netcdf_converter/grav2netcdf_converter_settings_ga_metadata: -------------------------------------------------------------------------------- 1 | gravity_metadata_list: 2 | #- 'ENO', 3 | - 'SURVEYID' 4 | - 'SURVEYNAME' 5 | - 'COUNTRYID' 6 | - 'STATEGROUP' 7 | - 'STATIONS' 8 | #- 'GRAVACC' - variable 9 | - ['GRAVDATUM', 'GRAVDATUMS'] #TODO always 'B'? Australian Absolute Gravity Datum 2007 (AAGD07) 10 | #- 'GNDELEVACC' - variable 11 | #- 'GNDELEVMETH' - variable 12 | #- 'GNDELEVDATUM' - variable - 6 outlyers 13 | #- RELIAB' variable - 5 outlyers 14 | - 'LAYOUT' # fuller descriptions of this are somewhere. 15 | #- 'ACCESS_CODE' filtered 16 | #- 'ENTRYDATE' 17 | #- 'ENTEREDBY' 18 | #- 'LASTUPDATE' 19 | #- 'UPDATEDBY' 20 | #- 'GRAVACCUNITS' #always um. put in grav acc var attribute - may be null sometimes 21 | #- 'GRAVACCMETHOD' variable 22 | - 'GNDELEVACCUNITS' # always m maybe some as null 23 | #- 'GNDELEVACCMETHOD' as variable 24 | - 'ELLIPSOIDHGTDATUM' #always - always GRS80 25 | 'ELLIPSOIDHGTMETH', 26 | #'ELLIPSOIDHGTACC', # as variable 27 | #'ELLIPSOIDHGTACCMETHOD',# as variable 28 | 'ELLIPSOIDHGTACCUOM', 29 | 'SURVEYTYPE', 30 | #'DATATYPES', 31 | #'UNO', 32 | 'OPERATOR', 33 | 'CONTRACTOR', 34 | 'PROCESSOR', 35 | 'CLIENT', # nulls 36 | 'OWNER', # nulls 37 | 'LEGISLATION', # nulls 38 | #'STATE', 39 | 'PROJ_LEADER', #nulls 40 | 'ON_OFF', 41 | 'STARTDATE', 42 | 'ENDDATE', 43 | 'VESSEL_TYPE', # nulls 44 | 'VESSEL', # nulls 45 | 'SPACEMIN', # can add uom which is metres 46 | 'SPACEMAX', 47 | #'LOCMETHOD', - variable 48 | 'ACCURACY', #??? 49 | #'GEODETIC_DATUM', 50 | 'PROJECTION', # nulls 51 | #'QA_CODE', 52 | 'RELEASEDATE', # not needed but open for discussion 53 | 'COMMENTS', # not needed but open for discussion 54 | #'DATA_ACTIVITY_CODE', 55 | #'NLAT', already in global attributes 56 | #'SLAT', already in global attributes 57 | #'ELONG', already in global attributes 58 | #'WLONG', already in global attributes 59 | #'ANO', 60 | #'QABY', 61 | #'QADATE', 62 | #'CONFID_UNTIL', 63 | ] -------------------------------------------------------------------------------- /geophys_utils/netcdf_converter/grav2netcdf_converter_sql_strings.yml: -------------------------------------------------------------------------------- 1 | sql_strings_dict: 2 | sql_get_surveyids: 3 | > 4 | select Surveyid from gravity.GRAVSURVEYS gs 5 | where exists (select o1.Surveyid from gravity.OBSERVATIONS o1 6 | left join gravity.OBSERVATIONS o2 7 | on 8 | o1.surveyid = o2.surveyid 9 | and (o1.entrydate > o2.entrydate OR(o1.entrydate = o2.entrydate and o1.obsno > o2.obsno)) 10 | and o1.geodetic_datum = o2.geodetic_datum 11 | and o1.dlat = o2.dlat 12 | and o1.dlong = o2.dlong 13 | and o1.access_code = o2.access_code 14 | and o1.status = o2.status 15 | where 16 | o1.surveyid = gs.surveyid 17 | and o1.status = 'A' 18 | and o1.access_code = 'O' 19 | and o1.grav is not null 20 | and o1.gndelev is not null 21 | and o1.meterhgt is not null 22 | and o1.nvalue is not null 23 | and o1.ellipsoidhgt is not null 24 | and o1.ellipsoidmeterhgt is not null 25 | and o1.eno in (select eno from a.surveys where countryid is null or countryid = 'AUS') 26 | and o2.obsno is null) 27 | -- and gs.surveyid = '201780' -- *** Use this for testing only *** 28 | order by gs.SURVEYID 29 | get_survey_metadata: 30 | > 31 | select * from gravity.GRAVSURVEYS gs 32 | inner join a.surveys using(eno) 33 | where 34 | gs.surveyid = {0} 35 | and exists 36 | (select o1.* from gravity.OBSERVATIONS o1 37 | left join gravity.OBSERVATIONS o2 38 | on 39 | o1.surveyid = o2.surveyid 40 | and (o1.entrydate > o2.entrydate OR(o1.entrydate = o2.entrydate and o1.obsno > o2.obsno)) 41 | and o1.geodetic_datum = o2.geodetic_datum 42 | and o1.dlat = o2.dlat 43 | and o1.dlong = o2.dlong 44 | and o1.access_code = o2.access_code 45 | and o1.status = o2.status 46 | where 47 | o1.surveyid = {0} 48 | and o1.status = 'A' 49 | and o1.access_code = 'O' 50 | and o1.dlat is not null 51 | and o1.dlong is not null 52 | and o1.grav is not null 53 | and o1.gndelev is not null 54 | and o1.meterhgt is not null 55 | and o1.nvalue is not null 56 | and o1.ellipsoidhgt is not null 57 | and o1.ellipsoidmeterhgt is not null 58 | and o1.eno in (select eno from a.surveys where countryid is null or countryid = 'AUS') 59 | and o2.obsno is null) 60 | get_data: 61 | > 62 | select NVL({0}, {1}) from gravity.OBSERVATIONS o1 63 | left join gravity.OBSERVATIONS o2 64 | on 65 | o1.surveyid = o2.surveyid 66 | and (o1.entrydate > o2.entrydate OR(o1.entrydate = o2.entrydate and o1.obsno > o2.obsno)) 67 | and o1.geodetic_datum = o2.geodetic_datum 68 | and o1.dlat = o2.dlat 69 | and o1.dlong = o2.dlong 70 | and o1.access_code = o2.access_code 71 | and o1.status = o2.status 72 | where 73 | o1.surveyid = {2} 74 | and o1.status = 'A' 75 | and o1.access_code = 'O' 76 | and o1.grav is not null 77 | and o1.gndelev is not null 78 | and o1.meterhgt is not null 79 | and o1.nvalue is not null 80 | and o1.ellipsoidhgt is not null 81 | and o1.ellipsoidmeterhgt is not null 82 | and o1.eno in (select eno from a.surveys where countryid is null or countryid = 'AUS') 83 | and o2.obsno is null 84 | order by o1.obsno 85 | get_field_description: 86 | > 87 | SELECT COMMENTS 88 | FROM ALL_COL_COMMENTS 89 | WHERE TABLE_NAME = 'OBSERVATIONS' 90 | AND COLUMN_NAME = '{}' 91 | get_dimensions: 92 | > 93 | select count(*) from gravity.OBSERVATIONS o1 94 | left join gravity.OBSERVATIONS o2 95 | on 96 | o1.surveyid = o2.surveyid 97 | and (o1.entrydate > o2.entrydate OR(o1.entrydate = o2.entrydate and o1.obsno > o2.obsno)) 98 | and o1.geodetic_datum = o2.geodetic_datum 99 | and o1.dlat = o2.dlat 100 | and o1.dlong = o2.dlong 101 | and o1.access_code = o2.access_code 102 | and o1.status = o2.status 103 | where 104 | o1.surveyid = {} 105 | and o1.status = 'A' 106 | and o1.access_code = 'O' 107 | and o1.grav is not null 108 | and o1.gndelev is not null 109 | and o1.meterhgt is not null 110 | and o1.nvalue is not null 111 | and o1.ellipsoidhgt is not null 112 | and o1.ellipsoidmeterhgt is not null 113 | and o1.eno in (select eno from a.surveys where countryid is null or countryid = 'AUS') 114 | and o2.obsno is null 115 | check_all_values_in_survey_consistent: 116 | > 117 | SELECT DISTINCT {0} 118 | FROM gravity.OBSERVATIONS o1 119 | WHERE o1.surveyid = {1} 120 | get_ellipsoidhgt_datums_lookup: 121 | > 122 | SELECT DISTINCT ELLIPSOIDHGTDATUM 123 | FROM gravity.OBSERVATIONS o1 -------------------------------------------------------------------------------- /geophys_utils/netcdf_converter/test_netcdf_converter.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # =============================================================================== 4 | # Copyright 2017 Geoscience Australia 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # =============================================================================== 18 | ''' 19 | TestNetCDFConverter concrete class for converting data to netCDF 20 | 21 | Created on 28Mar.2018 22 | 23 | @author: Alex Ip 24 | ''' 25 | from collections import OrderedDict 26 | 27 | import numpy as np 28 | 29 | from geophys_utils.netcdf_converter import ToNetCDFConverter, NetCDFVariable 30 | 31 | 32 | class TestNetCDFConverter(ToNetCDFConverter): 33 | ''' 34 | TestNetCDFConverter concrete class for converting CSV data to netCDF 35 | ''' 36 | 37 | def __init__(self, nc_out_path, netcdf_format='NETCDF4_CLASSIC'): 38 | ''' 39 | Concrete constructor for subclass TestNetCDFConverter 40 | Needs to initialise object with everything that is required for the other Concrete methods 41 | N.B: Make sure the base class constructor is called from the subclass constructor 42 | ''' 43 | ToNetCDFConverter.__init__(self, nc_out_path, netcdf_format) 44 | 45 | def get_global_attributes(self): 46 | ''' 47 | Concrete method to return dict of global attribute : pairs 48 | ''' 49 | return {'title': 'test dataset'} 50 | 51 | def get_dimensions(self): 52 | ''' 53 | Concrete method to return OrderedDict of : pairs 54 | ''' 55 | dimensions = OrderedDict() 56 | 57 | # Example lat/lon dimensions 58 | dimensions['lon'] = 509 59 | dimensions['lat'] = 639 60 | 61 | return dimensions 62 | 63 | def variable_generator(self): 64 | ''' 65 | Concrete generator to yield NetCDFVariable objects 66 | ''' 67 | # Example of latitude dimension variable creation 68 | yield self.build_dim_index_variable(dimension_name='lat', 69 | min_value=-22.9247209891964, 70 | max_value=-20.5641209891964, 71 | long_name='latitude', 72 | units='degrees north', 73 | standard_name='latitude', 74 | descending=True # Invert Y axis 75 | ) 76 | 77 | # Example of longitude dimension variable creation 78 | yield self.build_dim_index_variable(dimension_name='lon', 79 | min_value=121.122089060582, 80 | max_value=123.001689060582, 81 | long_name='longitude', 82 | units='degrees east', 83 | standard_name='longitude', 84 | descending=False 85 | ) 86 | 87 | # Example of crs variable creation for GDA94 88 | yield self.build_crs_variable('''\ 89 | GEOGCS["GDA94", 90 | DATUM["Geocentric_Datum_of_Australia_1994", 91 | SPHEROID["GRS 1980",6378137,298.257222101, 92 | AUTHORITY["EPSG","7019"]], 93 | TOWGS84[0,0,0,0,0,0,0], 94 | AUTHORITY["EPSG","6283"]], 95 | PRIMEM["Greenwich",0, 96 | AUTHORITY["EPSG","8901"]], 97 | UNIT["degree",0.0174532925199433, 98 | AUTHORITY["EPSG","9122"]], 99 | AUTHORITY["EPSG","4283"]] 100 | ''' 101 | ) 102 | 103 | yield NetCDFVariable(short_name='test_data', 104 | data=np.random.random((self.nc_output_dataset.dimensions['lat'].size, 105 | self.nc_output_dataset.dimensions['lon'].size)), 106 | dimensions=['lat', 'lon'], 107 | fill_value=0.0, 108 | attributes={'units': 'random crap', 109 | 'long_name': 'random numbers between 0 and 1' 110 | }, 111 | dtype=np.float32 112 | ) 113 | 114 | return 115 | 116 | 117 | def main(): 118 | nc_out_path = 'C:\\Temp\\test.nc' 119 | tnc = TestNetCDFConverter(nc_out_path) 120 | tnc.convert2netcdf() 121 | print('Finished writing netCDF file {}'.format(nc_out_path)) 122 | 123 | 124 | if __name__ == '__main__': 125 | main() 126 | -------------------------------------------------------------------------------- /geophys_utils/rechunk.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on 23 Feb 2020 3 | 4 | @author: alex 5 | ''' 6 | import argparse 7 | import logging 8 | import sys 9 | 10 | from geophys_utils._get_netcdf_util import get_netcdf_util 11 | 12 | logger = logging.getLogger() 13 | logger.setLevel(logging.INFO) # Initial logging level for this module 14 | 15 | 16 | def main(): 17 | ''' 18 | Main function for calling NetCDFUtils.copy function 19 | ''' 20 | # Define command line arguments 21 | parser = argparse.ArgumentParser() 22 | 23 | parser.add_argument("-f", "--format", 24 | help="NetCDF file format (one of 'NETCDF4', 'NETCDF4_CLASSIC', 'NETCDF3_CLASSIC', 'NETCDF3_64BIT_OFFSET' or 'NETCDF3_64BIT_DATA')", 25 | type=str, default='NETCDF4') 26 | parser.add_argument("--chunkspec", help="comma-separated list of / specifications", 27 | type=str) 28 | parser.add_argument("--complevel", help="Compression level for chunked variables as an integer 0-9. Default is 4", 29 | type=int, default=4) 30 | parser.add_argument('-d', '--debug', action='store_const', const=True, default=False, 31 | help='output debug information. Default is no debug info') 32 | parser.add_argument("input_path") 33 | parser.add_argument("output_path") 34 | 35 | args = parser.parse_args() 36 | 37 | if args.chunkspec: 38 | chunk_spec = {dim_name: int(chunk_size) 39 | for dim_name, chunk_size in 40 | [chunk_spec_string.strip().split('/') for chunk_spec_string in args.chunkspec.split(',')]} 41 | else: 42 | chunk_spec = None 43 | 44 | ncu = get_netcdf_util(args.input_path, 45 | debug=args.debug 46 | ) 47 | 48 | ncu.copy(args.output_path, 49 | # datatype_map_dict={}, 50 | # Compress all chunked variables 51 | variable_options_dict={variable_name: {'chunksizes': [chunk_spec.get(dimension) 52 | for dimension in variable.dimensions 53 | ], 54 | # 'zlib': bool(args.complevel), # depcrecated replaced with compression='zlib' 55 | 'compression': 'zlib' if args.complevel else 'None', 56 | 'complevel': args.complevel 57 | } 58 | for variable_name, variable in ncu.netcdf_dataset.variables.items() 59 | if (set(variable.dimensions) & set(chunk_spec.keys())) 60 | } if chunk_spec else {}, 61 | # dim_range_dict={'lat': (5,205),'lon': (5,305)}, 62 | # dim_mask_dict={}, 63 | nc_format=args.format, 64 | # limit_dim_size=False 65 | ) 66 | 67 | logger.debug('Copy complete') 68 | 69 | 70 | if __name__ == '__main__': 71 | console_handler = logging.StreamHandler(sys.stdout) 72 | # console_handler.setLevel(logging.INFO) 73 | console_handler.setLevel(logging.DEBUG) 74 | console_formatter = logging.Formatter('%(name)s: %(message)s') 75 | console_handler.setFormatter(console_formatter) 76 | 77 | if not logger.handlers: 78 | logger.addHandler(console_handler) 79 | logger.debug('Logging handlers set up for {}'.format(logger.name)) 80 | 81 | main() 82 | -------------------------------------------------------------------------------- /geophys_utils/test/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # =============================================================================== 4 | # Copyright 2017 Geoscience Australia 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # =============================================================================== 18 | """ 19 | Constructor for test module 20 | Unit tests for geophys_utils modules against NetCDF files 21 | 22 | Created on 7Oct.,2016 23 | 24 | @author: Alex Ip 25 | """ 26 | import logging 27 | 28 | logger = logging.getLogger(__name__) 29 | logger.setLevel(logging.INFO) # Logging level for this module 30 | -------------------------------------------------------------------------------- /geophys_utils/test/__main__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # =============================================================================== 4 | # Copyright 2017 Geoscience Australia 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # =============================================================================== 18 | """ 19 | Main unit for test module 20 | Unit tests for ncskosdump and ld_functions against a modified NetCDF file 21 | 22 | Created on 15/11/2016 23 | 24 | @author: Alex Ip 25 | """ 26 | from geophys_utils.test import \ 27 | test_array_pieces, \ 28 | test_crs_utils, \ 29 | test_data_stats, \ 30 | test_netcdf_grid_utils, \ 31 | test_netcdf_line_utils, \ 32 | test_netcdf_point_utils 33 | 34 | # Run all tests 35 | test_array_pieces.main() 36 | test_crs_utils.main() 37 | test_data_stats.main() 38 | test_netcdf_grid_utils.main() 39 | test_netcdf_line_utils.main() 40 | test_netcdf_point_utils.main() 41 | -------------------------------------------------------------------------------- /geophys_utils/test/test_array_pieces.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # =============================================================================== 4 | # Copyright 2017 Geoscience Australia 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # =============================================================================== 18 | """ 19 | Unit tests for geophys_utils._array_pieces module 20 | 21 | Created on 15/11/2016 22 | 23 | @author: Alex Ip 24 | """ 25 | import unittest 26 | import numpy as np 27 | from functools import reduce 28 | from geophys_utils._array_pieces import array_pieces 29 | 30 | 31 | class TestArrayPieces(unittest.TestCase): 32 | """Unit tests for geophys_utils._array_pieces module.""" 33 | 34 | def test_array_pieces(self): 35 | print('Testing array_pieces function') 36 | # create 100 x 100 array with unique elements 37 | test_array = np.reshape(np.arange(0, 10000, dtype=np.int16), (100, 100)) 38 | sixteenth_bytes = test_array.dtype.itemsize * reduce(lambda x, y: x * y / 16, test_array.shape) 39 | overlap = 10 40 | 41 | print('\tTesting single piece') 42 | array_pieces_results = {array_offset: piece_array for piece_array, array_offset in array_pieces(test_array)} 43 | 44 | assert len(array_pieces_results) == 1, 'Whole array not returned for large max_bytes' 45 | assert not np.any(test_array - list(array_pieces_results.values())[0]), 'Array contents changed' 46 | 47 | print('\tTesting sixteenth arrays') 48 | array_pieces_results = {array_offset: piece_array for piece_array, array_offset in array_pieces(test_array, 49 | max_bytes=sixteenth_bytes)} 50 | assert len(array_pieces_results) == 16, '16 arrays not returned for max_bytes=%d' % sixteenth_bytes 51 | for array_offset in sorted(array_pieces_results.keys()): 52 | piece_array = array_pieces_results[array_offset] 53 | 54 | assert piece_array.dtype.itemsize * reduce(lambda x, y: x * y / 16, 55 | piece_array.shape) <= sixteenth_bytes, 'piece_array too large' 56 | 57 | expected_shape = tuple([test_array.shape[dim_index] / 4 58 | for dim_index in range(2)]) 59 | assert piece_array.shape == expected_shape, 'piece_array is wrong shape' 60 | 61 | slices = tuple([slice(array_offset[dim_index], 62 | array_offset[dim_index] + piece_array.shape[dim_index] 63 | ) for dim_index in range(2)]) 64 | assert not np.any( 65 | piece_array - test_array[slices]), 'Array contents changed for array piece at %s' % array_offset 66 | 67 | print('\tTesting sixteenth arrays with overlap') 68 | array_pieces_results = {array_offset: piece_array for piece_array, array_offset in array_pieces(test_array, 69 | max_bytes=sixteenth_bytes, 70 | overlap=overlap)} 71 | 72 | assert len(array_pieces_results) == 16, '16 arrays not returned for overlap=%d, max_bytes=%d' % (overlap, 73 | sixteenth_bytes) 74 | for array_offset in sorted(array_pieces_results.keys()): 75 | piece_array = array_pieces_results[array_offset] 76 | 77 | expected_shape = tuple([test_array.shape[dim_index] / 4 + overlap 78 | if array_offset[dim_index] == 0 79 | or test_array.shape[dim_index] - array_offset[dim_index] - overlap <= 80 | test_array.shape[dim_index] / 4 81 | else test_array.shape[dim_index] / 4 + 2 * overlap 82 | for dim_index in range(2)] 83 | ) 84 | assert piece_array.shape == expected_shape, 'piece_array is wrong shape' 85 | 86 | slices = tuple([slice(array_offset[dim_index], 87 | array_offset[dim_index] + piece_array.shape[dim_index] 88 | ) for dim_index in range(2)]) 89 | assert not np.any( 90 | piece_array - test_array[slices]), 'Array contents changed for array piece at %s' % array_offset 91 | 92 | 93 | # Define test suites 94 | def test_suite(): 95 | """Returns a test suite of all the tests in this module.""" 96 | 97 | test_classes = [TestArrayPieces] 98 | 99 | suite_list = map(unittest.defaultTestLoader.loadTestsFromTestCase, 100 | test_classes) 101 | 102 | suite = unittest.TestSuite(suite_list) 103 | 104 | return suite 105 | 106 | 107 | # Define main function 108 | def main(): 109 | unittest.TextTestRunner(verbosity=2).run(test_suite()) 110 | 111 | 112 | if __name__ == '__main__': 113 | main() 114 | -------------------------------------------------------------------------------- /geophys_utils/test/test_crs_utils.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | #=============================================================================== 4 | # Copyright 2017 Geoscience Australia 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | #=============================================================================== 18 | """ 19 | Unit tests for geophys_utils._crs_utils against a NetCDF file 20 | 21 | Created on 15/11/2016 22 | 23 | @author: Alex Ip 24 | """ 25 | import unittest 26 | import numpy as np 27 | import re 28 | from osgeo.osr import CoordinateTransformation 29 | from geophys_utils._crs_utils import get_coordinate_transformation, get_utm_wkt, transform_coords 30 | 31 | COORD_ERROR_MARGIN = 1.0e-08 32 | 33 | class TestCRSUtils(unittest.TestCase): 34 | """Unit tests for geophys_utils._crs_utils module.""" 35 | 36 | EPSG4326_WKT = 'GEOGCS["WGS 84",DATUM["WGS_1984",SPHEROID["WGS 84",6378137,298.257223563,AUTHORITY["EPSG","7030"]],AUTHORITY["EPSG","6326"]],PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],UNIT["degree",0.0174532925199433,AUTHORITY["EPSG","9122"]],AXIS["Latitude",NORTH],AXIS["Longitude",EAST],AUTHORITY["EPSG","4326"]]' 37 | EPSG4326_EPSG = 'EPSG:4326' 38 | 39 | EPSG3577_WKT = 'PROJCS["GDA94 / Australian Albers",GEOGCS["GDA94",DATUM["Geocentric_Datum_of_Australia_1994",SPHEROID["GRS 1980",6378137,298.257222101,AUTHORITY["EPSG","7019"]],AUTHORITY["EPSG","6283"]],PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],UNIT["degree",0.0174532925199433,AUTHORITY["EPSG","9122"]],AUTHORITY["EPSG","4283"]],PROJECTION["Albers_Conic_Equal_Area"],PARAMETER["latitude_of_center",0],PARAMETER["longitude_of_center",132],PARAMETER["standard_parallel_1",-18],PARAMETER["standard_parallel_2",-36],PARAMETER["false_easting",0],PARAMETER["false_northing",0],UNIT["metre",1,AUTHORITY["EPSG","9001"]],AXIS["Easting",EAST],AXIS["Northing",NORTH],AUTHORITY["EPSG","3577"]]' 40 | EPSG3577_EPSG = 'EPSG:3577' 41 | 42 | #UTM_WKT = 'PROJCS["UTM Zone 55,Southern Hemisphere",GEOGCS["WGS 84",DATUM["WGS_1984",SPHEROID["WGS 84",6378137,298.257223563,AUTHORITY["EPSG","7030"]],AUTHORITY["EPSG","6326"]],PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],UNIT["degree",0.0174532925199433,AUTHORITY["EPSG","9122"]],AUTHORITY["EPSG","4326"]],PROJECTION["Transverse_Mercator"],PARAMETER["latitude_of_origin",0],PARAMETER["central_meridian",147],PARAMETER["scale_factor",0.9996],PARAMETER["false_easting",500000],PARAMETER["false_northing",10000000],UNIT["Meter",1]]' 43 | UTM_WKT = 'PROJCS["unnamed",GEOGCS["WGS 84",DATUM["WGS_1984",SPHEROID["WGS 84",6378137,298.257223563,AUTHORITY["EPSG","7030"]],AUTHORITY["EPSG","6326"]],PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],UNIT["degree",0.0174532925199433,AUTHORITY["EPSG","9122"]],AUTHORITY["EPSG","4326"]],PROJECTION["Transverse_Mercator"],PARAMETER["latitude_of_origin",0],PARAMETER["central_meridian",147],PARAMETER["scale_factor",0.9996],PARAMETER["false_easting",500000],PARAMETER["false_northing",10000000],UNIT["metre",1,AUTHORITY["EPSG","9001"]],AXIS["Easting",EAST],AXIS["Northing",NORTH]]' 44 | 45 | GDA94_WKT = 'GEOGCS["GDA94",DATUM["Geocentric_Datum_of_Australia_1994",SPHEROID["GRS 1980",6378137,298.257222101,AUTHORITY["EPSG","7019"]],AUTHORITY["EPSG","6283"]],PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],UNIT["degree",0.0174532925199433,AUTHORITY["EPSG","9122"]],AXIS["Latitude",NORTH],AXIS["Longitude",EAST],AUTHORITY["EPSG","4283"]]' 46 | GDA94_EPSG = 'EPSG:4283' 47 | 48 | GDA2020_WKT = 'GEOGCS["GDA2020",DATUM["Geocentric_Datum_of_Australia_2020",SPHEROID["GRS 1980",6378137,298.257222101,AUTHORITY["EPSG","7019"]],AUTHORITY["EPSG","1168"]],PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],UNIT["degree",0.0174532925199433,AUTHORITY["EPSG","9122"]],AXIS["Latitude",NORTH],AXIS["Longitude",EAST],AUTHORITY["EPSG","7844"]]' 49 | GDA2020_EPSG = 'EPSG:7844' 50 | 51 | EPSG4326_COORDS = [149.160, -35.306] 52 | #EPSG4326_COORD_ARRAY = np.array([[149.160, -35.306], [150, -35]]) 53 | EPSG4326_COORD_ARRAY = [[149.160, -35.306], [150, -35]] 54 | UTM_COORDS = [696382.56321712, 6090881.85849329] #(696382.5632178178, 6090881.858493158) 55 | UTM_COORD_ARRAY = [(696382.5632171195, 6090881.858493287), (773798.0963396085, 6122843.308355326)] 56 | 57 | def test_get_coordinate_transformation(self): 58 | print('Testing get_coordinate_transformation function') 59 | #TODO: Make this one test with different test cases 60 | coordinate_transformation = get_coordinate_transformation(TestCRSUtils.EPSG4326_WKT, 61 | TestCRSUtils.EPSG3577_WKT) 62 | assert coordinate_transformation is not None 63 | assert type(coordinate_transformation) == CoordinateTransformation 64 | 65 | coordinate_transformation = get_coordinate_transformation(TestCRSUtils.EPSG4326_WKT, 66 | TestCRSUtils.GDA2020_WKT) 67 | assert coordinate_transformation is not None 68 | assert type(coordinate_transformation) == CoordinateTransformation 69 | 70 | coordinate_transformation = get_coordinate_transformation(TestCRSUtils.GDA2020_WKT, 71 | TestCRSUtils.EPSG4326_WKT 72 | ) 73 | assert coordinate_transformation is not None 74 | assert type(coordinate_transformation) == CoordinateTransformation 75 | 76 | coordinate_transformation = get_coordinate_transformation(TestCRSUtils.EPSG4326_EPSG, 77 | TestCRSUtils.EPSG3577_EPSG) 78 | assert coordinate_transformation is not None 79 | assert type(coordinate_transformation) == CoordinateTransformation 80 | 81 | coordinate_transformation = get_coordinate_transformation(TestCRSUtils.EPSG4326_WKT, 82 | TestCRSUtils.EPSG4326_WKT) 83 | assert coordinate_transformation is None, 'Null transformation should return None' 84 | 85 | def test_get_utm_wkt(self): 86 | print('Testing get_utm_wkt function') 87 | utm_wkt = get_utm_wkt(TestCRSUtils.EPSG4326_COORDS, 88 | TestCRSUtils.EPSG4326_EPSG) 89 | utm_wkt = re.sub(',\s+', ',', re.sub('\s+', ' ', utm_wkt)) 90 | expected_wkt = re.sub(',\s+', ',', re.sub('\s+', ' ', TestCRSUtils.UTM_WKT)) 91 | assert utm_wkt == expected_wkt, 'Incorrect UTM CRS: {} instead of {}'.format(utm_wkt, expected_wkt) 92 | 93 | def test_transform_coords(self): 94 | print('Testing transform_coords function with single coordinate{}'.format(TestCRSUtils.EPSG4326_COORDS)) 95 | utm_coords = transform_coords(TestCRSUtils.EPSG4326_COORDS, TestCRSUtils.EPSG4326_WKT, TestCRSUtils.UTM_WKT) 96 | assert (abs(utm_coords - np.array(TestCRSUtils.UTM_COORDS)) < COORD_ERROR_MARGIN).all(), \ 97 | 'Incorrect UTM coordinates: {} instead of {}'.format(utm_coords, np.array(TestCRSUtils.UTM_COORDS)) 98 | 99 | print('Testing transform_coords function with multi coordinate {}'.format(TestCRSUtils.EPSG4326_COORD_ARRAY)) 100 | utm_coord_array = transform_coords(TestCRSUtils.EPSG4326_COORD_ARRAY, TestCRSUtils.EPSG4326_WKT, TestCRSUtils.UTM_WKT) 101 | assert (abs(utm_coord_array - TestCRSUtils.UTM_COORD_ARRAY) < COORD_ERROR_MARGIN).all(), \ 102 | 'Incorrect UTM coordinates: {} instead of {}'.format(utm_coord_array, TestCRSUtils.UTM_COORD_ARRAY) 103 | # Define test suites 104 | def test_suite(): 105 | """Returns a test suite of all the tests in this module.""" 106 | 107 | test_classes = [TestCRSUtils] 108 | 109 | suite_list = map(unittest.defaultTestLoader.loadTestsFromTestCase, 110 | test_classes) 111 | 112 | suite = unittest.TestSuite(suite_list) 113 | 114 | return suite 115 | 116 | 117 | # Define main function 118 | def main(): 119 | unittest.TextTestRunner(verbosity=2).run(test_suite()) 120 | 121 | if __name__ == '__main__': 122 | main() 123 | -------------------------------------------------------------------------------- /geophys_utils/test/test_data_stats.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # =============================================================================== 4 | # Copyright 2017 Geoscience Australia 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # =============================================================================== 18 | """ 19 | Unit tests for geophys_utils._data_stats against a NetCDF file 20 | 21 | Created on 15/11/2016 22 | 23 | @author: Alex Ip 24 | """ 25 | import os 26 | import unittest 27 | 28 | from geophys_utils._data_stats import DataStats 29 | 30 | 31 | class TestDataStats(unittest.TestCase): 32 | """Unit tests for geophys_utils._data_stats module.""" 33 | 34 | NC_PATH = 'test_grid.nc' 35 | MAX_BYTES = 1600 36 | MAX_ERROR = 0.000001 37 | EXPECTED_RESULT = {'y_size': 178, 38 | 'data_type': 'float32', 39 | 'min': -308.20889, 40 | 'max': 3412.063, 41 | # 'nc_path': u'C:\\Users\\u76345\\git\\geophys_utils\\geophys_utils\\test\\test_grid.nc', 42 | 'x_size': 79, 43 | 'mean': -15.958283494050137, 44 | 'nodata_value': -99999.0} 45 | 46 | def test_data_stats(self): 47 | print('Testing DataStats class') 48 | TestDataStats.EXPECTED_RESULT['nc_path'] = os.path.join(os.path.dirname(__file__), TestDataStats.NC_PATH) 49 | data_stats = DataStats(TestDataStats.EXPECTED_RESULT['nc_path'], 50 | max_bytes=TestDataStats.MAX_BYTES) 51 | for key in sorted(TestDataStats.EXPECTED_RESULT.keys()): 52 | try: 53 | error = data_stats.value(key) - TestDataStats.EXPECTED_RESULT[key] 54 | assert error <= TestDataStats.MAX_ERROR, 'Incorrect numeric value for {}\nExpected {}, got {}.'.format( 55 | key, 56 | TestDataStats.EXPECTED_RESULT[key], 57 | data_stats.value(key)) 58 | except TypeError: 59 | assert data_stats.value(key) == TestDataStats.EXPECTED_RESULT[key], 'Incorrect value for %s' % key 60 | 61 | 62 | # Define test suites 63 | def test_suite(): 64 | """Returns a test suite of all the tests in this module.""" 65 | 66 | test_classes = [TestDataStats] 67 | 68 | suite_list = map(unittest.defaultTestLoader.loadTestsFromTestCase, 69 | test_classes) 70 | 71 | suite = unittest.TestSuite(suite_list) 72 | 73 | return suite 74 | 75 | 76 | # Define main function 77 | def main(): 78 | unittest.TextTestRunner(verbosity=2).run(test_suite()) 79 | 80 | 81 | if __name__ == '__main__': 82 | main() 83 | -------------------------------------------------------------------------------- /geophys_utils/test/test_grid.nc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GeoscienceAustralia/geophys_utils/ecd8da91310eceb3aa6bf4f342b37ec275fb457a/geophys_utils/test/test_grid.nc -------------------------------------------------------------------------------- /geophys_utils/test/test_netcdf_grid_utils.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # =============================================================================== 4 | # Copyright 2017 Geoscience Australia 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # =============================================================================== 18 | """ 19 | Unit tests for geophys_utils._netcdf_grid_utils against a NetCDF file 20 | 21 | Created on 15/11/2016 22 | 23 | @author: Alex Ip 24 | """ 25 | import os 26 | import unittest 27 | 28 | import netCDF4 29 | import numpy as np 30 | from shapely.geometry.polygon import Polygon 31 | 32 | from geophys_utils._netcdf_grid_utils import NetCDFGridUtils 33 | 34 | netcdf_grid_utils = None 35 | 36 | NC_PATH = 'test_grid.nc' 37 | MAX_BYTES_SM = 100 38 | MAX_BYTES_LG = 5000 39 | MAX_ERROR = 0.000001 40 | TEST_COORDS = (148.213, -36.015) 41 | TEST_MULTI_COORDS = np.array([[148.213, -36.015], [148.516, -35.316]]) 42 | TEST_MANY_COORDS = np.array([[148.484, -35.352], 43 | [148.328, -35.428], [148.436, -35.744], [148.300, -35.436]]) 44 | TEST_INDICES = [1, 1] 45 | TEST_MULTI_INDICES = [[1, 1], [176, 77]] 46 | TEST_FRACTIONAL_INDICES = [1.25, 1.25] 47 | TEST_VALUE = -99999. 48 | TEST_MULTI_VALUES = [-99999.0, -134.711334229] 49 | TEST_MANY_VALS = [-136.13321, -31.7626, -58.755764, -90.484276] 50 | TEST_INTERPOLATED_VALUE = -99997.6171875 51 | 52 | 53 | class TestNetCDFGridUtilsConstructor(unittest.TestCase): 54 | """Unit tests for TestNetCDFGridUtils Constructor. 55 | N.B: This should be run first""" 56 | 57 | def test_netcdf_grid_utils_constructor(self): 58 | print('Testing NetCDFGridUtils constructor') 59 | global netcdf_grid_utils 60 | 61 | nc_path = os.path.join(os.path.dirname(__file__), NC_PATH) 62 | nc_dataset = netCDF4.Dataset(nc_path) 63 | netcdf_grid_utils = NetCDFGridUtils(nc_dataset) 64 | 65 | 66 | class TestNetCDFGridUtilsFunctions1(unittest.TestCase): 67 | """Unit tests for geophys_utils._netcdf_grid_utils functions""" 68 | 69 | def test_get_indices_from_coords(self): 70 | print('Testing get_indices_from_coords function with single coordinate {}'.format(TEST_COORDS)) 71 | indices = netcdf_grid_utils.get_indices_from_coords(TEST_COORDS) 72 | assert (indices == np.array(TEST_INDICES)).all, 'Incorrect indices: {} instead of {}'.format(indices, 73 | TEST_INDICES) 74 | 75 | print('Testing get_indices_from_coords function with multi coordinates {}'.format(TEST_MULTI_COORDS)) 76 | multi_indices = netcdf_grid_utils.get_indices_from_coords(TEST_MULTI_COORDS) 77 | assert (multi_indices == np.array(TEST_MULTI_INDICES)).all, 'Incorrect indices: {} instead of {}'.format( 78 | multi_indices, TEST_MULTI_INDICES) 79 | 80 | def test_get_fractional_indices_from_coords(self): 81 | print('Testing get_fractional_indices_from_coords function') 82 | indices = netcdf_grid_utils.get_fractional_indices_from_coords(TEST_COORDS) 83 | for ordinate_index in range(len(indices)): 84 | assert round(indices[ordinate_index], 6) == TEST_FRACTIONAL_INDICES[ 85 | ordinate_index], 'Fractional index incorrect' 86 | 87 | def test_get_value_at_coords(self): 88 | print('Testing get_value_at_coords function with single coordinate {}'.format(TEST_COORDS)) 89 | value = netcdf_grid_utils.get_value_at_coords(TEST_COORDS) 90 | assert value[0] == TEST_VALUE, 'Incorrect retrieved value: {} instead of {}'.format(value.data, TEST_VALUE) 91 | 92 | print('Testing get_value_at_coords function with multiple coordinates {}'.format(TEST_MULTI_COORDS)) 93 | multi_values = netcdf_grid_utils.get_value_at_coords(TEST_MULTI_COORDS) 94 | assert (np.abs(np.array(multi_values) - np.array( 95 | TEST_MULTI_VALUES)) < MAX_ERROR).all(), 'Incorrect retrieved value: {} instead of {}'.format(multi_values, 96 | TEST_MULTI_VALUES) 97 | 98 | print('Testing get_value_at_coords with long coordinate list {}'.format(TEST_MANY_COORDS)) 99 | many_values = netcdf_grid_utils.get_value_at_coords(TEST_MANY_COORDS) 100 | assert (np.abs(np.array(many_values) - np.array( 101 | TEST_MANY_VALS)) < MAX_ERROR).all(), 'Incorrect retrieved value: {} instead of {}'.format(many_values, 102 | TEST_MANY_VALS) 103 | 104 | print('Testing get_value_at_coords with long coordinate list {} and request size {} bytes'.format( 105 | TEST_MANY_COORDS, MAX_BYTES_SM)) 106 | many_values = netcdf_grid_utils.get_value_at_coords(TEST_MANY_COORDS, max_bytes=MAX_BYTES_SM) 107 | assert (np.abs(np.array(many_values) - np.array( 108 | TEST_MANY_VALS)) < MAX_ERROR).all(), 'Incorrect retrieved value: {} instead of {}'.format(many_values, 109 | TEST_MANY_VALS) 110 | 111 | print('Testing get_value_at_coords with long coordinate list {} and request size {} bytes'.format( 112 | TEST_MANY_COORDS, MAX_BYTES_LG)) 113 | many_values = netcdf_grid_utils.get_value_at_coords(TEST_MANY_COORDS, max_bytes=MAX_BYTES_LG) 114 | assert (np.abs(np.array(many_values) - np.array( 115 | TEST_MANY_VALS)) < MAX_ERROR).all(), 'Incorrect retrieved value: {} instead of {}'.format(many_values, 116 | TEST_MANY_VALS) 117 | 118 | def test_get_interpolated_value_at_coords(self): 119 | print('Testing get_interpolated_value_at_coords function') 120 | interpolated_value = netcdf_grid_utils.get_interpolated_value_at_coords(TEST_COORDS) 121 | assert interpolated_value == TEST_INTERPOLATED_VALUE, 'Incorrect interpolated value retrieved' 122 | 123 | def test_sample_transect(self): 124 | print('Testing sample_transect function') 125 | # TODO: Finish this! 126 | # transect_samples = netcdf_grid_utils.sample_transect(transect_vertices, crs=None, sample_metres=None) 127 | 128 | def test_concave_hull(self): 129 | print('Testing concave hull') 130 | assert isinstance(netcdf_grid_utils.get_concave_hull(), Polygon) 131 | 132 | 133 | # Define test suites 134 | def test_suite(): 135 | """Returns a test suite of all the tests in this module.""" 136 | 137 | test_classes = [TestNetCDFGridUtilsConstructor, 138 | TestNetCDFGridUtilsFunctions1] 139 | 140 | suite_list = map(unittest.defaultTestLoader.loadTestsFromTestCase, 141 | test_classes) 142 | 143 | suite = unittest.TestSuite(suite_list) 144 | 145 | return suite 146 | 147 | 148 | # Define main function 149 | def main(): 150 | unittest.TextTestRunner(verbosity=2).run(test_suite()) 151 | 152 | 153 | if __name__ == '__main__': 154 | main() 155 | -------------------------------------------------------------------------------- /geophys_utils/test/test_netcdf_line_utils.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # =============================================================================== 4 | # Copyright 2017 Geoscience Australia 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # =============================================================================== 18 | """ 19 | Unit tests for geophys_utils._netcdf_line_utils against a NetCDF line data file 20 | 21 | Created on 15/11/2016 22 | 23 | @author: Alex Ip 24 | """ 25 | import os 26 | import re 27 | import unittest 28 | 29 | import netCDF4 30 | import numpy as np 31 | from shapely.geometry.polygon import Polygon 32 | 33 | from geophys_utils._netcdf_line_utils import NetCDFLineUtils 34 | 35 | netcdf_line_utils = None 36 | 37 | NC_PATH = 'https://dapds00.nci.org.au/thredds/dodsC/iv65/Geoscience_Australia_Geophysics_Reference_Data_Collection/airborne_geophysics/SA/line/P1255/P1255-line-magnetic-Marree-AWAGS_MAG_2010.nc' 38 | NC_TITLE = 'Marree, SA, 2012 (P1255), magnetic line data, AWAGS levelled' 39 | 40 | TEST_BOUNDS = (137, -29, 138, -28) 41 | 42 | MAX_BYTES = 1600 43 | MAX_ERROR = 0.000001 44 | 45 | TEST_GET_LINE_MASK_RESULTS = ((190520, 5032), 46 | (190500, 4994) 47 | ) 48 | 49 | TEST_GET_LINE_RESULTS = ((190520, 7, 10064), 50 | (190500, 7, 9988) 51 | ) 52 | 53 | 54 | class TestNetCDFLineUtilsConstructor(unittest.TestCase): 55 | """Unit tests for TestNetCDFLineUtils Constructor. 56 | N.B: This should be run first""" 57 | 58 | def test_netcdf_line_utils_constructor(self): 59 | print('Testing NetCDFLineUtils constructor') 60 | global netcdf_line_utils 61 | 62 | if re.match('^http.*', NC_PATH): 63 | nc_path = NC_PATH 64 | else: 65 | nc_path = os.path.join(os.path.dirname(__file__), NC_PATH) 66 | print(nc_path) 67 | nc_dataset = netCDF4.Dataset(nc_path) 68 | netcdf_line_utils = NetCDFLineUtils(nc_dataset) 69 | 70 | # print(netcdf_line_utils.__dict__) 71 | assert nc_dataset.title == NC_TITLE, 'Invalid dataset title: "{}" != "{}"'.format(nc_dataset.title, NC_TITLE) 72 | 73 | 74 | class TestNetCDFLineUtilsFunctions1(unittest.TestCase): 75 | """Unit tests for geophys_utils._netcdf_line_utils functions""" 76 | 77 | def test_get_line_masks(self): 78 | print('Testing get_line_masks function') 79 | count = 0 80 | for line_number, line_mask in netcdf_line_utils.get_line_masks(): 81 | # print('Line {} has {} points'.format(line_number, np.count_nonzero(line_mask))) 82 | assert (line_number, np.count_nonzero(line_mask)) == TEST_GET_LINE_MASK_RESULTS[ 83 | count], "Invalid get_line_masks result" 84 | count += 1 85 | if count >= 2: 86 | break 87 | 88 | def test_concave_hull(self): 89 | print('Testing concave hull') 90 | assert isinstance(netcdf_line_utils.get_concave_hull(), Polygon) 91 | 92 | 93 | class TestNetCDFLineUtilsFunctions2(unittest.TestCase): 94 | """Unit tests for geophys_utils._netcdf_line_utils functions""" 95 | 96 | def test_get_lines(self): 97 | print('Testing get_lines function') 98 | count = 0 99 | for line_number, line_dict in netcdf_line_utils.get_lines(): 100 | # =================================================================== 101 | # print('Line {} has {} variables with {} points'.format(line_number, 102 | # len(line_dict)-1, 103 | # np.count_nonzero(line_dict['coordinates']) 104 | # ) 105 | # ) 106 | # =================================================================== 107 | assert (line_number, len(line_dict) - 1, np.count_nonzero(line_dict['coordinates'])) == \ 108 | TEST_GET_LINE_RESULTS[count], \ 109 | "Invalid get_lines result: Expected {}, got {}".format(TEST_GET_LINE_RESULTS[count], ( 110 | line_number, len(line_dict) - 1, np.count_nonzero(line_dict['coordinates']))) 111 | 112 | count += 1 113 | if count >= 2: 114 | break 115 | 116 | 117 | # Define test suites 118 | def test_suite(): 119 | """Returns a test suite of all the tests in this module.""" 120 | 121 | test_classes = [TestNetCDFLineUtilsConstructor, 122 | TestNetCDFLineUtilsFunctions1, 123 | TestNetCDFLineUtilsFunctions2 124 | ] 125 | 126 | suite_list = map(unittest.defaultTestLoader.loadTestsFromTestCase, 127 | test_classes) 128 | 129 | suite = unittest.TestSuite(suite_list) 130 | 131 | return suite 132 | 133 | 134 | # Define main function 135 | def main(): 136 | unittest.TextTestRunner(verbosity=2).run(test_suite()) 137 | 138 | 139 | if __name__ == '__main__': 140 | main() 141 | -------------------------------------------------------------------------------- /geophys_utils/test/test_netcdf_point_utils.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # =============================================================================== 4 | # Copyright 2017 Geoscience Australia 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # =============================================================================== 18 | """ 19 | Unit tests for geophys_utils._netcdf_point_utils against a NetCDF line data file 20 | 21 | Created on 15/11/2016 22 | 23 | @author: Alex Ip 24 | """ 25 | import os 26 | import re 27 | import unittest 28 | 29 | import netCDF4 30 | import numpy as np 31 | from shapely.geometry.polygon import Polygon 32 | 33 | from geophys_utils._netcdf_point_utils import NetCDFPointUtils 34 | 35 | netcdf_point_utils = None 36 | 37 | NC_PATH = 'https://dapds00.nci.org.au/thredds/dodsC/iv65/Geoscience_Australia_Geophysics_Reference_Data_Collection/ground_gravity/SA/point/P194830/P194830-point-gravity.nc' 38 | NC_TITLE = 'Leigh Creek Gravity (P194830), gravity point data' 39 | TEST_BOUNDS = (138.23, -30.3, 138.45, -30.125) 40 | GRID_RESOLUTION = 0.001 41 | RESAMPLING_METHOD = 'linear' # 'linear', 'nearest' or 'cubic'. See https://docs.scipy.org/doc/scipy/reference/generated/scipy.interpolate.griddata.html 42 | VARIABLES = 'bouguer' 43 | POINT_STEP = 1 44 | 45 | MAX_BYTES = 1600 46 | MAX_ERROR = 0.000001 47 | TEST_COORDS = (148.213, -36.015) 48 | TEST_INDICES = [1, 1] 49 | TEST_FRACTIONAL_INDICES = [1.25, 1.25] 50 | TEST_VALUE = 0.0 51 | TEST_INTERPOLATED_VALUE = -99997.6171875 52 | SPATIAL_MASK_COUNT = 158 53 | 54 | TEST_GRID_RESULTS = ( 55 | ( 56 | 'GEOGCS["GDA94",DATUM["Geocentric_Datum_of_Australia_1994",SPHEROID["GRS 1980",6378137,298.257222101,AUTHORITY["EPSG","7019"]],TOWGS84[0,0,0,0,0,0,0],AUTHORITY["EPSG","6283"]],PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],UNIT["degree",0.0174532925199433,AUTHORITY["EPSG","9122"]],AUTHORITY["EPSG","4283"]]', 57 | [138.29250000000002, 0.001, 0, -30.1185, 0, -0.001], 58 | (172, 168) 59 | ), 60 | ( 61 | 'GEOGCS["GDA94",DATUM["Geocentric_Datum_of_Australia_1994",SPHEROID["GRS 1980",6378137,298.257222101,AUTHORITY["EPSG","7019"]],TOWGS84[0,0,0,0,0,0,0],AUTHORITY["EPSG","6283"]],PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],UNIT["degree",0.0174532925199433,AUTHORITY["EPSG","9122"]],AUTHORITY["EPSG","4283"]]', 62 | [138.2295, 0.001, 0, -30.1245, 0, -0.001], 63 | (176, 221) 64 | ) 65 | ) 66 | 67 | EXPECTED_GML = 'POLYGON((138.3498 -30.2894, 138.3093 -30.1768, 138.2940 -30.1189, 138.3097 -30.1187, 138.3189 -30.1187, 138.3684 -30.1188, 138.4608 -30.2419, 138.4319 -30.2558, 138.3663 -30.2854, 138.3582 -30.2875, 138.3498 -30.2894))' 68 | 69 | 70 | class TestNetCDFPointUtilsConstructor(unittest.TestCase): 71 | """Unit tests for TestNetCDFPointUtils Constructor. 72 | N.B: This should be run first""" 73 | 74 | def test_netcdf_point_utils_constructor(self): 75 | print('Testing NetCDFPointUtils constructor') 76 | global netcdf_point_utils 77 | 78 | if re.match('^http.*', NC_PATH): 79 | nc_path = NC_PATH 80 | else: 81 | nc_path = os.path.join(os.path.dirname(__file__), NC_PATH) 82 | print(nc_path) 83 | nc_dataset = netCDF4.Dataset(nc_path) 84 | netcdf_point_utils = NetCDFPointUtils(nc_dataset) 85 | 86 | # print(netcdf_point_utils.__dict__) 87 | assert nc_dataset.title == NC_TITLE, 'Invalid dataset title: "{}" != "{}"'.format(nc_dataset.title, NC_TITLE) 88 | 89 | 90 | class TestNetCDFPointUtilsFunctions1(unittest.TestCase): 91 | """Unit tests for geophys_utils._netcdf_point_utils functions""" 92 | 93 | def test_get_polygon(self): 94 | print('Testing get_polygon function') 95 | polygon = netcdf_point_utils.get_polygon() 96 | assert polygon == EXPECTED_GML 97 | 98 | def test_get_spatial_mask(self): 99 | print('Testing get_spatial_mask function') 100 | spatial_mask = netcdf_point_utils.get_spatial_mask(TEST_BOUNDS) 101 | # print(spatial_mask) 102 | assert np.count_nonzero( 103 | spatial_mask) == SPATIAL_MASK_COUNT, f'Unexpected spatial mask count {np.count_nonzero(spatial_mask)} != {SPATIAL_MASK_COUNT}' 104 | 105 | def test_concave_hull(self): 106 | print('Testing concave hull') 107 | assert isinstance(netcdf_point_utils.get_concave_hull(), Polygon) 108 | 109 | 110 | class TestNetCDFPointUtilsGridFunctions(unittest.TestCase): 111 | """Unit tests for geophys_utils._netcdf_point_utils functions""" 112 | 113 | def test_grid_points(self): 114 | print('Testing grid_points function') 115 | grids, crs, geotransform = netcdf_point_utils.grid_points(grid_resolution=GRID_RESOLUTION, 116 | variables=VARIABLES, 117 | point_step=POINT_STEP) 118 | 119 | assert (crs, geotransform, grids.shape) == TEST_GRID_RESULTS[0], 'Invalid grid results: {} != {}'.format( 120 | (crs, geotransform, grids.shape), TEST_GRID_RESULTS[0]) 121 | 122 | def test_grid_points_bounded(self): 123 | print('Testing bounded grid_points function') 124 | grids, crs, geotransform = netcdf_point_utils.grid_points(grid_resolution=GRID_RESOLUTION, 125 | variables=VARIABLES, 126 | native_grid_bounds=TEST_BOUNDS, 127 | point_step=POINT_STEP) 128 | assert (crs, geotransform, grids.shape) == TEST_GRID_RESULTS[1], 'Invalid grid results: {} != {}'.format( 129 | (crs, geotransform, grids.shape), TEST_GRID_RESULTS[1]) 130 | 131 | 132 | # Define test suites 133 | def test_suite(): 134 | """Returns a test suite of all the tests in this module.""" 135 | 136 | test_classes = [TestNetCDFPointUtilsConstructor, 137 | TestNetCDFPointUtilsFunctions1, 138 | TestNetCDFPointUtilsGridFunctions 139 | ] 140 | 141 | suite_list = map(unittest.defaultTestLoader.loadTestsFromTestCase, 142 | test_classes) 143 | 144 | suite = unittest.TestSuite(suite_list) 145 | 146 | return suite 147 | 148 | 149 | # Define main function 150 | def main(): 151 | unittest.TextTestRunner(verbosity=2).run(test_suite()) 152 | 153 | 154 | if __name__ == '__main__': 155 | main() 156 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "geophys_utils" 3 | version = "0.1.0" 4 | authors = [ 5 | { name="Alex Ip", email="alex@trentham.net.au" }, 6 | { name="Andrew Turner", email="andrew.turner@ga.gov.au" }, 7 | ] 8 | maintainers = [ 9 | { name="Geoscience Australia", email="clientservices@ga.gov.au" }, 10 | ] 11 | description = "Geophysics data access utilities" 12 | readme = "README.md" 13 | requires-python = ">=3.9" 14 | classifiers = [ 15 | "Programming Language :: Python :: 3", 16 | "License :: OSI Approved :: Apache Software License", 17 | "Operating System :: OS Independent", 18 | ] 19 | dependencies = [ 20 | "numpy==1.24.3", 21 | "netCDF4==1.6.4", 22 | "gdal==3.6.2", 23 | "affine==2.3.0", 24 | "cartopy==0.18.0", 25 | "matplotlib==3.7.1", 26 | "numexpr==2.8.4", 27 | "owslib==0.24.1", 28 | "pyyaml==6.0", 29 | "requests==2.29.0", 30 | "scikit-image==0.19.3", 31 | "scipy==1.10.1", 32 | "shapely==2.0.1", 33 | "unidecode==1.3.6", 34 | "zipstream-new==1.1.8", 35 | ] 36 | [project.urls] 37 | "Homepage" = "https://github.com/geoscienceaustralia/geophys_utils" 38 | "Bug Tracker" = "https://github.com/geoscienceaustralia/geophys_utils/issues" 39 | [tool.setuptools] 40 | py-modules = [ 41 | 'geophys_utils', 42 | ] 43 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | numpy==1.24.3 2 | netCDF4==1.6.4 3 | gdal==3.6.2 4 | affine==2.3.0 5 | cartopy==0.18.0 6 | matplotlib==3.7.1 7 | numexpr==2.8.4 8 | owslib==0.24.1 9 | pyyaml==6.0 10 | requests==2.29.0 11 | scikit-image==0.19.3 12 | scipy==1.10.1 13 | shapely==2.0.1 14 | unidecode==1.3.6 15 | zipstream-new==1.1.8 -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import setuptools 4 | 5 | if __name__ == "__main__": 6 | setuptools.setup() --------------------------------------------------------------------------------