├── .github └── workflows │ ├── ci.yaml │ └── deploy-sdist.yaml ├── .gitignore ├── .pre-commit-config.yaml ├── .readthedocs.yml ├── CHANGELOG.md ├── LICENSE.txt ├── MANIFEST.in ├── README.md ├── RELEASING.md ├── changelog.rst ├── continuous_integration └── environment.yaml ├── doc ├── Makefile ├── rtd_environment.yaml └── source │ ├── _static │ └── .gitkeep │ ├── api.rst │ ├── conf.py │ ├── images │ └── noaa_class_preferences.png │ ├── index.rst │ ├── installation.rst │ ├── introduction.rst │ ├── legacy.rst │ ├── methods.rst │ └── usage.rst ├── etc └── pygac.cfg.template ├── gapfilled_tles └── TLE_noaa16.txt ├── pygac ├── __init__.py ├── calibration │ └── noaa.py ├── clock_offsets_converter.py ├── configuration.py ├── correct_tsm_issue.py ├── data │ └── calibration.json ├── gac_io.py ├── gac_klm.py ├── gac_pod.py ├── gac_reader.py ├── klm_reader.py ├── lac_klm.py ├── lac_pod.py ├── lac_reader.py ├── patmosx_coeff_reader.py ├── pod_reader.py ├── reader.py ├── runner.py ├── slerp.py ├── tests │ ├── __init__.py │ ├── test_angles.py │ ├── test_calibrate_klm.py │ ├── test_calibrate_pod.py │ ├── test_io.py │ ├── test_klm.py │ ├── test_noaa_calibration_coefficients.py │ ├── test_pod.py │ ├── test_reader.py │ ├── test_slerp.py │ ├── test_tsm.py │ ├── test_utils.py │ └── utils.py └── utils.py ├── pyproject.toml └── requirements.txt /.github/workflows/ci.yaml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | test: 7 | runs-on: ${{ matrix.os }} 8 | continue-on-error: ${{ matrix.experimental }} 9 | strategy: 10 | fail-fast: true 11 | matrix: 12 | os: ["ubuntu-latest"] 13 | python-version: ["3.11", "3.12", "3.13"] 14 | experimental: [false] 15 | 16 | env: 17 | PYTHON_VERSION: ${{ matrix.python-version }} 18 | OS: ${{ matrix.os }} 19 | ACTIONS_ALLOW_UNSECURE_COMMANDS: true 20 | PYGAC_CONFIG_FILE: etc/pygac.cfg.template 21 | 22 | steps: 23 | - name: Checkout source 24 | uses: actions/checkout@v2 25 | 26 | - name: Setup Conda Environment 27 | uses: conda-incubator/setup-miniconda@v2 28 | with: 29 | miniconda-version: "latest" 30 | python-version: ${{ matrix.python-version }} 31 | environment-file: continuous_integration/environment.yaml 32 | activate-environment: test-environment 33 | 34 | - name: Install pygac 35 | shell: bash -l {0} 36 | run: | 37 | pip install --no-deps -e . 38 | 39 | - name: Run unit tests 40 | shell: bash -l {0} 41 | run: | 42 | pytest --cov=pygac pygac/tests --cov-report=xml 43 | 44 | - name: Upload unittest coverage 45 | uses: codecov/codecov-action@v2 46 | with: 47 | file: ./coverage.xml 48 | env_vars: OS,PYTHON_VERSION 49 | -------------------------------------------------------------------------------- /.github/workflows/deploy-sdist.yaml: -------------------------------------------------------------------------------- 1 | name: Deploy sdist 2 | 3 | on: 4 | release: 5 | types: 6 | - published 7 | 8 | jobs: 9 | test: 10 | runs-on: ubuntu-latest 11 | 12 | steps: 13 | - name: Checkout source 14 | uses: actions/checkout@v4 15 | 16 | - name: Create sdist 17 | shell: bash -l {0} 18 | run: | 19 | python -m pip install -q build 20 | python -m build -s 21 | 22 | - name: Publish package to PyPI 23 | if: github.event.action == 'published' 24 | uses: pypa/gh-action-pypi-publish@v1.9.0 25 | with: 26 | user: __token__ 27 | password: ${{ secrets.pypi_password }} 28 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | build 2 | doc/build 3 | *.so 4 | *.pyc 5 | *~ 6 | dist 7 | pygac.egg-info 8 | etc/*.cfg 9 | gapfilled_tles/* 10 | pygac/version.py 11 | 12 | tmp 13 | .vscode 14 | .venv 15 | *.lock 16 | .python-version 17 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | exclude: '^$' 2 | fail_fast: false 3 | 4 | repos: 5 | - repo: https://github.com/astral-sh/ruff-pre-commit 6 | # Ruff version. 7 | rev: 'v0.6.2' 8 | hooks: 9 | - id: ruff 10 | args: [--fix] 11 | - repo: https://github.com/pre-commit/pre-commit-hooks 12 | rev: v4.5.0 13 | hooks: 14 | - id: trailing-whitespace 15 | - id: end-of-file-fixer 16 | -------------------------------------------------------------------------------- /.readthedocs.yml: -------------------------------------------------------------------------------- 1 | # Read the Docs configuration file 2 | # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details 3 | version: 2 4 | # Build documentation in the docs/ directory with Sphinx 5 | sphinx: 6 | configuration: doc/source/conf.py 7 | fail_on_warning: true 8 | 9 | build: 10 | os: "ubuntu-20.04" 11 | tools: 12 | python: "mambaforge-4.10" 13 | conda: 14 | environment: doc/rtd_environment.yaml 15 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ## Version 1.7.4 (2024/07/17) 2 | 3 | ### Issues Closed 4 | 5 | * [Issue 128](https://github.com/pytroll/pygac/issues/128) - Reading of files broken for numpy 2.0.0 6 | * [Issue 114](https://github.com/pytroll/pygac/issues/114) - Feature request: Overlap Area Crop just like pygac-fdr 7 | 8 | In this release 2 issues were closed. 9 | 10 | ### Pull Requests Merged 11 | 12 | #### Features added 13 | 14 | * [PR 126](https://github.com/pytroll/pygac/pull/126) - Switch to `pyproject.toml`entirely 15 | 16 | In this release 1 pull request was closed. 17 | 18 | 19 | ## Version 1.7.3 (2024/03/19) 20 | 21 | 22 | ### Pull Requests Merged 23 | 24 | #### Features added 25 | 26 | * [PR 125](https://github.com/pytroll/pygac/pull/125) - Support reading EO-SIP LAC data 27 | 28 | In this release 1 pull request was closed. 29 | 30 | 31 | ## Version 1.7.2 (2023/06/26) 32 | 33 | 34 | ### Pull Requests Merged 35 | 36 | #### Bugs fixed 37 | 38 | * [PR 120](https://github.com/pytroll/pygac/pull/120) - Compatibility with numpy v1.24 39 | 40 | #### Features added 41 | 42 | * [PR 122](https://github.com/pytroll/pygac/pull/122) - Don't use deprecated distutils module. 43 | 44 | #### Documentation changes 45 | 46 | * [PR 123](https://github.com/pytroll/pygac/pull/123) - Update supported data formats in documentation ([2494](https://github.com/pytroll/satpy/issues/2494)) 47 | 48 | In this release 3 pull requests were closed. 49 | 50 | 51 | ## Version 1.7.1 (2022/12/09) 52 | 53 | 54 | ### Pull Requests Merged 55 | 56 | #### Bugs fixed 57 | 58 | * [PR 119](https://github.com/pytroll/pygac/pull/119) - Fix the documentation to include FRAC and light cleanup 59 | 60 | #### Documentation changes 61 | 62 | * [PR 119](https://github.com/pytroll/pygac/pull/119) - Fix the documentation to include FRAC and light cleanup 63 | 64 | In this release 2 pull requests were closed. 65 | 66 | 67 | ## Version 1.7.0 (2022/10/31) 68 | 69 | ### Issues Closed 70 | 71 | * [Issue 112](https://github.com/pytroll/pygac/issues/112) - Usage documentation uses wrong function name for 'get_reader_class' ([PR 113](https://github.com/pytroll/pygac/pull/113) by [@mraspaud](https://github.com/mraspaud)) 72 | * [Issue 78](https://github.com/pytroll/pygac/issues/78) - Handle completely missing ICT or space counts ([PR 117](https://github.com/pytroll/pygac/pull/117) by [@mraspaud](https://github.com/mraspaud)) 73 | * [Issue 22](https://github.com/pytroll/pygac/issues/22) - Thermal calibration error ([PR 117](https://github.com/pytroll/pygac/pull/117) by [@mraspaud](https://github.com/mraspaud)) 74 | 75 | In this release 3 issues were closed. 76 | 77 | ### Pull Requests Merged 78 | 79 | #### Bugs fixed 80 | 81 | * [PR 117](https://github.com/pytroll/pygac/pull/117) - Fix calibration when 3b is totally deactivated ([78](https://github.com/pytroll/pygac/issues/78), [22](https://github.com/pytroll/pygac/issues/22)) 82 | * [PR 116](https://github.com/pytroll/pygac/pull/116) - Fix typos and deprecations 83 | * [PR 113](https://github.com/pytroll/pygac/pull/113) - Fix usage typo ([112](https://github.com/pytroll/pygac/issues/112)) 84 | 85 | #### Features added 86 | 87 | * [PR 118](https://github.com/pytroll/pygac/pull/118) - Bump up python versions 88 | * [PR 115](https://github.com/pytroll/pygac/pull/115) - Add support for frac data 89 | 90 | In this release 5 pull requests were closed. 91 | 92 | ## Version 1.6.0 (2022/08/04) 93 | 94 | ### Issues Closed 95 | 96 | * [Issue 107](https://github.com/pytroll/pygac/issues/107) - Fix sun-earth distance correction 97 | * [Issue 104](https://github.com/pytroll/pygac/issues/104) - Links to POD/KLM user guides are mixed up ([PR 105](https://github.com/pytroll/pygac/pull/105) by [@mraspaud](https://github.com/mraspaud)) 98 | * [Issue 103](https://github.com/pytroll/pygac/issues/103) - API is missing on readthedocs 99 | * [Issue 40](https://github.com/pytroll/pygac/issues/40) - Calculated sun-earth distance correction factor once and add as attribute 100 | 101 | In this release 4 issues were closed. 102 | 103 | ### Pull Requests Merged 104 | 105 | #### Features added 106 | 107 | * [PR 109](https://github.com/pytroll/pygac/pull/109) - Add new unpublished METOPC VIS calibration coefficients from Patmos-x 108 | 109 | #### Documentation changes 110 | 111 | * [PR 105](https://github.com/pytroll/pygac/pull/105) - Fix the pod and klm guide links ([104](https://github.com/pytroll/pygac/issues/104)) 112 | 113 | In this release 2 pull requests were closed. 114 | 115 | ## Version 1.5.0 (2022/01/10) 116 | 117 | ### Issues Closed 118 | 119 | * [Issue 94](https://github.com/pytroll/pygac/issues/94) - Method get_tle_lines() raises index error when tle lines are empty strings 120 | * [Issue 90](https://github.com/pytroll/pygac/issues/90) - Change in masked scanlines 121 | * [Issue 87](https://github.com/pytroll/pygac/issues/87) - Unit test discovery ([PR 98](https://github.com/pytroll/pygac/pull/98) by [@sfinkens](https://github.com/sfinkens)) 122 | * [Issue 85](https://github.com/pytroll/pygac/issues/85) - Update documentation 123 | * [Issue 80](https://github.com/pytroll/pygac/issues/80) - Reduce rounding error in POD reader adjust clock drift ([PR 84](https://github.com/pytroll/pygac/pull/84) by [@carloshorn](https://github.com/carloshorn)) 124 | * [Issue 76](https://github.com/pytroll/pygac/issues/76) - IndexError in PODReader._adjust_clock_drift ([PR 84](https://github.com/pytroll/pygac/pull/84) by [@carloshorn](https://github.com/carloshorn)) 125 | * [Issue 74](https://github.com/pytroll/pygac/issues/74) - Remove Cython dependency ([PR 83](https://github.com/pytroll/pygac/pull/83) by [@carloshorn](https://github.com/carloshorn)) 126 | * [Issue 46](https://github.com/pytroll/pygac/issues/46) - Pypi and presentation work 127 | * [Issue 43](https://github.com/pytroll/pygac/issues/43) - Links to POD/KLM user guides are broken 128 | * [Issue 39](https://github.com/pytroll/pygac/issues/39) - Update release in preparation for conda-forge package 129 | 130 | In this release 10 issues were closed. 131 | 132 | ### Pull Requests Merged 133 | 134 | #### Bugs fixed 135 | 136 | * [PR 91](https://github.com/pytroll/pygac/pull/91) - Invert bit position in quality flags 137 | * [PR 84](https://github.com/pytroll/pygac/pull/84) - Refactor clock drift ([82](https://github.com/pytroll/pygac/issues/82), [81](https://github.com/pytroll/pygac/issues/81), [80](https://github.com/pytroll/pygac/issues/80), [76](https://github.com/pytroll/pygac/issues/76)) 138 | * [PR 79](https://github.com/pytroll/pygac/pull/79) - Fix tests on bigendian platforms 139 | * [PR 75](https://github.com/pytroll/pygac/pull/75) - Typo 140 | 141 | #### Features added 142 | 143 | * [PR 98](https://github.com/pytroll/pygac/pull/98) - Switch to pytest for unit tests ([87](https://github.com/pytroll/pygac/issues/87)) 144 | * [PR 92](https://github.com/pytroll/pygac/pull/92) - Allow PathLike objects 145 | * [PR 89](https://github.com/pytroll/pygac/pull/89) - Add github workflow 146 | * [PR 86](https://github.com/pytroll/pygac/pull/86) - add METOP C coefficients 147 | * [PR 84](https://github.com/pytroll/pygac/pull/84) - Refactor clock drift ([82](https://github.com/pytroll/pygac/issues/82), [81](https://github.com/pytroll/pygac/issues/81), [80](https://github.com/pytroll/pygac/issues/80), [76](https://github.com/pytroll/pygac/issues/76)) 148 | * [PR 83](https://github.com/pytroll/pygac/pull/83) - Replace cython filter ([74](https://github.com/pytroll/pygac/issues/74)) 149 | 150 | In this release 10 pull requests were closed. 151 | 152 | ## Version 1.4.0 (2020/08/06) 153 | 154 | ### Issues Closed 155 | 156 | * [Issue 66](https://github.com/pytroll/pygac/issues/66) - Typos in calibration coefficients ([PR 67](https://github.com/pytroll/pygac/pull/67)) 157 | * [Issue 62](https://github.com/pytroll/pygac/issues/62) - Computation of Earth scene radiance ([PR 58](https://github.com/pytroll/pygac/pull/58)) 158 | * [Issue 60](https://github.com/pytroll/pygac/issues/60) - Improve readability of quality indicators bit unpacking ([PR 72](https://github.com/pytroll/pygac/pull/72)) 159 | * [Issue 57](https://github.com/pytroll/pygac/issues/57) - channel 4 BT to radiance conversion ([PR 67](https://github.com/pytroll/pygac/pull/67)) 160 | * [Issue 54](https://github.com/pytroll/pygac/issues/54) - Function check_file_version should not be part of pygac-run ([PR 55](https://github.com/pytroll/pygac/pull/55)) 161 | * [Issue 47](https://github.com/pytroll/pygac/issues/47) - Fix reading of renamed files 162 | 163 | In this release 6 issues were closed. 164 | 165 | ### Pull Requests Merged 166 | 167 | #### Bugs fixed 168 | 169 | * [PR 73](https://github.com/pytroll/pygac/pull/73) - Fix azimuth encoding 170 | * [PR 67](https://github.com/pytroll/pygac/pull/67) - Correct coefficients ([66](https://github.com/pytroll/pygac/issues/66), [57](https://github.com/pytroll/pygac/issues/57)) 171 | 172 | #### Features added 173 | 174 | * [PR 72](https://github.com/pytroll/pygac/pull/72) - Quality indicators ([60](https://github.com/pytroll/pygac/issues/60)) 175 | * [PR 71](https://github.com/pytroll/pygac/pull/71) - Expose new metadata 176 | * [PR 70](https://github.com/pytroll/pygac/pull/70) - Remove config dependency from reader class 177 | * [PR 58](https://github.com/pytroll/pygac/pull/58) - export coefficients to json file ([62](https://github.com/pytroll/pygac/issues/62)) 178 | * [PR 55](https://github.com/pytroll/pygac/pull/55) - Refactor gac-run ([54](https://github.com/pytroll/pygac/issues/54)) 179 | 180 | In this release 7 pull requests were closed. 181 | 182 | ## Version v1.3.1 (2020/02/07) 183 | 184 | ### Issues Closed 185 | 186 | * [Issue 52](https://github.com/pytroll/pygac/issues/52) - Allow gzip input files ([PR 53](https://github.com/pytroll/pygac/pull/53)) 187 | * [Issue 49](https://github.com/pytroll/pygac/issues/49) - Calibration Coeffs Patmos-X 2017 ([PR 50](https://github.com/pytroll/pygac/pull/50)) 188 | 189 | In this release 2 issues were closed. 190 | 191 | ### Pull Requests Merged 192 | 193 | #### Bugs fixed 194 | 195 | * [PR 50](https://github.com/pytroll/pygac/pull/50) - Fix typo in MetOp-B calibration ([49](https://github.com/pytroll/pygac/issues/49)) 196 | * [PR 48](https://github.com/pytroll/pygac/pull/48) - Update metadata when reading lonlat also 197 | 198 | #### Features added 199 | 200 | * [PR 53](https://github.com/pytroll/pygac/pull/53) - Allow gzip compressed files as input for klm and pod readers read method ([52](https://github.com/pytroll/pygac/issues/52)) 201 | * [PR 51](https://github.com/pytroll/pygac/pull/51) - Use the data format name to id ARS headered files 202 | 203 | In this release 4 pull requests were closed. 204 | 205 | 206 | ## Version 1.3.0 (2019/12/05) 207 | 208 | 209 | ### Pull Requests Merged 210 | 211 | #### Features added 212 | 213 | * [PR 45](https://github.com/pytroll/pygac/pull/45) - Add LAC support ([5](https://github.com/pytroll/pygac/issues/5)) 214 | * [PR 42](https://github.com/pytroll/pygac/pull/42) - Update documentation 215 | * [PR 41](https://github.com/pytroll/pygac/pull/41) - Add meta_data dictionary to reader class. 216 | 217 | In this release 3 pull requests were closed. 218 | 219 | 220 | ## Version 1.2.1 (2019/11/15) 221 | 222 | ### Pull Requests Merged 223 | 224 | #### Bugs fixed 225 | 226 | * [PR 37](https://github.com/pytroll/pygac/pull/37) - Fixing geotiepoints attribute error for python2.7 227 | * [PR 36](https://github.com/pytroll/pygac/pull/36) - Fix update of missing scanlines 228 | 229 | #### Features added 230 | 231 | * [PR 38](https://github.com/pytroll/pygac/pull/38) - Fix tests for python 3 232 | 233 | In this release 3 pull requests were closed. 234 | 235 | ## Version 1.2.0 (2019/10/17) 236 | 237 | ### Issues Closed 238 | 239 | * [Issue 33](https://github.com/pytroll/pygac/issues/33) - Make use of pytroll geotiepoints instead of using a deprecated copy 240 | * [Issue 7](https://github.com/pytroll/pygac/issues/7) - Project URL points to wrong domain 241 | 242 | In this release 2 issues were closed. 243 | 244 | ### Pull Requests Merged 245 | 246 | #### Bugs fixed 247 | 248 | * [PR 30](https://github.com/pytroll/pygac/pull/30) - Feature update angles computation 249 | 250 | #### Features added 251 | 252 | * [PR 35](https://github.com/pytroll/pygac/pull/35) - Changed azimuth angle range to ]-180, 180] 253 | * [PR 34](https://github.com/pytroll/pygac/pull/34) - Use the geotiepoints library 254 | * [PR 32](https://github.com/pytroll/pygac/pull/32) - Updated documentation about azimuth angles 255 | * [PR 31](https://github.com/pytroll/pygac/pull/31) - Refactor I/O 256 | 257 | In this release 5 pull requests were closed. 258 | 259 | ## Version 1.1.0 (2019/06/12) 260 | 261 | ### Issues Closed 262 | 263 | * [Issue 23](https://github.com/pytroll/pygac/issues/23) - Add support for Python3 264 | * [Issue 20](https://github.com/pytroll/pygac/issues/20) - IndexError if orbit time beyond TLE range ([PR 21](https://github.com/pytroll/pygac/pull/21)) 265 | * [Issue 18](https://github.com/pytroll/pygac/issues/18) - Error in pygac data zenith angle 266 | * [Issue 16](https://github.com/pytroll/pygac/issues/16) - Unit test failure 267 | 268 | In this release 4 issues were closed. 269 | 270 | ### Pull Requests Merged 271 | 272 | #### Bugs fixed 273 | 274 | * [PR 27](https://github.com/pytroll/pygac/pull/27) - Add get_times to angles computation 275 | * [PR 24](https://github.com/pytroll/pygac/pull/24) - Fix unit tests 276 | * [PR 21](https://github.com/pytroll/pygac/pull/21) - Fix TLE line identification ([20](https://github.com/pytroll/pygac/issues/20)) 277 | * [PR 17](https://github.com/pytroll/pygac/pull/17) - Python 3 compatibiity 278 | * [PR 15](https://github.com/pytroll/pygac/pull/15) - Fix selection of user defined scanlines #2 279 | * [PR 14](https://github.com/pytroll/pygac/pull/14) - Fix gac-run 280 | * [PR 9](https://github.com/pytroll/pygac/pull/9) - Fixes: Dependency and docs 281 | * [PR 3](https://github.com/pytroll/pygac/pull/3) - Feature clock fixed nan tsm scanline timestamp 282 | 283 | #### Features added 284 | 285 | * [PR 28](https://github.com/pytroll/pygac/pull/28) - Simplify TLE computation 286 | * [PR 26](https://github.com/pytroll/pygac/pull/26) - Python-3 Compatibility 287 | * [PR 25](https://github.com/pytroll/pygac/pull/25) - Fix style 288 | * [PR 19](https://github.com/pytroll/pygac/pull/19) - Add support for TIROS-N 289 | * [PR 13](https://github.com/pytroll/pygac/pull/13) - Update installation.rst with TLE files 290 | * [PR 11](https://github.com/pytroll/pygac/pull/11) - Add scanline timestamps to qualflags file 291 | * [PR 10](https://github.com/pytroll/pygac/pull/10) - Usability improvements 292 | * [PR 6](https://github.com/pytroll/pygac/pull/6) - Add new attributes to /how 293 | * [PR 4](https://github.com/pytroll/pygac/pull/4) - Improve interface of gac_run.py 294 | 295 | In this release 17 pull requests were closed. 296 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include etc/pygac.cfg.template 2 | include gapfilled_tles/TLE_*.txt 3 | include LICENSE.txt 4 | include README.md 5 | include pygac/version.py 6 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | pygac 2 | ===== 3 | 4 | [![Build](https://github.com/pytroll/pygac/actions/workflows/ci.yaml/badge.svg)](https://github.com/pytroll/pygac/actions/workflows/ci.yaml) 5 | [![Coverage](https://codecov.io/gh/pytroll/pygac/branch/main/graph/badge.svg?token=DQMgf2LxuM)](https://codecov.io/gh/pytroll/pygac) 6 | [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.5832775.svg)](https://doi.org/10.5281/zenodo.5832775) 7 | 8 | 9 | Pygac is a Python package to read, calibrate and navigate data from the AVHRR 10 | instrument onboard NOAA and MetOp satellites in GAC and LAC format. 11 | 12 | The documentation is available at https://pygac.readthedocs.io/. 13 | -------------------------------------------------------------------------------- /RELEASING.md: -------------------------------------------------------------------------------- 1 | # Releasing Pygac 2 | 3 | 1. checkout main branch 4 | 2. pull from repo 5 | 3. run the unittests 6 | 4. run `loghub`. Replace and with proper 7 | values. To get the previous version run `git tag` and select the most 8 | recent with highest version number. 9 | 10 | ``` 11 | loghub pytroll/pygac -u -st v -plg bug "Bugs fixed" -plg enhancement "Features added" -plg documentation "Documentation changes" -plg backwards-incompatibility "Backwards incompatible changes" 12 | ``` 13 | 14 | This command will create a CHANGELOG.temp file which need to be added 15 | to the top of the CHANGLOG.md file. The same content is also printed 16 | to terminal, so that can be copy-pasted, too. Remember to update also 17 | the version number to the same given in step 5. Don't forget to commit 18 | CHANGELOG.md! 19 | 20 | 5. Create a tag with the new version number, starting with a 'v', eg: 21 | 22 | ``` 23 | git tag -a v -m "Version " 24 | ``` 25 | 26 | For example if the previous tag was `v0.9.0` and the new release is a 27 | patch release, do: 28 | 29 | ``` 30 | git tag -a v0.9.1 -m "Version 0.9.1" 31 | ``` 32 | 33 | See [semver.org](http://semver.org/) on how to write a version number. 34 | 35 | 6. push changes to github `git push --follow-tags` 36 | 7. Verify github action unittests passed. 37 | 8. Create a "Release" on GitHub by going to 38 | https://github.com/pytroll/pygac/releases and clicking "Draft a new release". 39 | On the next page enter the newly created tag in the "Tag version" field, 40 | "Version X.Y.Z" in the "Release title" field, and paste the markdown from 41 | the changelog (the portion under the version section header) in the 42 | "Describe this release" box. Finally click "Publish release". 43 | 9. Verify the GitHub actions for deployment succeed and the release is on PyPI. 44 | -------------------------------------------------------------------------------- /changelog.rst: -------------------------------------------------------------------------------- 1 | Changelog 2 | ========= 3 | 4 | %%version%% (unreleased) 5 | ------------------------ 6 | 7 | - Update changelog. [Martin Raspaud] 8 | 9 | - Bump version: 1.0.0 → 1.0.1. [Martin Raspaud] 10 | 11 | - Use version number from version.py. [Martin Raspaud] 12 | 13 | - Update changelog. [Martin Raspaud] 14 | 15 | - Bump version: 0.1.0 → 1.0.0. [Martin Raspaud] 16 | 17 | - Do style cleanup for gac_pod. [Martin Raspaud] 18 | 19 | - Allow PYGAC_CONFIG_FILE to be missing. [Martin Raspaud] 20 | 21 | - Merge branch 'master' into develop. [Adam.Dybbroe] 22 | 23 | - Merge branch 'pre-master' [Adam.Dybbroe] 24 | 25 | - Merge branch 'feature-clock' into pre-master. [Adam.Dybbroe] 26 | 27 | - Time stamp for mid-morning orbit investigated. [Abhay Devasthale] 28 | 29 | - Time stamp for mid-night orbit investigated. [Abhay Devasthale] 30 | 31 | - The support for NOAA-6 and -8 added. [Abhay Devasthale] 32 | 33 | - Calibration support for NOAA-6 and -8 added. [Abhay Devasthale] 34 | 35 | - Merge branch 'feature-clock' into pre-master. [Martin Raspaud] 36 | 37 | - Changed noaa07 and noaa09 to noaa7 and noaa9 resp. [Abhay Devasthale] 38 | 39 | - I/O part updated. PyGAC will first cut the start and end of the orbit 40 | if invalid geolocation is present and THEN apply user provided start 41 | and end scanline numbers to output the orbit. [Abhay Devasthale] 42 | 43 | - Absolute value of minimum scanline number used to remove negative scan 44 | line numbers. [Abhay Devasthale] 45 | 46 | - Modifications to cut out orbits that contain invalid lat/lon values at 47 | the start and the end of the tracks. [Abhay Devasthale] 48 | 49 | - NOAA-11 IndexError messages fixed. Scan line number format changed 50 | from unsigned to signed integer. [Abhay Devasthale] 51 | 52 | - Update to handle out of range values of time stamps. [Abhay 53 | Devasthale] 54 | 55 | - Updated launch dates to match Andy's. Also changes calibration 56 | coefficients for MetOps. [Abhay Devasthale] 57 | 58 | - Bug fix while subsetting channel 2. [Abhay Devasthale] 59 | 60 | - Better handling of incorrect time stamps in the L1b data. Correct 61 | reorganizing of scanlines if the orbit is missing starting scanlines. 62 | Minor bug fixes in the header reading. [Abhay Devasthale] 63 | 64 | - Merge branch 'feature-clock' of github.com:adybbroe/pygac into 65 | feature-clock. [Martin Raspaud] 66 | 67 | - Changed handeling of buggy lat,lons and scanline numbers (Abhay) 68 | [Nina.Hakansson] 69 | 70 | - Bugfix: masking invalid brightness temperatures. [Nina.Hakansson] 71 | 72 | Before values set to -32001 where multiplied with 100 and saved as in16 73 | and got actual values -32768. 74 | 75 | 76 | - Merge branch 'feature-clock' of ssh://github.com/adybbroe/pygac into 77 | feature-clock. [Nina.Hakansson] 78 | 79 | - Rpy fix for pod-satellites (Abhay) [Nina.Hakansson] 80 | 81 | - Fixed the unit tests. [Martin Raspaud] 82 | 83 | - Don't use pod's fixed_error_correction rpy parameters. [Martin 84 | Raspaud] 85 | 86 | Use klm's constant_[roll,pitch,yaw]_attitude_error parameters instead. 87 | 88 | - Fix the rpy problem for older pod scenes. [Martin Raspaud] 89 | 90 | - Pep8ify. [Adam Dybbroe] 91 | 92 | - Updated lauch dates for several satellites. [Sara.Hornquist] 93 | 94 | - Quick fix to handle channel 3a/3b switch. [Nina.Hakansson] 95 | 96 | Maybe better to not interpolate. 97 | 98 | 99 | - Corrected launch date for metopb. [Sara.Hornquist] 100 | 101 | - Channel 3a/3b transition set to missing data. [Nina.Hakansson] 102 | 103 | - Adding metopb to gac_klm.py. [Nina.Hakansson] 104 | 105 | - Bugfix: apply earth-sun distance correction. [Nina.Hakansson] 106 | 107 | - If all pixels will be masked out, stop processing; i.e. do not produce 108 | output files. [Sara.Hornquist] 109 | 110 | - When using startline/endline, set times in the filenames that 111 | corresponds to the actual start and end time of the selected part of 112 | the orbit. Do not use the line numbers in the filenames any more. 113 | (changes from Abhay) [Sara.Hornquist] 114 | 115 | - Merge branch 'feature-clock' of ssh://github.com/adybbroe/pygac into 116 | feature-clock. [Sara.Hornquist] 117 | 118 | - Remove obsolete hdf5 files. [Adam Dybbroe] 119 | 120 | - Bug fix, one attribute was misspelled. [Sara.Hornquist] 121 | 122 | - Corrections in writing products, in order to match PPS v2014: - Make 123 | sure the string attribute has got the right format. - Changed nodata 124 | used for lat/lon. (the old nodata was inside valid range) 125 | [Sara.Hornquist] 126 | 127 | - Fix readme file. [Adam Dybbroe] 128 | 129 | - Gain factor applied to lat lon values. [Abhay Devasthale] 130 | 131 | - Minor correction in calibration file. NOAA-7 amd MetOp-A tested. 132 | [Abhay Devasthale] 133 | 134 | - Channel 3 BTs over the Antarctica corrected. [Abhay Devasthale] 135 | 136 | - Minor changes to output file name. [Abhay Devasthale] 137 | 138 | - Filenaming and quality flag changes. [Abhay Devasthale] 139 | 140 | * Filenaming convention is changed to harmonize with pps convention. 141 | * Quality flag file updated. 142 | 143 | 144 | - Merge branch 'feature-clock' of github.com:adybbroe/pygac into 145 | feature-clock. [Abhay Devasthale] 146 | 147 | Conflicts: 148 | pygac/gac_pod.py 149 | 150 | 151 | - Fix the clock drift correction. [Martin Raspaud] 152 | 153 | - Remove testing 2.6 in travis until scipy dependency is removed. 154 | [Martin Raspaud] 155 | 156 | - Try another hdf5 dev package. [Martin Raspaud] 157 | 158 | - Fix to allow h5py to build on 2.6. [Martin Raspaud] 159 | 160 | - Don't raise an error if PYGAC_CONFIG_FILE doesn't point to a file. 161 | [Martin Raspaud] 162 | 163 | - Fixing travis for 2.6. [Martin Raspaud] 164 | 165 | - Recent changes to GAC IO. [Abhay Devasthale] 166 | 167 | - Bugfixing. [Martin Raspaud] 168 | 169 | - Bugfix calibration coefficients. [Martin Raspaud] 170 | 171 | - Added missing calibration coefficients. [Martin Raspaud] 172 | 173 | - Add the gac reader generic class. [Martin Raspaud] 174 | 175 | - CI on 2.6 and add the PYGAC env var. [Martin Raspaud] 176 | 177 | - Completing calibration coefficients. [Martin Raspaud] 178 | 179 | - Finished factorizing, hopefully. [Martin Raspaud] 180 | 181 | - Add slerp tests. [Martin Raspaud] 182 | 183 | - Numpy 1.8.0 needed at least. [Martin Raspaud] 184 | 185 | - Revamped tests. [Martin Raspaud] 186 | 187 | - Implemented clock drift for pod. [Martin Raspaud] 188 | 189 | - Add slerp computations. [Martin Raspaud] 190 | 191 | - Add a simple clock drift adjustment (line shifting) [Martin Raspaud] 192 | 193 | - WIP: Update calibration coeffs. [Martin Raspaud] 194 | 195 | - Finish factorizing code for calibration. Some calibration coeffs 196 | missing. [Martin Raspaud] 197 | 198 | - WIP: Clock drift and refactoring. [Martin Raspaud] 199 | 200 | - Cleaning, and beginning of refactoring. [Martin Raspaud] 201 | 202 | - Supplements A, B and C added. [abhaydd] 203 | 204 | - Updating documentation. [abhaydd] 205 | 206 | - Updating pygac api documentation. [abhaydd] 207 | 208 | - Updated text on command-line usage. [abhaydd] 209 | 210 | - Update usage.rst. [abhaydd] 211 | 212 | - Update usage.rst. [abhaydd] 213 | 214 | - Bugfix. [Adam Dybbroe] 215 | 216 | - Added for scipy dependency. [Adam Dybbroe] 217 | 218 | - Added requirements file, for Travis... [Adam Dybbroe] 219 | 220 | - Added support for travis. [Adam Dybbroe] 221 | 222 | - Added buttons on readme page for code health etc. [Adam Dybbroe] 223 | 224 | - Added customization support for Landscape. [Adam Dybbroe] 225 | 226 | - Smoothing window for thermal channel calibration adjusted. 227 | [Abhay.Devasthale] 228 | 229 | - Updates on time information in output files. No 10th seconds, and 230 | seconds-since-1970 is now properly set. [Sara.Hornquist] 231 | 232 | - Merge branch 'pre-master' of github.com:adybbroe/pygac into pre- 233 | master. [Sara.Hornquist] 234 | 235 | - Dumping of debugging info on screen is avoided in gac_pod.py. 236 | [Abhay.Devasthale] 237 | 238 | - Update in output files: attribute what/time do not have tenth-of- 239 | second any more. [Sara.Hornquist] 240 | 241 | - Updated documentation on filenames. [Sara.Hornquist] 242 | 243 | - Negative reflectances replaced by MISSING_DATA. [Abhay.Devasthale] 244 | 245 | - Replaced nighttime reflectances with MISSING_DATA. [Abhay.Devasthale] 246 | 247 | - POD: Refined the tle search to get the nearest match. [Martin Raspaud] 248 | 249 | In the case of old satellites, the tle data can be quite scarse. For that 250 | reason, the find_tle_index function was enhanced to provide the closest 251 | match to the required date. 252 | 253 | - Bugfix in pod, and cleanup. [Martin Raspaud] 254 | 255 | - A correct determination of which sensor was generating each prt has been 256 | implemented, allowing the data to miss scanlines. It is based on the 257 | scanline numbers provided in the data 258 | - The pod data is also cleaned up before after reading. 259 | - The code has been cleaned up a little, to follow python standards. 260 | 261 | - Remove astronomy.py, depend on pyorbital instead. [Martin Raspaud] 262 | 263 | - Added h5py as a requirement in setup. [Adam Dybbroe] 264 | 265 | - Merge branch 'pre-master' of github.com:adybbroe/pygac into pre- 266 | master. [Adam Dybbroe] 267 | 268 | - Add some test scripts, and remove test data. [Martin Raspaud] 269 | 270 | - Added documentation. [Abhay.Devasthale] 271 | 272 | - Update api.rst. [abhaydd] 273 | 274 | - Update api.rst. [abhaydd] 275 | 276 | - Update api.rst. [abhaydd] 277 | 278 | - Update api.rst. [abhaydd] 279 | 280 | - Minor editorial. [Adam Dybbroe] 281 | 282 | - Fixing Manifest and setup. [Adam Dybbroe] 283 | 284 | - Updated usage docs. [Adam Dybbroe] 285 | 286 | - Adding a bit of documentation and the test case. [Adam Dybbroe] 287 | 288 | - Add empty (sphinx) docs. [Adam Dybbroe] 289 | 290 | - Adding configuration and logging. [Adam Dybbroe] 291 | 292 | - Merge branch 'master' into develop. [Adam Dybbroe] 293 | 294 | - Changed readme. [Adam Dybbroe] 295 | 296 | - Making a python package out of it. [Adam Dybbroe] 297 | 298 | - Initial commit. [Adam Dybbroe] 299 | 300 | 301 | -------------------------------------------------------------------------------- /continuous_integration/environment.yaml: -------------------------------------------------------------------------------- 1 | name: test-environment 2 | channels: 3 | - conda-forge 4 | dependencies: 5 | - numpy 6 | - scipy 7 | - bottleneck 8 | - python-dateutil 9 | - hdf5 10 | - h5py 11 | - packaging 12 | - pytest 13 | - pytest-cov 14 | - xarray 15 | - pip 16 | - pip: 17 | - docutils 18 | - pyorbital 19 | - python-geotiepoints 20 | - trollimage 21 | - pyspectral 22 | - pyorbital 23 | - georeferencer 24 | -------------------------------------------------------------------------------- /doc/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = build 9 | 10 | # Internal variables. 11 | PAPEROPT_a4 = -D latex_paper_size=a4 12 | PAPEROPT_letter = -D latex_paper_size=letter 13 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source 14 | 15 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest 16 | 17 | help: 18 | @echo "Please use \`make ' where is one of" 19 | @echo " html to make standalone HTML files" 20 | @echo " dirhtml to make HTML files named index.html in directories" 21 | @echo " singlehtml to make a single large HTML file" 22 | @echo " pickle to make pickle files" 23 | @echo " json to make JSON files" 24 | @echo " htmlhelp to make HTML files and a HTML help project" 25 | @echo " qthelp to make HTML files and a qthelp project" 26 | @echo " devhelp to make HTML files and a Devhelp project" 27 | @echo " epub to make an epub" 28 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 29 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 30 | @echo " text to make text files" 31 | @echo " man to make manual pages" 32 | @echo " changes to make an overview of all changed/added/deprecated items" 33 | @echo " linkcheck to check all external links for integrity" 34 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 35 | 36 | clean: 37 | -rm -rf $(BUILDDIR)/* 38 | 39 | html: 40 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 41 | @echo 42 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 43 | 44 | dirhtml: 45 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 46 | @echo 47 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 48 | 49 | singlehtml: 50 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 51 | @echo 52 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 53 | 54 | pickle: 55 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 56 | @echo 57 | @echo "Build finished; now you can process the pickle files." 58 | 59 | json: 60 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 61 | @echo 62 | @echo "Build finished; now you can process the JSON files." 63 | 64 | htmlhelp: 65 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 66 | @echo 67 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 68 | ".hhp project file in $(BUILDDIR)/htmlhelp." 69 | 70 | qthelp: 71 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 72 | @echo 73 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 74 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 75 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/pygac.qhcp" 76 | @echo "To view the help file:" 77 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/pygac.qhc" 78 | 79 | devhelp: 80 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 81 | @echo 82 | @echo "Build finished." 83 | @echo "To view the help file:" 84 | @echo "# mkdir -p $$HOME/.local/share/devhelp/pygac" 85 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/pygac" 86 | @echo "# devhelp" 87 | 88 | epub: 89 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 90 | @echo 91 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 92 | 93 | latex: 94 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 95 | @echo 96 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 97 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 98 | "(use \`make latexpdf' here to do that automatically)." 99 | 100 | latexpdf: 101 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 102 | @echo "Running LaTeX files through pdflatex..." 103 | make -C $(BUILDDIR)/latex all-pdf 104 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 105 | 106 | text: 107 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 108 | @echo 109 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 110 | 111 | man: 112 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 113 | @echo 114 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 115 | 116 | changes: 117 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 118 | @echo 119 | @echo "The overview file is in $(BUILDDIR)/changes." 120 | 121 | linkcheck: 122 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 123 | @echo 124 | @echo "Link check complete; look for any errors in the above output " \ 125 | "or in $(BUILDDIR)/linkcheck/output.txt." 126 | 127 | doctest: 128 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 129 | @echo "Testing of doctests in the sources finished, look at the " \ 130 | "results in $(BUILDDIR)/doctest/output.txt." 131 | -------------------------------------------------------------------------------- /doc/rtd_environment.yaml: -------------------------------------------------------------------------------- 1 | name: readthedocs 2 | channels: 3 | - conda-forge 4 | dependencies: 5 | - numpy 6 | - scipy 7 | - bottleneck 8 | - python-dateutil 9 | - hdf5 10 | - h5py 11 | - graphviz 12 | - setuptools 13 | - setuptools_scm 14 | - setuptools_scm_git_archive 15 | - sphinx 16 | - sphinx_rtd_theme 17 | - sphinxcontrib-apidoc 18 | - pip 19 | - pip: 20 | - docutils 21 | - pyorbital 22 | - python-geotiepoints 23 | - trollimage 24 | - pyspectral 25 | - pyorbital 26 | - .. # relative path to the pygac project 27 | -------------------------------------------------------------------------------- /doc/source/_static/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pytroll/pygac/fe43070f12ca395a0e1e41f1f444bb54d94dee9a/doc/source/_static/.gitkeep -------------------------------------------------------------------------------- /doc/source/api.rst: -------------------------------------------------------------------------------- 1 | The Pygac API 2 | ============= 3 | 4 | .. inheritance-diagram:: pygac.gac_pod.GACPODReader pygac.gac_klm.GACKLMReader 5 | pygac.lac_pod.LACPODReader pygac.lac_klm.LACKLMReader 6 | 7 | 8 | Reader Base Classes 9 | ------------------- 10 | 11 | Common functionality shared by multiple readers. 12 | 13 | Reader 14 | ~~~~~~ 15 | 16 | .. automodule:: pygac.reader 17 | :members: 18 | :undoc-members: 19 | 20 | GAC format reader 21 | ~~~~~~~~~~~~~~~~~ 22 | 23 | .. automodule:: pygac.gac_reader 24 | :members: 25 | :undoc-members: 26 | 27 | 28 | LAC format reader 29 | ~~~~~~~~~~~~~~~~~ 30 | 31 | .. automodule:: pygac.lac_reader 32 | :members: 33 | :undoc-members: 34 | 35 | 36 | POD series reader 37 | ~~~~~~~~~~~~~~~~~ 38 | 39 | .. automodule:: pygac.pod_reader 40 | :members: 41 | :undoc-members: 42 | :exclude-members: tsm_affected_intervals 43 | 44 | 45 | KLM series reader 46 | ~~~~~~~~~~~~~~~~~ 47 | 48 | .. automodule:: pygac.klm_reader 49 | :members: 50 | :undoc-members: 51 | :exclude-members: tsm_affected_intervals 52 | 53 | 54 | Actual Reader Implementations 55 | ----------------------------- 56 | 57 | Actual reader implementations building upon the base classes. 58 | 59 | GAC POD reader 60 | ~~~~~~~~~~~~~~ 61 | 62 | .. automodule:: pygac.gac_pod 63 | :members: 64 | :undoc-members: 65 | 66 | 67 | GAC KLM reader 68 | ~~~~~~~~~~~~~~ 69 | 70 | .. automodule:: pygac.gac_klm 71 | :members: 72 | :undoc-members: 73 | 74 | 75 | LAC POD reader 76 | ~~~~~~~~~~~~~~ 77 | 78 | .. automodule:: pygac.lac_pod 79 | :members: 80 | :undoc-members: 81 | 82 | 83 | LAC KLM reader 84 | ~~~~~~~~~~~~~~ 85 | 86 | .. automodule:: pygac.lac_klm 87 | :members: 88 | :undoc-members: 89 | 90 | Calibration 91 | ----------- 92 | 93 | .. automodule:: pygac.calibration 94 | :members: 95 | :undoc-members: 96 | -------------------------------------------------------------------------------- /doc/source/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # pygac documentation build configuration file, created by 4 | # sphinx-quickstart on Thu May 1 12:56:29 2014. 5 | # 6 | # This file is execfile()d with the current directory set to its containing dir. 7 | # 8 | # Note that not all possible configuration values are present in this 9 | # autogenerated file. 10 | # 11 | # All configuration values have a default; values that are commented out 12 | # serve to show the default. 13 | 14 | 15 | # If extensions (or modules to document with autodoc) are in another directory, 16 | # add these directories to sys.path here. If the directory is relative to the 17 | # documentation root, use os.path.abspath to make it absolute, like shown here. 18 | #sys.path.insert(0, os.path.abspath('.')) 19 | 20 | # -- General configuration ----------------------------------------------------- 21 | 22 | # If your documentation needs a minimal Sphinx version, state it here. 23 | #needs_sphinx = '1.0' 24 | 25 | # Add any Sphinx extension module names here, as strings. They can be extensions 26 | # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. 27 | extensions = ["sphinx.ext.autodoc", "sphinx.ext.doctest", "sphinx.ext.todo", 28 | "sphinx.ext.inheritance_diagram", "sphinx.ext.napoleon"] 29 | 30 | # Add any paths that contain templates here, relative to this directory. 31 | templates_path = ["_templates"] 32 | 33 | # The suffix of source filenames. 34 | source_suffix = ".rst" 35 | 36 | # The encoding of source files. 37 | #source_encoding = 'utf-8-sig' 38 | 39 | # The master toctree document. 40 | master_doc = "index" 41 | 42 | # General information about the project. 43 | project = u"pygac" 44 | copyright = u"2014, Abhay Devasthale, Martin Raspaud and Adam Dybbroe" 45 | 46 | # The version info for the project you're documenting, acts as replacement for 47 | # |version| and |release|, also used in various other places throughout the 48 | # built documents. 49 | # 50 | # The short X.Y version. 51 | version = "0.1" 52 | # The full version, including alpha/beta/rc tags. 53 | release = "0.1" 54 | 55 | # The language for content autogenerated by Sphinx. Refer to documentation 56 | # for a list of supported languages. 57 | #language = None 58 | 59 | # There are two options for replacing |today|: either, you set today to some 60 | # non-false value, then it is used: 61 | #today = '' 62 | # Else, today_fmt is used as the format for a strftime call. 63 | #today_fmt = '%B %d, %Y' 64 | 65 | # List of patterns, relative to source directory, that match files and 66 | # directories to ignore when looking for source files. 67 | exclude_patterns = [] 68 | 69 | # The reST default role (used for this markup: `text`) to use for all documents. 70 | #default_role = None 71 | 72 | # If true, '()' will be appended to :func: etc. cross-reference text. 73 | #add_function_parentheses = True 74 | 75 | # If true, the current module name will be prepended to all description 76 | # unit titles (such as .. function::). 77 | #add_module_names = True 78 | 79 | # If true, sectionauthor and moduleauthor directives will be shown in the 80 | # output. They are ignored by default. 81 | #show_authors = False 82 | 83 | # The name of the Pygments (syntax highlighting) style to use. 84 | pygments_style = "sphinx" 85 | 86 | # A list of ignored prefixes for module index sorting. 87 | #modindex_common_prefix = [] 88 | 89 | 90 | # -- Options for HTML output --------------------------------------------------- 91 | 92 | # The theme to use for HTML and HTML Help pages. See the documentation for 93 | # a list of builtin themes. 94 | html_theme = "sphinx_rtd_theme" 95 | 96 | # Theme options are theme-specific and customize the look and feel of a theme 97 | # further. For a list of options available for each theme, see the 98 | # documentation. 99 | #html_theme_options = {} 100 | 101 | # Add any paths that contain custom themes here, relative to this directory. 102 | #html_theme_path = [] 103 | 104 | # The name for this set of Sphinx documents. If None, it defaults to 105 | # " v documentation". 106 | #html_title = None 107 | 108 | # A shorter title for the navigation bar. Default is the same as html_title. 109 | #html_short_title = None 110 | 111 | # The name of an image file (relative to this directory) to place at the top 112 | # of the sidebar. 113 | #html_logo = None 114 | 115 | # The name of an image file (within the static path) to use as favicon of the 116 | # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 117 | # pixels large. 118 | #html_favicon = None 119 | 120 | # Add any paths that contain custom static files (such as style sheets) here, 121 | # relative to this directory. They are copied after the builtin static files, 122 | # so a file named "default.css" will overwrite the builtin "default.css". 123 | html_static_path = ["_static"] 124 | 125 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, 126 | # using the given strftime format. 127 | #html_last_updated_fmt = '%b %d, %Y' 128 | 129 | # If true, SmartyPants will be used to convert quotes and dashes to 130 | # typographically correct entities. 131 | #html_use_smartypants = True 132 | 133 | # Custom sidebar templates, maps document names to template names. 134 | #html_sidebars = {} 135 | 136 | # Additional templates that should be rendered to pages, maps page names to 137 | # template names. 138 | #html_additional_pages = {} 139 | 140 | # If false, no module index is generated. 141 | #html_domain_indices = True 142 | 143 | # If false, no index is generated. 144 | #html_use_index = True 145 | 146 | # If true, the index is split into individual pages for each letter. 147 | #html_split_index = False 148 | 149 | # If true, links to the reST sources are added to the pages. 150 | #html_show_sourcelink = True 151 | 152 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 153 | #html_show_sphinx = True 154 | 155 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. 156 | #html_show_copyright = True 157 | 158 | # If true, an OpenSearch description file will be output, and all pages will 159 | # contain a tag referring to it. The value of this option must be the 160 | # base URL from which the finished HTML is served. 161 | #html_use_opensearch = '' 162 | 163 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 164 | #html_file_suffix = None 165 | 166 | # Output file base name for HTML help builder. 167 | htmlhelp_basename = "pygacdoc" 168 | 169 | 170 | # -- Options for LaTeX output -------------------------------------------------- 171 | 172 | # The paper size ('letter' or 'a4'). 173 | #latex_paper_size = 'letter' 174 | 175 | # The font size ('10pt', '11pt' or '12pt'). 176 | #latex_font_size = '10pt' 177 | 178 | # Grouping the document tree into LaTeX files. List of tuples 179 | # (source start file, target name, title, author, documentclass [howto/manual]). 180 | latex_documents = [ 181 | ("index", "pygac.tex", u"pygac Documentation", 182 | u"Abhay Devasthale, Martin Raspaud and Adam Dybbroe", "manual"), 183 | ] 184 | 185 | # The name of an image file (relative to this directory) to place at the top of 186 | # the title page. 187 | #latex_logo = None 188 | 189 | # For "manual" documents, if this is true, then toplevel headings are parts, 190 | # not chapters. 191 | #latex_use_parts = False 192 | 193 | # If true, show page references after internal links. 194 | #latex_show_pagerefs = False 195 | 196 | # If true, show URL addresses after external links. 197 | #latex_show_urls = False 198 | 199 | # Additional stuff for the LaTeX preamble. 200 | #latex_preamble = '' 201 | 202 | # Documents to append as an appendix to all manuals. 203 | #latex_appendices = [] 204 | 205 | # If false, no module index is generated. 206 | #latex_domain_indices = True 207 | 208 | 209 | # -- Options for manual page output -------------------------------------------- 210 | 211 | # One entry per manual page. List of tuples 212 | # (source start file, name, description, authors, manual section). 213 | man_pages = [ 214 | ("index", "pygac", u"pygac Documentation", 215 | [u"Abhay Devasthale, Martin Raspaud and Adam Dybbroe"], 1) 216 | ] 217 | -------------------------------------------------------------------------------- /doc/source/images/noaa_class_preferences.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pytroll/pygac/fe43070f12ca395a0e1e41f1f444bb54d94dee9a/doc/source/images/noaa_class_preferences.png -------------------------------------------------------------------------------- /doc/source/index.rst: -------------------------------------------------------------------------------- 1 | .. pygac documentation master file, created by 2 | sphinx-quickstart on Thu May 1 12:56:29 2014. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Pygac's documentation 7 | ===================== 8 | 9 | Pygac is a Python package to read, calibrate and navigate data from the AVHRR 10 | instrument onboard NOAA and MetOp satellites in GAC and LAC format. 11 | 12 | 13 | Table of Contents 14 | ----------------- 15 | 16 | .. toctree:: 17 | :maxdepth: 2 18 | 19 | introduction 20 | installation 21 | usage 22 | methods 23 | api 24 | legacy 25 | 26 | 27 | 28 | Indices and tables 29 | ================== 30 | 31 | * :ref:`genindex` 32 | * :ref:`modindex` 33 | * :ref:`search` 34 | 35 | -------------------------------------------------------------------------------- /doc/source/installation.rst: -------------------------------------------------------------------------------- 1 | Installation 2 | ------------ 3 | 4 | You can install the latest stable release of the software via the python package index (pypi) 5 | 6 | .. code-block:: bash 7 | 8 | pip install pygac 9 | 10 | 11 | TLE files 12 | ~~~~~~~~~ 13 | The pygac package requires Two-Line Element files stored per-satellite 14 | in files with names such as TLE_noaa19.txt. The contents should be the 15 | historical TLEs, i.e. a concatenation of just lines 1 and 2 without the 16 | satellite name. For example 17 | 18 | .. code-block:: 19 | 20 | 1 23455U 94089A 01122.93455091 .00000622 00000-0 36103-3 0 7210 21 | 2 23455 99.1771 113.3063 0008405 277.6106 82.4106 14.12671703326608 22 | 1 23455U 94089A 01326.97611660 .00000739 00000-0 42245-3 0 9806 23 | 2 23455 99.1886 322.4670 0009980 66.2863 293.9354 14.12871991355419 24 | etc 25 | 26 | These can be downloaded from CelesTrak via the `special data request form`_. 27 | 28 | .. _special data request form: 29 | https://celestrak.com/NORAD/archives/request.php 30 | 31 | 32 | Development 33 | ~~~~~~~~~~~ 34 | 35 | For development clone the repository from github and install pygac in editable mode. 36 | 37 | .. code-block:: bash 38 | 39 | git clone git://github.com/pytroll/pygac 40 | cd pygac 41 | pip install -e .[dev] 42 | 43 | It is recommended to activate `pre-commit`_ checks. 44 | 45 | .. code-block:: bash 46 | 47 | pre-commit install 48 | 49 | The test suite can be run using pytest. 50 | 51 | .. code-block:: bash 52 | 53 | pytest -vs pygac/tests 54 | 55 | 56 | .. _pre-commit: 57 | https://pre-commit.com/ 58 | -------------------------------------------------------------------------------- /doc/source/introduction.rst: -------------------------------------------------------------------------------- 1 | Introduction 2 | ============ 3 | 4 | Supported Data Format 5 | --------------------- 6 | 7 | Pygac reads AVHRR GAC (Global Area Coverage), LAC (Local Area Coverage) and FRAC 8 | (Full Resolution Area Coverage) level 1b data from NOAA, which is described in 9 | the `POD`_ (NOAA-14 and before) and `KLM`_ (NOAA-15 and following) user guides. 10 | The data can be obtained from `NOAA CLASS`_, where you can also find a 11 | comprehensive `introduction`_. 12 | 13 | .. note:: 14 | 15 | Pygac can only read AVHRR data with 10 bits/pixel. This can be specified in 16 | your NOAA CLASS order or user preferences (see screenshot below). 17 | 18 | .. image:: images/noaa_class_preferences.png 19 | :width: 350 20 | :alt: Screenshot of AVHRR data extraction preferences 21 | 22 | .. _NOAA CLASS: 23 | https://www.class.noaa.gov/ 24 | .. _POD: 25 | https://github.com/user-attachments/files/17407676/POD-Guide.pdf 26 | .. _KLM: 27 | https://github.com/user-attachments/files/17361330/NOAA_KLM_Users_Guide.pdf 28 | .. _introduction: 29 | https://www.class.noaa.gov/release/data_available/avhrr/index.htm 30 | 31 | 32 | Supported Sensors 33 | ----------------- 34 | Pygac currently supports AVHRR generations 1-3 onboard NOAA (TIROS-N, NOAA-6 35 | and onwards) and MetOp satellites. 36 | 37 | 38 | .. _here: 39 | https://www.avl.class.noaa.gov/release/data_available/avhrr/index.htm 40 | 41 | 42 | Related Projects 43 | ---------------- 44 | 45 | - `pygac-fdr`_: Generate a fundamental data record of AVHRR GAC data using 46 | Pygac. 47 | - `level1c4pps`_: Prepare AVHRR GAC data for `NWCSAF/PPS`_ using Pygac. 48 | 49 | .. _level1c4pps: https://github.com/foua-pps/level1c4pps 50 | .. _NWCSAF/PPS: https://www.nwcsaf.org/16 51 | .. _pygac-fdr: https://github.com/pytroll/pygac-fdr 52 | -------------------------------------------------------------------------------- /doc/source/methods.rst: -------------------------------------------------------------------------------- 1 | Methods 2 | ======= 3 | 4 | Calibration 5 | ----------- 6 | 7 | At present, calibration coefficients provided by Andrew Heidinger 8 | (NOAA) under SCOPE-CM project are applied for all satellites. 9 | 10 | Pygac comes with one default calibration file, which can be found in 11 | `pygac/data/calibration.json`_. The current version is *PATMOS-x, v2017r1*, 12 | including provisional coefficients for MetOp-C. A record of all versions is 13 | kept in :class:`pygac.calibration.Calibrator.version_hashs`. Alternatively, it 14 | is possible to pass custom coefficients to the reader, see 15 | :class:`pygac.reader.Reader`. This could also be a previous default version. 16 | 17 | .. _pygac/data/calibration.json: 18 | https://github.com/pytroll/pygac/blob/main/pygac/data/calibration.json 19 | 20 | The solar channel calibration (Channels 1 and 2, and Channel 3a if available) 21 | takes into account inter-satellite differences and is derived using 22 | amalgamation of different calibration references including the most recent 23 | MODIS Collection 6 data, in-situ targets, and simultaneous nadir 24 | observations. The detailed methodology is presented in Heidinger et al. 25 | (2010). The resulting (inter)calibration coefficients are of highest quality 26 | and follow the Global Climate Observing System (GCOS) standards for 27 | deriving fundamental climate data records. 28 | 29 | The reflectances are normalized by a factor (as a function of day of a year) 30 | to account for changing Earth-Sun distance. However, it is left to the 31 | user to apply further normalization using cosine of solar zenith 32 | angle for example (depending on application in question). 33 | 34 | The thermal channel intercalibration is done from scratch, starting from 35 | obtaining Platinum Resistance Thermometer (PRT), space and Internal 36 | Calibration Target (ICT, blackbody) counts. ICT temperatures are obtained 37 | from PRT counts based on coefficients provided in the POD and KLM Data User 38 | Guides (Kidwell, 2000). For each thermal channel, a smoothing window of 51 39 | successive PRT, ICT and space counts is used to obtain robust gain values and 40 | to dampen undue high frequency fluctuations in the count data (Trishchenko, 41 | 2002). Section 7.1.2.5 of `KLM User Guide`_ presents the summary of equations 42 | implemented in Pygac to calibrate thermal channels, including non-linearity 43 | correction (Walton et al. 1998). 44 | 45 | .. _KLM User Guide: 46 | https://www.ncei.noaa.gov/pub/data/satellite/publications/podguides/TIROS-N%20thru%20N-14/ 47 | 48 | The PRT readings are supposed to be present in a specific order, for example, 49 | the reset value followed by four readings from the PRTs. However, this may 50 | not always be the case for orbits that contain data gaps or due to any other 51 | unexplained reason. If not taken into account properly, this irregularity 52 | could result in the underestimation of brightness temperatures, when 53 | calibration information is smoothed over many scanlines. In Pygac, this 54 | inconsistency is handled properly while calibrating thermal channels. 55 | 56 | In some cases it was found that, apart from the reset values, even the 57 | readings from any one of the four PRTs could also have very low suspicious 58 | values. This could also seriously affect the computation of brightness 59 | temperatures. Pygac detects such anomalies and corrects them using 60 | interpolation of nearby valid PRT readings. 61 | 62 | 63 | Geolocation 64 | ----------- 65 | 66 | GAC 67 | **** 68 | 69 | Each GAC row has total 409 pixels. But lat-lon values are provided for every 70 | eigth pixel starting from pixel 5 and ending at pixel 405. Using Numpy, Scipy 71 | and Pyresample packages, inter- and extrapolation is carried out to obtain 72 | geolocation for each pixel along the scanline. 73 | 74 | If the GAC data belongs to POD family, then clock drift errors are used to 75 | adjust existing Lat-Lon information. Here, Pygac makes use of `PyOrbital`_ 76 | package. Pygac interpolates the clock offset and adjusts the nominal scan 77 | times to the actual scan times. Since the geolocation was computed using the 78 | nominal scan times, Pygac interpolates the latitudes and longitudes to the 79 | actual scan times using spherical linear interpolation, aka slerp. However, 80 | in the case of a clock drift error greater than the scan rate of the dataset, 81 | the latitude and longitude for each pixel of the scan lines that cannot have 82 | an interpolated geolocation (typically at the start or end of the dataset) 83 | are recomputed. This is done using pyorbital, which in turn uses TLEs to 84 | compute the position of the satellite at each scan time and the instrument 85 | geometry compute the longitude and latitude of each pixel of the dataset. 86 | Since this operation can be quite costly, the interpolation is preferred 87 | whenever possible. 88 | 89 | .. _PyOrbital: 90 | https://pyorbital.readthedocs.io 91 | 92 | LAC and FRAC 93 | ************ 94 | 95 | For LAC and FRAC data, the procedure for geolocation is similar to the GAC 96 | data, except that the full resolution navigation is used. 97 | 98 | Recomputation of Longitudes and Latitudes 99 | ***************************************** 100 | 101 | The default method for producing navigation information for every pixel is to 102 | interpolate longitudes and latitudes provided in the level 1b file. However, 103 | PyGAC also has the possibility to recompute this information from the metadata 104 | (line numbers and scan times) and TLEs. This is activated when the keyword 105 | `compute_lonlats_from_tles` is set to True in the reader instantiation. 106 | 107 | Georeferencing from reference image 108 | *********************************** 109 | 110 | If provided a reference image as a keyword in the reader instantiation (as a 111 | path to `reference_image`), the `Georeferencer` package will be used to compute 112 | displacement from dynamically computed ground control points (GCPs) and thus 113 | recompute a more accurate navigation for the swath. At the time of writing, 114 | this is only supported for LAC and FRAC data. 115 | 116 | .. Georeferencer 117 | https://github.com/pytroll/georeferencer 118 | 119 | 120 | Computation of Angles 121 | --------------------- 122 | 123 | The azimuth angles are calculated using `get_alt_az`_ and `get_observer_look`_ 124 | from pyorbital. The azimuth described in the link is measured as clockwise 125 | from North instead of counter-clockwise from South. Counter clockwise from 126 | south would be the standard for a right-handed orthogonal coordinate system. 127 | Pygac was updated to use the same definition for angles as pyorbital (2019, 128 | September, version > 1.1.0). Previous versions used azimuth +/-180 degrees, 129 | which correspond to degrees clockwise from south. All angles are converted to 130 | degrees. All azimuth angles are converted to range ]-180, 180] (2019 October 131 | version > 1.1.0 ). Note that ]-180, 180] is an open interval. 132 | 133 | 134 | .. _get_alt_az: 135 | https://pyorbital.readthedocs.io/en/latest/#pyorbital.astronomy.get_alt_az 136 | .. _get_observer_look: 137 | https://pyorbital.readthedocs.io/en/latest/#pyorbital.orbital.Orbital.get_observer_look 138 | 139 | 140 | Correction of Satellite Location 141 | -------------------------------- 142 | 143 | Whenever possible, Pygac uses RPY corrections along with other orbital 144 | parameters to compute accurate satellite location (e.g. instead of assuming 145 | constant altitude). However, RPY corrections are not available for all NOAA 146 | satellites. In case of the majority of the POD family satellites, these 147 | corrections are set to zero. 148 | 149 | 150 | Correction of Scanline Timestamps 151 | --------------------------------- 152 | 153 | The geolocation in Pygac depends on accurate scanline timestamps. However, 154 | these may be corrupt, especially for older sensors. Assuming a constant 155 | scanning rate, Pygac attempts to fix them using extrapolation based on the scan 156 | line number and a reference time. 157 | 158 | Finding the right reference time is difficult due to the multitude of 159 | possible timestamp corruptions. But the combination of the following three 160 | options proved to be a robust reference in many situations: 161 | Timestamp of the first scanline, median time offset of all scanlines and header 162 | timestamp. See 163 | :meth:`pygac.reader.Reader.correct_times_median` and 164 | :meth:`pygac.reader.Reader.correct_times_thresh` 165 | for details. 166 | 167 | Finally, not only timestamps but also scanline numbers may be corrupt. 168 | Therefor lines with erroneous scanline numbers are removed before 169 | extrapolation, see :meth:`pygac.reader.Reader.correct_scan_line_numbers`. 170 | 171 | 172 | Scan-Motor-Issue 173 | ---------------- 174 | 175 | Between 2001 and 2004 GAC data from NOAA-14, NOAA-15, and NOAA-16 frequently 176 | contain a significant amount of noise towards an edge of the swath. As 177 | reported by `Schlundt et al (2017)`_, section 5.2, this is probably caused by a 178 | temporary scan-motor issue. Pygac tries to identify and mask affected pixels. 179 | 180 | .. _Schlundt et al (2017): 181 | https://climate.esa.int/media/documents/Cloud_Technical-Report-AVHRR-GAC-FCDR-generation_v1.0.pdf 182 | -------------------------------------------------------------------------------- /doc/source/usage.rst: -------------------------------------------------------------------------------- 1 | Usage 2 | ----- 3 | 4 | Through Satpy 5 | ~~~~~~~~~~~~~ 6 | 7 | The preferred way of using Pygac is through `Satpy`_. Results are returned as 8 | dask-friendly `xarray`_ DataArrays with proper dataset/coordinate names and 9 | additional metadata. It is also possible to select a user-defined range of 10 | scanlines. Furthermore, Satpy provides many options for resampling, 11 | visualizing and saving the data. 12 | 13 | .. code-block:: python 14 | 15 | import satpy 16 | 17 | # Channel set for KLM satellites. For POD satellites the channels are 18 | # ['1', '2', '3', '4', '5']. 19 | channels = ['1', '2', '3a', '3b', '4', '5'] 20 | ancillary = ['solar_zenith_angle', 21 | 'sensor_zenith_angle', 22 | 'solar_azimuth_angle', 23 | 'sensor_azimuth_angle', 24 | 'sun_sensor_azimuth_difference_angle', 25 | 'qual_flags', 26 | 'latitude', 27 | 'longitude'] 28 | 29 | scene = satpy.Scene(filenames=['NSS.GHRR.NP.D15361.S0121.E0315.B3547172.SV'], 30 | reader='avhrr_l1b_gaclac', 31 | reader_kwargs={'tle_dir': '/path/to/tle/', 32 | 'tle_name': 'TLE_%(satname)s.txt'}) 33 | scene.load(channels + ancillary) 34 | 35 | 36 | For a list of Satpy reader keyword arguments see `satpy.readers.avhrr_l1b_gaclac`_ 37 | and for further Pygac reader keyword arguments see :class:`pygac.reader.Reader`. 38 | Especially it is possible to choose a different version of calibration 39 | coefficients or even specify your own. 40 | 41 | .. _Satpy: https://satpy.readthedocs.io 42 | .. _xarray: https://xarray.pydata.org 43 | .. _satpy.readers.avhrr_l1b_gaclac: 44 | https://satpy.readthedocs.io/en/stable/api/satpy.readers.avhrr_l1b_gaclac.html?highlight=avhrr_l1b_gaclac 45 | .. _example notebook: 46 | https://github.com/pytroll/pytroll-examples/blob/main/satpy/avhrr_l1b_gaclac.ipynb 47 | 48 | 49 | Direct Usage 50 | ~~~~~~~~~~~~ 51 | 52 | Alternatively you can also use Pygac directly. 53 | 54 | .. code-block:: python 55 | 56 | from pygac import get_reader_class 57 | 58 | filename = 'NSS.GHRR.NP.D15361.S0121.E0315.B3547172.SV' 59 | reader_cls = get_reader_class(filename) 60 | reader = reader_cls(tle_dir='/path/to/tle', tle_name='TLE_%(satname)s.txt') 61 | reader.read(filename) 62 | 63 | channels = reader.get_calibrated_channels() 64 | lons, lats = reader.get_lonlat() 65 | scanline_times = reader.get_times() 66 | bad_quality_lines = reader.mask 67 | 68 | 69 | Legacy CLI 70 | ~~~~~~~~~~ 71 | 72 | .. note:: 73 | 74 | Usage of the legacy command line program ``pygac-run`` is deprecated in 75 | favour of the above options. 76 | 77 | There is also a legacy command line program ``pygac-run`` which saves the 78 | results to HDF5 and requires a configuration file. 79 | 80 | Copy the template file ``etc/pygac.cfg.template`` to ``pygac.cfg`` and place 81 | it in a directory as you please. Set the environment variable ``PYGAC_CONFIG_FILE`` 82 | pointing to the file. e.g. 83 | 84 | .. code-block:: bash 85 | 86 | PYGAC_CONFIG_FILE=/home/user/pygac.cfg; export PYGAC_CONFIG_FILE 87 | 88 | Also adapt the configuration file to your needs. The ``tledir`` parameter should 89 | be set to where your Two Line Element (TLE) files are located. 90 | 91 | Then call ``pygac-run`` on a GAC/LAC file. 92 | 93 | .. code-block:: bash 94 | 95 | pygac-run testdata/NSS.GHRR.NL.D02187.S1904.E2058.B0921517.GC 0 0 96 | 97 | The last two digits are the start and end scanline numbers, thus specifying the 98 | portion of the GAC orbit that user wants to process. The first scanline number 99 | starts at 0. If zeroes are specified at both locations, then the entire orbit 100 | will be processed. 101 | 102 | The result will be three hdf5 files, one with the calibrated AVHRR data, 103 | the other with sun-satellite viewing geometry data and this third with 104 | scanline quality information. 105 | 106 | 107 | -------------------------------------------------------------------------------- /etc/pygac.cfg.template: -------------------------------------------------------------------------------- 1 | [tle] 2 | tledir = /path/to/gapfilled/tles 3 | tlename = TLE_%(satname)s.txt 4 | 5 | [output] 6 | output_dir = /tmp 7 | output_file_prefix = ECC_GAC 8 | -------------------------------------------------------------------------------- /pygac/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # Copyright (c) 2014 Adam.Dybbroe 5 | 6 | # Author(s): 7 | 8 | # Adam.Dybbroe 9 | # Carlos Horn 10 | 11 | # This program is free software: you can redistribute it and/or modify 12 | # it under the terms of the GNU General Public License as published by 13 | # the Free Software Foundation, either version 3 of the License, or 14 | # (at your option) any later version. 15 | 16 | # This program is distributed in the hope that it will be useful, 17 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 18 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 19 | # GNU General Public License for more details. 20 | 21 | # You should have received a copy of the GNU General Public License 22 | # along with this program. If not, see . 23 | 24 | try: 25 | from pygac.version import version as __version__ # noqa 26 | except ModuleNotFoundError: 27 | raise ModuleNotFoundError( 28 | "No module named pygac.version. This could mean " 29 | "you didn't install 'pygac' properly. Try reinstalling ('pip " 30 | "install').") 31 | 32 | import logging 33 | 34 | from pygac.configuration import get_config, read_config_file # noqa 35 | from pygac.runner import get_reader_class, process_file # noqa 36 | 37 | # add a NullHandler to prevent messages in sys.stderr if the using application does 38 | # not use logging, but pygac makes logging calls of severity WARNING and greater. 39 | # See https://docs.python.org/3/howto/logging.html (Configuring Logging for a Library) 40 | logging.getLogger("pygac").addHandler(logging.NullHandler()) 41 | -------------------------------------------------------------------------------- /pygac/clock_offsets_converter.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # Copyright (c) 2014 Martin Raspaud 5 | 6 | # Author(s): 7 | 8 | # Martin Raspaud 9 | 10 | # This program is free software: you can redistribute it and/or modify 11 | # it under the terms of the GNU General Public License as published by 12 | # the Free Software Foundation, either version 3 of the License, or 13 | # (at your option) any later version. 14 | 15 | # This program is distributed in the hope that it will be useful, 16 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 17 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 18 | # GNU General Public License for more details. 19 | 20 | # You should have received a copy of the GNU General Public License 21 | # along with this program. If not, see . 22 | 23 | """Converts text to python 24 | """ 25 | from datetime import datetime 26 | 27 | txt = {"noaa14": 28 | """95013 223633 -0.37 95213 233350 0.59 29 | 95214 011409 -0.93 95365 225835 0.04 30 | 95365 211723 0.53 96079 234708 1.07 31 | 96080 012736 -0.43 96222 230133 0.59 32 | 96223 004223 0.07 96289 223559 0.58 33 | 96290 001652 -0.93 97126 223140 0.69 34 | 97127 001236 0.18 97182 001344 0.64 35 | 97182 015422 1.63 97196 230953 1.75 36 | 97197 005044 0.25 97231 232830 0.55 37 | 97232 010911 0.03 97302 000502 0.63 38 | 97302 014537 -0.90 98111 234827 0.55 39 | 98112 012906 0.04 98181 223817 0.62 40 | 98181 001812 -0.88 98342 230000 0.45 41 | 98343 000000 -0.05 98365 202823 0.14 42 | 98365 220000 -0.34 99131 205700 0.72 43 | 99132 002800 0.22 99222 235900 0.97 44 | 99223 010000 -0.52 99310 235900 0.15 45 | 99311 000000 -0.65 00018 235900 -0.02 46 | 00019 000000 -0.50 00238 235900 0.70""", 47 | 48 | "noaa12": 49 | """92001 100717 -0.214742484714 92085 201923 0.659529247721 50 | 92085 201923 0.659529247721 92182 201100 0.379947500166 51 | 92182 201100 0.379947500166 92143 020248 -0.0575787620387 52 | 92143 020248 -0.0575787620387 92183 025048 -0.529106798212 53 | 92183 025048 -0.529106798212 92210 235849 -0.448157864915 54 | 92210 235849 -0.448157864915 92211 025739 0.356900665981 55 | 92211 025739 0.356900665981 92300 055344 0.114043171222 56 | 92300 055344 0.114043171222 92324 004603 -0.549446071001 57 | 92324 004603 -0.549446071001 93079 003814 0.186526323477 58 | 93079 003814 0.186526323477 93144 220815 -0.292140236798 59 | 93144 220815 -0.292140236798 93181 235452 -0.0720106859295 60 | 93181 235452 -0.0720106859295 93182 011048 -0.0832968119208 61 | 93182 011048 -0.0832968119208 93292 111904 -0.227945993953 62 | 93292 111904 -0.227945993953 93214 234544 -0.353398580408 63 | 93214 234544 -0.353398580408 93240 122200 -0.246247707826 64 | 93240 122200 -0.246247707826 93262 162300 -0.482771980563 65 | 93262 162300 -0.482771980563 93308 222757 -0.305833686394 66 | 93308 222757 -0.305833686394 93309 033915 -0.287643764912 67 | 93309 033915 -0.287643764912 93359 191324 -0.364354167998 68 | 93359 191324 -0.364354167998 94059 154014 0.102864588298 69 | 94059 154014 0.102864588298 94061 163911 0.169217421643 70 | 94061 163911 0.169217421643 94110 043012 0.274001589582 71 | 94110 043012 0.274001589582 94117 001244 0.0576556437725 72 | 94117 001244 0.0576556437725 94174 214941 -0.0565555674317 73 | 94174 214941 -0.0565555674317 94181 224623 0.0311954692773 74 | 94181 224623 0.0311954692773 94182 000600 0.18871970849 75 | 94182 000600 0.18871970849 94197 000425 -0.084719088514 76 | 94197 000425 -0.084719088514 95070 174136 0.0151876009177 77 | 95070 174136 0.0151876009177 94340 193700 -0.263355299772 78 | 94340 193700 -0.263355299772 95092 024421 -0.476681468304 79 | 95092 024421 -0.476681468304 95129 230531 -0.749001042571 80 | 95129 230531 -0.749001042571 95130 002554 0.229911650151 81 | 95130 002554 0.229911650151 95224 201530 -0.0575744014939 82 | 95224 201530 -0.0575744014939 95194 173300 0.101028125701 83 | 95194 173300 0.101028125701 95259 172842 0.0668472664936 84 | 95259 172842 0.0668472664936 95365 070459 -0.176565005252 85 | 95365 070459 -0.176565005252 96001 011610 0.0500187493715 86 | 96001 011610 0.0500187493715 96163 005747 -0.693724861601 87 | 96163 005747 -0.693724861601 96313 093417 0.0453934348631 88 | 96313 093417 0.0453934348631 97093 101211 0.0168789226547 89 | 97093 101211 0.0168789226547 97093 101211 0.0168789226547 90 | 97093 101211 0.0168789226547 97181 230200 0.07963445895 91 | 97181 230200 0.07963445895 97182 003500 -0.764519227361 92 | 97182 003500 -0.764519227361 97282 020852 -0.0541122818126 93 | 97282 020852 -0.0541122818126 97235 232100 -0.745022654355 94 | 97235 232100 -0.745022654355 97334 234704 -0.121024921026 95 | 97334 234704 -0.121024921026 97335 031035 -0.131177187257 96 | 97335 031035 -0.131177187257 98083 052322 -0.288495446794 97 | 98083 052322 -0.288495446794 98256 093733 -0.175770660865 98 | 98256 093733 -0.175770660865 98236 085900 0.19721669783 99 | 98236 085900 0.19721669783 98237 065201 0.379949295221 100 | 98237 065201 0.379949295221 98348 225656 -0.347935041683""", 101 | 102 | "noaa11": 103 | """88270 175315 0.10 88362 225730 0.55 104 | 88363 003825 -0.45 89108 223157 0.55 105 | 89109 001247 -1.21 89283 223149 0.92 106 | 89284 001246 -0.99 90001 005132 0.19 107 | 90001 062225 -0.81 90086 225343 0.51 108 | 90087 003437 -0.48 90157 232813 0.67 109 | 90158 010906 -0.33 90200 185021 0.40 110 | 90206 004808 0.68 90210 000412 0.76 111 | 90214 063112 0.10 90233 230231 0.53 112 | 90234 004321 -0.47 90296 231326 0.66 113 | 90297 005426 -0.35 90345 223316 0.54 114 | 90346 072437 -0.44 91037 234244 0.63 115 | 91038 012353 -0.38 91078 224447 0.42 116 | 91079 002546 -0.59 91127 233159 0.36 117 | 91128 011307 -0.64 91176 204252 0.35 118 | 91177 001500 -0.67 91232 233429 0.47 119 | 91233 011521 -0.54 91281 223014 0.44 120 | 91282 001116 -0.57 91330 230237 0.45 121 | 91331 004335 -0.56 92014 233119 0.47 122 | 92015 011217 -0.53 92056 233822 0.36 123 | 92057 011908 -0.64 92105 235823 0.46 124 | 92106 013910 -0.55 92147 235749 0.38 125 | 92148 013925 -0.63 92210 230706 0.74 126 | 92211 004718 -0.26 92252 230538 0.70 127 | 92253 004621 -0.32 92308 215437 0.96 128 | 92309 082622 -0.05 92336 225954 0.59 129 | 92337 003847 -0.42 93012 225434 0.56 130 | 93013 003543 -0.46 93047 223341 0.36 131 | 93048 001415 -0.67 93089 222657 0.33 132 | 93090 014845 -0.68 93138 223524 0.47 133 | 93139 001555 -0.55 93181 235524 0.51 134 | 93182 013608 -0.50 93234 231252 0.78 135 | 93235 005344 -0.21 93271 221249 0.68 136 | 93272 000413 -0.33 93313 235403 0.70 137 | 93314 013043 -0.33 93355 234331 0.73 138 | 93356 012432 -0.28 94032 233238 0.77 139 | 94033 011326 -0.24 94067 230533 0.64 140 | 94068 004655 -0.38 94109 224303 0.69 141 | 94110 003436 -0.34 94144 221719 0.57 142 | 94145 000747 -0.43 94181 230407 0.50 143 | 94182 005611 -1.51 94209 173715 -0.81 144 | 94209 204026 0.23 94221 232635 0.54 145 | 94222 010716 -0.45 94256 224431 0.45 146 | 94257 003856 -0.59 94298 223205 0.53 147 | 94299 002843 -0.47 94340 221659 0.65 148 | 94341 001032 -0.38 95012 164514 0.60 149 | 95013 194752 -0.36 95052 231636 0.67 150 | 95053 010842 -0.33 95094 225950 0.77 151 | 95095 005202 -0.23 95208 163447 2.81 152 | 95208 163450 2.81 96004 165442 7.36""", 153 | 154 | "noaa7": 155 | """81179 000000 0.1 82003 235959 0.8 156 | 82004 000000 0.8 82053 235959 1.1 157 | 82054 000000 -0.5 82181 235959 0.1 158 | 82182 000000 -0.6 83025 235959 0.3 159 | 83026 000000 -0.1 83181 235959 0.5 160 | 83182 000000 1.5 83193 235959 1.6 161 | 83194 000000 -0.7 84081 235959 0.3 162 | 84082 000000 -1.1 84342 235959 -0.1 163 | 84343 000000 -0.9 84366 235959 -0.8 164 | 85001 000000 0.0 85107 235959 0.4 165 | 85108 000000 0.2 85136 235959 0.3 166 | 85137 000000 -1.1 85181 235959 -1.0 167 | 85182 000000 -0.0 85304 235959 0.4 168 | 85305 000000 -0.3 86148 235959 0.3 169 | 86149 000000 -1.3 86158 235959 -1.3""", 170 | 171 | "noaa9": 172 | """86014 183754 -0.53 86051 090207 -0.80 173 | 86051 184443 0.45 86190 092344 -0.76 174 | 86190 190707 0.46 86308 195123 -0.80 175 | 86309 081339 0.47 87048 210941 -0.84 176 | 87049 075241 0.40 87139 195622 -0.85 177 | 87140 081825 0.39 87216 210921 -0.75 178 | 87217 075309 0.48 87293 204300 -0.77 179 | 87294 090756 0.48 87365 225918 -0.79 180 | 88001 004020 0.21 88061 234258 -0.89 181 | 88062 012352 0.36 88125 000338 -0.87 182 | 88125 014432 0.39 88187 224121 -0.87 183 | 88188 002232 0.40 88250 225846 -0.93 184 | 88251 022040 0.32 88306 224954 -0.89 185 | 88307 003033 0.35 88363 001536 -0.88 186 | 88363 015925 0.37 89023 110528 -0.25 187 | 89024 091318 0.75 89094 111829 -0.91 188 | 89095 035705 0.31 89143 031928 -0.85 189 | 89144 030810 0.36 89192 111852 -0.79 190 | 89193 035610 0.46 89242 232725 -0.80 191 | 89243 024905 0.45 89290 035408 -0.79 192 | 89291 020143 0.45 89338 101232 -0.81 193 | 89339 025034 0.41 89365 212451 -0.30 194 | 90001 023815 0.69 90058 102159 -0.87 195 | 90059 030319 0.36 90107 105326 -0.99 196 | 90108 014914 0.25 90149 221415 -0.93 197 | 90150 033528 0.31 90191 222735 -0.85 198 | 90192 033935 0.36 90233 110326 -0.81 199 | 90234 034148 0.40 90275 110402 -0.79 200 | 90276 020138 0.42 90314 095839 -0.72 201 | 90320 232628 0.33 90365 224854 -1.03 202 | 91001 022659 -0.07 91022 214259 -0.71 203 | 91023 030313 0.52 91064 231826 -0.76 204 | 91065 025737 0.47 91106 044021 -0.82 205 | 91107 024821 0.39 91142 223525 -0.72 206 | 91142 224834 0.16 91176 034520 -0.88 207 | 91177 033300 0.31 91211 031322 -0.78 208 | 91213 043140 0.39 91231 225648 -0.21 209 | 91270 042933 -0.24 91288 224028 -0.86 210 | 91289 053517 0.36 91331 000157 -1.03 211 | 91331 051428 0.44 92007 233938 -0.95 212 | 92008 063157 0.52 92049 231312 -0.90 213 | 92050 042700 0.58 92090 195517 -0.84 214 | 92091 055256 0.75 92133 221715 -0.77 215 | 92134 065146 0.70 92182 234047 -0.99 216 | 92183 134449 -0.01 92207 120034 -0.87 217 | 92211 124842 0.45 92236 141404 -0.42 218 | 92246 233007 0.71 92287 231141 -0.78 219 | 92288 060512 0.68 92329 143633 -0.79 220 | 92330 002139 0.69 93005 234707 -0.87 221 | 93006 064126 0.63 93047 231538 -0.91 222 | 93048 060831 0.57 93089 175433 -0.96 223 | 93090 002050 0.52 93136 193915 -1.26 224 | 93137 020351 0.25 93166 195650 -0.88 225 | 93167 004310 0.62 93182 060334 0.00 226 | 93182 131313 1.02 93213 181559 -0.18 227 | 93243 011321 1.22 93292 194948 -0.72 228 | 93293 021545 0.77 93341 210417 -1.15 229 | 93343 013534 0.83 94025 203714 -1.04 230 | 94026 012315 0.42 94060 194824 -0.95 231 | 94061 072640 0.53 94101 020148 -1.03 232 | 94103 013547 0.35 94137 200139 -1.02 233 | 94138 022803 0.48 94181 235541 -1.28 234 | 94182 030236 0.20 94209 193554 -0.94 235 | 94210 020305 0.07 94235 204323 -1.01 236 | 94236 012744 0.53 94277 214407 -1.15 237 | 94278 022908 0.30 94305 204327 -0.82 238 | 94306 012723 0.67 94347 150149 -1.06 239 | 94348 004630 0.42 95019 202513 -1.11 240 | 95020 010905 0.41 95052 195749 -0.97 241 | 95053 004426 0.50 95087 190612 -0.90 242 | 95088 013146 0.56 95115 212721 -0.62 243 | 95116 021019 0.89 95150 203430 -0.57 244 | 95151 030113 0.91 95201 193323 -1.24 245 | 95202 020016 0.74 95214 200458 0.21"""} 246 | 247 | sat = None 248 | 249 | 250 | def get_offsets(sat): 251 | """Get the clock drift offsets for sat. 252 | """ 253 | errors = [] 254 | offsets = txt[sat] 255 | for line in offsets.split("\n"): 256 | elts = line.split() 257 | 258 | errors.append((datetime.strptime("".join(elts[:2]), "%y%j%H%M%S"), 259 | float(elts[2]))) 260 | errors.append((datetime.strptime("".join(elts[3:5]), "%y%j%H%M%S"), 261 | float(elts[5]))) 262 | return zip(*errors) 263 | -------------------------------------------------------------------------------- /pygac/configuration.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # Copyright (c) 2020 Pygac Developers 5 | 6 | # Author(s): 7 | 8 | # Carlos Horn 9 | 10 | # This program is free software: you can redistribute it and/or modify 11 | # it under the terms of the GNU General Public License as published by 12 | # the Free Software Foundation, either version 3 of the License, or 13 | # (at your option) any later version. 14 | 15 | # This program is distributed in the hope that it will be useful, 16 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 17 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 18 | # GNU General Public License for more details. 19 | 20 | # You should have received a copy of the GNU General Public License 21 | # along with this program. If not, see . 22 | 23 | """Module configuration class. 24 | 25 | Read and manage module configuration. 26 | """ 27 | import configparser 28 | import logging 29 | import os 30 | 31 | LOG = logging.getLogger(__name__) 32 | 33 | 34 | class Configuration(configparser.ConfigParser): 35 | """Configuration container for pygac.""" 36 | 37 | config_file = "" 38 | 39 | def read(self, config_file): 40 | """Read and parse the configuration file 41 | 42 | Args: 43 | config_file (str): path to the config file 44 | 45 | Note: 46 | In contrast to the parent method, this implementation 47 | only accepts a single file and raises an exception if 48 | the given file is not readable. 49 | """ 50 | if not os.path.isfile(config_file): 51 | raise FileNotFoundError( 52 | 'Given config path "%s" is not a file!' % config_file) 53 | try: 54 | super().read(config_file) 55 | except configparser.NoSectionError: 56 | LOG.error('Failed reading configuration file: "%s"' 57 | % config_file) 58 | raise 59 | self.config_file = config_file 60 | 61 | 62 | _config = Configuration() 63 | 64 | 65 | def get_config(initialized=True): 66 | """Retrun the module configuration. 67 | 68 | Args: 69 | initialized (bool): if true ensure that the configuration has 70 | been initialized (default: True) 71 | """ 72 | global _config 73 | if initialized and not _config.sections(): 74 | LOG.info('Configuration has not been initialized. Use' 75 | ' environment variable "PYGAC_CONFIG_FILE"') 76 | try: 77 | config_file = os.environ["PYGAC_CONFIG_FILE"] 78 | except KeyError: 79 | LOG.error("Environment variable PYGAC_CONFIG_FILE not set!") 80 | raise 81 | _config.read(config_file) 82 | return _config 83 | 84 | 85 | def read_config_file(config_file): 86 | """Read a given config file.""" 87 | config = get_config(initialized=False) 88 | config.read(config_file) 89 | -------------------------------------------------------------------------------- /pygac/gac_klm.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # Copyright (c) 2014 Abhay Devasthale and Martin Raspaud 5 | 6 | # Author(s): 7 | 8 | # Abhay Devasthale 9 | # Martin Raspaud 10 | # Adam Dybbroe 11 | 12 | # This program is free software: you can redistribute it and/or modify 13 | # it under the terms of the GNU General Public License as published by 14 | # the Free Software Foundation, either version 3 of the License, or 15 | # (at your option) any later version. 16 | 17 | # This program is distributed in the hope that it will be useful, 18 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 19 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 20 | # GNU General Public License for more details. 21 | 22 | # You should have received a copy of the GNU General Public License 23 | # along with this program. If not, see . 24 | 25 | """Reader for GAC KLM data.""" 26 | 27 | from __future__ import print_function 28 | 29 | import logging 30 | 31 | import numpy as np 32 | 33 | from pygac.gac_reader import GACReader 34 | from pygac.klm_reader import KLMReader, main_klm 35 | 36 | LOG = logging.getLogger(__name__) 37 | 38 | # video data object 39 | 40 | scanline = np.dtype([("scan_line_number", ">u2"), 41 | ("scan_line_year", ">u2"), 42 | ("scan_line_day_of_year", ">u2"), 43 | ("satellite_clock_drift_delta", ">i2"), 44 | ("scan_line_utc_time_of_day", ">u4"), 45 | ("scan_line_bit_field", ">u2"), 46 | ("zero_fill0", ">i2", (5, )), 47 | # QUALITY INDICATORS 48 | ("quality_indicator_bit_field", ">u4"), 49 | ("scan_line_quality_flags", [("reserved", ">u1"), 50 | ("time_problem_code", ">u1"), 51 | ("calibration_problem_code", ">u1"), 52 | ("earth_location_problem_code", ">u1")]), 53 | ("calibration_quality_flags", ">u2", (3, )), 54 | ("count_of_bit_errors_in_frame_sync", ">u2"), 55 | ("zero_fill1", ">i4", (2, )), 56 | # CALIBRATION COEFFICIENTS 57 | ("visible_operational_cal_ch_1_slope_1", ">i4"), 58 | ("visible_operational_cal_ch_1_intercept_1", ">i4"), 59 | ("visible_operational_cal_ch_1_slope_2", ">i4"), 60 | ("visible_operational_cal_ch_1_intercept_2", ">i4"), 61 | ("visible_operational_cal_ch_1_intersection", ">i4"), 62 | ("visible_test_cal_ch_1_slope_1", ">i4"), 63 | ("visible_test_cal_ch_1_intercept_1", ">i4"), 64 | ("visible_test_cal_ch_1_slope_2", ">i4"), 65 | ("visible_test_cal_ch_1_intercept_2", ">i4"), 66 | ("visible_test_cal_ch_1_intersection", ">i4"), 67 | ("visible_prelaunch_cal_ch_1_slope_1", ">i4"), 68 | ("visible_prelaunch_cal_ch_1_intercept_1", ">i4"), 69 | ("visible_prelaunch_cal_ch_1_slope_2", ">i4"), 70 | ("visible_prelaunch_cal_ch_1_intercept_2", ">i4"), 71 | ("visible_prelaunch_cal_ch_1_intersection", ">i4"), 72 | ("visible_operational_cal_ch_2_slope_1", ">i4"), 73 | ("visible_operational_cal_ch_2_intercept_1", ">i4"), 74 | ("visible_operational_cal_ch_2_slope_2", ">i4"), 75 | ("visible_operational_cal_ch_2_intercept_2", ">i4"), 76 | ("visible_operational_cal_ch_2_intersection", ">i4"), 77 | ("visible_test_cal_ch_2_slope_1", ">i4"), 78 | ("visible_test_cal_ch_2_intercept_1", ">i4"), 79 | ("visible_test_cal_ch_2_slope_2", ">i4"), 80 | ("visible_test_cal_ch_2_intercept_2", ">i4"), 81 | ("visible_test_cal_ch_2_intersection", ">i4"), 82 | ("visible_prelaunch_cal_ch_2_slope_1", ">i4"), 83 | ("visible_prelaunch_cal_ch_2_intercept_1", ">i4"), 84 | ("visible_prelaunch_cal_ch_2_slope_2", ">i4"), 85 | ("visible_prelaunch_cal_ch_2_intercept_2", ">i4"), 86 | ("visible_prelaunch_cal_ch_2_intersection", ">i4"), 87 | ("visible_operational_cal_ch_3a_slope_1", ">i4"), 88 | ("visible_operational_cal_ch_3a_intercept_1", ">i4"), 89 | ("visible_operational_cal_ch_3a_slope_2", ">i4"), 90 | ("visible_operational_cal_ch_3a_intercept_2", ">i4"), 91 | ("visible_operational_cal_ch_3a_intersection", ">i4"), 92 | ("visible_test_cal_ch_3a_slope_1", ">i4"), 93 | ("visible_test_cal_ch_3a_intercept_1", ">i4"), 94 | ("visible_test_cal_ch_3a_slope_2", ">i4"), 95 | ("visible_test_cal_ch_3a_intercept_2", ">i4"), 96 | ("visible_test_cal_ch_3a_intersection", ">i4"), 97 | ("visible_prelaunch_cal_ch_3a_slope_1", ">i4"), 98 | ("visible_prelaunch_cal_ch_3a_intercept_1", ">i4"), 99 | ("visible_prelaunch_cal_ch_3a_slope_2", ">i4"), 100 | ("visible_prelaunch_cal_ch_3a_intercept_2", ">i4"), 101 | ("visible_prelaunch_cal_ch_3a_intersection", ">i4"), 102 | ("ir_operational_cal_ch_3b_coefficient_1", ">i4"), 103 | ("ir_operational_cal_ch_3b_coefficient_2", ">i4"), 104 | ("ir_operational_cal_ch_3b_coefficient_3", ">i4"), 105 | ("ir_test_cal_ch_3b_coefficient_1", ">i4"), 106 | ("ir_test_cal_ch_3b_coefficient_2", ">i4"), 107 | ("ir_test_cal_ch_3b_coefficient_3", ">i4"), 108 | ("ir_operational_cal_ch_4_coefficient_1", ">i4"), 109 | ("ir_operational_cal_ch_4_coefficient_2", ">i4"), 110 | ("ir_operational_cal_ch_4_coefficient_3", ">i4"), 111 | ("ir_test_cal_ch_4_coefficient_1", ">i4"), 112 | ("ir_test_cal_ch_4_coefficient_2", ">i4"), 113 | ("ir_test_cal_ch_4_coefficient_3", ">i4"), 114 | ("ir_operational_cal_ch_5_coefficient_1", ">i4"), 115 | ("ir_operational_cal_ch_5_coefficient_2", ">i4"), 116 | ("ir_operational_cal_ch_5_coefficient_3", ">i4"), 117 | ("ir_test_cal_ch_5_coefficient_1", ">i4"), 118 | ("ir_test_cal_ch_5_coefficient_2", ">i4"), 119 | ("ir_test_cal_ch_5_coefficient_3", ">i4"), 120 | # NAVIGATION 121 | ("computed_yaw_steering", ">i2", (3,)), # only in version 5 122 | ("total_applied_attitude_correction", 123 | ">i2", (3,)), # only in version 5 124 | ("navigation_status_bit_field", ">u4"), 125 | ("time_associated_with_tip_euler_angles", ">u4"), 126 | ("tip_euler_angles", ">i2", (3, )), 127 | ("spacecraft_altitude_above_reference_ellipsoid", ">u2"), 128 | ("angular_relationships", ">i2", (153, )), 129 | ("zero_fill3", ">i2", (3, )), 130 | ("earth_location", [("lats", ">i4"), 131 | ("lons", ">i4")], (51,)), 132 | ("zero_fill4", ">i4", (2, )), 133 | # HRPT MINOR FRAME TELEMETRY 134 | ("frame_sync", ">u2", (6, )), 135 | ("id", ">u2", (2, )), 136 | ("time_code", ">u2", (4, )), 137 | ("telemetry", [("ramp_calibration", ">u2", (5, )), 138 | ("PRT", ">u2", (3, )), 139 | ("ch3_patch_temp", ">u2"), 140 | ("spare", ">u2"), ]), 141 | ("back_scan", ">u2", (30, )), 142 | ("space_data", ">u2", (50, )), 143 | ("sync_delta", ">u2"), 144 | ("zero_fill5", ">i2"), 145 | # AVHRR SENSOR DATA 146 | ("sensor_data", ">u4", (682, )), 147 | ("zero_fill6", ">i4", (2, )), 148 | # DIGITAL B TELEMETRY 149 | ("invalid_word_bit_flags1", ">u2"), 150 | ("avhrr_digital_b_data", ">u2"), 151 | ("zero_fill7", ">i4", (3, )), 152 | # ANALOG HOUSEKEEPING DATA (TIP) 153 | ("invalid_word_bit_flags2", ">u4"), 154 | ("patch_temperature_range", ">u1"), 155 | ("patch_temperature_extended", ">u1"), 156 | ("patch_power", ">u1"), 157 | ("radiator_temperature", ">u1"), 158 | ("blackbody_temperature_1", ">u1"), 159 | ("blackbody_temperature_2", ">u1"), 160 | ("blackbody_temperature_3", ">u1"), 161 | ("blackbody_temperature_4", ">u1"), 162 | ("electronics_current", ">u1"), 163 | ("motor_current", ">u1"), 164 | ("earth_shield_position", ">u1"), 165 | ("electronics_temperature", ">u1"), 166 | ("cooler_housing_temperature", ">u1"), 167 | ("baseplate_temperature", ">u1"), 168 | ("motor_housing_temperature", ">u1"), 169 | ("a/d_converter_temperature", ">u1"), 170 | ("detector_#4_bias_voltage", ">u1"), 171 | ("detector_#5_bias_voltage", ">u1"), 172 | ("blackbody_temperature_channel3b", ">u1"), 173 | ("blackbody_temperature_channel4", ">u1"), 174 | ("blackbody_temperature_channel5", ">u1"), 175 | ("reference_voltage", ">u1"), 176 | ("zero_fill8", ">i2", (3, )), 177 | # CLOUDS FROM AVHRR (CLAVR) 178 | ("reserved0", ">u4"), 179 | ("reserved1", ">u4"), 180 | ("reserved2", ">u2", (52, )), 181 | # FILLER 182 | ("zero_fill9", ">i4", (112, ))]) 183 | 184 | 185 | class GACKLMReader(GACReader, KLMReader): 186 | """The GAC KLM reader class. 187 | 188 | The offset attribute tells where in the file the scanline data starts. 189 | """ 190 | lonlat_sample_points = np.arange(4.5, 405, 8) 191 | 192 | def __init__(self, *args, **kwargs): 193 | """Init the GAC KLM reader.""" 194 | GACReader.__init__(self, *args, **kwargs) 195 | self.scanline_type = scanline 196 | self.offset = 4608 197 | 198 | 199 | def main(filename, start_line, end_line): 200 | """Generate a l1c file.""" 201 | return main_klm(GACKLMReader, filename, start_line, end_line) 202 | 203 | 204 | if __name__ == "__main__": 205 | import sys 206 | main(sys.argv[1], sys.argv[2], sys.argv[3]) 207 | -------------------------------------------------------------------------------- /pygac/gac_pod.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # Copyright (c) 2014-2016. 3 | # 4 | 5 | # Author(s): 6 | 7 | # Abhay Devasthale 8 | # Adam Dybbroe 9 | # Sajid Pareeth 10 | # Martin Raspaud 11 | 12 | # This work was done in the framework of ESA-CCI-Clouds phase I 13 | 14 | 15 | # This program is free software: you can redistribute it and/or modify 16 | # it under the terms of the GNU General Public License as published by 17 | # the Free Software Foundation, either version 3 of the License, or 18 | # (at your option) any later version. 19 | 20 | # This program is distributed in the hope that it will be useful, 21 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 22 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 23 | # GNU General Public License for more details. 24 | 25 | # You should have received a copy of the GNU General Public License 26 | # along with this program. If not, see . 27 | """Reader for GAC POD data.""" 28 | 29 | import logging 30 | 31 | import numpy as np 32 | 33 | from pygac.gac_reader import GACReader 34 | from pygac.pod_reader import PODReader, main_pod 35 | 36 | LOG = logging.getLogger(__name__) 37 | 38 | scanline = np.dtype([("scan_line_number", ">i2"), 39 | ("time_code", ">u2", (3, )), 40 | ("quality_indicators", ">u4"), 41 | ("calibration_coefficients", ">i4", (10, )), 42 | ("number_of_meaningful_zenith_angles_and_earth_location_appended", 43 | ">u1"), 44 | ("solar_zenith_angles", "i1", (51, )), 45 | ("earth_location", [("lats", ">i2"), 46 | ("lons", ">i2")], (51,)), 47 | ("telemetry", ">u4", (35, )), 48 | ("sensor_data", ">u4", (682, )), 49 | ("add_on_zenith", ">u2", (10, )), 50 | ("clock_drift_delta", ">u2"), 51 | ("spare3", "u2", (11, ))]) 52 | 53 | 54 | class GACPODReader(GACReader, PODReader): 55 | """The GAC POD reader class. 56 | 57 | The `scan_points` attributes provides the position of the longitude and latitude points to 58 | compute relative to the full swath width. 59 | 60 | The offset attribute tells where in the file the scanline data starts. 61 | """ 62 | 63 | def __init__(self, *args, **kwargs): 64 | """Init the GAC POD reader.""" 65 | GACReader.__init__(self, *args, **kwargs) 66 | self.scanline_type = scanline 67 | # GAC POD files were originally written to tapes using 6440 byte physical records 68 | # with two 3220 logical records per physical record. As the header is in the first 69 | # physical record the scanline data will start at offset 6440 in the filestream. 70 | # This means the second logical record (at offset 3220) is not used and contains 71 | # junk / padding data. The data often appears to be a valid scanline, but it is a 72 | # duplicate of a data appearing later in the file and should not be used. 73 | self.offset = 6440 74 | self.scan_points = np.arange(3.5, 2048, 5) 75 | 76 | 77 | def main(filename, start_line, end_line): 78 | """Generate a l1c file.""" 79 | return main_pod(GACPODReader, filename, start_line, end_line) 80 | 81 | 82 | if __name__ == "__main__": 83 | import sys 84 | main(sys.argv[1], sys.argv[2], sys.argv[3]) 85 | -------------------------------------------------------------------------------- /pygac/gac_reader.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # Copyright (c) 2014, 2015 Martin Raspaud 5 | 6 | # Author(s): 7 | 8 | # Martin Raspaud 9 | # Carlos Horn 10 | 11 | # This program is free software: you can redistribute it and/or modify 12 | # it under the terms of the GNU General Public License as published by 13 | # the Free Software Foundation, either version 3 of the License, or 14 | # (at your option) any later version. 15 | 16 | # This program is distributed in the hope that it will be useful, 17 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 18 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 19 | # GNU General Public License for more details. 20 | 21 | # You should have received a copy of the GNU General Public License 22 | # along with this program. If not, see . 23 | 24 | """Generic reader for GAC data. 25 | 26 | Can't be used as is, has to be subclassed to add specific read functions. 27 | """ 28 | 29 | import logging 30 | import warnings 31 | 32 | import numpy as np 33 | try: 34 | from pyorbital.geoloc_instrument_definitions import avhrr_gac_from_times 35 | except ImportError: 36 | # In pyorbital 1.9.2 and earlier avhrr_gac returned LAC geometry 37 | from pyorbital.geoloc_instrument_definitions import avhrr_gac 38 | def avhrr_gac_from_times(times, points): 39 | return avhrr_gac(times, points*5+3.5) 40 | warnings.warn('pyorbital version does not support avhrr_gac_from_times. ' + 41 | 'Computation of missing longitude/latitudes may be incorrect.') 42 | 43 | 44 | from pygac.reader import Reader, ReaderError 45 | 46 | LOG = logging.getLogger(__name__) 47 | 48 | 49 | class GACReader(Reader): 50 | """Reader for GAC data.""" 51 | 52 | # Scanning frequency (scanlines per millisecond) 53 | scan_freq = 2.0 / 1000.0 54 | # Max scanlines 55 | max_scanlines = 15000 56 | lonlat_sample_points = np.arange(4, 405, 8) 57 | 58 | def __init__(self, *args, **kwargs): 59 | """Init the GAC reader.""" 60 | super().__init__(*args, **kwargs) 61 | self.scan_width = 409 62 | self.geoloc_definition = avhrr_gac_from_times 63 | 64 | @classmethod 65 | def _validate_header(cls, header): 66 | """Check if the header belongs to this reader.""" 67 | # call super to enter the Method Resolution Order (MRO) 68 | super(GACReader, cls)._validate_header(header) 69 | LOG.debug("validate header") 70 | data_set_name = header["data_set_name"].decode() 71 | # split header into parts 72 | creation_site, transfer_mode, platform_id = ( 73 | data_set_name.split(".")[:3]) 74 | if transfer_mode != "GHRR": 75 | raise ReaderError('Improper transfer mode "%s"!' % transfer_mode) 76 | -------------------------------------------------------------------------------- /pygac/lac_klm.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # Copyright (c) 2014-2019 3 | # 4 | 5 | # Author(s): 6 | 7 | # Abhay Devasthale 8 | # Sajid Pareeth 9 | # Martin Raspaud 10 | # Adam Dybbroe 11 | 12 | # This program is free software: you can redistribute it and/or modify 13 | # it under the terms of the GNU General Public License as published by 14 | # the Free Software Foundation, either version 3 of the License, or 15 | # (at your option) any later version. 16 | 17 | # This program is distributed in the hope that it will be useful, 18 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 19 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 20 | # GNU General Public License for more details. 21 | 22 | # You should have received a copy of the GNU General Public License 23 | # along with this program. If not, see . 24 | 25 | """Reader for LAC KLM data.""" 26 | 27 | import logging 28 | 29 | import numpy as np 30 | 31 | from pygac.klm_reader import KLMReader, main_klm 32 | from pygac.lac_reader import LACReader 33 | 34 | LOG = logging.getLogger(__name__) 35 | 36 | 37 | # video data object 38 | 39 | scanline = np.dtype([("scan_line_number", ">u2"), 40 | ("scan_line_year", ">u2"), 41 | ("scan_line_day_of_year", ">u2"), 42 | ("satellite_clock_drift_delta", ">i2"), 43 | ("scan_line_utc_time_of_day", ">u4"), 44 | ("scan_line_bit_field", ">u2"), 45 | ("zero_fill0", ">i2", (5, )), 46 | # QUALITY INDICATORS 47 | ("quality_indicator_bit_field", ">u4"), 48 | ("scan_line_quality_flags", [("reserved", ">u1"), 49 | ("time_problem_code", ">u1"), 50 | ("calibration_problem_code", ">u1"), 51 | ("earth_location_problem_code", ">u1")]), 52 | ("calibration_quality_flags", ">u2", (3, )), 53 | ("count_of_bit_errors_in_frame_sync", ">u2"), 54 | ("zero_fill1", ">i4", (2, )), 55 | # CALIBRATION COEFFICIENTS 56 | ("visible_operational_cal_ch_1_slope_1", ">i4"), 57 | ("visible_operational_cal_ch_1_intercept_1", ">i4"), 58 | ("visible_operational_cal_ch_1_slope_2", ">i4"), 59 | ("visible_operational_cal_ch_1_intercept_2", ">i4"), 60 | ("visible_operational_cal_ch_1_intersection", ">i4"), 61 | ("visible_test_cal_ch_1_slope_1", ">i4"), 62 | ("visible_test_cal_ch_1_intercept_1", ">i4"), 63 | ("visible_test_cal_ch_1_slope_2", ">i4"), 64 | ("visible_test_cal_ch_1_intercept_2", ">i4"), 65 | ("visible_test_cal_ch_1_intersection", ">i4"), 66 | ("visible_prelaunch_cal_ch_1_slope_1", ">i4"), 67 | ("visible_prelaunch_cal_ch_1_intercept_1", ">i4"), 68 | ("visible_prelaunch_cal_ch_1_slope_2", ">i4"), 69 | ("visible_prelaunch_cal_ch_1_intercept_2", ">i4"), 70 | ("visible_prelaunch_cal_ch_1_intersection", ">i4"), 71 | ("visible_operational_cal_ch_2_slope_1", ">i4"), 72 | ("visible_operational_cal_ch_2_intercept_1", ">i4"), 73 | ("visible_operational_cal_ch_2_slope_2", ">i4"), 74 | ("visible_operational_cal_ch_2_intercept_2", ">i4"), 75 | ("visible_operational_cal_ch_2_intersection", ">i4"), 76 | ("visible_test_cal_ch_2_slope_1", ">i4"), 77 | ("visible_test_cal_ch_2_intercept_1", ">i4"), 78 | ("visible_test_cal_ch_2_slope_2", ">i4"), 79 | ("visible_test_cal_ch_2_intercept_2", ">i4"), 80 | ("visible_test_cal_ch_2_intersection", ">i4"), 81 | ("visible_prelaunch_cal_ch_2_slope_1", ">i4"), 82 | ("visible_prelaunch_cal_ch_2_intercept_1", ">i4"), 83 | ("visible_prelaunch_cal_ch_2_slope_2", ">i4"), 84 | ("visible_prelaunch_cal_ch_2_intercept_2", ">i4"), 85 | ("visible_prelaunch_cal_ch_2_intersection", ">i4"), 86 | ("visible_operational_cal_ch_3a_slope_1", ">i4"), 87 | ("visible_operational_cal_ch_3a_intercept_1", ">i4"), 88 | ("visible_operational_cal_ch_3a_slope_2", ">i4"), 89 | ("visible_operational_cal_ch_3a_intercept_2", ">i4"), 90 | ("visible_operational_cal_ch_3a_intersection", ">i4"), 91 | ("visible_test_cal_ch_3a_slope_1", ">i4"), 92 | ("visible_test_cal_ch_3a_intercept_1", ">i4"), 93 | ("visible_test_cal_ch_3a_slope_2", ">i4"), 94 | ("visible_test_cal_ch_3a_intercept_2", ">i4"), 95 | ("visible_test_cal_ch_3a_intersection", ">i4"), 96 | ("visible_prelaunch_cal_ch_3a_slope_1", ">i4"), 97 | ("visible_prelaunch_cal_ch_3a_intercept_1", ">i4"), 98 | ("visible_prelaunch_cal_ch_3a_slope_2", ">i4"), 99 | ("visible_prelaunch_cal_ch_3a_intercept_2", ">i4"), 100 | ("visible_prelaunch_cal_ch_3a_intersection", ">i4"), 101 | ("ir_operational_cal_ch_3b_coefficient_1", ">i4"), 102 | ("ir_operational_cal_ch_3b_coefficient_2", ">i4"), 103 | ("ir_operational_cal_ch_3b_coefficient_3", ">i4"), 104 | ("ir_test_cal_ch_3b_coefficient_1", ">i4"), 105 | ("ir_test_cal_ch_3b_coefficient_2", ">i4"), 106 | ("ir_test_cal_ch_3b_coefficient_3", ">i4"), 107 | ("ir_operational_cal_ch_4_coefficient_1", ">i4"), 108 | ("ir_operational_cal_ch_4_coefficient_2", ">i4"), 109 | ("ir_operational_cal_ch_4_coefficient_3", ">i4"), 110 | ("ir_test_cal_ch_4_coefficient_1", ">i4"), 111 | ("ir_test_cal_ch_4_coefficient_2", ">i4"), 112 | ("ir_test_cal_ch_4_coefficient_3", ">i4"), 113 | ("ir_operational_cal_ch_5_coefficient_1", ">i4"), 114 | ("ir_operational_cal_ch_5_coefficient_2", ">i4"), 115 | ("ir_operational_cal_ch_5_coefficient_3", ">i4"), 116 | ("ir_test_cal_ch_5_coefficient_1", ">i4"), 117 | ("ir_test_cal_ch_5_coefficient_2", ">i4"), 118 | ("ir_test_cal_ch_5_coefficient_3", ">i4"), 119 | # NAVIGATION 120 | ("computed_yaw_steering", ">i2", (3,)), # only in version 5 121 | ("total_applied_attitude_correction", 122 | ">i2", (3,)), # only in version 5 123 | ("navigation_status_bit_field", ">u4"), 124 | ("time_associated_with_tip_euler_angles", ">u4"), 125 | ("tip_euler_angles", ">i2", (3, )), 126 | ("spacecraft_altitude_above_reference_ellipsoid", ">u2"), 127 | ("angular_relationships", ">i2", (153, )), 128 | ("zero_fill2", ">i2", (3, )), 129 | ("earth_location", [("lats", ">i4"), 130 | ("lons", ">i4")], (51,)), 131 | ("zero_fill3", ">i4", (2, )), 132 | # HRPT MINOR FRAME TELEMETRY 133 | ("frame_sync", ">u2", (6, )), 134 | ("id", ">u2", (2, )), 135 | ("time_code", ">u2", (4, )), 136 | ("telemetry", [("ramp_calibration", ">u2", (5, )), 137 | ("PRT", ">u2", (3, )), 138 | ("ch3_patch_temp", ">u2"), 139 | ("spare", ">u2"), ]), 140 | ("back_scan", ">u2", (30, )), 141 | ("space_data", ">u2", (50, )), 142 | ("sync_delta", ">u2"), 143 | ("zero_fill4", ">i2"), 144 | # EARTH OBSERVATIONS 145 | ("sensor_data", ">u4", (3414,)), 146 | ("zero_fill5", ">i4", (2,)), 147 | # DIGITAL B HOUSEKEEPING TELEMETRY 148 | ("digital_b_telemetry_update_flags", ">u2"), 149 | ("avhrr_digital_b_data", ">u2"), 150 | ("zero_fill6", ">i4", (3,)), 151 | # ANALOG HOUSEKEEPING DATA (TIP) 152 | ("analog_telemetry_update_flags", ">u4"), 153 | ("patch_temperature_range", ">u1"), 154 | ("patch_temperature_extended", ">u1"), 155 | ("patch_power", ">u1"), 156 | ("radiator_temperature", ">u1"), 157 | ("blackbody_temperature_1", ">u1"), 158 | ("blackbody_temperature_2", ">u1"), 159 | ("blackbody_temperature_3", ">u1"), 160 | ("blackbody_temperature_4", ">u1"), 161 | ("electronics_current", ">u1"), 162 | ("motor_current", ">u1"), 163 | ("earth_shield_position", ">u1"), 164 | ("electronics_temperature", ">u1"), 165 | ("cooler_housing_temperature", ">u1"), 166 | ("baseplate_temperature", ">u1"), 167 | ("motor_housing_temperature", ">u1"), 168 | ("a/d_converter_temperature", ">u1"), 169 | ("detector_#4_bias_voltage", ">u1"), 170 | ("detector_#5_bias_voltage", ">u1"), 171 | ("blackbody_temperature_channel3b", ">u1"), 172 | ("blackbody_temperature_channel4", ">u1"), 173 | ("blackbody_temperature_channel5", ">u1"), 174 | ("reference_voltage", ">u1"), 175 | ("zero_fill7", ">i2", (3,)), 176 | # CLOUDS FROM AVHRR (CLAVR) 177 | ("reserved_clavr_status_bit_field", ">u4"), 178 | ("reserved_clavr", ">u4"), 179 | ("reserved_clavr_ccm", ">u2", (256,)), 180 | # FILLER 181 | ("zero_fill8", ">i4", (94,))]) 182 | 183 | 184 | class LACKLMReader(LACReader, KLMReader): 185 | """The LAC KLM reader. 186 | 187 | The offset attribute tells where in the file the scanline data starts. 188 | """ 189 | 190 | def __init__(self, *args, **kwargs): 191 | """Init the LAC KLM reader.""" 192 | LACReader.__init__(self, *args, **kwargs) 193 | self.scanline_type = scanline 194 | self.offset = 15872 195 | # packed: 15872 196 | # self.offset = 22528 197 | # unpacked: 22528 198 | 199 | 200 | def main(filename, start_line, end_line): 201 | """Generate a l1c file.""" 202 | return main_klm(LACKLMReader, filename, start_line, end_line) 203 | 204 | 205 | if __name__ == "__main__": 206 | import sys 207 | main(sys.argv[1], sys.argv[2], sys.argv[3]) 208 | -------------------------------------------------------------------------------- /pygac/lac_pod.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # Copyright (c) 2014-2019. 3 | # 4 | 5 | # Author(s): 6 | 7 | # Abhay Devasthale 8 | # Adam Dybbroe 9 | # Sajid Pareeth 10 | # Martin Raspaud 11 | 12 | # This program is free software: you can redistribute it and/or modify 13 | # it under the terms of the GNU General Public License as published by 14 | # the Free Software Foundation, either version 3 of the License, or 15 | # (at your option) any later version. 16 | 17 | # This program is distributed in the hope that it will be useful, 18 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 19 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 20 | # GNU General Public License for more details. 21 | 22 | # You should have received a copy of the GNU General Public License 23 | # along with this program. If not, see . 24 | 25 | """Reader for LAC POD data.""" 26 | 27 | import logging 28 | 29 | import numpy as np 30 | 31 | from pygac.lac_reader import LACReader 32 | from pygac.pod_reader import PODReader, main_pod 33 | 34 | LOG = logging.getLogger(__name__) 35 | 36 | scanline = np.dtype([("scan_line_number", ">i2"), 37 | ("time_code", ">u2", (3, )), 38 | ("quality_indicators", ">u4"), 39 | ("calibration_coefficients", ">i4", (10, )), 40 | ("number_of_meaningful_zenith_angles_and_earth_location_appended", ">u1"), 41 | ("solar_zenith_angles", "i1", (51, )), 42 | ("earth_location", [("lats", ">i2"), 43 | ("lons", ">i2")], (51,)), 44 | ("telemetry", ">u4", (35, )), 45 | ("sensor_data", ">u4", (3414, )), 46 | ("add_on_zenith", ">u2", (10, )), 47 | ("clock_drift_delta", ">u2"), # only in newest version 48 | ("spare3", "u2", (337, ))]) 49 | 50 | 51 | class LACPODReader(LACReader, PODReader): 52 | """The LAC POD reader. 53 | 54 | The `scan_points` attributes provides the position of the longitude and latitude points to 55 | compute relative to the full swath width. 56 | 57 | The offset attribute tells where in the file the scanline data starts. 58 | """ 59 | 60 | def __init__(self, *args, **kwargs): 61 | """Init the LAC POD reader.""" 62 | LACReader.__init__(self, *args, **kwargs) 63 | self.scanline_type = scanline 64 | self.offset = 14800 65 | self.scan_points = np.arange(2048) 66 | 67 | 68 | def main(filename, start_line, end_line): 69 | """Generate a l1c file.""" 70 | return main_pod(LACPODReader, filename, start_line, end_line) 71 | 72 | 73 | if __name__ == "__main__": 74 | import sys 75 | main(sys.argv[1], sys.argv[2], sys.argv[3]) 76 | -------------------------------------------------------------------------------- /pygac/lac_reader.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # Copyright (c) 2014-2019. 3 | # 4 | 5 | # Author(s): 6 | 7 | # Abhay Devasthale 8 | # Adam Dybbroe 9 | # Sajid Pareeth 10 | # Martin Raspaud 11 | # Carlos Horn 12 | 13 | # This program is free software: you can redistribute it and/or modify 14 | # it under the terms of the GNU General Public License as published by 15 | # the Free Software Foundation, either version 3 of the License, or 16 | # (at your option) any later version. 17 | 18 | # This program is distributed in the hope that it will be useful, 19 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 20 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 21 | # GNU General Public License for more details. 22 | 23 | # You should have received a copy of the GNU General Public License 24 | # along with this program. If not, see . 25 | """The LAC reader.""" 26 | 27 | import logging 28 | 29 | import numpy as np 30 | try: 31 | from pyorbital.geoloc_instrument_definitions import avhrr_from_times 32 | except ImportError: 33 | # In pyorbital 1.9.2 and earlier avhrr_gac returned LAC geometry 34 | from pyorbital.geoloc_instrument_definitions import avhrr_gac as avhrr_from_times 35 | 36 | from pygac.reader import Reader, ReaderError 37 | 38 | LOG = logging.getLogger(__name__) 39 | 40 | 41 | class LACReader(Reader): 42 | """Reader for LAC data.""" 43 | 44 | # Scanning frequency (scanlines per millisecond) 45 | scan_freq = 6.0 / 1000.0 46 | # Max scanlines 47 | max_scanlines = 65535 48 | lonlat_sample_points = np.arange(24, 2048, 40) 49 | 50 | def __init__(self, *args, **kwargs): 51 | """Init the LAC reader.""" 52 | super(LACReader, self).__init__(*args, **kwargs) 53 | self.scan_width = 2048 54 | self.geoloc_definition = avhrr_from_times 55 | 56 | @classmethod 57 | def _validate_header(cls, header): 58 | """Check if the header belongs to this reader.""" 59 | # call super to enter the Method Resolution Order (MRO) 60 | super(LACReader, cls)._validate_header(header) 61 | LOG.debug("validate header") 62 | data_set_name = header["data_set_name"].decode() 63 | # split header into parts 64 | creation_site, transfer_mode, platform_id = ( 65 | data_set_name.split(".")[:3]) 66 | if transfer_mode not in ["LHRR", "HRPT", "FRAC"]: 67 | raise ReaderError('Improper transfer mode "%s"!' % transfer_mode) 68 | -------------------------------------------------------------------------------- /pygac/patmosx_coeff_reader.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # Author(s): 4 | 5 | # Carlos Horn 6 | 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License as published by 9 | # the Free Software Foundation, either version 3 of the License, or 10 | # (at your option) any later version. 11 | 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | 17 | # You should have received a copy of the GNU General Public License 18 | # along with this program. If not, see . 19 | 20 | """Convert PATMOS-x calibration file tarballs to PyGAC calibration json format. 21 | 22 | The official tarballs are available on the PATMOS-x webpage "https://cimss.ssec.wisc.edu/patmosx/avhrr_cal.html". 23 | """ 24 | 25 | import argparse 26 | import datetime as dt 27 | import json 28 | import logging 29 | import pathlib 30 | import re 31 | import tarfile 32 | 33 | from scipy.optimize import bisect 34 | 35 | 36 | class PatmosxReader: 37 | """Read PATMOS-x coefficient files tarballs.""" 38 | # regular expression with named capturing groups to read an entire patmosx file 39 | regex = re.compile( 40 | r"\s*(?P\w+)[^\n]*\n" 41 | r"\s*(?P[eE0-9\.-]+)[^\n]*\n" 42 | r"\s*(?P[eE0-9\.-]+)[^\n]*\n" 43 | r'\s*(?P[eE0-9\.-]+),?\s*(?P[eE0-9\.-]+),?\s*(?P[eE0-9\.-]+),?\s*(?P[eE0-9\.-]+)[^\n]*\n' # noqa 44 | r'\s*(?P[eE0-9\.-]+),?\s*(?P[eE0-9\.-]+),?\s*(?P[eE0-9\.-]+),?\s*(?P[eE0-9\.-]+)[^\n]*\n' # noqa 45 | r'\s*(?P[eE0-9\.-]+),?\s*(?P[eE0-9\.-]+),?\s*(?P[eE0-9\.-]+),?\s*(?P[eE0-9\.-]+)[^\n]*\n' # noqa 46 | r"\s*(?P[eE0-9\.-]+)[^\n]*\n" 47 | r"\s*(?P[eE0-9\.-]+)[^\n]*\n" 48 | r"\s*(?P[eE0-9\.-]+)[^\n]*\n" 49 | r"\s*(?P[eE0-9\.-]+)[^\n]*\n" 50 | r"\s*(?P[eE0-9\.-]+)[^\n]*\n" 51 | r"\s*(?P[eE0-9\.-]+)[^\n]*\n" 52 | r"\s*(?P[eE0-9\.-]+)[^\n]*\n" 53 | r"\s*(?P[eE0-9\.-]+)[^\n]*\n" 54 | r"\s*(?P[eE0-9\.-]+)[^\n]*\n" 55 | r"\s*(?P[eE0-9\.-]+)[^\n]*\n" 56 | r"\s*(?P[eE0-9\.-]+)[^\n]*\n" 57 | r"\s*(?P[eE0-9\.-]+)[^\n]*\n" 58 | r"(?:[a-z]+[^\n]*\n)?" 59 | r'\s*(?P[eE0-9\.-]+)\s*(?P[eE0-9\.-]+)\s*(?P[eE0-9\.-]+)[^\n]*\n' # noqa 60 | r'\s*(?P[eE0-9\.-]+)\s*(?P[eE0-9\.-]+)\s*(?P[eE0-9\.-]+)[^\n]*\n' # noqa 61 | r'\s*(?P[eE0-9\.-]+)\s*(?P[eE0-9\.-]+)\s*(?P[eE0-9\.-]+)[^\n]*\n' # noqa 62 | r'\s*(?P[eE0-9\.-]+)\s*(?P[eE0-9\.-]+)\s*(?P[eE0-9\.-]+)[^\n]*\n' # noqa 63 | r'\s*(?P[eE0-9\.-]+)\s*(?P[eE0-9\.-]+)\s*(?P[eE0-9\.-]+)[^\n]*\n' # noqa 64 | r'\s*(?P[eE0-9\.-]+)\s*(?P[eE0-9\.-]+)\s*(?P[eE0-9\.-]+)[^\n]*\n' # noqa 65 | r"\s*(?P[eE0-9\.-]+)[^\n]*\n" 66 | r"\s*(?P[eE0-9\.-]+)[^\n]*\n" 67 | r"\s*(?P[eE0-9\.-]+)[^\n]*\n" 68 | r"\s*(?P[eE0-9\.-]+)[^\n]*\n" 69 | r'\s*(?P[eE0-9\.-]+)\,*\s*(?P[eE0-9\.-]+)\,*\s*(?P[eE0-9\.-]+)\,*\s*(?P[eE0-9\.-]+)\,*\s*(?P[eE0-9\.-]+)[^\n]*\n' # noqa 70 | r'\s*(?P[eE0-9\.-]+)\,*\s*(?P[eE0-9\.-]+)\,*\s*(?P[eE0-9\.-]+)\,*\s*(?P[eE0-9\.-]+)\,*\s*(?P[eE0-9\.-]+)[^\n]*\n' # noqa 71 | r'\s*(?P[eE0-9\.-]+)\,*\s*(?P[eE0-9\.-]+)\,*\s*(?P[eE0-9\.-]+)\,*\s*(?P[eE0-9\.-]+)\,*\s*(?P[eE0-9\.-]+)[^\n]*\n' # noqa 72 | r'\s*(?P[eE0-9\.-]+)\,*\s*(?P[eE0-9\.-]+)\,*\s*(?P[eE0-9\.-]+)\,*\s*(?P[eE0-9\.-]+)\,*\s*(?P[eE0-9\.-]+)[^\n]*\n' # noqa 73 | r'\s*(?P[eE0-9\.-]+)\s*(?P[eE0-9\.-]+)\s*(?P[eE0-9\.-]+)\s*(?P[eE0-9\.-]+)[^\n]*\n' # noqa 74 | r'(?:\s*(?P[eE0-9\.-]+),\s*(?P[eE0-9\.-]+),\s*(?P[eE0-9\.-]+),\s*(?P[eE0-9\.-]+)[^\n]*\n)?' # noqa 75 | r'(?:\s*(?P[eE0-9\.-]+),\s*(?P[eE0-9\.-]+),\s*(?P[eE0-9\.-]+),\s*(?P[eE0-9\.-]+)[^\n]*\n)?' # noqa 76 | r"(?:\![^v][^\n]*\n)*" 77 | r"(?:\!(?Pv\w+))?" 78 | ) 79 | 80 | def __init__(self, tarball): 81 | self.tarball = tarball 82 | self.coeffs = [] 83 | with tarfile.open(tarball) as archive: 84 | for tarinfo in archive: 85 | if tarinfo.isfile(): 86 | # open satellite specific coefficient file 87 | filename = tarinfo.name 88 | fileobj = archive.extractfile(filename) 89 | content = fileobj.read().decode() 90 | match = self.regex.match(content) 91 | sat_coeffs = {key: self.parse_types(value) for key, value in match.groupdict().items()} 92 | self.coeffs.append(sat_coeffs) 93 | 94 | def __iter__(self): 95 | yield from self.coeffs # noqa 96 | 97 | @staticmethod 98 | def parse_types(value): 99 | """parse the types of coefficients""" 100 | try: 101 | try: 102 | _value = int(value) 103 | except ValueError: 104 | _value = float(value) 105 | except ValueError: 106 | _value = value 107 | except TypeError: 108 | _value = None 109 | return _value 110 | 111 | 112 | class Translator: 113 | """Translate PATMOS-x coefficients to PyGAC format.""" 114 | sat_names = {"m01": "metopb", "m02": "metopa", "n05": "tirosn", "m03": "metopc"} 115 | sat_names.update({"n{0:02d}".format(i): "noaa{0}".format(i) for i in range(6,20)}) 116 | description = { 117 | "visible": { 118 | "channels": ["1", "2", "3a"], 119 | "coefficients": { 120 | "dark_count": "instrument counts under dark conditions []", 121 | "gain_switch": "dual-gain switch count, set to 'null' for single-gain instruments []", 122 | "s0": "single-gain calibration slope at launch date [%]", 123 | "s1": "linear single-gain calibration slope parameter [% years^{-1}]", 124 | "s2": "quadratic single-gain calibration slope parameter [% years^{-2}]", 125 | "date_of_launch": "timestamp of launch date [UTC]" 126 | }, 127 | "method": 'Heidinger, A.K., W.C. Straka III, C.C. Molling, J.T. Sullivan, and X. Wu, 2010: Deriving an inter-sensor consistent calibration for the AVHRR solar reflectance data record. International Journal of Remote Sensing, 31:6493-6517' # noqa 128 | }, 129 | "thermal": { 130 | "channels": ["3b", "4", "5"], 131 | "coefficients": { 132 | "centroid_wavenumber": "centroid wavenumber [cm^{-1}]", 133 | "b0": "constant non-linear radiance correction coefficient [mW m^{-2} sr cm^{-1}]", 134 | "b1": "linear non-linear radiance correction coefficient []", 135 | "b2": "quadratic non-linear radiance correction coefficient [(mW^{-1} m^2 sr^{-1} cm)]", 136 | "space_radiance": "radiance of space [mW m^{-2} sr cm^{-1}]", 137 | "to_eff_blackbody_intercept": "thermal channel temperature to effective blackbody temperature intercept [K]", # noqa 138 | "to_eff_blackbody_slope": "thermal channel temperature to effective blackbody temperature slope []", 139 | "d0": "constant thermometer counts to temperature conversion coefficient [K]", 140 | "d1": "linear thermometer counts to temperature conversion coefficient [K]", 141 | "d2": "quadratic thermometer counts to temperature conversion coefficient [K]", 142 | "d3": "cubic thermometer counts to temperature conversion coefficient [K]", 143 | "d4": "quartic thermometer counts to temperature conversion coefficient [K]" 144 | }, 145 | "method": "Goodrum, G., Kidwell, K.B. and W. Winston, 2000: NOAA KLM User's Guide. U.S. Department of Commerce, National Oceanic and Atmospheric Administration, National Environmental Satellite, Data and Information Service; Walton, C. C., J. T. Sullivan, C. R. N. Rao, and M. P. Weinreb, 1998: Corrections for detector nonlinearities and calibration inconsistencies of the infrared channels of the Advanced Very High Resolution Radiometer. J. Geophys. Res., 103, 3323-3337; Trishchenko, A.P., 2002: Removing Unwanted Fluctuations in the AVHRR Thermal Calibration Data Using Robust Techniques. Journal of Atmospheric and Oceanic Technology, 19:1939-1954" # noqa 146 | } 147 | } 148 | 149 | def __init__(self, patmosx_coeffs): 150 | self.coeffs = {"description": self.description} 151 | for patmosx_sat_coeffs in patmosx_coeffs: 152 | sat_name = self.sat_names[patmosx_sat_coeffs["sat_name"]] 153 | pygac_sat_coeffs = self.convert(patmosx_sat_coeffs) 154 | self.coeffs[sat_name] = pygac_sat_coeffs 155 | 156 | @classmethod 157 | def convert(cls, patmosx_sat_coeffs): 158 | pygac_sat_coeffs = {} 159 | # visible calibration 160 | for ch in ("1", "2", "3a"): 161 | s0l = patmosx_sat_coeffs["ch{0}_low_gain_S0".format(ch)] 162 | s0h = patmosx_sat_coeffs["ch{0}_high_gain_S0".format(ch)] 163 | if s0l == s0h: 164 | gain_switch = None 165 | s0 = s0l 166 | else: 167 | gain_switch = patmosx_sat_coeffs["ch{0}_gain_switches_count".format(ch)] 168 | s0 = cls.find_s0(s0l, s0h, ch) 169 | pygac_sat_coeffs["channel_{0}".format(ch)] = { 170 | "dark_count": float(patmosx_sat_coeffs["ch{0}_dark_count".format(ch)]), 171 | "gain_switch": gain_switch, 172 | "s0": s0, 173 | "s1": patmosx_sat_coeffs["ch{0}_high_gain_S1".format(ch)], 174 | "s2": patmosx_sat_coeffs["ch{0}_high_gain_S2".format(ch)] 175 | } 176 | date_of_launch = cls.float2date(patmosx_sat_coeffs["date_of_launch"]) 177 | pygac_sat_coeffs["date_of_launch"] = date_of_launch.strftime("%Y-%m-%dT%H:%M:%S.%fZ") 178 | # thermal channels 179 | for ch in ("3b", "4", "5"): 180 | pygac_sat_coeffs["channel_{0}".format(ch)] = { 181 | "b0": patmosx_sat_coeffs["ch{0}_b0".format(ch)], 182 | "b1": patmosx_sat_coeffs["ch{0}_b1".format(ch)], 183 | "b2": patmosx_sat_coeffs["ch{0}_b2".format(ch)], 184 | "centroid_wavenumber": patmosx_sat_coeffs["nu_{0}".format(ch)], 185 | "space_radiance": patmosx_sat_coeffs["ch{0}_Ns".format(ch)], 186 | "to_eff_blackbody_intercept": (-patmosx_sat_coeffs["a1_{0}".format(ch)] 187 | / patmosx_sat_coeffs["a2_{0}".format(ch)]), 188 | "to_eff_blackbody_slope": 1/patmosx_sat_coeffs["a2_{0}".format(ch)] 189 | } 190 | for t in range(1, 5): 191 | pygac_sat_coeffs["thermometer_{0}".format(t)] = { 192 | "d{0}".format(d): float(patmosx_sat_coeffs["PRT{0}_{1}".format(t, d)]) 193 | for d in range(5) 194 | } 195 | return pygac_sat_coeffs 196 | 197 | @staticmethod 198 | def find_s0(s0_low, s0_high, ch): 199 | """Find the single-gain calibration slope at launch date. 200 | 201 | Arguments 202 | s0_low - low gain calibration slope at launch date 203 | s0_high - high gain calibration slope at launch date 204 | ch - channel name ("1", "2", "3a") 205 | 206 | Note: 207 | In case of a single-gain instrument, s0_low is equal to s0_high. 208 | """ 209 | if s0_low == s0_high: 210 | # single gain case 211 | return s0_low 212 | if ch == "3a": 213 | g_low, g_high = 0.25, 1.75 214 | else: 215 | g_low, g_high = 0.5, 1.5 216 | 217 | # Note: the PATMOS-x coefficients are rounded to three decimals. 218 | def diff(s0): return s0_low - round(s0*g_low, 3) + s0_high - round(s0*g_high, 3) 219 | 220 | s0_l = s0_low / g_low 221 | s0_h = s0_high / g_high 222 | if diff(s0_l) == 0: 223 | s0 = s0_l 224 | elif diff(s0_h) == 0: 225 | s0 = s0_h 226 | else: 227 | s0 = bisect(diff, s0_l, s0_h) 228 | return s0 229 | 230 | @staticmethod 231 | def float2date(date_float): 232 | """Convert date float to date. 233 | 234 | Argument 235 | date_float (float) - date as year 236 | 237 | Return 238 | date (datetime.datetime) - date 239 | 240 | Note 241 | This is the reverse function of date2float. 242 | """ 243 | year = int(date_float) 244 | days_in_year = (dt.datetime(year+1, 1, 1) - dt.datetime(year, 1, 1)).days 245 | seconds = date_float*days_in_year*24*3600 - year*days_in_year*24*3600 246 | diff = dt.timedelta(seconds=seconds) 247 | date = dt.datetime(year, 1, 1) + diff 248 | return date 249 | 250 | def save(self, filepath): 251 | """Save coefficients as PyGAC json file.""" 252 | with open(filepath, mode="w") as json_file: 253 | json.dump(self.coeffs, json_file, indent=4, sort_keys=True) 254 | 255 | 256 | def main(): 257 | """The main function.""" 258 | parser = argparse.ArgumentParser(description=__doc__) 259 | parser.add_argument("tarball", type=str, help="path to PATMOS-x coefficients tarball") 260 | parser.add_argument("-o", "--output", type=str, metavar="JSON", 261 | help='path to PyGAC json file, defaults to tarball path with suffix ".json"') 262 | parser.add_argument("-v", "--verbose", action="store_true", help="explain what is being done") 263 | args = parser.parse_args() 264 | if args.verbose: 265 | loglevel = logging.INFO 266 | else: 267 | loglevel = logging.WARNING 268 | logging.basicConfig(level=loglevel, format="[%(asctime)s] %(message)s") 269 | tarball = pathlib.Path(args.tarball) 270 | logging.info('Read PATMOS-x tarball "%s".', tarball) 271 | patmosx_coeffs = PatmosxReader(tarball) 272 | logging.info("Translate PATMOS-x coefficients to PyGAC format.") 273 | pygac_coeffs = Translator(patmosx_coeffs) 274 | output = args.output or tarball.with_suffix(".json") 275 | logging.info('Write PyGAC calibration json file "%s".', output) 276 | pygac_coeffs.save(output) 277 | logging.info("Done!") 278 | 279 | if __name__ == "__main__": 280 | main() 281 | -------------------------------------------------------------------------------- /pygac/runner.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # Copyright (c) 2014, 2019 Pygac Developers 4 | 5 | # Author(s): 6 | 7 | # Carlos Horn 8 | 9 | # This program is free software: you can redistribute it and/or modify 10 | # it under the terms of the GNU General Public License as published by 11 | # the Free Software Foundation, either version 3 of the License, or 12 | # (at your option) any later version. 13 | 14 | # This program is distributed in the hope that it will be useful, 15 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 16 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 17 | # GNU General Public License for more details. 18 | 19 | # You should have received a copy of the GNU General Public License 20 | # along with this program. If not, see . 21 | 22 | """Processing utilities for GAC/LAC KLM/POD files. 23 | 24 | Functions: 25 | process_file: allows to process a given input file 26 | get_reader_class: allows to select the appropriate reader 27 | class for a given file. 28 | """ 29 | 30 | import datetime 31 | import logging 32 | import os 33 | 34 | from pygac.configuration import get_config 35 | from pygac.gac_klm import GACKLMReader 36 | from pygac.gac_pod import GACPODReader 37 | from pygac.lac_klm import LACKLMReader 38 | from pygac.lac_pod import LACPODReader 39 | from pygac.utils import file_opener 40 | 41 | LOG = logging.getLogger(__name__) 42 | 43 | _reader_classes = [GACKLMReader, LACKLMReader, GACPODReader, LACPODReader] 44 | 45 | 46 | def get_reader_class(filename, fileobj=None): 47 | """Return the reader class that can read the GAC/LAC KLM/POD file. 48 | 49 | Args: 50 | filename (str): Path to GAC/LAC file 51 | fileobj: An open file object to read from. (optional) 52 | """ 53 | found_reader = False 54 | for index, Reader in enumerate(_reader_classes): 55 | if Reader.can_read(filename, fileobj=fileobj): 56 | LOG.debug("%s can read the file." % Reader.__name__) 57 | found_reader = True 58 | break 59 | if not found_reader: 60 | raise ValueError('Unable to read the file "%s"' % filename) 61 | # Move the Reader in front of _reader_classes. Chance is high that the 62 | # next file is of the same kind. 63 | _reader_classes.insert(0, _reader_classes.pop(index)) 64 | return Reader 65 | 66 | 67 | def process_file(filename, start_line, end_line, fileobj=None): 68 | """Read, calibrate and navigate NOAA AVHRR GAC/LAC POD/KLM data. 69 | It creates three hdf5 files in the output location given by the pygac 70 | config file. The three files contain the avhrr data, quality flags, 71 | and sunsatangles. 72 | 73 | Argsuments 74 | filename (str): Path to GAC/LAC file 75 | start_line (int): First scanline to be processed (0-based) 76 | end_line (int): Last scanline to be processed (0-based), 77 | set to 0 for the last available scanline 78 | fileobj: An open file object to read from. (optional) 79 | 80 | Note 81 | This function expects an initialized config file. 82 | """ 83 | tic = datetime.datetime.now() 84 | LOG.info("Process file: %s", str(filename)) 85 | 86 | # reader specific values 87 | config = get_config() 88 | tle_dir = config.get("tle", "tledir", raw=True) 89 | tle_name = config.get("tle", "tlename", raw=True) 90 | coeffs_file = config.get("calibration", "coeffs_file", fallback="") 91 | # output specific values 92 | output_dir = config.get("output", "output_dir", raw=True) 93 | output_file_prefix = config.get("output", "output_file_prefix", raw=True) 94 | avhrr_dir = os.environ.get("SM_AVHRR_DIR") 95 | qual_dir = os.environ.get("SM_AVHRR_DIR") 96 | sunsatangles_dir = os.environ.get("SM_SUNSATANGLES_DIR") 97 | 98 | # Keep the file open while searching for the reader class and later 99 | # creation of the instance. 100 | with file_opener(fileobj or filename) as open_file: 101 | reader_cls = get_reader_class(filename, fileobj=open_file) 102 | reader = reader_cls( 103 | tle_dir=tle_dir, tle_name=tle_name, 104 | calibration_file=coeffs_file 105 | ) 106 | reader.read(filename, fileobj=fileobj) 107 | reader.save( 108 | start_line, end_line, 109 | output_file_prefix=output_file_prefix, 110 | output_dir=output_dir, 111 | avhrr_dir=avhrr_dir, qual_dir=qual_dir, 112 | sunsatangles_dir=sunsatangles_dir 113 | ) 114 | LOG.info("Processing took: %s", str(datetime.datetime.now() - tic)) 115 | -------------------------------------------------------------------------------- /pygac/slerp.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # Copyright (c) 2014 Martin Raspaud 5 | 6 | # Author(s): 7 | 8 | # Martin Raspaud 9 | 10 | # This program is free software: you can redistribute it and/or modify 11 | # it under the terms of the GNU General Public License as published by 12 | # the Free Software Foundation, either version 3 of the License, or 13 | # (at your option) any later version. 14 | 15 | # This program is distributed in the hope that it will be useful, 16 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 17 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 18 | # GNU General Public License for more details. 19 | 20 | # You should have received a copy of the GNU General Public License 21 | # along with this program. If not, see . 22 | 23 | """slerp implementation in numpy""" 24 | 25 | import numpy as np 26 | 27 | 28 | def tocart(lon, lat): 29 | rlat = np.deg2rad(lat) 30 | rlon = np.deg2rad(lon) 31 | x__ = np.cos(rlat) * np.cos(rlon) 32 | y__ = np.cos(rlat) * np.sin(rlon) 33 | z__ = np.sin(rlat) 34 | return np.dstack((x__, y__, z__)) 35 | 36 | 37 | def toll(arr): 38 | lat = np.arcsin(arr.take(2, -1)) 39 | lon = np.arctan2(arr.take(1, -1), arr.take(0, -1)) 40 | return np.rad2deg(np.dstack((lon.squeeze(), lat.squeeze()))) 41 | 42 | 43 | def dot(a, b): 44 | return np.sum(a * b, -1) 45 | 46 | 47 | def slerp(lon0, lat0, lon1, lat1, t): 48 | cp0 = tocart(lon0, lat0) 49 | cp1 = tocart(lon1, lat1) 50 | 51 | dot_product = np.clip(dot(cp0, cp1), -1.0, 1.0) 52 | identical_mask = np.isclose(dot_product, 1.0) 53 | 54 | omega = np.arccos(dot_product)[:, :, np.newaxis] 55 | sin_omega = np.sin(omega) 56 | 57 | interp = ( 58 | np.sin((1 - t) * omega) / np.where(sin_omega == 0, 1, sin_omega) * cp0 59 | + np.sin(t * omega) / np.where(sin_omega == 0, 1, sin_omega) * cp1 60 | ) 61 | 62 | interp[identical_mask] = cp0[identical_mask] 63 | return toll(interp) 64 | -------------------------------------------------------------------------------- /pygac/tests/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # Copyright (c) 2013, 2014 Martin Raspaud 5 | 6 | # Author(s): 7 | 8 | # Martin Raspaud 9 | # Carlos Horn 10 | 11 | # This program is free software: you can redistribute it and/or modify 12 | # it under the terms of the GNU General Public License as published by 13 | # the Free Software Foundation, either version 3 of the License, or 14 | # (at your option) any later version. 15 | 16 | # This program is distributed in the hope that it will be useful, 17 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 18 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 19 | # GNU General Public License for more details. 20 | 21 | # You should have received a copy of the GNU General Public License 22 | # along with this program. If not, see . 23 | 24 | """The tests package.""" 25 | -------------------------------------------------------------------------------- /pygac/tests/test_angles.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # Copyright (c) 2014-2019 Pytroll Developers 5 | 6 | # Author(s): 7 | 8 | # Nina Hakansson 9 | # Adam Dybbroe 10 | # Carlos Horn 11 | 12 | # This program is free software: you can redistribute it and/or modify 13 | # it under the terms of the GNU General Public License as published by 14 | # the Free Software Foundation, either version 3 of the License, or 15 | # (at your option) any later version. 16 | 17 | # This program is distributed in the hope that it will be useful, 18 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 19 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 20 | # GNU General Public License for more details. 21 | 22 | # You should have received a copy of the GNU General Public License 23 | # along with this program. If not, see . 24 | 25 | """Test function for the angle calculation.""" 26 | 27 | import unittest 28 | 29 | import numpy as np 30 | 31 | from pygac.utils import centered_modulus, get_absolute_azimuth_angle_diff 32 | 33 | 34 | class TestAngles(unittest.TestCase): 35 | """Test function for the angle calculation.""" 36 | 37 | def test_azidiff_angle(self): 38 | """Test function for the azidiff angle.""" 39 | sat_az = np.ma.array([[48.0, 56.0, 64.0, 72.0], 40 | [80.0, 88.0, 96.0, 104.0], 41 | [-80.0, -88.0, -96.0, -104.0], 42 | [-180.0, -188.0, -196.0, -204.0]], mask=False) 43 | sun_az = np.ma.array([[148.0, 156.0, 164.0, 172.0], 44 | [180.0, 188.0, 196.0, 204.0], 45 | [180.0, 188.0, 196.0, 204.0], 46 | [185.0, 193.0, 201.0, 209.0]], mask=False) 47 | 48 | res = np.ma.array([[100., 100., 100., 100.], 49 | [100., 100., 100., 100.], 50 | [100., 84., 68., 52.], 51 | [5., 21., 37., 53.]], 52 | mask=False) 53 | rel_azi = get_absolute_azimuth_angle_diff(sat_az, sun_az) 54 | 55 | np.testing.assert_allclose(rel_azi, res) 56 | 57 | def test_centered_modulus(self): 58 | """Test centered_modulus.""" 59 | angles = np.ma.array( 60 | [[180.0, -180.0, -179.9, 201.0], 61 | [80.0, 360.0, -360.0, 604.0], 62 | [-80.0, -88.0, -796.0, -104.0], 63 | [-3.0, -188.0, -196.0, -204.0]], mask=False) 64 | expected = np.ma.array( 65 | [[180.0, 180.0, -179.9, -159.0], 66 | [80.0, 0.0, 0.0, -116.0], 67 | [-80.0, -88.0, -76.0, -104.0], 68 | [-3.0, 172.0, 164.0, 156.0]], mask=False) 69 | transformed = centered_modulus(angles, 360.0) 70 | np.testing.assert_allclose(transformed, expected) 71 | -------------------------------------------------------------------------------- /pygac/tests/test_calibrate_klm.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # Copyright (c) 2014-2019 Pytroll Developers 5 | 6 | # Author(s): 7 | 8 | # Martin Raspaud 9 | 10 | # This program is free software: you can redistribute it and/or modify 11 | # it under the terms of the GNU General Public License as published by 12 | # the Free Software Foundation, either version 3 of the License, or 13 | # (at your option) any later version. 14 | 15 | # This program is distributed in the hope that it will be useful, 16 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 17 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 18 | # GNU General Public License for more details. 19 | 20 | # You should have received a copy of the GNU General Public License 21 | # along with this program. If not, see . 22 | 23 | """Test function for the POD calibration. 24 | """ 25 | 26 | 27 | import unittest 28 | 29 | import numpy as np 30 | 31 | from pygac.calibration.noaa import Calibrator, calibrate_solar, calibrate_thermal 32 | 33 | 34 | class TestGenericCalibration(unittest.TestCase): 35 | 36 | def test_calibration_vis(self): 37 | 38 | counts = np.array([[0, 0, 0, 0, 0, 39 | 512, 512, 512, 512, 512, 40 | 1023, 1023, 1023, 1023, 1023], 41 | [41, 41, 41, 41, 41, 42 | 150, 150, 150, 150, 150, 43 | 700, 700, 700, 700, 700]]) 44 | year = 2010 45 | jday = 1 46 | spacecraft_id = "noaa19" 47 | cal = Calibrator(spacecraft_id) 48 | corr = 1 49 | channel = 0 50 | 51 | ref1 = calibrate_solar(counts[:, channel::5], channel, year, jday, cal, corr) 52 | 53 | channel = 1 54 | 55 | ref2 = calibrate_solar(counts[:, channel::5], channel, year, jday, cal, corr) 56 | 57 | channel = 2 58 | 59 | data = np.ma.array(counts[:, channel::5], mask=True) 60 | 61 | ref3 = calibrate_solar(data, channel, year, jday, cal, corr) 62 | 63 | expected = (np.array([[np.nan, 27.32328509, 110.84050459], 64 | [0.1191198, 6.02096454, 58.0497768]]), 65 | np.array([[np.nan, 3.04160070e+01, 1.24374292e+02], 66 | [1.22580933e-01, 6.80324179e+00, 6.49838301e+01]]), 67 | np.array([[0., 524.33117, 1035.33117], 68 | [41., 150., 712.33117]])) 69 | np.testing.assert_allclose(ref1, expected[0]) 70 | np.testing.assert_allclose(ref2, expected[1]) 71 | np.testing.assert_allclose(ref3, expected[2]) 72 | 73 | def test_calibration_ir(self): 74 | counts = np.array([[0, 0, 612, 0, 0, 75 | 512, 512, 487, 512, 512, 76 | 923, 923, 687, 923, 923], 77 | [41, 41, 634, 41, 41, 78 | 150, 150, 461, 150, 150, 79 | 700, 700, 670, 700, 700], 80 | [241, 241, 656, 241, 241, 81 | 350, 350, 490, 350, 350, 82 | 600, 600, 475, 600, 600]]) 83 | prt_counts = np.array([0, 230, 230]) 84 | ict_counts = np.array([[745.3, 397.9, 377.8], 85 | [744.8, 398.1, 378.4], 86 | [745.7, 398., 378.3]]) 87 | space_counts = np.array([[987.3, 992.5, 989.4], 88 | [986.9, 992.8, 989.6], 89 | [986.3, 992.3, 988.9]]) 90 | 91 | spacecraft_id = "noaa19" 92 | cal = Calibrator(spacecraft_id) 93 | ch3 = calibrate_thermal(counts[:, 2::5], 94 | prt_counts, 95 | ict_counts[:, 0], 96 | space_counts[:, 0], 97 | line_numbers=np.array([1, 2, 3]), 98 | channel=3, 99 | cal=cal) 100 | 101 | expected_ch3 = np.array([[298.36742, 305.248478, 293.238328], 102 | [296.960275, 306.493766, 294.488956], 103 | [295.476935, 305.101309, 305.829827]]) 104 | 105 | np.testing.assert_allclose(expected_ch3, ch3) 106 | 107 | ch4 = calibrate_thermal(counts[:, 3::5], 108 | prt_counts, 109 | ict_counts[:, 1], 110 | space_counts[:, 1], 111 | line_numbers=np.array([1, 2, 3]), 112 | channel=4, 113 | cal=cal) 114 | 115 | expected_ch4 = np.array([[326.576534, 275.348988, 197.688755], 116 | [323.013104, 313.207077, 249.36352], 117 | [304.58091, 293.579308, 264.0631]]) 118 | 119 | np.testing.assert_allclose(expected_ch4, ch4) 120 | 121 | ch5 = calibrate_thermal(counts[:, 4::5], 122 | prt_counts, 123 | ict_counts[:, 2], 124 | space_counts[:, 2], 125 | line_numbers=np.array([1, 2, 3]), 126 | channel=5, 127 | cal=cal) 128 | 129 | expected_ch5 = np.array([[326.96161, 272.090164, 188.267991], 130 | [323.156317, 312.673269, 244.184452], 131 | [303.439383, 291.649444, 259.973091]]) 132 | 133 | np.testing.assert_allclose(expected_ch5, ch5) 134 | -------------------------------------------------------------------------------- /pygac/tests/test_calibrate_pod.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # Copyright (c) 2014, 2015 Martin Raspaud 5 | 6 | # Author(s): 7 | 8 | # Martin Raspaud 9 | 10 | # This program is free software: you can redistribute it and/or modify 11 | # it under the terms of the GNU General Public License as published by 12 | # the Free Software Foundation, either version 3 of the License, or 13 | # (at your option) any later version. 14 | 15 | # This program is distributed in the hope that it will be useful, 16 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 17 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 18 | # GNU General Public License for more details. 19 | 20 | # You should have received a copy of the GNU General Public License 21 | # along with this program. If not, see . 22 | 23 | """Test function for the POD calibration. 24 | """ 25 | 26 | 27 | import unittest 28 | 29 | import numpy as np 30 | 31 | from pygac.calibration.noaa import Calibrator, calibrate_solar, calibrate_thermal 32 | 33 | 34 | class TestGenericCalibration(unittest.TestCase): 35 | 36 | def test_calibration_vis(self): 37 | 38 | counts = np.array([[0, 0, 0, 0, 0, 39 | 512, 512, 512, 512, 512, 40 | 1023, 1023, 1023, 1023, 1023], 41 | [41, 41, 41, 41, 41, 42 | 150, 150, 150, 150, 150, 43 | 700, 700, 700, 700, 700]]) 44 | year = 1997 45 | jday = 196 46 | spacecraft_id = "noaa14" 47 | cal = Calibrator(spacecraft_id) 48 | corr = 1 49 | 50 | channel = 0 51 | 52 | ref1 = calibrate_solar(counts[:, channel::5], channel, year, jday, cal, corr) 53 | 54 | channel = 1 55 | 56 | ref2 = calibrate_solar(counts[:, channel::5], channel, year, jday, cal, corr) 57 | 58 | channel = 2 59 | 60 | data = np.ma.array(counts[:, channel::5], mask=True) 61 | 62 | ref3 = calibrate_solar(data, channel, year, jday, cal, corr) 63 | 64 | expected = (np.array([[np.nan, 60.91525491, 127.00377987], 65 | [0., 14.0971609, 85.22962417]]), 66 | np.array([[np.nan, 72.51635437, 151.19121018], 67 | [0., 16.7819164, 101.46131111]]), 68 | np.array([[-32001., -32001., -32001.], 69 | [-32001., -32001., -32001.]])) 70 | 71 | np.testing.assert_allclose(ref1, expected[0]) 72 | np.testing.assert_allclose(ref2, expected[1]) 73 | np.testing.assert_allclose(ref3.filled(-32001), expected[2]) 74 | 75 | def test_calibration_ir(self): 76 | counts = np.array([[0, 0, 612, 0, 0, 77 | 512, 512, 487, 512, 512, 78 | 923, 923, 687, 923, 923], 79 | [41, 41, 634, 41, 41, 80 | 150, 150, 461, 150, 150, 81 | 700, 700, 670, 700, 700], 82 | [241, 241, 656, 241, 241, 83 | 350, 350, 490, 350, 350, 84 | 600, 600, 475, 600, 600]]) 85 | prt_counts = np.array([0, 230, 230]) 86 | ict_counts = np.array([[745.3, 397.9, 377.8], 87 | [744.8, 398.1, 378.4], 88 | [745.7, 398., 378.3]]) 89 | space_counts = np.array([[987.3, 992.5, 989.4], 90 | [986.9, 992.8, 989.6], 91 | [986.3, 992.3, 988.9]]) 92 | 93 | spacecraft_id = "noaa14" 94 | cal = Calibrator(spacecraft_id) 95 | ch3 = calibrate_thermal(counts[:, 2::5], 96 | prt_counts, 97 | ict_counts[:, 0], 98 | space_counts[:, 0], 99 | line_numbers=np.array([1, 2, 3]), 100 | channel=3, 101 | cal=cal) 102 | 103 | expected_ch3 = np.array([[298.28466, 305.167571, 293.16182], 104 | [296.878502, 306.414234, 294.410224], 105 | [295.396779, 305.020259, 305.749526]]) 106 | 107 | np.testing.assert_allclose(expected_ch3, ch3) 108 | 109 | ch4 = calibrate_thermal(counts[:, 3::5], 110 | prt_counts, 111 | ict_counts[:, 1], 112 | space_counts[:, 1], 113 | line_numbers=np.array([1, 2, 3]), 114 | channel=4, 115 | cal=cal) 116 | 117 | expected_ch4 = np.array([[325.828062, 275.414804, 196.214709], 118 | [322.359517, 312.785057, 249.380649], 119 | [304.326806, 293.490822, 264.148021]]) 120 | 121 | np.testing.assert_allclose(expected_ch4, ch4) 122 | 123 | ch5 = calibrate_thermal(counts[:, 4::5], 124 | prt_counts, 125 | ict_counts[:, 2], 126 | space_counts[:, 2], 127 | line_numbers=np.array([1, 2, 3]), 128 | channel=5, 129 | cal=cal) 130 | 131 | expected_ch5 = np.array([[326.460316, 272.146547, 187.434456], 132 | [322.717606, 312.388155, 244.241633], 133 | [303.267012, 291.590832, 260.05426]]) 134 | 135 | np.testing.assert_allclose(expected_ch5, ch5) 136 | -------------------------------------------------------------------------------- /pygac/tests/test_io.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # Author(s): 5 | 6 | # Stephan Finkensieper 7 | 8 | # This program is free software: you can redistribute it and/or modify 9 | # it under the terms of the GNU General Public License as published by 10 | # the Free Software Foundation, either version 3 of the License, or 11 | # (at your option) any later version. 12 | 13 | # This program is distributed in the hope that it will be useful, 14 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 15 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 | # GNU General Public License for more details. 17 | 18 | # You should have received a copy of the GNU General Public License 19 | # along with this program. If not, see . 20 | """Test the I/O.""" 21 | 22 | import unittest 23 | from unittest import mock 24 | 25 | import numpy as np 26 | import numpy.testing 27 | 28 | import pygac.gac_io as gac_io 29 | import pygac.utils as utils 30 | 31 | 32 | class TestIO(unittest.TestCase): 33 | """Test the gac_io module.""" 34 | 35 | longMessage = True 36 | 37 | def test_strip_invalid_lat(self): 38 | """Test stripping the invalid lats.""" 39 | lats = np.array([np.nan, 1, np.nan, 2, np.nan]) 40 | start, end = utils.strip_invalid_lat(lats) 41 | self.assertEqual(start, 1) 42 | self.assertEqual(end, 3) 43 | 44 | def test_update_scanline(self): 45 | """Test updating the scanlines.""" 46 | test_data = [{"new_start_line": 100, "new_end_line": 200, 47 | "scanline": 110, "scanline_exp": 10}, 48 | {"new_start_line": 100, "new_end_line": 200, 49 | "scanline": 90, "scanline_exp": None}, 50 | {"new_start_line": 100, "new_end_line": 200, 51 | "scanline": 210, "scanline_exp": None}] 52 | for t in test_data: 53 | scanline_exp = t.pop("scanline_exp") 54 | scanline = utils._update_scanline(**t) 55 | self.assertEqual(scanline, scanline_exp) 56 | 57 | def test_update_missing_scanlines(self): 58 | """Test updating the missing scanlines.""" 59 | qual_flags = np.array([[1, 2, 4, 5, 6, 8, 9, 11, 12]]).transpose() 60 | miss_lines = np.array([3, 7, 10]) 61 | test_data = [{"start_line": 0, "end_line": 8, 62 | "miss_lines_exp": [3, 7, 10]}, 63 | {"start_line": 3, "end_line": 6, 64 | "miss_lines_exp": [1, 2, 3, 4, 7, 10, 11, 12]}] 65 | for t in test_data: 66 | miss_lines_exp = t.pop("miss_lines_exp") 67 | miss_lines = utils._update_missing_scanlines( 68 | miss_lines=miss_lines, qual_flags=qual_flags, **t) 69 | numpy.testing.assert_array_equal(miss_lines, miss_lines_exp) 70 | 71 | # If intersection of miss_lines and qual_flags is not empty 72 | # (here: extra "1" in miss_lines), make sure that items are 73 | # not added twice. 74 | miss_lines = utils._update_missing_scanlines( 75 | miss_lines=np.array([1, 3, 7, 10]), 76 | qual_flags=qual_flags, 77 | start_line=3, end_line=6) 78 | numpy.testing.assert_array_equal(miss_lines, 79 | [1, 2, 3, 4, 7, 10, 11, 12]) 80 | 81 | def test_slice(self): 82 | """Test slices.""" 83 | ch = np.array([[1, 2, 3, 4, 5]]).transpose() 84 | sliced_exp = np.array([[2, 3, 4]]).transpose() 85 | 86 | # Without update 87 | sliced = utils._slice(ch, start_line=1, end_line=3) 88 | numpy.testing.assert_array_equal(sliced, sliced_exp) 89 | 90 | # With update 91 | sliced, updated = utils._slice(ch, start_line=1, end_line=3, 92 | update=[0, 2, 4, None]) 93 | numpy.testing.assert_array_equal(sliced, sliced_exp) 94 | numpy.testing.assert_array_equal(updated, [None, 1, None, None]) 95 | 96 | # Make sure slice is a copy 97 | ch += 1 98 | numpy.testing.assert_array_equal(sliced, sliced_exp) 99 | 100 | def test_slice_channel(self): 101 | """Test selection of user defined scanlines. 102 | 103 | Scanline Nr: 1 2 3 4 5 6 7 8 9 10 11 12 13 14 104 | Missing: X X X X 105 | Inv. lat/lon: X X X X 106 | 107 | Before stripping of invalid lats 108 | -------------------------------- 109 | idx: 0 1 2 3 4 5 6 7 8 9 110 | data: 1 2 3 4 5 6 7 8 9 10 111 | 112 | After stripping of invalid lats 113 | ------------------------------- 114 | idx: 0 1 2 3 4 5 115 | data: 3 4 5 6 7 8 116 | 117 | 118 | => All missing lines: [1, 2, 3, 4, 11, 12, 13, 14] 119 | """ 120 | # Define test data 121 | ch = np.array([[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]]).transpose() 122 | miss_lines = np.array([1, 4, 11, 12]) # never recorded 123 | qualflags = np.array( 124 | [[2, 3, 5, 6, 7, 8, 9, 10, 13, 14]]).transpose() # scanline number 125 | midn_line = 3 126 | first_valid_lat = 2 127 | last_valid_lat = 7 128 | start_line = 1 129 | end_line = 4 130 | 131 | # Without lat stripping 132 | sliced_ref = np.array([[2, 3, 4, 5]]).transpose() 133 | sliced, miss_lines_new, midn_line_new = utils.slice_channel( 134 | ch, start_line=start_line, end_line=end_line, 135 | miss_lines=miss_lines, midnight_scanline=midn_line, 136 | qual_flags=qualflags) 137 | numpy.testing.assert_array_equal(sliced, sliced_ref) 138 | numpy.testing.assert_array_equal(miss_lines_new, miss_lines) 139 | self.assertEqual(midn_line_new, 2) 140 | 141 | # With lat stripping 142 | sliced_ref = np.array([[4, 5, 6, 7]]).transpose() 143 | miss_lines_ref = np.array([1, 2, 3, 4, 11, 12, 13, 14]) 144 | sliced, miss_lines_new, midn_line_new = utils.slice_channel( 145 | ch, start_line=start_line, end_line=end_line, 146 | first_valid_lat=first_valid_lat, last_valid_lat=last_valid_lat, 147 | miss_lines=miss_lines, midnight_scanline=midn_line, 148 | qual_flags=qualflags) 149 | numpy.testing.assert_array_equal(sliced, sliced_ref) 150 | numpy.testing.assert_array_equal(miss_lines_new, miss_lines_ref) 151 | self.assertEqual(midn_line_new, 0) 152 | 153 | def test_check_user_scanlines(self): 154 | """Check the scanlines.""" 155 | # All scanlines 156 | start, end = utils.check_user_scanlines(0, 0, 100, 200) 157 | self.assertEqual(start, 0) 158 | self.assertEqual(end, 100) 159 | 160 | start, end = utils.check_user_scanlines(0, 0, None, None, 100) 161 | self.assertEqual(start, 0) 162 | self.assertEqual(end, 99) 163 | 164 | # Valid scanlines 165 | start, end = utils.check_user_scanlines(10, 20, 100, 200) 166 | self.assertEqual(start, 10) 167 | self.assertEqual(end, 20) 168 | 169 | start, end = utils.check_user_scanlines(10, 20, None, None, 100) 170 | self.assertEqual(start, 10) 171 | self.assertEqual(end, 20) 172 | 173 | # Invalid scanlines 174 | start, end = utils.check_user_scanlines(10, 110, 100, 200) 175 | self.assertEqual(start, 10) 176 | self.assertEqual(end, 100) 177 | 178 | start, end = utils.check_user_scanlines(10, 110, None, None, 100) 179 | self.assertEqual(start, 10) 180 | self.assertEqual(end, 99) 181 | 182 | self.assertRaises(ValueError, gac_io.check_user_scanlines, 183 | 110, 120, 100, 200) 184 | self.assertRaises(ValueError, gac_io.check_user_scanlines, 185 | 110, 120, None, None, 100) 186 | 187 | @mock.patch("pygac.gac_io.strip_invalid_lat") 188 | @mock.patch("pygac.gac_io.avhrrGAC_io") 189 | @mock.patch("pygac.gac_io.slice_channel") 190 | @mock.patch("pygac.gac_io.check_user_scanlines") 191 | def test_save_gac(self, check_user_scanlines, slice_channel, avhrr_gac_io, 192 | strip_invalid_lat): 193 | """Test saving.""" 194 | # Test scanline selection 195 | mm = mock.MagicMock() 196 | kwargs = dict( 197 | satellite_name=mm, 198 | xutcs=mm, 199 | lats=mm, 200 | lons=mm, 201 | ref1=mm, 202 | ref2=mm, 203 | ref3=mm, 204 | bt3=mm, 205 | bt4=mm, 206 | bt5=mm, 207 | sun_zen=mm, 208 | sat_zen=mm, 209 | sun_azi=mm, 210 | sat_azi=mm, 211 | rel_azi=mm, 212 | qual_flags=mm, 213 | gac_file=mm, 214 | meta_data=mm, 215 | output_file_prefix=mm, 216 | avhrr_dir=mm, 217 | qual_dir=mm, 218 | sunsatangles_dir=mm 219 | ) 220 | slice_channel.return_value = mm, "miss", "midnight" 221 | strip_invalid_lat.return_value = 0, 0 222 | check_user_scanlines.return_value = "start", "end" 223 | 224 | gac_io.save_gac(start_line=0, end_line=0, **kwargs) 225 | slice_channel.assert_called_with(mock.ANY, 226 | start_line="start", end_line="end", 227 | first_valid_lat=mock.ANY, 228 | last_valid_lat=mock.ANY 229 | ) 230 | expected_args = [ 231 | mock.ANY, 232 | mock.ANY, 233 | mock.ANY, 234 | mock.ANY, 235 | mock.ANY, 236 | mock.ANY, 237 | mock.ANY, 238 | mock.ANY, 239 | mock.ANY, 240 | mock.ANY, 241 | mock.ANY, 242 | mock.ANY, 243 | mock.ANY, 244 | mock.ANY, 245 | mock.ANY, 246 | mock.ANY, 247 | mock.ANY, 248 | mock.ANY, 249 | mock.ANY, 250 | mock.ANY, 251 | mock.ANY, 252 | mock.ANY, 253 | mock.ANY, 254 | mock.ANY, 255 | mock.ANY, 256 | mock.ANY, 257 | "midnight", 258 | "miss", 259 | mock.ANY, 260 | mock.ANY, 261 | mock.ANY, 262 | mock.ANY 263 | ] 264 | avhrr_gac_io.assert_called_with(*expected_args) 265 | -------------------------------------------------------------------------------- /pygac/tests/test_klm.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # Copyright (c) 2013, 2014 Martin Raspaud 5 | 6 | # Author(s): 7 | 8 | # Martin Raspaud 9 | # Carlos Horn 10 | 11 | # This program is free software: you can redistribute it and/or modify 12 | # it under the terms of the GNU General Public License as published by 13 | # the Free Software Foundation, either version 3 of the License, or 14 | # (at your option) any later version. 15 | 16 | # This program is distributed in the hope that it will be useful, 17 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 18 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 19 | # GNU General Public License for more details. 20 | 21 | # You should have received a copy of the GNU General Public License 22 | # along with this program. If not, see . 23 | 24 | """Test the GAC KLM reader.""" 25 | 26 | import datetime as dt 27 | from unittest import mock 28 | 29 | import numpy as np 30 | import numpy.testing 31 | import xarray as xr 32 | 33 | from pygac.gac_klm import GACKLMReader 34 | from pygac.klm_reader import header 35 | from pygac.lac_klm import LACKLMReader, scanline 36 | from pygac.tests.utils import CalledWithArray 37 | 38 | 39 | class TestKLM: 40 | """Test the klm reader.""" 41 | 42 | def setup_method(self): 43 | """Set up the tests.""" 44 | self.reader = GACKLMReader() 45 | 46 | def test_get_lonlat(self): 47 | """Test readout of lon/lat coordinates.""" 48 | lons_exp = np.array([[2, 4], 49 | [6, 8]]) 50 | lats_exp = np.array([[1, 3], 51 | [5, 7]]) 52 | 53 | self.reader.scans = {"earth_location": {"lats": lats_exp * 1e4, 54 | "lons": lons_exp * 1e4}} 55 | 56 | lons, lats = self.reader._get_lonlat_from_file() 57 | numpy.testing.assert_array_equal(lons, lons_exp) 58 | numpy.testing.assert_array_equal(lats, lats_exp) 59 | 60 | def test_get_header_timestamp(self): 61 | """Test readout of header timestamp.""" 62 | self.reader.head = { 63 | "start_of_data_set_year": np.array([2019]), 64 | "start_of_data_set_day_of_year": np.array([123]), 65 | "start_of_data_set_utc_time_of_day": np.array([123456]) 66 | } 67 | time = self.reader.get_header_timestamp() 68 | assert time == dt.datetime(2019, 5, 3, 0, 2, 3, 456000) 69 | 70 | def test_get_times(self): 71 | """Test readout of scanline timestamps.""" 72 | self.reader.scans = {"scan_line_year": 1, 73 | "scan_line_day_of_year": 2, 74 | "scan_line_utc_time_of_day": 3} 75 | assert self.reader._get_times_from_file() == (1, 2, 3) 76 | 77 | def test_get_ch3_switch(self): 78 | """Test channel 3 identification.""" 79 | self.reader.scans = { 80 | "scan_line_bit_field": np.array([1, 2, 3, 4, 5, 6])} 81 | switch_exp = np.array([1, 2, 3, 0, 1, 2]) 82 | numpy.testing.assert_array_equal( 83 | self.reader.get_ch3_switch(), switch_exp) 84 | 85 | def test_postproc(self): 86 | """Test KLM specific postprocessing.""" 87 | self.reader.scans = { 88 | "scan_line_bit_field": np.array([0, 1, 2])} 89 | channels = np.array([[[1., 2., 3., 4.], 90 | [1., 2., 3., 4.]], 91 | [[1., 2., 3., 4.], 92 | [1., 2., 3., 4.]], 93 | [[1., 2., 3, 4.], 94 | [1., 2., 3, 4.]]]) # (lines, pixels, channels) 95 | 96 | masked_exp = np.array([[[1., 2., np.nan, 4.], 97 | [1., 2., np.nan, 4.]], 98 | [[1., 2., 3., np.nan], 99 | [1., 2., 3., np.nan]], 100 | [[1., 2., np.nan, np.nan], 101 | [1., 2., np.nan, np.nan]]]) 102 | channels = xr.DataArray(channels, dims=["scan_line_index", "columns", "channel_name"], 103 | coords=dict(channel_name=["1", "2", "3a", "3b"] )) 104 | ds = xr.Dataset(dict(channels=channels)) 105 | self.reader.postproc(ds) # masks in-place 106 | numpy.testing.assert_array_equal(channels, masked_exp) 107 | 108 | def test_quality_indicators(self): 109 | """Test the quality indicator unpacking.""" 110 | reader = self.reader 111 | QFlag = reader.QFlag 112 | 113 | dtype = np.uint32 114 | 115 | quality_indicators = np.array([ 116 | 0, # nothing flagged 117 | np.iinfo(dtype).max, # everything flagged 118 | QFlag.CALIBRATION | QFlag.NO_EARTH_LOCATION, 119 | QFlag.TIME_ERROR | QFlag.DATA_GAP, 120 | QFlag.FATAL_FLAG 121 | ], dtype=np.int64) 122 | reader.scans = {self.reader._quality_indicators_key: quality_indicators} 123 | # test mask, i.e. QFlag.FATAL_FLAG | QFlag.CALIBRATION | QFlag.NO_EARTH_LOCATION 124 | expected_mask = np.array([False, True, True, False, True], dtype=bool) 125 | numpy.testing.assert_array_equal(reader.mask, expected_mask) 126 | # test individual flags 127 | assert reader._get_corrupt_mask(flags=QFlag.FATAL_FLAG).any() 128 | # count the occurence (everything flagged and last entrance => 2) 129 | assert reader._get_corrupt_mask(flags=QFlag.FATAL_FLAG).sum() == 2 130 | 131 | 132 | class TestGACKLM: 133 | """Tests for gac klm.""" 134 | 135 | def setup_method(self): 136 | """Set up the tests.""" 137 | self.reader = GACKLMReader() 138 | 139 | @mock.patch("pygac.klm_reader.get_tsm_idx") 140 | def test_get_tsm_pixels(self, get_tsm_idx): 141 | """Test channel set used for TSM correction.""" 142 | ones = np.ones((409, 100)) 143 | zeros = np.zeros(ones.shape) 144 | ch1 = 1*ones 145 | ch2 = 2*ones 146 | ch4 = 4*ones 147 | ch5 = 5*ones 148 | channels = np.dstack((ch1, ch2, zeros, zeros, ch4, ch5)) 149 | self.reader.get_tsm_pixels(channels) 150 | get_tsm_idx.assert_called_with(CalledWithArray(ch1), 151 | CalledWithArray(ch2), 152 | CalledWithArray(ch4), 153 | CalledWithArray(ch5)) 154 | 155 | 156 | class TestLACKLM: 157 | """Tests for lac klm.""" 158 | 159 | def setup_method(self): 160 | """Set up the tests.""" 161 | self.reader = LACKLMReader() 162 | self.reader.scans = np.ones(100, dtype=scanline) 163 | self.reader.scans["scan_line_year"] = 2001 164 | self.reader.scans["sensor_data"] = 2047 165 | self.reader.head = np.ones(1, dtype=header)[0] 166 | self.reader.spacecraft_id = 12 167 | self.reader.head["noaa_spacecraft_identification_code"] = self.reader.spacecraft_id 168 | self.reader.spacecraft_name = "metopa" 169 | self.reader.scans["scan_line_number"] = np.arange(100) 170 | # PRT 171 | self.reader.scans["telemetry"]["PRT"] = 400 172 | self.reader.scans["telemetry"]["PRT"][0::5, :] = 0 173 | self.reader.tle_lines = ("first line", "second line") 174 | 175 | def test_get_ch3_switch(self): 176 | """Test channel 3 identification.""" 177 | self.reader.scans = { 178 | "scan_line_bit_field": np.array([1, 2, 3, 4, 5, 6])} 179 | switch_exp = np.array([1, 2, 3, 0, 1, 2]) 180 | numpy.testing.assert_array_equal( 181 | self.reader.get_ch3_switch(), switch_exp) 182 | 183 | def test_calibrate_channels(self): 184 | """Test channel calibration.""" 185 | # ICT 186 | self.reader.scans["back_scan"] = 400 187 | self.reader.scans["back_scan"][0::5, :] = 0 188 | # Space 189 | self.reader.scans["space_data"] = 400 190 | self.reader.scans["space_data"][0::5, :] = 0 191 | 192 | assert np.any(np.isfinite(self.reader.get_calibrated_channels())) 193 | 194 | def test_calibrate_inactive_3b(self): 195 | """Test calibration of an inactive 3b.""" 196 | channels = self.reader.get_calibrated_channels() 197 | assert np.all(np.isnan(channels[:, :, 3])) 198 | 199 | 200 | def test_gac_scanline_dtype(): 201 | """Test the gac scanline size.""" 202 | from pygac.gac_klm import scanline 203 | assert scanline.itemsize == 4608 204 | 205 | def test_lac_scanline_dtype(): 206 | """Test the lac scanline size.""" 207 | from pygac.lac_klm import scanline 208 | assert scanline.itemsize == 15872 209 | -------------------------------------------------------------------------------- /pygac/tests/test_noaa_calibration_coefficients.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # Copyright (c) 2014-2020 Pytroll Developers 4 | 5 | # Author(s): 6 | 7 | # Carlos Horn 8 | 9 | # This program is free software: you can redistribute it and/or modify 10 | # it under the terms of the GNU General Public License as published by 11 | # the Free Software Foundation, either version 3 of the License, or 12 | # (at your option) any later version. 13 | 14 | # This program is distributed in the hope that it will be useful, 15 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 16 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 17 | # GNU General Public License for more details. 18 | 19 | # You should have received a copy of the GNU General Public License 20 | # along with this program. If not, see . 21 | 22 | """Test function for the calibration coeffictions handling. 23 | """ 24 | 25 | import sys 26 | import unittest 27 | from unittest import mock 28 | 29 | import numpy as np 30 | 31 | from pygac.calibration.noaa import Calibrator, CoeffStatus, calibrate_solar 32 | 33 | # dummy user json file including only noaa19 data with changed channel 1 coefficients 34 | user_json_file = b"""{ 35 | "noaa19": { 36 | "channel_1": { 37 | "dark_count": 0, 38 | "gain_switch": 1000, 39 | "s0": 2, 40 | "s1": 0, 41 | "s2": 0 42 | }, 43 | "channel_2": { 44 | "dark_count": 39.0, 45 | "gain_switch": 500.37, 46 | "s0": 0.122, 47 | "s1": 0.95, 48 | "s2": -0.039 49 | }, 50 | "channel_3a": { 51 | "dark_count": 39.4, 52 | "gain_switch": 496.11, 53 | "s0": 0.1, 54 | "s1": 0.0, 55 | "s2": 0.0 56 | }, 57 | "channel_3b": { 58 | "b0": 0.0, 59 | "b1": 0.0, 60 | "b2": 0.0, 61 | "centroid_wavenumber": 2670.2425, 62 | "space_radiance": 0.0, 63 | "to_eff_blackbody_intercept": 1.6863857, 64 | "to_eff_blackbody_slope": 0.9974112191806167 65 | }, 66 | "channel_4": { 67 | "b0": 5.7, 68 | "b1": -0.11187000000000002, 69 | "b2": 0.00054668, 70 | "centroid_wavenumber": 927.92374, 71 | "space_radiance": -5.49, 72 | "to_eff_blackbody_intercept": 0.39419031, 73 | "to_eff_blackbody_slope": 0.9986718662850276 74 | }, 75 | "channel_5": { 76 | "b0": 3.58, 77 | "b1": -0.05991000000000002, 78 | "b2": 0.00024985, 79 | "centroid_wavenumber": 831.28619, 80 | "space_radiance": -3.39, 81 | "to_eff_blackbody_intercept": 0.2636462, 82 | "to_eff_blackbody_slope": 0.9990463103920997 83 | }, 84 | "date_of_launch": "2009-02-05T00:57:36.000000Z", 85 | "thermometer_1": { 86 | "d0": 276.6067, 87 | "d1": 0.051111, 88 | "d2": 1.405783e-06, 89 | "d3": 0.0, 90 | "d4": 0.0 91 | }, 92 | "thermometer_2": { 93 | "d0": 276.6119, 94 | "d1": 0.05109, 95 | "d2": 1.496037e-06, 96 | "d3": 0.0, 97 | "d4": 0.0 98 | }, 99 | "thermometer_3": { 100 | "d0": 276.6311, 101 | "d1": 0.051033, 102 | "d2": 1.49699e-06, 103 | "d3": 0.0, 104 | "d4": 0.0 105 | }, 106 | "thermometer_4": { 107 | "d0": 276.6268, 108 | "d1": 0.051058, 109 | "d2": 1.49311e-06, 110 | "d3": 0.0, 111 | "d4": 0.0 112 | } 113 | } 114 | }""" 115 | 116 | 117 | class TestCalibrationCoefficientsHandling(unittest.TestCase): 118 | 119 | @mock.patch("pygac.calibration.noaa.open", mock.mock_open(read_data=user_json_file)) 120 | def test_user_coefficients_file(self): 121 | if sys.version_info.major < 3: 122 | cal = Calibrator("noaa19", coeffs_file="/path/to/unknow/defaults.json") 123 | else: 124 | with self.assertWarnsRegex(RuntimeWarning, 125 | "Unknown calibration coefficients version!"): 126 | cal = Calibrator("noaa19", coeffs_file="/path/to/unknow/defaults.json") 127 | 128 | self.assertEqual(cal.dark_count[0], 0) 129 | self.assertEqual(cal.gain_switch[0], 1000) 130 | self.assertEqual(cal.s0[0], 2) 131 | self.assertEqual(cal.s1[0], 0) 132 | self.assertEqual(cal.s2[0], 0) 133 | # check that the version is set to None if an unknown file is used 134 | if sys.version_info.major > 2: 135 | self.assertIsNone(cal.version) 136 | 137 | def test_custom_coefficients(self): 138 | custom_coeffs = { 139 | "channel_1": { 140 | "dark_count": 0, 141 | "gain_switch": 1000, 142 | "s0": 2, 143 | "s1": 0, 144 | "s2": 0 145 | } 146 | } 147 | # The coefficients are choosen to preserve the counts of channel 1 if counts are less than 1000 148 | counts = np.arange(10) 149 | year = 2010 150 | jday = 1 151 | spacecraft_id = "noaa19" 152 | channel = 0 153 | cal = Calibrator(spacecraft_id, custom_coeffs=custom_coeffs) 154 | scaled_radiance = calibrate_solar(counts, channel, year, jday, cal) 155 | np.testing.assert_allclose(scaled_radiance, counts) 156 | # check that the version is set to None if custom coeffs are used 157 | if sys.version_info.major > 2: 158 | self.assertIsNone(cal.version) 159 | 160 | def test_vis_deprecation_warning(self): 161 | counts = np.arange(10) 162 | year = 2010 163 | jday = 1 164 | spacecraft_id = "noaa19" 165 | channel = 0 166 | corr = 2 167 | message = ( 168 | "Using the 'corr' argument is depricated in favor of making the units" 169 | " of the function result clear. Please make any unit conversion outside this function." 170 | ) 171 | cal = Calibrator(spacecraft_id) 172 | with self.assertWarnsRegex(DeprecationWarning, message): 173 | calibrate_solar(counts, channel, year, jday, cal, corr=corr) 174 | # check that the version is set in this case 175 | self.assertIsNotNone(cal.version) 176 | 177 | def test_default_coeffs(self): 178 | """Test identification of default coefficients.""" 179 | _, version = Calibrator.read_coeffs(None) 180 | self.assertIsNotNone(version) 181 | 182 | def test_read_coeffs_warnings(self): 183 | """Test warnings issued by Calibrator.read_coeffs.""" 184 | version_dicts = [ 185 | # Non-nominal coefficients 186 | {"name": "v123", 187 | "status": CoeffStatus.PROVISIONAL}, 188 | # Unknown coefficients 189 | {"name": None, 190 | "status": None} 191 | ] 192 | with mock.patch.object(Calibrator, "version_hashs") as version_hashs: 193 | for version_dict in version_dicts: 194 | version_hashs.get.return_value = version_dict 195 | with self.assertWarns(RuntimeWarning): 196 | Calibrator.read_coeffs(None) 197 | -------------------------------------------------------------------------------- /pygac/tests/test_pod.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # Author(s): 5 | 6 | # Stephan Finkensieper 7 | # Carlos Horn 8 | 9 | # This program is free software: you can redistribute it and/or modify 10 | # it under the terms of the GNU General Public License as published by 11 | # the Free Software Foundation, either version 3 of the License, or 12 | # (at your option) any later version. 13 | 14 | # This program is distributed in the hope that it will be useful, 15 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 16 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 17 | # GNU General Public License for more details. 18 | 19 | # You should have received a copy of the GNU General Public License 20 | # along with this program. If not, see . 21 | """Test module for the pod reading.""" 22 | 23 | import datetime as dt 24 | import sys 25 | import unittest 26 | from unittest import mock 27 | 28 | import numpy as np 29 | import numpy.testing 30 | 31 | from pygac.clock_offsets_converter import txt as clock_offsets_txt 32 | from pygac.gac_pod import GACPODReader 33 | from pygac.lac_pod import LACPODReader 34 | from pygac.reader import NoTLEData, ReaderError 35 | from pygac.tests.utils import CalledWithArray 36 | 37 | 38 | class TestPOD(unittest.TestCase): 39 | """Test the POD GAC reader.""" 40 | 41 | longMessage = True 42 | 43 | def setUp(self): 44 | """Set up the test.""" 45 | self.reader = GACPODReader() 46 | # python 2 compatibility 47 | if sys.version_info.major < 3: 48 | self.assertRaisesRegex = self.assertRaisesRegexp 49 | 50 | def test__validate_header(self): 51 | """Test the header validation""" 52 | filename = b"NSS.GHRR.TN.D80001.S0332.E0526.B0627173.WI" 53 | head = {"data_set_name": filename} 54 | GACPODReader._validate_header(head) 55 | # wrong name pattern 56 | with self.assertRaisesRegex(ReaderError, 57 | "Data set name .* does not match!"): 58 | head = {"data_set_name": b"abc.txt"} 59 | GACPODReader._validate_header(head) 60 | # wrong platform 61 | name = b"NSS.GHRR.NL.D02187.S1904.E2058.B0921517.GC" 62 | with self.assertRaisesRegex(ReaderError, 63 | 'Improper platform id "NL"!'): 64 | head = {"data_set_name": name} 65 | GACPODReader._validate_header(head) 66 | # wrong transfer mode 67 | name = filename.replace(b"GHRR", b"LHRR") 68 | with self.assertRaisesRegex(ReaderError, 69 | 'Improper transfer mode "LHRR"!'): 70 | head = {"data_set_name": name} 71 | GACPODReader._validate_header(head) 72 | # other change reader 73 | head = {"data_set_name": name} 74 | LACPODReader._validate_header(head) 75 | 76 | @mock.patch("pygac.reader.Reader.get_calibrated_channels") 77 | def test__get_calibrated_channels_uniform_shape(self, get_channels): 78 | """Test the uniform shape as required by gac_io.save_gac.""" 79 | channels = np.arange(2*2*5, dtype=float).reshape((2, 2, 5)) 80 | get_channels.return_value = channels 81 | uniform_channels = self.reader._get_calibrated_channels_uniform_shape() 82 | self.assertTrue(np.isnan(uniform_channels[:, :, 2]).all()) 83 | self.assertTrue(uniform_channels[:, :, [0, 1, 3, 4, 5]].sum() 84 | == channels.sum()) 85 | 86 | def test_decode_timestamps(self): 87 | """Test POD timestamp decoding.""" 88 | # Reference timestamps, one before 2000 one after 2000 89 | t2000_ref = (2001, 335, 53644260) 90 | t1900_ref = (1983, 336, 35058207) 91 | 92 | # Encoded version 93 | t2000_enc = np.array([847, 818, 35812]) 94 | t1900_enc = np.array([42832, 534, 61983]) 95 | 96 | # Test whether PODReader decodes them correctly 97 | self.assertEqual(GACPODReader.decode_timestamps(t2000_enc), t2000_ref, 98 | msg="Timestamp after 2000 was decoded incorrectly") 99 | self.assertEqual(GACPODReader.decode_timestamps(t1900_enc), t1900_ref, 100 | msg="Timestamp before 2000 was decoded incorrectly") 101 | 102 | @mock.patch("pygac.gac_pod.GACPODReader.decode_timestamps") 103 | def test_get_header_timestamp(self, decode_timestamps): 104 | """Test readout of header timestamp.""" 105 | self.reader.head = {"start_time": 123} 106 | decode_timestamps.return_value = np.array( 107 | [2019]), np.array([123]), np.array([123456]) 108 | time = self.reader.get_header_timestamp() 109 | decode_timestamps.assert_called_with(123) 110 | self.assertEqual(time, dt.datetime(2019, 5, 3, 0, 2, 3, 456000)) 111 | 112 | @mock.patch("pygac.gac_pod.GACPODReader.decode_timestamps") 113 | def test_get_times(self, decode_timestamps): 114 | """Test getting times.""" 115 | self.reader.scans = {"time_code": 123} 116 | self.reader._get_times_from_file() 117 | decode_timestamps.assert_called_with(123) 118 | 119 | def test_get_lonlat(self): 120 | """Test readout of lon/lat coordinates.""" 121 | lons_exp = np.array([[2, 4], 122 | [6, 8]]) 123 | lats_exp = np.array([[1, 3], 124 | [5, 7]]) 125 | 126 | self.reader.scans = {"earth_location": {"lats": lats_exp * 128, 127 | "lons": lons_exp * 128}} 128 | 129 | lons, lats = self.reader._get_lonlat_from_file() 130 | numpy.testing.assert_array_equal(lons, lons_exp) 131 | numpy.testing.assert_array_equal(lats, lats_exp) 132 | 133 | @mock.patch("pygac.pod_reader.get_tsm_idx") 134 | def test_get_tsm_pixels(self, get_tsm_idx): 135 | """Test channel set used for TSM correction.""" 136 | ones = np.ones((409, 100)) 137 | zeros = np.zeros(ones.shape) 138 | ch1 = 1*ones 139 | ch2 = 2*ones 140 | ch4 = 4*ones 141 | ch5 = 5*ones 142 | channels = np.dstack((ch1, ch2, zeros, ch4, ch5)) 143 | self.reader.get_tsm_pixels(channels) 144 | get_tsm_idx.assert_called_with(CalledWithArray(ch1), 145 | CalledWithArray(ch2), 146 | CalledWithArray(ch4), 147 | CalledWithArray(ch5)) 148 | 149 | def test_quality_indicators(self): 150 | """Test the quality indicator unpacking.""" 151 | reader = self.reader 152 | QFlag = reader.QFlag 153 | quality_indicators = np.array([ 154 | 1, # 00...001 155 | QFlag.FATAL_FLAG, # 100...00 156 | QFlag.CALIBRATION | QFlag.NO_EARTH_LOCATION, 157 | QFlag.TIME_ERROR | QFlag.DATA_GAP, 158 | ], dtype=">u4") 159 | # check if the bits look as expected 160 | bits = np.unpackbits(quality_indicators.view(np.uint8)).reshape((-1, 32)) 161 | # For a big endian integer, the number 1 fills only the last of the 32 bits 162 | self.assertEqual(bits[0].sum(), 1) # only one bit is filled 163 | self.assertEqual(bits[0][-1], 1) # the last bit is filled 164 | # The fatal flag fills only the first bit 165 | self.assertEqual(bits[1].sum(), 1) # only one bit is filled 166 | self.assertEqual(bits[1][0], 1) # the first bit is filled 167 | 168 | # setup reader and test 169 | reader.scans = {self.reader._quality_indicators_key: quality_indicators} 170 | 171 | # default mask is QFlag.FATAL_FLAG | QFlag.CALIBRATION | QFlag.NO_EARTH_LOCATION 172 | expected_mask = np.array([False, True, True, False], dtype=bool) 173 | numpy.testing.assert_array_equal(reader.mask, expected_mask) 174 | 175 | # test individual flags 176 | expected_mask = np.array([False, False, False, True], dtype=bool) 177 | numpy.testing.assert_array_equal( 178 | reader._get_corrupt_mask(flags=QFlag.TIME_ERROR), 179 | expected_mask 180 | ) 181 | # test combination of flags 182 | expected_mask = np.array([False, False, True, True], dtype=bool) 183 | flags = QFlag.DATA_GAP | QFlag.NO_EARTH_LOCATION 184 | numpy.testing.assert_array_equal( 185 | reader._get_corrupt_mask(flags=flags), 186 | expected_mask 187 | ) 188 | 189 | @mock.patch("pygac.reader.get_lonlatalt") 190 | @mock.patch("pygac.reader.compute_pixels") 191 | @mock.patch("pygac.reader.Reader.get_tle_lines") 192 | @mock.patch("pygac.gac_reader.avhrr_gac_from_times") 193 | def test__adjust_clock_drift(self, avhrr_gac, get_tle_lines, 194 | compute_pixels, get_lonlatalt): 195 | """Test the clock drift adjustment.""" 196 | sat_name = "fake_sat" 197 | reader = self.reader 198 | 199 | # We construct the following input 200 | # the scan lines do not start with 1 and have a gap 201 | scan_lines = np.array([15, 16, 17, 18, 22, 23, 24, 25]) 202 | scan_rate = 0.5 # seconds/line 203 | # the first valid scans starts 1980-01-01 (without clock correction) 204 | # which leads to the following utcs for the given scan lines 205 | # ['1980-01-01T00:00:00.000', '1980-01-01T00:00:00.500', 206 | # '1980-01-01T00:00:01.000', '1980-01-01T00:00:01.500', 207 | # '1980-01-01T00:00:03.500', '1980-01-01T00:00:04.000', 208 | # '1980-01-01T00:00:04.500', '1980-01-01T00:00:05.000'] 209 | scan_utcs = ( 210 | (1000 * scan_rate * (scan_lines - scan_lines[0])).astype("timedelta64[ms]") 211 | + np.datetime64("1980", "ms") 212 | ) 213 | # For the geolocations, we assume an artificial swath of two pixels width 214 | # from north to south with constant lon, lat difference of 3deg for simplicity 215 | # lons = [[0, 3], [0, 3], [0, 3], [0, 3], 216 | # [0, 3], [0, 3], [0, 3], [0, 3]], 217 | # lats = [[45, 45], [48, 48], [51, 51], [54, 54], 218 | # [66, 66], [69, 69], [72, 72], [75, 75]] 219 | scan_angle = 3.0 # deg 220 | scan_lons, scan_lats = np.meshgrid(scan_angle*np.arange(2), scan_angle*scan_lines) 221 | 222 | # we assume a constant clock offset of 3.75 seconds 223 | # which should lead to the following adjustment on utcs 224 | # ['1979-12-31T23:59:56.250', '1979-12-31T23:59:56.750', 225 | # '1979-12-31T23:59:57.250', '1979-12-31T23:59:57.750', 226 | # '1979-12-31T23:59:59.750', '1980-01-01T00:00:00.250', 227 | # '1980-01-01T00:00:00.750', '1980-01-01T00:00:01.250'] 228 | offset = 3.75 229 | scan_offsets = offset*np.ones_like(scan_lines, dtype=float) # seconds 230 | expected_utcs = scan_utcs - (1000*scan_offsets).astype("timedelta64[ms]") 231 | 232 | # the adjustment of geolocations should keep the lons unchanged, 233 | # but should shift the lats by scan_angel * offset / scan_rate 234 | # = 3deg/line * 3.75sec / 0.5sec/line = 22.5deg 235 | # [[22.5, 22.5], [25.5, 25.5], [28.5, 28.5], [31.5, 31.5], 236 | # [43.5, 43.5], [46.5, 46.5], [49.5, 49.5], [52.5, 52.5]] 237 | expected_lons = scan_lons 238 | lats_shift = scan_angle*offset/scan_rate 239 | expected_lats = scan_lats - lats_shift 240 | 241 | # prepare the reader 242 | reader.scans = {"scan_line_number": scan_lines} 243 | reader._times_as_np_datetime64 = scan_utcs 244 | reader.lons = scan_lons 245 | reader.lats = scan_lats 246 | reader.spacecraft_name = sat_name 247 | 248 | # prepare offsets 249 | clock_offsets_txt[sat_name] = ("75001 000000 {offset} " 250 | "85001 000000 {offset}").format(offset=offset) 251 | # set attitude coeffs 252 | reader._rpy = np.zeros(3) 253 | 254 | # set mocks for reader._compute_missing_lonlat call 255 | sgeom = mock.Mock() 256 | sgeom.times.return_value = [None] 257 | avhrr_gac.return_value = sgeom 258 | get_tle_lines.return_value = [None, None] 259 | compute_pixels.return_value = None 260 | # for the following mock, we need to know the missing values in advanced, 261 | # which we do, because we can convert the offset seconds into line number. 262 | offset_lines = offset / scan_rate 263 | min_line = np.floor(scan_lines[0] - offset_lines).astype(int) 264 | max_line = scan_lines[-1] 265 | missed_lines = np.setdiff1d(np.arange(min_line, max_line+1), scan_lines) 266 | n_missed = len(missed_lines) 267 | missed_lons = np.tile([0., scan_angle], n_missed) 268 | missed_lats = np.repeat(scan_angle*missed_lines, 2) 269 | get_lonlatalt.return_value = [missed_lons, missed_lats] 270 | 271 | # adjust clock drift 272 | reader.lonlat_sample_points = [0, 2] 273 | 274 | reader._adjust_clock_drift() 275 | 276 | # check output 277 | # use allclose for geolocations, because the slerp interpolation 278 | # includes a transormation to cartesian coordinates and back to lon, lats. 279 | numpy.testing.assert_array_equal(reader._times_as_np_datetime64, expected_utcs) 280 | numpy.testing.assert_allclose(reader.lons, expected_lons) 281 | numpy.testing.assert_allclose(reader.lats, expected_lats) 282 | 283 | # undo changes to clock_offsets_txt 284 | clock_offsets_txt.pop(sat_name) 285 | 286 | @mock.patch("pygac.pod_reader.get_offsets") 287 | @mock.patch("pygac.reader.Reader.get_tle_lines") 288 | def test__adjust_clock_drift_without_tle(self, get_tle_lines, get_offsets): 289 | """Test that clockdrift adjustment can handle missing TLE data.""" 290 | reader = self.reader 291 | reader._times_as_np_datetime64 = np.zeros(10, dtype="datetime64[ms]") 292 | reader.scans = {"scan_line_number": np.arange(10)} 293 | get_offsets.return_value = np.zeros(10), np.zeros(10) 294 | get_tle_lines.side_effect = NoTLEData("No TLE data available") 295 | reader._adjust_clock_drift() # should pass without errors 296 | 297 | 298 | class TestPOD_truncate(unittest.TestCase): 299 | """Test the POD GAC reader.""" 300 | 301 | longMessage = True 302 | 303 | def setUp(self): 304 | """Set up the test.""" 305 | self.reader = GACPODReader() 306 | 307 | def test__two_logical_record(self): 308 | """Test that truncate_padding_record is correctly dropping end padding""" 309 | reader = self.reader 310 | # GAC POD should have two logical records per physical 311 | self.assertEqual(reader.offset // reader.scanline_type.itemsize, 2) 312 | 313 | def test__truncate_padding_record_even_correct(self): 314 | """Test that truncate_padding_record is correctly dropping end padding""" 315 | reader = self.reader 316 | # Even number of scans, correct file length 317 | reader.head = {"number_of_scans": 65534} 318 | buffer = reader._truncate_padding_record(bytes(65534*3220)) 319 | self.assertEqual(len(buffer), 65534*3220) 320 | 321 | 322 | def test__truncate_padding_record_odd_correct(self): 323 | """Test that truncate_padding_record is correctly dropping end padding""" 324 | reader = self.reader 325 | # Odd number of scans, correct file length (should truncate) 326 | reader.head = {"number_of_scans": 65533} 327 | buffer = reader._truncate_padding_record(bytes(65534*3220)) 328 | self.assertEqual(len(buffer), 65533*3220) 329 | 330 | 331 | def test__truncate_padding_record_odd_nopadding(self): 332 | """Test that truncate_padding_record is correctly dropping end padding""" 333 | reader = self.reader 334 | # Odd number of scans, padding record is missing 335 | with self.assertWarnsRegex(RuntimeWarning, "incomplete physical record"): 336 | reader.head = {"number_of_scans": 65533} 337 | buffer = reader._truncate_padding_record(bytes(65533*3220)) 338 | self.assertEqual(len(buffer), 65533*3220) 339 | 340 | def test__truncate_padding_record_shortfile(self): 341 | """Test that truncate_padding_record is correctly dropping end padding""" 342 | reader = self.reader 343 | # File is too short (should do nothing) 344 | reader.head = {"number_of_scans": 65533} 345 | buffer = reader._truncate_padding_record(bytes(65532*3220)) 346 | self.assertEqual(len(buffer), 65532*3220) 347 | 348 | def test__truncate_padding_record_longfile(self): 349 | """Test that truncate_padding_record is correctly dropping end padding""" 350 | reader = self.reader 351 | # File is too long (should do nothing) 352 | reader.head = {"number_of_scans": 65533} 353 | buffer = reader._truncate_padding_record(bytes(65535*3220)) 354 | self.assertEqual(len(buffer), 65535*3220) 355 | -------------------------------------------------------------------------------- /pygac/tests/test_slerp.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # Copyright (c) 2014 Martin Raspaud 5 | 6 | # Author(s): 7 | 8 | # Martin Raspaud 9 | 10 | # This program is free software: you can redistribute it and/or modify 11 | # it under the terms of the GNU General Public License as published by 12 | # the Free Software Foundation, either version 3 of the License, or 13 | # (at your option) any later version. 14 | 15 | # This program is distributed in the hope that it will be useful, 16 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 17 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 18 | # GNU General Public License for more details. 19 | 20 | # You should have received a copy of the GNU General Public License 21 | # along with this program. If not, see . 22 | 23 | """Test the slerp module 24 | """ 25 | 26 | 27 | import unittest 28 | 29 | import numpy as np 30 | 31 | from pygac.slerp import slerp 32 | 33 | 34 | class TestSlerp(unittest.TestCase): 35 | 36 | def test_slerp(self): 37 | lon0, lat0 = (0, 0) 38 | lon1, lat1 = (0, 1) 39 | self.assertTrue( 40 | np.allclose(slerp(lon0, lat0, lon1, lat1, 0.5), (0, 0.5))) 41 | 42 | def test_slerp_datum(self): 43 | lon0, lat0 = (183, 0) 44 | lon1, lat1 = (179, 0) 45 | res = slerp(lon0, lat0, lon1, lat1, 0.5) 46 | res %= 360 47 | self.assertTrue( 48 | np.allclose(res, (181, 0))) 49 | 50 | def test_slerp_pole(self): 51 | lon0, lat0 = (0, 89) 52 | lon1, lat1 = (180, 89) 53 | res = slerp(lon0, lat0, lon1, lat1, 0.5) 54 | self.assertTrue( 55 | np.allclose(res[:, :, 1], 90)) 56 | 57 | lon0, lat0 = (-90, 89) 58 | lon1, lat1 = (90, 89) 59 | res = slerp(lon0, lat0, lon1, lat1, 0.5) 60 | self.assertTrue( 61 | np.allclose(res[:, :, 1], 90)) 62 | 63 | lon0, lat0 = (0, 89) 64 | lon1, lat1 = (180, 87) 65 | res = slerp(lon0, lat0, lon1, lat1, 0.5) 66 | self.assertTrue( 67 | np.allclose(res, (180, 89))) 68 | 69 | def test_slerp_vec(self): 70 | lon0 = np.array([[0, 0], 71 | [0, 0]]) 72 | lat0 = np.array([[0, 0], 73 | [0, 0]]) 74 | lon1 = np.array([[0, 0], 75 | [0, 0]]) 76 | lat1 = np.array([[1, 1], 77 | [1, 1]]) 78 | 79 | res = slerp(lon0, lat0, lon1, lat1, 0.5) 80 | self.assertTrue(np.allclose(res[:, :, 0], 0)) 81 | self.assertTrue(np.allclose(res[:, :, 1], 0.5)) 82 | 83 | lon0 = np.array([[183, 0], 84 | [-90, 0]]) 85 | lat0 = np.array([[0, 89], 86 | [89, 89]]) 87 | lon1 = np.array([[179, 180], 88 | [90, 180]]) 89 | lat1 = np.array([[0, 89], 90 | [89, 87]]) 91 | 92 | res = slerp(lon0, lat0, lon1, lat1, 0.5) 93 | 94 | self.assertTrue(np.allclose(res[0, 0, :] % 360, (181, 0))) 95 | self.assertTrue(np.allclose(res[0, 1, 1], 90)) 96 | self.assertTrue(np.allclose(res[1, 0, 1], 90)) 97 | self.assertTrue(np.allclose(res[1, 1, :], (180, 89))) 98 | 99 | def test_slerp_tvec(self): 100 | lon0 = np.array([[0, 0], 101 | [0, 0], 102 | [0, 0], 103 | [0, 0], 104 | [0, 0], 105 | [0, 0], 106 | [0, 0]]) 107 | lat0 = np.array([[0, 0], 108 | [5, 0], 109 | [10, 0], 110 | [15, 0], 111 | [20, 0], 112 | [25, 0], 113 | [30, 0]]) 114 | lon1 = np.array([[0, 0], 115 | [0, 0], 116 | [0, 0], 117 | [0, 0], 118 | [0, 0], 119 | [0, 0], 120 | [0, 0]]) 121 | lat1 = np.array([[45, 45], 122 | [45, 45], 123 | [45, 45], 124 | [45, 45], 125 | [45, 45], 126 | [45, 45], 127 | [45, 45]]) 128 | 129 | t = np.array([[0.5, 0, 0.2, 0.4, 0.6, 0.8, 1]]).T 130 | t = t[:, :, np.newaxis] 131 | res = slerp(lon0, lat0, lon1, lat1, t) 132 | expected = np.array([[22.5, 22.5], 133 | [5., 0.], 134 | [17., 9.], 135 | [27., 18.], 136 | [35., 27.], 137 | [41., 36.], 138 | [45., 45.]]) 139 | 140 | self.assertTrue(np.allclose(res[:, :, 0], 0)) 141 | self.assertTrue(np.allclose(res[:, :, 1], expected)) 142 | -------------------------------------------------------------------------------- /pygac/tests/test_tsm.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # Author(s): 5 | 6 | # Stephan Finkensieper 7 | 8 | # This program is free software: you can redistribute it and/or modify 9 | # it under the terms of the GNU General Public License as published by 10 | # the Free Software Foundation, either version 3 of the License, or 11 | # (at your option) any later version. 12 | 13 | # This program is distributed in the hope that it will be useful, 14 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 15 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 | # GNU General Public License for more details. 17 | 18 | # You should have received a copy of the GNU General Public License 19 | # along with this program. If not, see . 20 | 21 | """Test TSM module.""" 22 | 23 | import unittest 24 | 25 | import numpy as np 26 | import numpy.testing 27 | 28 | import pygac.correct_tsm_issue as tsm 29 | 30 | 31 | class TSMTest(unittest.TestCase): 32 | """Test TSM module.""" 33 | 34 | def test_get_tsm_idx(self): 35 | """Test identification of TSM affected pixels.""" 36 | cols = np.arange(409) 37 | rows = np.arange(3000) 38 | mcols, mrows = np.meshgrid(cols, rows) 39 | 40 | # Create dummy reflectances in [0, 1] and BTs [170, 350] 41 | refl = np.sin(mrows / float(rows.size)) * \ 42 | np.cos(mcols / float(cols.size)) 43 | bt = np.cos(mrows / float(rows.size)) * \ 44 | np.sin(mcols / float(cols.size)) 45 | bt = 170 + (350 - 170) * bt 46 | 47 | ch1 = refl.copy() 48 | ch2 = refl.copy() 49 | ch4 = bt.copy() 50 | ch5 = bt.copy() 51 | 52 | # Add rectangle with noise to channel 1 and 4 53 | noise_cols, noise_rows = np.meshgrid(np.arange(200, 300), 54 | np.arange(1000, 2000)) 55 | rng = np.random.default_rng() 56 | for ch in [ch1, ch4]: 57 | ch[noise_rows, noise_cols] += 1000 + 1000*rng.random(size=noise_rows.shape) 58 | 59 | # We expect the filter to also detect the edges of the noisy 60 | # rectangle: At the edges there is a transition from zero to 61 | # nonzero channel diff which leads to an increased 3x3 standard 62 | # deviation. 63 | noise_cols_exp, noise_rows_exp = np.meshgrid(np.arange(199, 301), 64 | np.arange(999, 2001)) 65 | idx = tsm.get_tsm_idx(ch1, ch2, ch4, ch5) 66 | numpy.testing.assert_array_equal(idx[0], noise_rows_exp.ravel()) 67 | numpy.testing.assert_array_equal(idx[1], noise_cols_exp.ravel()) 68 | 69 | def test_std_filter(self): 70 | """Test standard deviation filter.""" 71 | # Define test data 72 | data = np.array([ 73 | [2., 2., 2., 2.], 74 | [2., np.nan, np.nan, 2.], 75 | [np.nan, 3., 3., np.nan], 76 | [3., 2., 2., 3.], 77 | ]) 78 | 79 | # Define reference 80 | filtered_ref = np.sqrt(np.array([ 81 | [0., 0., 0., 0.], 82 | [0.1875, 2./9., 2./9., 0.1875], 83 | [0.25, 0.25, 0.25, 0.25], 84 | [2./9., 0.24, 0.24, 2./9.] 85 | ])) 86 | 87 | # Apply filter and compare results against reference 88 | filtered = tsm.std_filter(data=data, box_size=3) 89 | numpy.testing.assert_array_equal(filtered, filtered_ref) 90 | -------------------------------------------------------------------------------- /pygac/tests/test_utils.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # Copyright (c) 2013, 2014 Martin Raspaud 5 | 6 | # Author(s): 7 | 8 | # Carlos Horn 9 | 10 | # This program is free software: you can redistribute it and/or modify 11 | # it under the terms of the GNU General Public License as published by 12 | # the Free Software Foundation, either version 3 of the License, or 13 | # (at your option) any later version. 14 | 15 | # This program is distributed in the hope that it will be useful, 16 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 17 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 18 | # GNU General Public License for more details. 19 | 20 | # You should have received a copy of the GNU General Public License 21 | # along with this program. If not, see . 22 | 23 | """Test pygac.utils module 24 | """ 25 | 26 | import gzip 27 | import io 28 | import os 29 | import unittest 30 | from unittest import mock 31 | 32 | import numpy as np 33 | 34 | from pygac.utils import calculate_sun_earth_distance_correction, file_opener 35 | 36 | 37 | class TestUtils(unittest.TestCase): 38 | """Test pygac.utils functions""" 39 | 40 | longMessage = True 41 | 42 | @mock.patch("pygac.utils.open", mock.MagicMock(return_value=io.BytesIO(b"file content"))) 43 | def test_file_opener_1(self): 44 | """Test if a file is redirected correctly through file_opener.""" 45 | with file_opener("path/to/file") as f: 46 | content = f.read() 47 | self.assertEqual(content, b"file content") 48 | 49 | def test_file_opener_2(self): 50 | """Test file_opener with file objects and compression""" 51 | # prepare test 52 | normal_message = b"normal message" 53 | gzip_message_decoded = b"gzip message" 54 | with io.BytesIO() as f: 55 | with gzip.open(f, mode="wb") as g: 56 | g.write(gzip_message_decoded) 57 | f.seek(0) 58 | gzip_message_encoded = f.read() 59 | # on normal file (check also if it remains open) 60 | with io.BytesIO(normal_message) as f: 61 | with file_opener(f) as g: 62 | message = g.read() 63 | self.assertFalse(f.closed) 64 | self.assertEqual(message, normal_message) 65 | # on gzip file 66 | with io.BytesIO(gzip_message_encoded) as f: 67 | with file_opener(f) as g: 68 | message = g.read() 69 | self.assertEqual(message, gzip_message_decoded) 70 | 71 | @mock.patch("pygac.utils.open", mock.MagicMock(side_effect=FileNotFoundError)) 72 | def test_file_opener_3(self): 73 | """Test file_opener with PathLike object""" 74 | # prepare test 75 | class RawBytes(os.PathLike): 76 | def __init__(self, filename, raw_bytes): 77 | self.filename = str(filename) 78 | self.raw_bytes = raw_bytes 79 | 80 | def __fspath__(self): 81 | return self.filename 82 | 83 | def open(self): 84 | return io.BytesIO(self.raw_bytes) 85 | 86 | filename = "/path/to/file" 87 | file_bytes = b"TestTestTest" 88 | test_pathlike = RawBytes(filename, file_bytes) 89 | with file_opener(test_pathlike) as f: 90 | content = f.read() 91 | self.assertEqual(content, file_bytes) 92 | 93 | # test with lazy loading open method (open only in context) 94 | class RawBytesLazy(RawBytes): 95 | def open(self): 96 | self.lazy_opener_mock = mock.MagicMock() 97 | self.lazy_opener_mock.__enter__.return_value = io.BytesIO(self.raw_bytes) 98 | return self.lazy_opener_mock 99 | 100 | test_pathlike = RawBytesLazy(filename, file_bytes) 101 | with file_opener(test_pathlike) as f: 102 | content = f.read() 103 | self.assertEqual(content, file_bytes) 104 | test_pathlike.lazy_opener_mock.__exit__.assert_called_once_with(None, None, None) 105 | 106 | def test_calculate_sun_earth_distance_correction(self): 107 | """Test function for the sun distance corretction.""" 108 | corr = calculate_sun_earth_distance_correction(3) 109 | np.testing.assert_almost_equal(corr, 0.96660494, decimal=7) 110 | -------------------------------------------------------------------------------- /pygac/tests/utils.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # Author(s): 5 | 6 | # Stephan Finkensieper 7 | 8 | # This program is free software: you can redistribute it and/or modify 9 | # it under the terms of the GNU General Public License as published by 10 | # the Free Software Foundation, either version 3 of the License, or 11 | # (at your option) any later version. 12 | 13 | # This program is distributed in the hope that it will be useful, 14 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 15 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 | # GNU General Public License for more details. 17 | 18 | # You should have received a copy of the GNU General Public License 19 | # along with this program. If not, see . 20 | 21 | """Test utilities.""" 22 | 23 | import numpy as np 24 | 25 | 26 | class CalledWithArray(object): 27 | """Adapter for arrays in mock.assert_called_with().""" 28 | 29 | def __init__(self, array): 30 | self.array = array 31 | 32 | def __repr__(self): 33 | return repr(self.array) 34 | 35 | def __eq__(self, other): 36 | return np.all(self.array == other) 37 | -------------------------------------------------------------------------------- /pygac/utils.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # Author(s): 5 | 6 | # Stephan Finkensieper 7 | # Carlos Horn 8 | 9 | # This program is free software: you can redistribute it and/or modify 10 | # it under the terms of the GNU General Public License as published by 11 | # the Free Software Foundation, either version 3 of the License, or 12 | # (at your option) any later version. 13 | 14 | # This program is distributed in the hope that it will be useful, 15 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 16 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 17 | # GNU General Public License for more details. 18 | 19 | # You should have received a copy of the GNU General Public License 20 | # along with this program. If not, see . 21 | 22 | import gzip 23 | import io 24 | import logging 25 | from contextlib import contextmanager, nullcontext 26 | 27 | import numpy as np 28 | 29 | LOG = logging.getLogger(__name__) 30 | 31 | 32 | def gzip_inspected(open_file): 33 | """Try to gzip decompress the file object if applicable.""" 34 | try: 35 | file_object = gzip.GzipFile(mode="rb", fileobj=open_file) 36 | file_object.read(1) 37 | except OSError: 38 | file_object = open_file 39 | finally: 40 | file_object.seek(0) 41 | return file_object 42 | 43 | 44 | @contextmanager 45 | def file_opener(file): 46 | if isinstance(file, io.IOBase) and file.seekable(): 47 | # avoid closing the file using nullcontext 48 | open_file = nullcontext(file) 49 | elif hasattr(file, "open"): 50 | try: 51 | open_file = file.open(mode="rb") 52 | except TypeError: 53 | open_file = file.open() 54 | else: 55 | open_file = open(file, mode="rb") 56 | # set open_file into context in case of lazy loading in __enter__ method. 57 | with open_file as file_object: 58 | yield gzip_inspected(file_object) 59 | 60 | 61 | def get_absolute_azimuth_angle_diff(sat_azi, sun_azi): 62 | """Calculates absolute azimuth difference angle. """ 63 | rel_azi = abs(sat_azi - sun_azi) 64 | rel_azi = rel_azi % 360 65 | # Not using np.where to avoid copying array 66 | rel_azi[rel_azi > 180] = 360.0 - rel_azi[rel_azi > 180] 67 | return rel_azi 68 | 69 | 70 | def centered_modulus(array, divisor): 71 | """Transform array to half open range ]-divisor/2, divisor/2].""" 72 | arr = array % divisor 73 | arr[arr > divisor / 2] -= divisor 74 | return arr 75 | 76 | 77 | def calculate_sun_earth_distance_correction(jday): 78 | """Calculate the sun earth distance correction. 79 | 80 | In 2008 3-4 different equations of ESD were considered. 81 | This one was chosen as it at the time gave reflectances most closely 82 | matching the PATMOS-x data provided then by Andy Heidinger. 83 | 84 | Formula might need to be reconsidered if jday is updated to a float. 85 | 86 | """ 87 | # Earth-Sun distance correction factor 88 | corr = 1.0 - 0.0334 * np.cos(2.0 * np.pi * (jday - 2) / 365.25) 89 | return corr 90 | 91 | 92 | def check_user_scanlines(start_line, end_line, first_valid_lat=None, 93 | last_valid_lat=None, along_track=None): 94 | """Check user-defined scanlines. 95 | 96 | Can be used by both pygac and satpy. 97 | 98 | Args: 99 | start_line: User-defined start line (afer stripping, if enabled) 100 | end_line: User-defined end line (afer stripping, if enabled) 101 | first_valid_lat: First scanline with valid latitudes 102 | last_valid_lat: Last scanline with valid latitudes 103 | along_track: Number of scanlines (only needed if stripping 104 | is disabled) 105 | """ 106 | if first_valid_lat is not None and last_valid_lat is not None: 107 | num_valid_lines = last_valid_lat - first_valid_lat + 1 108 | else: 109 | if along_track is None: 110 | raise ValueError("Need along_track") 111 | num_valid_lines = along_track 112 | 113 | start_line = int(start_line) 114 | end_line = int(end_line) 115 | if end_line == 0: 116 | # If the user specifies 0 as the last scanline, process all 117 | # scanlines with valid coordinates 118 | end_line = num_valid_lines - 1 119 | elif end_line >= num_valid_lines: 120 | end_line = num_valid_lines - 1 121 | LOG.warning("Given end line exceeds scanline range, resetting " 122 | "to {}".format(end_line)) 123 | if start_line > num_valid_lines: 124 | raise ValueError("Given start line {} exceeds scanline range {}" 125 | .format(start_line, num_valid_lines)) 126 | return start_line, end_line 127 | 128 | 129 | def strip_invalid_lat(lats): 130 | """Strip invalid latitudes at the end and beginning of the orbit.""" 131 | no_wrong_lat = np.where(np.logical_not(np.isnan(lats))) 132 | return min(no_wrong_lat[0]), max(no_wrong_lat[0]) 133 | 134 | 135 | def slice_channel(ch, start_line, end_line, first_valid_lat=None, 136 | last_valid_lat=None, midnight_scanline=None, 137 | miss_lines=None, qual_flags=None): 138 | """Slice channel data using user-defined start/end line. 139 | 140 | If valid_lat_start/end are given, strip scanlines with invalid 141 | coordinates at the beginning and end of the orbit. 142 | 143 | Can be used by both pygac and satpy. 144 | 145 | Args: 146 | ch: Channel data 147 | start_line: User-defined start line (afer stripping, if enabled) 148 | end_line: User-defined end line (after stripping, if enabled) 149 | first_valid_lat: First scanline with valid latitudes 150 | last_valid_lat: Last scanline with valid latitudes. 151 | midnight_scanline: If given, update midnight scanline to the new 152 | scanline range. 153 | miss_lines: If given, update list of missing lines with the ones 154 | that have been stripped due to invalid coordinates 155 | qual_flags: Quality flags, needed to updated missing lines. 156 | """ 157 | if first_valid_lat is not None and last_valid_lat is not None: 158 | # Strip invalid coordinates and update midnight scanline as well as 159 | # user-defined start/end lines 160 | ch, updated = _slice(ch, 161 | start_line=first_valid_lat, 162 | end_line=last_valid_lat, 163 | update=[midnight_scanline]) 164 | midnight_scanline = updated[0] 165 | 166 | # Reset user-defined end line, if it has been removed 167 | end_line = min(end_line, ch.shape[0] - 1) 168 | start_line = min(start_line, ch.shape[0] - 1) 169 | 170 | # Update missing scanlines 171 | if miss_lines is not None: 172 | miss_lines = _update_missing_scanlines( 173 | miss_lines=miss_lines, 174 | qual_flags=qual_flags, 175 | start_line=first_valid_lat, 176 | end_line=last_valid_lat) 177 | 178 | # Slice data using user-defined start/end lines 179 | ch_slc, updated = _slice(ch, start_line=start_line, end_line=end_line, 180 | update=[midnight_scanline]) 181 | midnight_scanline = updated[0] 182 | 183 | return ch_slc, miss_lines, midnight_scanline 184 | 185 | 186 | def _slice(ch, start_line, end_line, update=None): 187 | """Slice the given channel. 188 | 189 | Args: 190 | start_line: New start line 191 | end_line: New end line 192 | update: List of scanlines to be updated to the new range 193 | """ 194 | # Slice data using new start/end lines 195 | if len(ch.shape) == 1: 196 | ch_slc = ch[start_line:end_line + 1].copy() 197 | else: 198 | ch_slc = ch[start_line:end_line + 1, :].copy() 199 | 200 | if update: 201 | updated = [_update_scanline(line, start_line, end_line) 202 | if line is not None else None 203 | for line in update] 204 | return ch_slc, updated 205 | 206 | return ch_slc 207 | 208 | 209 | def _update_scanline(scanline, new_start_line, new_end_line): 210 | """Update the given scanline to the new range. 211 | 212 | Set scanline to None if it lies outside the new range. 213 | """ 214 | scanline -= new_start_line 215 | num_lines = new_end_line - new_start_line + 1 216 | if scanline < 0 or scanline >= num_lines: 217 | scanline = None 218 | return scanline 219 | 220 | 221 | def _update_missing_scanlines(miss_lines, qual_flags, start_line, end_line): 222 | """Add scanlines excluded by slicing to the list of missing scanlines. 223 | 224 | Args: 225 | miss_lines: List of missing scanlines 226 | qual_flags: Quality flags 227 | start_line: New start line of the slice 228 | end_line: New end line of the slice 229 | """ 230 | return np.sort(np.unique( 231 | qual_flags[0:start_line, 0].tolist() + 232 | miss_lines.tolist() + 233 | qual_flags[end_line + 1:, 0].tolist() 234 | )) 235 | 236 | 237 | def plot_correct_times_thresh(res, filename=None): 238 | """Visualize results of GACReader.correct_times_thresh.""" 239 | import matplotlib.pyplot as plt 240 | 241 | t = res["t"] 242 | tcorr = res.get("tcorr") 243 | n = res["n"] 244 | offsets = res.get("offsets") 245 | t0_head = res.get("t0_head") 246 | max_diff_from_t0_head = res.get("max_diff_from_t0_head") 247 | fail_reason = res.get("fail_reason", "Failed for unknown reason") 248 | 249 | # Setup figure 250 | along_track = np.arange(t.size) 251 | _, (ax0, ax1, ax2) = plt.subplots(nrows=3, sharex=True, 252 | figsize=(8, 10)) 253 | 254 | # Plot original vs corrected timestamps 255 | ax0.plot(along_track, t, "b-", label="original") 256 | if tcorr is not None: 257 | ax0.plot(along_track, tcorr, color="red", linestyle="--", 258 | label="corrected") 259 | else: 260 | ax0.set_title(fail_reason) 261 | 262 | ax0.set_ylabel("Time") 263 | ax0.set_ylim(t.min() - np.timedelta64(30, "m"), 264 | t.max() + np.timedelta64(30, "m")) 265 | ax0.legend(loc="best") 266 | 267 | # Plot offset (original time - ideal time) 268 | if offsets is not None: 269 | ax1.plot(along_track, offsets) 270 | ax1.fill_between( 271 | along_track, 272 | t0_head - np.ones(along_track.size) * max_diff_from_t0_head, 273 | t0_head + np.ones(along_track.size) * max_diff_from_t0_head, 274 | facecolor="g", alpha=0.33) 275 | ax1.axhline(y=t0_head, color="g", linestyle="--", 276 | label="Header timestamp") 277 | ax1.set_ylim(t0_head - 5 * max_diff_from_t0_head, 278 | t0_head + 5 * max_diff_from_t0_head) 279 | ax1.set_ylabel("Offset t-tn [ms]") 280 | ax1.legend(loc="best") 281 | 282 | # Plot scanline number 283 | ax2.plot(along_track, n) 284 | ax2.set_ylabel("Scanline number") 285 | ax2.set_xlabel("Along Track") 286 | 287 | if filename: 288 | plt.savefig(filename, bbox_inches="tight", dpi=100) 289 | else: 290 | plt.show() 291 | 292 | 293 | def plot_correct_scanline_numbers(res, filename=None): 294 | """Visualize results of GACReader.correct_scanline_numbers.""" 295 | import matplotlib.pyplot as plt 296 | 297 | along_track = res["along_track"] 298 | n_orig = res["n_orig"] 299 | n_corr = res["n_corr"] 300 | within_range = res["within_range"] 301 | thresh = res["thresh"] 302 | diffs = res["diffs"] 303 | nz_diffs = res["nz_diffs"] 304 | 305 | # Setup figure 306 | _, (ax0, ax1) = plt.subplots(nrows=2) 307 | 308 | # Plot original vs corrected scanline numbers 309 | ax0.plot(along_track, n_orig, "b-", label="original") 310 | along_track_corr = along_track.copy() 311 | along_track_corr = along_track_corr[within_range] 312 | along_track_corr = along_track_corr[diffs <= thresh] 313 | ax0.plot(along_track_corr, n_corr, "r--", label="corrected") 314 | ax0.set_ylabel("Scanline Number") 315 | ax0.set_xlabel("Along Track") 316 | ax0.legend(loc="best") 317 | 318 | # Plot difference from ideal 319 | ax1.plot(np.arange(len(nz_diffs)), nz_diffs) 320 | ax1.axhline(thresh, color="r", label="thresh={0:.2f}" 321 | .format(thresh)) 322 | ax1.set_xlabel("Index") 323 | ax1.set_ylabel("nonzero |n - n'|") 324 | ax1.legend() 325 | 326 | plt.tight_layout() 327 | 328 | if filename: 329 | plt.savefig(filename, bbox_inches="tight") 330 | else: 331 | plt.show() 332 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "pygac" 3 | dynamic = ["version"] 4 | description = "NOAA AVHRR GAC/LAC/FRAC reader and calibration" 5 | authors = [ 6 | { name = "The Pytroll Team", email = "pytroll@googlegroups.com" } 7 | ] 8 | dependencies = [ 9 | "bottleneck>=1.0.0", 10 | "docutils>=0.3", 11 | "h5py>=2.0.1", 12 | "numpy>=1.8.0", 13 | "pyorbital>=1.10.0", 14 | "python-geotiepoints>=1.1.8", 15 | "scipy>=0.8.0", 16 | "packaging", 17 | "xarray>=2024.1.0" 18 | ] 19 | readme = "README.md" 20 | requires-python = ">=3.10" 21 | license = { text = "GPLv3" } 22 | classifiers = [ 23 | "Programming Language :: Python :: 3", 24 | "Operating System :: OS Independent", 25 | "Development Status :: 4 - Beta", 26 | "Intended Audience :: Science/Research", 27 | "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)", 28 | "Operating System :: OS Independent", 29 | "Topic :: Scientific/Engineering" 30 | ] 31 | 32 | [project.urls] 33 | Homepage = "https://github.com/pytroll/pygac" 34 | "Bug Tracker" = "https://github.com/pytroll/pygac/issues" 35 | Documentation = "https://pygac.readthedocs.io/en/stable/" 36 | "Source Code" = "https://github.com/pytroll/pygac" 37 | Organization = "https://pytroll.github.io/" 38 | Slack = "https://pytroll.slack.com/" 39 | "Release Notes" = "https://github.com/pytroll/pygac/blob/main/CHANGELOG.md" 40 | 41 | [project.optional-dependencies] 42 | dev = ['pytest', 'pre-commit', 'ruff'] 43 | georef = [ 44 | "georeferencer", 45 | ] 46 | 47 | [project.scripts] 48 | pygac-convert-patmosx-coefficients = "pygac.patmosx_coeff_reader:main" 49 | 50 | [project.entry-points."pygac.calibration"] 51 | noaa = "pygac.calibration.noaa:calibrate" 52 | 53 | [build-system] 54 | requires = ["hatchling", "hatch-vcs"] 55 | build-backend = "hatchling.build" 56 | 57 | [tool.hatch.metadata] 58 | allow-direct-references = true 59 | 60 | [tool.hatch.build.targets.wheel] 61 | packages = ["pygac"] 62 | 63 | [tool.hatch.version] 64 | source = "vcs" 65 | 66 | [tool.hatch.build.hooks.vcs] 67 | version-file = "pygac/version.py" 68 | 69 | [tool.ruff] 70 | line-length = 120 71 | 72 | [tool.ruff.lint] 73 | select = ["E", "W", "F", "I", "Q", "NPY"] 74 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | numpy>=1.6.1 2 | h5py>=2.0.1 3 | scipy>=0.8.0 4 | python-geotiepoints>=1.1.8 5 | packaging>=14.0 6 | xarray>=2024.1.0 7 | --------------------------------------------------------------------------------