├── .flake8 ├── .github └── workflows │ └── runflake8.yml ├── .gitignore ├── .readthedocs.yaml ├── CreateDistribution.txt ├── INSTALL.txt ├── LICENSE.txt ├── MANIFEST.in ├── README.md ├── bin ├── fmask_sentinel2Stacked.py ├── fmask_sentinel2makeAnglesImage.py ├── fmask_usgsLandsatMakeAnglesImage.py ├── fmask_usgsLandsatSaturationMask.py ├── fmask_usgsLandsatStacked.py └── fmask_usgsLandsatTOA.py ├── c_src ├── fillminima.c └── valueindexes.c ├── doc ├── Makefile ├── make.bat ├── requirements.txt └── source │ ├── conf.py │ ├── fmask_config.rst │ ├── fmask_fillminima.rst │ ├── fmask_fmask.rst │ ├── fmask_fmaskerrors.rst │ ├── fmask_landsatTOA.rst │ ├── fmask_landsatangles.rst │ ├── fmask_saturationcheck.rst │ ├── fmask_valueindexes.rst │ ├── fmask_zerocheck.rst │ ├── index.rst │ └── releasenotes.rst ├── fmask ├── __init__.py ├── cmdline │ ├── __init__.py │ ├── sentinel2Stacked.py │ ├── sentinel2makeAnglesImage.py │ ├── usgsLandsatMakeAnglesImage.py │ ├── usgsLandsatSaturationMask.py │ ├── usgsLandsatStacked.py │ └── usgsLandsatTOA.py ├── config.py ├── fillminima.py ├── fmask.py ├── fmaskerrors.py ├── landsatTOA.py ├── landsatangles.py ├── saturationcheck.py ├── sen2meta.py ├── valueindexes.py └── zerocheck.py ├── pyproject.toml └── setup.py /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | exclude = .git,__pycache__,build,dist 3 | # Error codes we should ignore 4 | # W291 trailing whitespace 5 | # W293 blank line contains whitespace 6 | # W391 blank line at end of file 7 | # W504 line break after binary operator 8 | # W605 invalid escape sequence \* NOTE: This \ is required by readthedocs 9 | # E128 continuation line under-indented for visual indent 10 | # E225 missing whitespace around operator 11 | # E228 missing whitespace around modulo operator 12 | # E501 line too long (??? > 79 characters) 13 | ignore = W291,W293,W391,W504,W605,E128,E225,E228,E501 14 | -------------------------------------------------------------------------------- /.github/workflows/runflake8.yml: -------------------------------------------------------------------------------- 1 | name: Flake8 Lint 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | do-flake8: 7 | 8 | runs-on: ubuntu-latest 9 | 10 | steps: 11 | - name: Checkout 12 | uses: actions/checkout@v2 13 | - name: Setup Python 14 | uses: actions/setup-python@v1 15 | - name: Lint with flake8 16 | uses: py-actions/flake8@v2 17 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *.img 3 | *.kea 4 | -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | # .readthedocs.yaml 2 | # Read the Docs configuration file 3 | # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details 4 | 5 | # Required 6 | version: 2 7 | 8 | # Set the version of Python and other tools you might need 9 | build: 10 | os: ubuntu-22.04 11 | tools: 12 | python: "3.11" 13 | 14 | # Build documentation in the doc/ directory with Sphinx 15 | sphinx: 16 | configuration: doc/source/conf.py 17 | 18 | python: 19 | install: 20 | - requirements: doc/requirements.txt 21 | -------------------------------------------------------------------------------- /CreateDistribution.txt: -------------------------------------------------------------------------------- 1 | How to create a distribution of python-fmask. 2 | 3 | 1. Ensure that you have pulled and committed everything which needs to go in. 4 | 2. Change the version number in the fmask/__init__.py. Version number 5 | is of the form a.b.c, as discussed below. 6 | 3. Update the release notes page in the doco (doc/source/releasenotes.rst), 7 | by going through the change logs since the last release, and noting 8 | what has been done. 9 | DON'T FORGET TO COMMIT THIS, BEFORE THE NEXT STEP!!!! 10 | 4. Push the changes to github with "git push" and create a PR for it and merge it. 11 | 5. Check out a clean copy of the repository into /tmp or 12 | somewhere similar and 'cd' into it. 13 | 6. Create the distribution tarball, using 14 | python -m build 15 | This creates both a tar.gz and a wheel, under a subdirectory called dist 16 | 7. Create a checksum of the tar.gz, e.g. 17 | sha256sum python-fmask-0.5.9.tar.gz > python-fmask-0.5.9.tar.gz.sha256 18 | 8. Go to the https://github.com/ubarsc/python-fmask/releases page, and create a 19 | new release by pressing "Draft a new release". 20 | You should fill in the following: 21 | Tag version: pythonfmask-A.B.C 22 | Release Title: Version A.B.C 23 | Description: Add a brief description (a few lines at most) explaining 24 | the key points about this release. 25 | Upload files: Add the tar.gz and zip files. 26 | Click "Publish release" 27 | 28 | 29 | Version Numbers. 30 | The python-fmask version number is structured as A.B.C. We follow the conventions 31 | outlined in Semantic Versioning [https://semver.org] 32 | - The A number should change for major alterations, most particularly those 33 | which break backward compatability, or which involve major restructuring of 34 | code or data structures. 35 | - The B number should change for introduction of significant new features 36 | - The C number should change for bug fixes or very minor changes. 37 | -------------------------------------------------------------------------------- /INSTALL.txt: -------------------------------------------------------------------------------- 1 | 2 | WARNING: To install python-fmask it is strongly recommended to use the conda forge 3 | pre-built binaries. Only install from source if you absolutely must. 4 | 5 | The code requires RIOS, GDAL, numpy and scipy. These must be installed, preferrably 6 | using a package manager such as conda. 7 | 8 | 9 | To install python-fmask from the source code bundle, use the following commands 10 | 11 | First unpack the bundle. For the tar.gz file, this would be 12 | tar xfz python-fmask-0.4.2.tar.gz 13 | For the zip file this would be 14 | unzip -q python-fmask-0.4.2.zip 15 | 16 | The installation uses setuptools packaging. This is best driven by pip 17 | (direct use of setup.py is now deprecated by the setuptools people), so 18 | the following commands are fairly standard. 19 | 20 | Build the code 21 | cd python-fmask-0.4.2 22 | pip install . 23 | 24 | 25 | If you wish to install in a non-default location, use 26 | pip install . --prefix=/yourChosenDirectory 27 | 28 | If installed in a non-default location, you will then need to ensure that the 29 | right environment variables are set. For simple bash syntax, this would be 30 | something like: 31 | export PATH="/yourChosenDirectory/bin:$PATH" 32 | export PYTHONPATH="/yourChosenDirectory/lib/pythonX.X/site-packages:$PYTHONPATH" 33 | 34 | Note that the pythonX.X sub-directory needs to match your version of python. 35 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE.txt 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Implementation in Python of the cloud and shadow algorithms known collectively as Fmask, 2 | as published in: 3 | 4 | Zhu, Z. and Woodcock, C.E. (2012). Object-based cloud and cloud shadow detection in Landsat imagery Remote Sensing of Environment 118 (2012) 83-94. 5 | 6 | and 7 | 8 | Zhu, Z., Wang, S. and Woodcock, C.E. (2015). Improvement and expansion of the Fmask algorithm: cloud, cloud shadow, and snow detection for Landsats 4-7, 8, and Sentinel 2 images Remote Sensing of Environment 159 (2015) 269-277. 9 | 10 | Please visit the main web page at: [www.pythonfmask.org](http://www.pythonfmask.org/) 11 | -------------------------------------------------------------------------------- /bin/fmask_sentinel2Stacked.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # This file is part of 'python-fmask' - a cloud masking module 3 | # Copyright (C) 2015 Neil Flood 4 | # 5 | # This program is free software; you can redistribute it and/or 6 | # modify it under the terms of the GNU General Public License 7 | # as published by the Free Software Foundation; either version 3 8 | # of the License, or (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with this program; if not, write to the Free Software 17 | # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 18 | 19 | from fmask.cmdline import sentinel2Stacked 20 | import warnings 21 | warnings.warn("Future versions of fmask may remove the .py extension from this script name", DeprecationWarning) 22 | 23 | if __name__ == '__main__': 24 | sentinel2Stacked.mainRoutine() 25 | 26 | -------------------------------------------------------------------------------- /bin/fmask_sentinel2makeAnglesImage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # This file is part of 'python-fmask' - a cloud masking module 3 | # Copyright (C) 2015 Neil Flood 4 | # 5 | # This program is free software; you can redistribute it and/or 6 | # modify it under the terms of the GNU General Public License 7 | # as published by the Free Software Foundation; either version 3 8 | # of the License, or (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with this program; if not, write to the Free Software 17 | # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 18 | 19 | from fmask.cmdline import sentinel2makeAnglesImage 20 | import warnings 21 | warnings.warn("Future versions of fmask may remove the .py extension from this script name", DeprecationWarning) 22 | 23 | if __name__ == "__main__": 24 | sentinel2makeAnglesImage.mainRoutine() 25 | 26 | -------------------------------------------------------------------------------- /bin/fmask_usgsLandsatMakeAnglesImage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # This file is part of 'python-fmask' - a cloud masking module 3 | # Copyright (C) 2015 Neil Flood 4 | # 5 | # This program is free software; you can redistribute it and/or 6 | # modify it under the terms of the GNU General Public License 7 | # as published by the Free Software Foundation; either version 3 8 | # of the License, or (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with this program; if not, write to the Free Software 17 | # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 18 | 19 | from fmask.cmdline import usgsLandsatMakeAnglesImage 20 | import warnings 21 | warnings.warn("Future versions of fmask may remove the .py extension from this script name", DeprecationWarning) 22 | 23 | if __name__ == "__main__": 24 | usgsLandsatMakeAnglesImage.mainRoutine() 25 | -------------------------------------------------------------------------------- /bin/fmask_usgsLandsatSaturationMask.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # This file is part of 'python-fmask' - a cloud masking module 3 | # Copyright (C) 2015 Neil Flood 4 | # 5 | # This program is free software; you can redistribute it and/or 6 | # modify it under the terms of the GNU General Public License 7 | # as published by the Free Software Foundation; either version 3 8 | # of the License, or (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with this program; if not, write to the Free Software 17 | # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 18 | 19 | from fmask.cmdline import usgsLandsatSaturationMask 20 | import warnings 21 | warnings.warn("Future versions of fmask may remove the .py extension from this script name", DeprecationWarning) 22 | 23 | if __name__ == '__main__': 24 | usgsLandsatSaturationMask.mainRoutine() 25 | 26 | 27 | -------------------------------------------------------------------------------- /bin/fmask_usgsLandsatStacked.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # This file is part of 'python-fmask' - a cloud masking module 3 | # Copyright (C) 2015 Neil Flood 4 | # 5 | # This program is free software; you can redistribute it and/or 6 | # modify it under the terms of the GNU General Public License 7 | # as published by the Free Software Foundation; either version 3 8 | # of the License, or (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with this program; if not, write to the Free Software 17 | # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 18 | 19 | from fmask.cmdline import usgsLandsatStacked 20 | import warnings 21 | warnings.warn("Future versions of fmask may remove the .py extension from this script name", DeprecationWarning) 22 | 23 | if __name__ == '__main__': 24 | usgsLandsatStacked.mainRoutine() 25 | 26 | -------------------------------------------------------------------------------- /bin/fmask_usgsLandsatTOA.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # This file is part of 'python-fmask' - a cloud masking module 3 | # Copyright (C) 2015 Neil Flood 4 | # 5 | # This program is free software; you can redistribute it and/or 6 | # modify it under the terms of the GNU General Public License 7 | # as published by the Free Software Foundation; either version 3 8 | # of the License, or (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with this program; if not, write to the Free Software 17 | # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 18 | 19 | from fmask.cmdline import usgsLandsatTOA 20 | import warnings 21 | warnings.warn("Future versions of fmask may remove the .py extension from this script name", DeprecationWarning) 22 | 23 | if __name__ == '__main__': 24 | usgsLandsatTOA.mainRoutine() 25 | 26 | 27 | -------------------------------------------------------------------------------- /c_src/fillminima.c: -------------------------------------------------------------------------------- 1 | /* This file is part of 'python-fmask' - a cloud masking module 2 | * Copyright (C) 2015 Neil Flood 3 | * 4 | * This program is free software; you can redistribute it and/or 5 | * modify it under the terms of the GNU General Public License 6 | * as published by the Free Software Foundation; either version 3 7 | * of the License, or (at your option) any later version. 8 | * 9 | * This program is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with this program; if not, write to the Free Software 16 | * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 17 | */ 18 | /* 19 | Module to implement filling of local minima in a raster surface. 20 | 21 | The algorithm is from 22 | Soille, P., and Gratin, C. (1994). An efficient algorithm for drainage network 23 | extraction on DEMs. J. Visual Communication and Image Representation. 24 | 5(2). 181-189. 25 | 26 | The algorithm is intended for hydrological processing of a DEM, but is used by the 27 | Fmask cloud shadow algorithm as part of its process for finding local minima which 28 | represent potential shadow objects. 29 | */ 30 | #include 31 | #include "numpy/arrayobject.h" 32 | #include 33 | 34 | /* An exception object for this module */ 35 | /* created in the init function */ 36 | struct FillMinimaState 37 | { 38 | PyObject *error; 39 | }; 40 | 41 | #if PY_MAJOR_VERSION >= 3 42 | #define GETSTATE(m) ((struct FillMinimaState*)PyModule_GetState(m)) 43 | #else 44 | #define GETSTATE(m) (&_state) 45 | static struct FillMinimaState _state; 46 | #endif 47 | 48 | /* Routines for handling the hierarchical pixel queue which the 49 | algorithm requires. 50 | */ 51 | typedef struct PQstruct { 52 | int i, j; 53 | struct PQstruct *next; 54 | } PQel; 55 | 56 | typedef struct { 57 | PQel *first, *last; 58 | int n; 59 | } PQhdr; 60 | 61 | typedef struct PQ { 62 | int hMin; 63 | int numLevels; 64 | PQhdr *q; 65 | } PixelQueue; 66 | 67 | /* A new pixel structure */ 68 | static PQel *newPix(int i, int j) { 69 | PQel *p; 70 | 71 | p = (PQel *)calloc(1, sizeof(PQel)); 72 | p->i = i; 73 | p->j = j; 74 | p->next = NULL; 75 | if (i>20000) { 76 | printf("i=%d\\n", i); 77 | exit(1); 78 | } 79 | return p; 80 | } 81 | 82 | /* Initialize pixel queue */ 83 | static PixelQueue *PQ_init(int hMin, int hMax) { 84 | PixelQueue *pixQ; 85 | int numLevels, i; 86 | 87 | pixQ = (PixelQueue *)calloc(1, sizeof(PixelQueue)); 88 | numLevels = hMax - hMin + 1; 89 | pixQ->hMin = hMin; 90 | pixQ->numLevels = numLevels; 91 | pixQ->q = (PQhdr *)calloc(numLevels, sizeof(PQhdr)); 92 | for (i=0; iq[i].first = NULL; 94 | pixQ->q[i].last = NULL; 95 | pixQ->q[i].n = 0; 96 | } 97 | return pixQ; 98 | } 99 | 100 | /* Add a pixel at level h */ 101 | static void PQ_add(PixelQueue *pixQ, PQel *p, int h) { 102 | int ndx; 103 | PQel *current, *newP; 104 | PQhdr *thisQ; 105 | 106 | /* Take a copy of the pixel structure */ 107 | newP = newPix(p->i, p->j); 108 | 109 | ndx = h - pixQ->hMin; 110 | if (ndx > pixQ->numLevels) { 111 | printf("Level h=%d too large. ndx=%d, numLevels=%d\\n", h, ndx, pixQ->numLevels); 112 | exit(1); 113 | } 114 | if (ndx < 0) { 115 | printf("Ndx is negative, which is not allowed. ndx=%d, h=%d, hMin=%d\n", ndx, h, pixQ->hMin); 116 | exit(1); 117 | } 118 | thisQ = &(pixQ->q[ndx]); 119 | /* Add to end of queue at this level */ 120 | current = thisQ->last; 121 | if (current != NULL) { 122 | current->next = newP; 123 | } 124 | thisQ->last = newP; 125 | thisQ->n++; 126 | /* If head of queue is NULL, make the new one the head */ 127 | if (thisQ->first == NULL) { 128 | thisQ->first = newP; 129 | } 130 | } 131 | 132 | /* Return TRUE if queue at level h is empty */ 133 | static int PQ_empty(PixelQueue *pixQ, int h) { 134 | int ndx, empty, n; 135 | PQel *current; 136 | 137 | ndx = h - pixQ->hMin; 138 | current = pixQ->q[ndx].first; 139 | n = pixQ->q[ndx].n; 140 | empty = (current == NULL); 141 | if (empty && (n != 0)) { 142 | printf("Empty, but n=%d\\n", n); 143 | exit(1); 144 | } 145 | if ((n == 0) && (! empty)) { 146 | printf("n=0, but not empty\\n"); 147 | while (current != NULL) { 148 | printf(" h=%d i=%d j=%d\\n", h, current->i, current->j); 149 | current = current->next; 150 | } 151 | exit(1); 152 | } 153 | return empty; 154 | } 155 | 156 | /* Return the first element in the queue at level h, and remove it 157 | from the queue */ 158 | static PQel *PQ_first(PixelQueue *pixQ, int h) { 159 | int ndx; 160 | PQel *current; 161 | PQhdr *thisQ; 162 | 163 | ndx = h - pixQ->hMin; 164 | thisQ = &(pixQ->q[ndx]); 165 | current = thisQ->first; 166 | /* Remove from head of queue */ 167 | if (current != NULL) { 168 | thisQ->first = current->next; 169 | if (thisQ->first == NULL) { 170 | thisQ->last = NULL; 171 | } 172 | thisQ->n--; 173 | if (thisQ->n < 0) { 174 | printf("n=%d in PQ_first()\\n", thisQ->n); 175 | exit(1); 176 | } else if (thisQ->n == 0) { 177 | if (thisQ->first != NULL) { 178 | printf("n=0, but 'first' != NULL. first(i,j) = %d,%d\\n", 179 | thisQ->first->i, thisQ->first->j); 180 | } 181 | } 182 | } 183 | return current; 184 | } 185 | 186 | static int di[4] = { 0, 0, -1, 1 }; 187 | static int dj[4] = { -1, 1, 0, 0 }; 188 | 189 | /* Return a list of neighbouring pixels to given pixel p. */ 190 | static PQel *neighbours(PQel *p, int nRows, int nCols) { 191 | int n, i, j; 192 | PQel *pl, *pNew; 193 | 194 | pl = NULL; 195 | for (n=0; n<4; n++) { 196 | i = p->i + di[n]; 197 | j = p->j + dj[n]; 198 | if ((i >= 0) && (i < nRows) && (j >= 0) && (j < nCols)) { 199 | pNew = newPix(i, j); 200 | pNew->next = pl; 201 | pl = pNew; 202 | } 203 | } 204 | return pl; 205 | } 206 | 207 | #define max(a,b) ((a) > (b) ? (a) : (b)) 208 | 209 | static PyObject *fillminima_fillMinima(PyObject *self, PyObject *args) 210 | { 211 | PyArrayObject *pimg, *pimg2, *pBoundaryRows, *pBoundaryCols, *pNullMask; 212 | int hMin, hMax; 213 | double dBoundaryVal; 214 | 215 | npy_int64 r, c; 216 | npy_intp i, nRows, nCols; 217 | npy_int16 imgval, img2val; 218 | PixelQueue *pixQ; 219 | PQel *p, *nbrs, *pNbr, *pNext; 220 | int hCrt; 221 | 222 | if( !PyArg_ParseTuple(args, "OOiiOdOO:fillMinima", &pimg, &pimg2, &hMin, &hMax, 223 | &pNullMask, &dBoundaryVal, &pBoundaryRows, &pBoundaryCols)) 224 | return NULL; 225 | 226 | if( !PyArray_Check(pimg) || !PyArray_Check(pimg2) || !PyArray_Check(pBoundaryRows) || 227 | !PyArray_Check(pBoundaryCols) || !PyArray_Check(pNullMask) ) 228 | { 229 | PyErr_SetString(GETSTATE(self)->error, "parameters 0, 1, 4, 6 and 7 must be numpy arrays"); 230 | return NULL; 231 | } 232 | 233 | if( (PyArray_TYPE(pimg) != NPY_INT16) || (PyArray_TYPE(pimg2) != NPY_INT16)) 234 | { 235 | PyErr_SetString(GETSTATE(self)->error, "parameters 0 and 1 must be int16 arrays"); 236 | return NULL; 237 | } 238 | 239 | if( PyArray_TYPE(pNullMask) != NPY_BOOL ) 240 | { 241 | PyErr_SetString(GETSTATE(self)->error, "parameter 4 must be a bool array"); 242 | return NULL; 243 | } 244 | 245 | if( (PyArray_TYPE(pBoundaryRows) != NPY_INT64) || (PyArray_TYPE(pBoundaryCols) != NPY_INT64) ) 246 | { 247 | PyErr_SetString(GETSTATE(self)->error, "parameters 6 and 7 must be int64 arrays"); 248 | return NULL; 249 | } 250 | 251 | nRows = PyArray_DIMS(pimg)[0]; 252 | nCols = PyArray_DIMS(pimg)[1]; 253 | 254 | pixQ = PQ_init(hMin, hMax); 255 | 256 | /* Initialize the boundary */ 257 | for (i=0; ii; 279 | c = pNbr->j; 280 | /* Exclude null area of original image */ 281 | if (! *((npy_bool*)PyArray_GETPTR2(pNullMask, r, c))) { 282 | imgval = *((npy_int16*)PyArray_GETPTR2(pimg, r, c)); 283 | img2val = *((npy_int16*)PyArray_GETPTR2(pimg2, r, c)); 284 | if (img2val == hMax) { 285 | img2val = max(hCrt, imgval); 286 | *((npy_int16*)PyArray_GETPTR2(pimg2, r, c)) = img2val; 287 | PQ_add(pixQ, pNbr, img2val); 288 | } 289 | } 290 | pNext = pNbr->next; 291 | free(pNbr); 292 | pNbr = pNext; 293 | } 294 | free(p); 295 | } 296 | hCrt++; 297 | } while (hCrt < hMax); 298 | 299 | free(pixQ); 300 | 301 | Py_RETURN_NONE; 302 | } 303 | 304 | 305 | // Our list of functions in this module 306 | static PyMethodDef FillMinimaMethods[] = { 307 | {"fillMinima", fillminima_fillMinima, METH_VARARGS, 308 | "function to implement filling of local minima in a raster surface:\n" 309 | "call signature: fillminima(inarray, outarray, hMin, hMax, nullmask, boundaryval, boundaryRows, boundaryCols)\n" 310 | "where:\n" 311 | " inarray is the input array\n" 312 | " outarray is the output array\n" 313 | " hMin is the minimum of the input image (excluding null values)\n" 314 | " hMax is the maximum of the input image (excluding null values)\n" 315 | " nullmask is the mask of where null values exist in inarray\n" 316 | " boundaryVal is the input boundar value\n" 317 | " boundaryRows and boundaryCols specify the boundary of the search\n"}, 318 | {NULL} /* Sentinel */ 319 | }; 320 | 321 | #if PY_MAJOR_VERSION >= 3 322 | 323 | static int fillminima_traverse(PyObject *m, visitproc visit, void *arg) 324 | { 325 | Py_VISIT(GETSTATE(m)->error); 326 | return 0; 327 | } 328 | 329 | static int fillminima_clear(PyObject *m) 330 | { 331 | Py_CLEAR(GETSTATE(m)->error); 332 | return 0; 333 | } 334 | 335 | static struct PyModuleDef moduledef = { 336 | PyModuleDef_HEAD_INIT, 337 | "_fillminima", 338 | NULL, 339 | sizeof(struct FillMinimaState), 340 | FillMinimaMethods, 341 | NULL, 342 | fillminima_traverse, 343 | fillminima_clear, 344 | NULL 345 | }; 346 | 347 | #define INITERROR return NULL 348 | 349 | PyMODINIT_FUNC 350 | PyInit__fillminima(void) 351 | 352 | #else 353 | #define INITERROR return 354 | 355 | PyMODINIT_FUNC 356 | init_fillminima(void) 357 | #endif 358 | { 359 | PyObject *pModule; 360 | struct FillMinimaState *state; 361 | 362 | // initialize the numpy stuff 363 | import_array(); 364 | 365 | #if PY_MAJOR_VERSION >= 3 366 | pModule = PyModule_Create(&moduledef); 367 | #else 368 | pModule = Py_InitModule("_fillminima", FillMinimaMethods); 369 | #endif 370 | if( pModule == NULL ) 371 | INITERROR; 372 | 373 | state = GETSTATE(pModule); 374 | 375 | // Create and add our exception type 376 | state->error = PyErr_NewException("_fillminima.error", NULL, NULL); 377 | if( state->error == NULL ) 378 | { 379 | Py_DECREF(pModule); 380 | INITERROR; 381 | } 382 | 383 | #if PY_MAJOR_VERSION >= 3 384 | return pModule; 385 | #endif 386 | } 387 | 388 | 389 | -------------------------------------------------------------------------------- /c_src/valueindexes.c: -------------------------------------------------------------------------------- 1 | /* This file is part of 'python-fmask' - a cloud masking module 2 | * Copyright (C) 2015 Neil Flood 3 | * 4 | * This program is free software; you can redistribute it and/or 5 | * modify it under the terms of the GNU General Public License 6 | * as published by the Free Software Foundation; either version 3 7 | * of the License, or (at your option) any later version. 8 | * 9 | * This program is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with this program; if not, write to the Free Software 16 | * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 17 | */ 18 | 19 | #include 20 | #include "numpy/arrayobject.h" 21 | #include 22 | 23 | /* An exception object for this module */ 24 | /* created in the init function */ 25 | struct ValueIndexesState 26 | { 27 | PyObject *error; 28 | }; 29 | 30 | #if PY_MAJOR_VERSION >= 3 31 | #define GETSTATE(m) ((struct ValueIndexesState*)PyModule_GetState(m)) 32 | #else 33 | #define GETSTATE(m) (&_state) 34 | static struct ValueIndexesState _state; 35 | #endif 36 | 37 | /* 2^32 - 1 */ 38 | #define MAXUINT32 4294967295 39 | 40 | static PyObject *valueIndexes_valndxFunc(PyObject *self, PyObject *args) 41 | { 42 | PyArrayObject *pInput, *pIndexes, *pValLU, *pCurrentIdx; 43 | npy_int64 nMin, nMax, arrVal = 0; 44 | int nDone = 0, nFound = 0, nDim, nType, i, idx; 45 | npy_uint32 j, m; 46 | npy_intp *pDims, *pCurrIdx; 47 | 48 | if( !PyArg_ParseTuple(args, "OOLLOO:valndxFunc", &pInput, &pIndexes, &nMin, &nMax, &pValLU, &pCurrentIdx)) 49 | return NULL; 50 | 51 | if( !PyArray_Check(pInput) || !PyArray_Check(pIndexes) || !PyArray_Check(pValLU) || !PyArray_Check(pCurrentIdx) ) 52 | { 53 | PyErr_SetString(GETSTATE(self)->error, "parameters 0, 1, 4 and 5 must be numpy arrays"); 54 | return NULL; 55 | } 56 | 57 | if( (PyArray_TYPE(pIndexes) != NPY_UINT32) || (PyArray_TYPE(pValLU) != NPY_UINT32) || (PyArray_TYPE(pCurrentIdx) != NPY_UINT32)) 58 | { 59 | PyErr_SetString(GETSTATE(self)->error, "parameters 1, 4 and 5 must be uint32 arrays"); 60 | return NULL; 61 | } 62 | 63 | nDim = PyArray_NDIM(pInput); 64 | pDims = PyArray_DIMS(pInput); 65 | nType = PyArray_TYPE(pInput); 66 | 67 | if( !PyTypeNum_ISINTEGER(nType) ) 68 | { 69 | PyErr_SetString(GETSTATE(self)->error, "parameter 0 must be an integer array"); 70 | return NULL; 71 | } 72 | 73 | pCurrIdx = (npy_intp*)calloc(nDim, sizeof(npy_intp)); 74 | 75 | while( !nDone ) 76 | { 77 | /* Get the value at pCurrIdx */ 78 | switch(nType) 79 | { 80 | case NPY_INT8: arrVal = (npy_uint64) *((npy_int8*)PyArray_GetPtr(pInput, pCurrIdx)); break; 81 | case NPY_UINT8: arrVal = (npy_uint64) *((npy_uint8*)PyArray_GetPtr(pInput, pCurrIdx)); break; 82 | case NPY_INT16: arrVal = (npy_uint64) *((npy_int16*)PyArray_GetPtr(pInput, pCurrIdx)); break; 83 | case NPY_UINT16: arrVal = (npy_uint64) *((npy_uint16*)PyArray_GetPtr(pInput, pCurrIdx)); break; 84 | case NPY_INT32: arrVal = (npy_uint64) *((npy_int32*)PyArray_GetPtr(pInput, pCurrIdx)); break; 85 | case NPY_UINT32: arrVal = (npy_uint64) *((npy_uint32*)PyArray_GetPtr(pInput, pCurrIdx)); break; 86 | case NPY_INT64: arrVal = (npy_uint64) *((npy_int64*)PyArray_GetPtr(pInput, pCurrIdx)); break; 87 | case NPY_UINT64: arrVal = (npy_uint64) *((npy_uint64*)PyArray_GetPtr(pInput, pCurrIdx)); break; 88 | } 89 | 90 | nFound = 0; 91 | j = 0; 92 | if( (arrVal >= nMin) && (arrVal <= nMax)) 93 | { 94 | j = *((npy_uint32*)PyArray_GETPTR1(pValLU, arrVal - nMin)); 95 | nFound = (j < MAXUINT32); 96 | } 97 | 98 | if( nFound ) 99 | { 100 | m = *((npy_uint32*)PyArray_GETPTR1(pCurrentIdx, j)); 101 | for( i = 0; i < nDim; i++ ) 102 | { 103 | *((npy_uint32*)PyArray_GETPTR2(pIndexes, m, i)) = pCurrIdx[i]; 104 | } 105 | *((npy_uint32*)PyArray_GETPTR1(pCurrentIdx, j)) = m + 1; 106 | } 107 | 108 | /* code that updates curridx - incs the next dim 109 | if we have done all the elements in the current 110 | dim */ 111 | idx = nDim - 1; 112 | while( idx >= 0 ) 113 | { 114 | pCurrIdx[idx]++; 115 | if( pCurrIdx[idx] >= pDims[idx] ) 116 | { 117 | pCurrIdx[idx] = 0; 118 | idx--; 119 | } 120 | else 121 | { 122 | break; 123 | } 124 | } 125 | 126 | /* if we are done we have run out of dims */ 127 | nDone = (idx < 0); 128 | } 129 | 130 | free(pCurrIdx); 131 | 132 | Py_RETURN_NONE; 133 | } 134 | 135 | 136 | // Our list of functions in this module 137 | static PyMethodDef ValueIndexesMethods[] = { 138 | {"valndxFunc", valueIndexes_valndxFunc, METH_VARARGS, 139 | "function to go through an array and create a lookup table of the array indexes for each distinct value in the data array:\n" 140 | "call signature: valndxFunc(input, indexes, min, max, ValLU, CurrentIdx)\n" 141 | "where:\n" 142 | " input is the input array\n" 143 | " indexes is the output array which will be filled with the indexes of each value\n" 144 | " min is the minimum value of input\n" 145 | " max is the minimum value of input\n" 146 | " ValLU is the lookup array to index into indexes and CurrentIdx\n" 147 | " CurrentIdx has the current index\n" 148 | " \n"}, 149 | {NULL} /* Sentinel */ 150 | }; 151 | 152 | #if PY_MAJOR_VERSION >= 3 153 | 154 | static int valueIndexes_traverse(PyObject *m, visitproc visit, void *arg) 155 | { 156 | Py_VISIT(GETSTATE(m)->error); 157 | return 0; 158 | } 159 | 160 | static int valueIndexes_clear(PyObject *m) 161 | { 162 | Py_CLEAR(GETSTATE(m)->error); 163 | return 0; 164 | } 165 | 166 | static struct PyModuleDef moduledef = { 167 | PyModuleDef_HEAD_INIT, 168 | "_valueindexes", 169 | NULL, 170 | sizeof(struct ValueIndexesState), 171 | ValueIndexesMethods, 172 | NULL, 173 | valueIndexes_traverse, 174 | valueIndexes_clear, 175 | NULL 176 | }; 177 | 178 | #define INITERROR return NULL 179 | 180 | PyMODINIT_FUNC 181 | PyInit__valueindexes(void) 182 | 183 | #else 184 | #define INITERROR return 185 | 186 | PyMODINIT_FUNC 187 | init_valueindexes(void) 188 | #endif 189 | { 190 | PyObject *pModule; 191 | struct ValueIndexesState *state; 192 | 193 | /* initialize the numpy stuff */ 194 | import_array(); 195 | 196 | #if PY_MAJOR_VERSION >= 3 197 | pModule = PyModule_Create(&moduledef); 198 | #else 199 | pModule = Py_InitModule("_valueindexes", ValueIndexesMethods); 200 | #endif 201 | if( pModule == NULL ) 202 | INITERROR; 203 | 204 | state = GETSTATE(pModule); 205 | 206 | /* Create and add our exception type */ 207 | state->error = PyErr_NewException("_valueindexes.error", NULL, NULL); 208 | if( state->error == NULL ) 209 | { 210 | Py_DECREF(pModule); 211 | INITERROR; 212 | } 213 | 214 | #if PY_MAJOR_VERSION >= 3 215 | return pModule; 216 | #endif 217 | } 218 | 219 | 220 | -------------------------------------------------------------------------------- /doc/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = build 9 | 10 | # User-friendly check for sphinx-build 11 | ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) 12 | $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) 13 | endif 14 | 15 | # Internal variables. 16 | PAPEROPT_a4 = -D latex_paper_size=a4 17 | PAPEROPT_letter = -D latex_paper_size=letter 18 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source 19 | # the i18n builder cannot share the environment and doctrees with the others 20 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source 21 | 22 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext 23 | 24 | help: 25 | @echo "Please use \`make ' where is one of" 26 | @echo " html to make standalone HTML files" 27 | @echo " dirhtml to make HTML files named index.html in directories" 28 | @echo " singlehtml to make a single large HTML file" 29 | @echo " pickle to make pickle files" 30 | @echo " json to make JSON files" 31 | @echo " htmlhelp to make HTML files and a HTML help project" 32 | @echo " qthelp to make HTML files and a qthelp project" 33 | @echo " devhelp to make HTML files and a Devhelp project" 34 | @echo " epub to make an epub" 35 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 36 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 37 | @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" 38 | @echo " text to make text files" 39 | @echo " man to make manual pages" 40 | @echo " texinfo to make Texinfo files" 41 | @echo " info to make Texinfo files and run them through makeinfo" 42 | @echo " gettext to make PO message catalogs" 43 | @echo " changes to make an overview of all changed/added/deprecated items" 44 | @echo " xml to make Docutils-native XML files" 45 | @echo " pseudoxml to make pseudoxml-XML files for display purposes" 46 | @echo " linkcheck to check all external links for integrity" 47 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 48 | 49 | clean: 50 | rm -rf $(BUILDDIR)/* 51 | 52 | html: 53 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 54 | @echo 55 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 56 | 57 | dirhtml: 58 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 59 | @echo 60 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 61 | 62 | singlehtml: 63 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 64 | @echo 65 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 66 | 67 | pickle: 68 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 69 | @echo 70 | @echo "Build finished; now you can process the pickle files." 71 | 72 | json: 73 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 74 | @echo 75 | @echo "Build finished; now you can process the JSON files." 76 | 77 | htmlhelp: 78 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 79 | @echo 80 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 81 | ".hhp project file in $(BUILDDIR)/htmlhelp." 82 | 83 | qthelp: 84 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 85 | @echo 86 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 87 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 88 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/RIOS.qhcp" 89 | @echo "To view the help file:" 90 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/RIOS.qhc" 91 | 92 | devhelp: 93 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 94 | @echo 95 | @echo "Build finished." 96 | @echo "To view the help file:" 97 | @echo "# mkdir -p $$HOME/.local/share/devhelp/RIOS" 98 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/RIOS" 99 | @echo "# devhelp" 100 | 101 | epub: 102 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 103 | @echo 104 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 105 | 106 | latex: 107 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 108 | @echo 109 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 110 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 111 | "(use \`make latexpdf' here to do that automatically)." 112 | 113 | latexpdf: 114 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 115 | @echo "Running LaTeX files through pdflatex..." 116 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 117 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 118 | 119 | latexpdfja: 120 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 121 | @echo "Running LaTeX files through platex and dvipdfmx..." 122 | $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja 123 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 124 | 125 | text: 126 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 127 | @echo 128 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 129 | 130 | man: 131 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 132 | @echo 133 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 134 | 135 | texinfo: 136 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 137 | @echo 138 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 139 | @echo "Run \`make' in that directory to run these through makeinfo" \ 140 | "(use \`make info' here to do that automatically)." 141 | 142 | info: 143 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 144 | @echo "Running Texinfo files through makeinfo..." 145 | make -C $(BUILDDIR)/texinfo info 146 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 147 | 148 | gettext: 149 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 150 | @echo 151 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 152 | 153 | changes: 154 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 155 | @echo 156 | @echo "The overview file is in $(BUILDDIR)/changes." 157 | 158 | linkcheck: 159 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 160 | @echo 161 | @echo "Link check complete; look for any errors in the above output " \ 162 | "or in $(BUILDDIR)/linkcheck/output.txt." 163 | 164 | doctest: 165 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 166 | @echo "Testing of doctests in the sources finished, look at the " \ 167 | "results in $(BUILDDIR)/doctest/output.txt." 168 | 169 | xml: 170 | $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml 171 | @echo 172 | @echo "Build finished. The XML files are in $(BUILDDIR)/xml." 173 | 174 | pseudoxml: 175 | $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml 176 | @echo 177 | @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." 178 | -------------------------------------------------------------------------------- /doc/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | REM Command file for Sphinx documentation 4 | 5 | if "%SPHINXBUILD%" == "" ( 6 | set SPHINXBUILD=sphinx-build 7 | ) 8 | set BUILDDIR=build 9 | set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% source 10 | set I18NSPHINXOPTS=%SPHINXOPTS% source 11 | if NOT "%PAPER%" == "" ( 12 | set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% 13 | set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% 14 | ) 15 | 16 | if "%1" == "" goto help 17 | 18 | if "%1" == "help" ( 19 | :help 20 | echo.Please use `make ^` where ^ is one of 21 | echo. html to make standalone HTML files 22 | echo. dirhtml to make HTML files named index.html in directories 23 | echo. singlehtml to make a single large HTML file 24 | echo. pickle to make pickle files 25 | echo. json to make JSON files 26 | echo. htmlhelp to make HTML files and a HTML help project 27 | echo. qthelp to make HTML files and a qthelp project 28 | echo. devhelp to make HTML files and a Devhelp project 29 | echo. epub to make an epub 30 | echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter 31 | echo. text to make text files 32 | echo. man to make manual pages 33 | echo. texinfo to make Texinfo files 34 | echo. gettext to make PO message catalogs 35 | echo. changes to make an overview over all changed/added/deprecated items 36 | echo. xml to make Docutils-native XML files 37 | echo. pseudoxml to make pseudoxml-XML files for display purposes 38 | echo. linkcheck to check all external links for integrity 39 | echo. doctest to run all doctests embedded in the documentation if enabled 40 | goto end 41 | ) 42 | 43 | if "%1" == "clean" ( 44 | for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i 45 | del /q /s %BUILDDIR%\* 46 | goto end 47 | ) 48 | 49 | 50 | %SPHINXBUILD% 2> nul 51 | if errorlevel 9009 ( 52 | echo. 53 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 54 | echo.installed, then set the SPHINXBUILD environment variable to point 55 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 56 | echo.may add the Sphinx directory to PATH. 57 | echo. 58 | echo.If you don't have Sphinx installed, grab it from 59 | echo.http://sphinx-doc.org/ 60 | exit /b 1 61 | ) 62 | 63 | if "%1" == "html" ( 64 | %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html 65 | if errorlevel 1 exit /b 1 66 | echo. 67 | echo.Build finished. The HTML pages are in %BUILDDIR%/html. 68 | goto end 69 | ) 70 | 71 | if "%1" == "dirhtml" ( 72 | %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml 73 | if errorlevel 1 exit /b 1 74 | echo. 75 | echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. 76 | goto end 77 | ) 78 | 79 | if "%1" == "singlehtml" ( 80 | %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml 81 | if errorlevel 1 exit /b 1 82 | echo. 83 | echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. 84 | goto end 85 | ) 86 | 87 | if "%1" == "pickle" ( 88 | %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle 89 | if errorlevel 1 exit /b 1 90 | echo. 91 | echo.Build finished; now you can process the pickle files. 92 | goto end 93 | ) 94 | 95 | if "%1" == "json" ( 96 | %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json 97 | if errorlevel 1 exit /b 1 98 | echo. 99 | echo.Build finished; now you can process the JSON files. 100 | goto end 101 | ) 102 | 103 | if "%1" == "htmlhelp" ( 104 | %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp 105 | if errorlevel 1 exit /b 1 106 | echo. 107 | echo.Build finished; now you can run HTML Help Workshop with the ^ 108 | .hhp project file in %BUILDDIR%/htmlhelp. 109 | goto end 110 | ) 111 | 112 | if "%1" == "qthelp" ( 113 | %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp 114 | if errorlevel 1 exit /b 1 115 | echo. 116 | echo.Build finished; now you can run "qcollectiongenerator" with the ^ 117 | .qhcp project file in %BUILDDIR%/qthelp, like this: 118 | echo.^> qcollectiongenerator %BUILDDIR%\qthelp\RIOS.qhcp 119 | echo.To view the help file: 120 | echo.^> assistant -collectionFile %BUILDDIR%\qthelp\RIOS.ghc 121 | goto end 122 | ) 123 | 124 | if "%1" == "devhelp" ( 125 | %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp 126 | if errorlevel 1 exit /b 1 127 | echo. 128 | echo.Build finished. 129 | goto end 130 | ) 131 | 132 | if "%1" == "epub" ( 133 | %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub 134 | if errorlevel 1 exit /b 1 135 | echo. 136 | echo.Build finished. The epub file is in %BUILDDIR%/epub. 137 | goto end 138 | ) 139 | 140 | if "%1" == "latex" ( 141 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 142 | if errorlevel 1 exit /b 1 143 | echo. 144 | echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. 145 | goto end 146 | ) 147 | 148 | if "%1" == "latexpdf" ( 149 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 150 | cd %BUILDDIR%/latex 151 | make all-pdf 152 | cd %BUILDDIR%/.. 153 | echo. 154 | echo.Build finished; the PDF files are in %BUILDDIR%/latex. 155 | goto end 156 | ) 157 | 158 | if "%1" == "latexpdfja" ( 159 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 160 | cd %BUILDDIR%/latex 161 | make all-pdf-ja 162 | cd %BUILDDIR%/.. 163 | echo. 164 | echo.Build finished; the PDF files are in %BUILDDIR%/latex. 165 | goto end 166 | ) 167 | 168 | if "%1" == "text" ( 169 | %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text 170 | if errorlevel 1 exit /b 1 171 | echo. 172 | echo.Build finished. The text files are in %BUILDDIR%/text. 173 | goto end 174 | ) 175 | 176 | if "%1" == "man" ( 177 | %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man 178 | if errorlevel 1 exit /b 1 179 | echo. 180 | echo.Build finished. The manual pages are in %BUILDDIR%/man. 181 | goto end 182 | ) 183 | 184 | if "%1" == "texinfo" ( 185 | %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo 186 | if errorlevel 1 exit /b 1 187 | echo. 188 | echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. 189 | goto end 190 | ) 191 | 192 | if "%1" == "gettext" ( 193 | %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale 194 | if errorlevel 1 exit /b 1 195 | echo. 196 | echo.Build finished. The message catalogs are in %BUILDDIR%/locale. 197 | goto end 198 | ) 199 | 200 | if "%1" == "changes" ( 201 | %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes 202 | if errorlevel 1 exit /b 1 203 | echo. 204 | echo.The overview file is in %BUILDDIR%/changes. 205 | goto end 206 | ) 207 | 208 | if "%1" == "linkcheck" ( 209 | %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck 210 | if errorlevel 1 exit /b 1 211 | echo. 212 | echo.Link check complete; look for any errors in the above output ^ 213 | or in %BUILDDIR%/linkcheck/output.txt. 214 | goto end 215 | ) 216 | 217 | if "%1" == "doctest" ( 218 | %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest 219 | if errorlevel 1 exit /b 1 220 | echo. 221 | echo.Testing of doctests in the sources finished, look at the ^ 222 | results in %BUILDDIR%/doctest/output.txt. 223 | goto end 224 | ) 225 | 226 | if "%1" == "xml" ( 227 | %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml 228 | if errorlevel 1 exit /b 1 229 | echo. 230 | echo.Build finished. The XML files are in %BUILDDIR%/xml. 231 | goto end 232 | ) 233 | 234 | if "%1" == "pseudoxml" ( 235 | %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml 236 | if errorlevel 1 exit /b 1 237 | echo. 238 | echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml. 239 | goto end 240 | ) 241 | 242 | :end 243 | -------------------------------------------------------------------------------- /doc/requirements.txt: -------------------------------------------------------------------------------- 1 | sphinx==7 2 | -------------------------------------------------------------------------------- /doc/source/conf.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | # 4 | # RIOS documentation build configuration file, created by 5 | # sphinx-quickstart on Tue Sep 23 19:01:36 2014. 6 | # 7 | # This file is execfile()d with the current directory set to its 8 | # containing dir. 9 | # 10 | # Note that not all possible configuration values are present in this 11 | # autogenerated file. 12 | # 13 | # All configuration values have a default; values that are commented out 14 | # serve to show the default. 15 | 16 | import sys 17 | import os 18 | import importlib 19 | 20 | sys.path.insert(0, os.path.abspath('../..')) 21 | # for version info 22 | import fmask # noqa: E402 23 | 24 | # List of modules we will mock, but only if they are not genuinely present 25 | MOCK_MODULES = ['numpy', 'scipy', 'scipy.ndimage', 'scipy.constants', 26 | 'scipy.stats', 'osgeo', 'gdal', 'osgeo.gdal', 'rios', 'fmask._fillminima', 27 | 'fmask._valueindexes'] 28 | # Check which ones are not present, and add them to the mock list 29 | autodoc_mock_imports = [] 30 | for mod_name in MOCK_MODULES: 31 | try: 32 | importlib.import_module(mod_name) 33 | except ImportError: 34 | autodoc_mock_imports.append(mod_name) 35 | 36 | # If extensions (or modules to document with autodoc) are in another directory, 37 | # add these directories to sys.path here. If the directory is relative to the 38 | # documentation root, use os.path.abspath to make it absolute, like shown here. 39 | # sys.path.insert(0, os.path.abspath('.')) 40 | 41 | # -- General configuration ------------------------------------------------ 42 | 43 | # If your documentation needs a minimal Sphinx version, state it here. 44 | # needs_sphinx = '1.0' 45 | 46 | # Add any Sphinx extension module names here, as strings. They can be 47 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 48 | # ones. 49 | extensions = [ 50 | 'sphinx.ext.autodoc', 51 | 'sphinx.ext.viewcode', 52 | ] 53 | 54 | # Add any paths that contain templates here, relative to this directory. 55 | templates_path = ['_templates'] 56 | 57 | # The suffix of source filenames. 58 | source_suffix = '.rst' 59 | 60 | # The encoding of source files. 61 | # source_encoding = 'utf-8-sig' 62 | 63 | # The master toctree document. 64 | master_doc = 'index' 65 | 66 | # General information about the project. 67 | project = 'pythonfmask' 68 | copyright = '2015, Neil Flood & Sam Gillingham' 69 | 70 | # The version info for the project you're documenting, acts as replacement for 71 | # |version| and |release|, also used in various other places throughout the 72 | # built documents. 73 | # 74 | # The short X.Y version. 75 | version = fmask.__version__ 76 | # The full version, including alpha/beta/rc tags. 77 | release = fmask.__version__ 78 | 79 | # The language for content autogenerated by Sphinx. Refer to documentation 80 | # for a list of supported languages. 81 | # language = None 82 | 83 | # There are two options for replacing |today|: either, you set today to some 84 | # non-false value, then it is used: 85 | # today = '' 86 | # Else, today_fmt is used as the format for a strftime call. 87 | # today_fmt = '%B %d, %Y' 88 | 89 | # List of patterns, relative to source directory, that match files and 90 | # directories to ignore when looking for source files. 91 | exclude_patterns = [] 92 | 93 | # The reST default role (used for this markup: `text`) to use for all 94 | # documents. 95 | # default_role = None 96 | 97 | # If true, '()' will be appended to :func: etc. cross-reference text. 98 | # add_function_parentheses = True 99 | 100 | # If true, the current module name will be prepended to all description 101 | # unit titles (such as .. function::). 102 | # add_module_names = True 103 | 104 | # If true, sectionauthor and moduleauthor directives will be shown in the 105 | # output. They are ignored by default. 106 | # show_authors = False 107 | 108 | # The name of the Pygments (syntax highlighting) style to use. 109 | pygments_style = 'sphinx' 110 | 111 | # A list of ignored prefixes for module index sorting. 112 | # modindex_common_prefix = [] 113 | 114 | # If true, keep warnings as "system message" paragraphs in the built documents. 115 | # keep_warnings = False 116 | 117 | 118 | # -- Options for HTML output ---------------------------------------------- 119 | 120 | # The theme to use for HTML and HTML Help pages. See the documentation for 121 | # a list of builtin themes. 122 | html_theme = 'classic' 123 | 124 | # Theme options are theme-specific and customize the look and feel of a theme 125 | # further. For a list of options available for each theme, see the 126 | # documentation. 127 | # html_theme_options = {} 128 | 129 | # Add any paths that contain custom themes here, relative to this directory. 130 | # html_theme_path = [] 131 | 132 | # The name for this set of Sphinx documents. If None, it defaults to 133 | # " v documentation". 134 | # html_title = None 135 | 136 | # A shorter title for the navigation bar. Default is the same as html_title. 137 | # html_short_title = None 138 | 139 | # The name of an image file (relative to this directory) to place at the top 140 | # of the sidebar. 141 | # html_logo = None 142 | 143 | # The name of an image file (within the static path) to use as favicon of the 144 | # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 145 | # pixels large. 146 | # html_favicon = None 147 | 148 | # Add any paths that contain custom static files (such as style sheets) here, 149 | # relative to this directory. They are copied after the builtin static files, 150 | # so a file named "default.css" will overwrite the builtin "default.css". 151 | html_static_path = ['_static'] 152 | 153 | # Add any extra paths that contain custom files (such as robots.txt or 154 | # .htaccess) here, relative to this directory. These files are copied 155 | # directly to the root of the documentation. 156 | # html_extra_path = [] 157 | 158 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, 159 | # using the given strftime format. 160 | # html_last_updated_fmt = '%b %d, %Y' 161 | 162 | # If true, SmartyPants will be used to convert quotes and dashes to 163 | # typographically correct entities. 164 | # html_use_smartypants = True 165 | 166 | # Custom sidebar templates, maps document names to template names. 167 | # html_sidebars = {} 168 | 169 | # Additional templates that should be rendered to pages, maps page names to 170 | # template names. 171 | # html_additional_pages = {} 172 | 173 | # If false, no module index is generated. 174 | # html_domain_indices = True 175 | 176 | # If false, no index is generated. 177 | # html_use_index = True 178 | 179 | # If true, the index is split into individual pages for each letter. 180 | # html_split_index = False 181 | 182 | # If true, links to the reST sources are added to the pages. 183 | # html_show_sourcelink = True 184 | 185 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 186 | # html_show_sphinx = True 187 | 188 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. 189 | # html_show_copyright = True 190 | 191 | # If true, an OpenSearch description file will be output, and all pages will 192 | # contain a tag referring to it. The value of this option must be the 193 | # base URL from which the finished HTML is served. 194 | # html_use_opensearch = '' 195 | 196 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 197 | # html_file_suffix = None 198 | 199 | # Output file base name for HTML help builder. 200 | htmlhelp_basename = 'Fmaskdoc' 201 | 202 | 203 | # -- Options for LaTeX output --------------------------------------------- 204 | 205 | latex_elements = { 206 | # The paper size ('letterpaper' or 'a4paper'). 207 | # 'papersize': 'letterpaper', 208 | 209 | # The font size ('10pt', '11pt' or '12pt'). 210 | # 'pointsize': '10pt', 211 | 212 | # Additional stuff for the LaTeX preamble. 213 | # 'preamble': '', 214 | } 215 | 216 | # Grouping the document tree into LaTeX files. List of tuples 217 | # (source start file, target name, title, 218 | # author, documentclass [howto, manual, or own class]). 219 | latex_documents = [ 220 | ('index', 'fmask.tex', 'PythonFmask Documentation', 221 | 'Neil Flood, Sam Gillingham', 'manual'), 222 | ] 223 | 224 | # The name of an image file (relative to this directory) to place at the top of 225 | # the title page. 226 | # latex_logo = None 227 | 228 | # For "manual" documents, if this is true, then toplevel headings are parts, 229 | # not chapters. 230 | # latex_use_parts = False 231 | 232 | # If true, show page references after internal links. 233 | # latex_show_pagerefs = False 234 | 235 | # If true, show URL addresses after external links. 236 | # latex_show_urls = False 237 | 238 | # Documents to append as an appendix to all manuals. 239 | # latex_appendices = [] 240 | 241 | # If false, no module index is generated. 242 | # latex_domain_indices = True 243 | 244 | 245 | # -- Options for manual page output --------------------------------------- 246 | 247 | # One entry per manual page. List of tuples 248 | # (source start file, name, description, authors, manual section). 249 | man_pages = [ 250 | ('index', 'fmask', 'PythonFmask Documentation', 251 | ['Neil Flood, Sam Gillingham'], 1) 252 | ] 253 | 254 | # If true, show URL addresses after external links. 255 | # man_show_urls = False 256 | 257 | 258 | # -- Options for Texinfo output ------------------------------------------- 259 | 260 | # Grouping the document tree into Texinfo files. List of tuples 261 | # (source start file, target name, title, author, 262 | # dir menu entry, description, category) 263 | texinfo_documents = [ 264 | ('index', 'fmask', 'PythonFmask Documentation', 265 | 'Neil Flood, Sam Gillingham', 'fmask', 'One line description of project.', 266 | 'Miscellaneous'), 267 | ] 268 | 269 | # Documents to append as an appendix to all manuals. 270 | # texinfo_appendices = [] 271 | 272 | # If false, no module index is generated. 273 | # texinfo_domain_indices = True 274 | 275 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 276 | # texinfo_show_urls = 'footnote' 277 | 278 | # If true, do not generate a @detailmenu in the "Top" node's menu. 279 | # texinfo_no_detailmenu = False 280 | autodoc_member_order = 'groupwise' 281 | -------------------------------------------------------------------------------- /doc/source/fmask_config.rst: -------------------------------------------------------------------------------- 1 | config 2 | ========= 3 | .. automodule:: fmask.config 4 | :members: 5 | :undoc-members: 6 | 7 | * :ref:`genindex` 8 | * :ref:`modindex` 9 | * :ref:`search` 10 | -------------------------------------------------------------------------------- /doc/source/fmask_fillminima.rst: -------------------------------------------------------------------------------- 1 | fillminima 2 | ========== 3 | .. automodule:: fmask.fillminima 4 | :members: 5 | :undoc-members: 6 | 7 | * :ref:`genindex` 8 | * :ref:`modindex` 9 | * :ref:`search` 10 | -------------------------------------------------------------------------------- /doc/source/fmask_fmask.rst: -------------------------------------------------------------------------------- 1 | fmask 2 | ========= 3 | .. automodule:: fmask.fmask 4 | :members: 5 | :undoc-members: 6 | 7 | * :ref:`genindex` 8 | * :ref:`modindex` 9 | * :ref:`search` 10 | -------------------------------------------------------------------------------- /doc/source/fmask_fmaskerrors.rst: -------------------------------------------------------------------------------- 1 | fmaskerrors 2 | =========== 3 | .. automodule:: fmask.fmaskerrors 4 | :members: 5 | :undoc-members: 6 | 7 | * :ref:`genindex` 8 | * :ref:`modindex` 9 | * :ref:`search` 10 | -------------------------------------------------------------------------------- /doc/source/fmask_landsatTOA.rst: -------------------------------------------------------------------------------- 1 | landsatTOA 2 | ========== 3 | .. automodule:: fmask.landsatTOA 4 | :members: 5 | :undoc-members: 6 | 7 | * :ref:`genindex` 8 | * :ref:`modindex` 9 | * :ref:`search` 10 | -------------------------------------------------------------------------------- /doc/source/fmask_landsatangles.rst: -------------------------------------------------------------------------------- 1 | landsatangles 2 | ============= 3 | .. automodule:: fmask.landsatangles 4 | :members: 5 | :undoc-members: 6 | 7 | * :ref:`genindex` 8 | * :ref:`modindex` 9 | * :ref:`search` 10 | -------------------------------------------------------------------------------- /doc/source/fmask_saturationcheck.rst: -------------------------------------------------------------------------------- 1 | saturationcheck 2 | =============== 3 | .. automodule:: fmask.saturationcheck 4 | :members: 5 | :undoc-members: 6 | 7 | * :ref:`genindex` 8 | * :ref:`modindex` 9 | * :ref:`search` 10 | -------------------------------------------------------------------------------- /doc/source/fmask_valueindexes.rst: -------------------------------------------------------------------------------- 1 | valueindexes 2 | ============ 3 | .. automodule:: fmask.valueindexes 4 | :members: 5 | :undoc-members: 6 | 7 | * :ref:`genindex` 8 | * :ref:`modindex` 9 | * :ref:`search` 10 | -------------------------------------------------------------------------------- /doc/source/fmask_zerocheck.rst: -------------------------------------------------------------------------------- 1 | zerocheck 2 | ========= 3 | .. automodule:: fmask.zerocheck 4 | :members: 5 | :undoc-members: 6 | 7 | * :ref:`genindex` 8 | * :ref:`modindex` 9 | * :ref:`search` 10 | -------------------------------------------------------------------------------- /doc/source/index.rst: -------------------------------------------------------------------------------- 1 | .. _contents: 2 | 3 | Python Fmask 4 | ======================================================== 5 | 6 | Introduction 7 | ------------ 8 | 9 | A set of command line utilities and Python modules that implement 10 | the 'fmask' algorithm as published in: 11 | 12 | Zhu, Z. and Woodcock, C.E. (2012). 13 | Object-based cloud and cloud shadow detection in Landsat imagery 14 | Remote Sensing of Environment 118 (2012) 83-94. 15 | 16 | and 17 | 18 | Zhu, Z., Wang, S. and Woodcock, C.E. (2015). 19 | Improvement and expansion of the Fmask algorithm: cloud, cloud 20 | shadow, and snow detection for Landsats 4-7, 8, and Sentinel 2 images 21 | Remote Sensing of Environment 159 (2015) 269-277. 22 | 23 | Also includes optional extension for Sentinel-2 from 24 | Frantz, D., Hass, E., Uhl, A., Stoffels, J., & Hill, J. (2018). 25 | Improvement of the Fmask algorithm for Sentinel-2 images: Separating clouds 26 | from bright surfaces based on parallax effects. 27 | Remote Sensing of Environment 215, 471-481. 28 | 29 | Installation requires `Python `_, `numpy `_, `scipy `_, 30 | `GDAL `_ and `RIOS `_ and the ability to compile C extensions for Python. 31 | It is licensed under GPL 3. 32 | 33 | Originally developed by Neil Flood at `DSITI `_ 34 | and additional work funded by `Landcare Research `_. 35 | 36 | Later Fmask4 Methods 37 | -------------------- 38 | A later publication by Qiu et al, 2019, suggests a number of further changes to Fmask, and 39 | gives this newer version the name of Fmask4. We have done some testing of these suggested 40 | methods, using Landsat and Sentinel-2 data over large areas of Australia. We found that 41 | overall, the losses outweighed the benefits, and so are unsure whether to 42 | implement these changes. Currently we have chosen not to. 43 | 44 | Qiu, S., Zhu, Z., & He, B. (2019). Fmask 4.0: Improved cloud and cloud shadow 45 | detection in Landsats 4-8 and Sentinel-2 imagery. Remote Sensing of Environment, 231, 111205. 46 | 47 | 48 | 49 | Disclaimer and Acknowledgement 50 | ------------------------------ 51 | This Python implementation has followed the work of the authors cited above, and we 52 | offer our thanks for their work. None of them were involved in the creation of 53 | this Python implementation, and all errors made are our own responsibility. 54 | 55 | 56 | Philosophy 57 | ---------- 58 | This package implements the Fmask algorithm as a Python module. It is intended that this 59 | can be wrapped in a variety of main programs which can handle the local details of how 60 | the image files are named and organised, and is intended to provide maximum flexibility. It 61 | should not be tied to expecting the imagery to be layed out in a particular manner. 62 | 63 | This modular design also simplifies the use of the same core algorithm on either Landsat and 64 | Sentinel imagery. The wrapper programs take care of differences in file organisation and 65 | metadata formats, while the core algorithm is the same for both. 66 | 67 | However, we have also supplied some example wrapper scripts, based around the image organisation 68 | as supplied by the usual distributors of the imagery. In the case of Landsat, we have supplied 69 | main programs which can cope with the data as it comes from USGS, while in the case of Sentinel-2 70 | we have supplied wrappers to deal with the data as supplied by ESA. 71 | 72 | It is expected that some users will use these directly, while larger organisations will wish to 73 | create their own wrappers specific to their own file naming and layout conventions. 74 | 75 | The output from the core algorithm module is a single thematic raster, with integer 76 | codes representing null, clear, cloud, shadow, snow, water respectively. 77 | 78 | The examples shown below use the given example wrappers. 79 | 80 | Command Line Examples 81 | --------------------- 82 | 83 | All the commandline programs given use argparse to handle commandline arguments, and hence will 84 | respond sensibly to the -h option by printing their own help. 85 | Some have options to modify their behaviour. 86 | 87 | Please note that the output format used is defined by `RIOS `_. This defaults to HFA (.img). 88 | See `RIOS documentation `_ 89 | for more information and how to change this using the environment variable $RIOS_DFLT_DRIVER. 90 | 91 | USGS Landsat 92 | ^^^^^^^^^^^^ 93 | 94 | **Update:** Since the USGS released their Collection-1 data set (completed globally in 2017), 95 | they now distribute cloud, shadow 96 | and snow masks included in their QA layer. These are calculated using CFMask, which should, 97 | in principle, be equivalent to the results of this code. Therefore, when processing USGS 98 | Collection-1 data, users may prefer the USGS-supplied masks. 99 | See `USGS QA Layer `_. 100 | 101 | The command line scripts supplied can process an untarred USGS Landsat scene. 102 | Here is an example of how to do this. This command will take a given scene directory, 103 | find the right images, and create an output file called cloud.img:: 104 | 105 | fmask_usgsLandsatStacked -o cloud.img --scenedir LC08_L1TP_150033_20150413_20170410_01_T1 106 | 107 | If the thermal band is empty (for Landsat-8 with the SSM anomaly, after 2015-11-01) then it 108 | is ignored gracefully. 109 | 110 | There are command line options to modify many aspects of the algorithm's behaviour. 111 | 112 | There are four options which are now obsolete, for manually specifying a pre-stacked file 113 | of reflectance bands, thermal bands, a saturation mask and the image of angles. 114 | These should be considered obsolete, and are 115 | replaced with the --scenedir option, which takes care of all internally. 116 | 117 | Sentinel2 118 | ^^^^^^^^^ 119 | 120 | The command line scripts supplied can process a Sentinel2 Level C granule from the image directory. 121 | Here is an example of how to do this. This example works at 20m resolution, but the 122 | recipe can be varied as required. Be warned, processing at 10m resolution would be considerably 123 | slower, and is unlikely to be any more accurate. 124 | 125 | This command will take a given .SAFE directory, find the right images, and create an 126 | output file called cloud.img:: 127 | 128 | fmask_sentinel2Stacked -o cloud.img --safedir S2B_MSIL1C_20180918T235239_N0206_R130_T56JNQ_20180919T011001.SAFE 129 | 130 | When working with the old ESA zipfile format, which packed multiple tiles into a single SAFE-format 131 | zipfile, this approach will not work, as it won't know which tile to process. So, instead, use 132 | the option to specify the granule directory, as follows:: 133 | 134 | fmask_sentinel2Stacked -o cloud.img --granuledir S2A_OPER_PRD_MSIL1C_PDMC_20160111T072442_R030_V20160111T000425_20160111T000425.SAFE/GRANULE/S2A_OPER_MSI_L1C_TL_SGS__20160111T051031_A002887_T56JNQ_N02.01 135 | 136 | This would also work on a new-format directory, but specifying the top .SAFE directory is easier. 137 | 138 | There are command line options to modify many aspects of the algorithm's behaviour. 139 | 140 | There are two options which are now obsolete, for manually specifying a pre-stacked file 141 | of reflectance bands, and the image of angles. These should be considered obsolete, and are 142 | replaced with the --safedir or --granuledir option, which take care of all internally. 143 | 144 | Re-wrapping and Re-configuring 145 | ------------------------------ 146 | To build a different set of wrappers, and configure things differently, the default 147 | wrappers are a good place to start. The configuration is mainly handled by the 148 | :class:`fmask.config.FmaskConfig` class. For example, one would 149 | call :func:`fmask.config.FmaskConfig.setReflectiveBand` to change which layer of the stack 150 | corresponds to which wavelength band. 151 | 152 | Downloads 153 | --------- 154 | This code relies heavily on GDAL, numpy, scipy and RIOS. These packages must be 155 | installed for anything to work. For this reason, we recommend that you install using 156 | a package manager, which will build a complete environment with everything required. 157 | 158 | Pre-built binary `Conda `_ packages are available 159 | under the 'conda-forge' channel. Once you have installed 160 | `Conda `_, run the following commands on the 161 | command line to install python-fmask:: 162 | 163 | conda config --prepend channels conda-forge 164 | conda config --set channel_priority strict 165 | conda create -n myenv python-fmask 166 | conda activate myenv 167 | 168 | For those using the `Spack package manager `_, 169 | python-fmask is also available as:: 170 | 171 | spack install py-python-fmask 172 | 173 | The source can be downloaded directly as a bundle from 174 | `GitHub `_. 175 | Release notes for each version can be read in :doc:`releasenotes`. To install from source, 176 | read the INSTALL.txt file included inside the source bundle. 177 | 178 | Please note that python-fmask is *not* available from the PyPI repository. This is 179 | because it depends on the GDAL library, which is also not available there, and must 180 | be installed by some other means, such as conda. If one is using conda for GDAL, one 181 | may as well use it for python-fmask, too. While it is technically possible to bundle 182 | the GDAL binaries into a PyPI distribution, this carries grave risks of version 183 | conflicts if any other package does the same thing, and is best avoided. 184 | 185 | Applications that use python-fmask 186 | ---------------------------------- 187 | 188 | * `Cloud Masking `_: It is a Qgis plugin for cloud masking 189 | the Landsat (4, 5, 7 and 8) products using different process and filters such as Fmask, Blue Band, 190 | Cloud QA, Aerosol and Pixel QA. 191 | 192 | Issues 193 | ------ 194 | 195 | Please log bugs encountered with the `Issue Tracker `_. 196 | 197 | 198 | Python developer documentation 199 | ------------------------------ 200 | 201 | .. toctree:: 202 | :maxdepth: 1 203 | 204 | Running the fmask algorithm 205 | Configuring the fmask run 206 | Creating Top of Atmosphere rasters for Landsat 207 | fmask_saturationcheck 208 | fmask_landsatangles 209 | fmask_zerocheck 210 | fmask_fillminima 211 | fmask_valueindexes 212 | fmask_fmaskerrors 213 | 214 | * :ref:`modindex` 215 | * :ref:`search` 216 | 217 | .. codeauthor:: Neil Flood & Sam Gillingham 218 | -------------------------------------------------------------------------------- /doc/source/releasenotes.rst: -------------------------------------------------------------------------------- 1 | Release Notes 2 | ============= 3 | 4 | Version 0.5.10 (2024-06-19) 5 | --------------------------- 6 | 7 | Bug Fixes 8 | * Cope with numpy-2.0 changes (https://github.com/ubarsc/python-fmask/pull/79) 9 | 10 | Version 0.5.9 (2024-03-13) 11 | -------------------------- 12 | 13 | Bug Fixes: 14 | * Convert from bool to uint8 before writing a mask as latest RIOS no 15 | longer supports writing bool arrays 16 | 17 | Enhancements: 18 | * Add support for entrypoints using "pip install". These entrypoints 19 | cannot have an extension so to reduce confusion the documentation 20 | has been updated to cover using these new entrypoints (no '.py'). 21 | The conda package will be similarly updated. The old scripts remain 22 | as they are, although they now emit a warning. 23 | 24 | Version 0.5.8 (2022-12-22) 25 | -------------------------- 26 | 27 | Bug Fixes 28 | * Cope with numpy-1.24 removal of deprecated type symbols like numpy.bool 29 | * Better handling of gdal exception use/non-use 30 | 31 | Enhancements 32 | * Use gdal.Driver.Delete to remove temporary files in Sentinel-2 cmdline 33 | script 34 | * Use gdal internal function calls to avoid use of os.system() 35 | 36 | 37 | Version 0.5.7 (2022-02-11) 38 | -------------------------- 39 | 40 | Enhancements 41 | * Minor changes to support Landsat-9 from USGS 42 | 43 | Version 0.5.6 (2021-10-19) 44 | -------------------------- 45 | 46 | Enhancements 47 | * Cope with ESA's sudden inclusion of radiometric offsets in their 48 | Sentinel-2 reflectance imagery. Using earlier python-fmask versions 49 | with the new ESA files will result in incorrect answers. 50 | 51 | Version 0.5.5 (2020-09-01) 52 | -------------------------- 53 | 54 | Bug Fixes 55 | * Cope with the axis swapping behavour of GDAL 3.x. 56 | * Display proper error messages if missing gdalwarp or gdal_merge.py. 57 | 58 | Enhancements 59 | * Improve readability of code in masking step. 60 | * Allow sentinel2Stacked to be called directly from Python. 61 | 62 | 63 | Version 0.5.4 (2019-08-06) 64 | -------------------------- 65 | 66 | Bug Fixes 67 | * Removed the "darkness" test, suggested by Franz (2015), which had inadvertently been 68 | added from a test branch into version 0.5.0. This test aims to remove a certain 69 | type of false positive cloud, but larger tests suggest that it removes more true 70 | positives, and so should not have been included. 71 | * Added new logic to cope with cases of nearly 100% cloud cover. In cases where only 72 | very small amounts of land are visible, the dynamic threshold for the "clear land 73 | probability" is too contaminated to be usable. If less than 3% of the image is 74 | clear land, then a fallback threshold is used instead. This avoids images which 75 | are entirely covered in cloud from being classed as almost entirely cloud-free. 76 | This particularly affected Sentinel-2, where the thermal is not available to 77 | catch these cases anyway. 78 | 79 | Documentation 80 | * Added a disclaimer to the front page, emphasizing that all errors are ours, and 81 | the authors of the original papers bear no responsibility. 82 | * Added a note to the front page that we have done some testing of the proposed 83 | Fmask4 changes from Qiu et al, 2019, and are unsure whether they help or not, 84 | so have not implemented them. 85 | 86 | Version 0.5.3 (2019-01-15) 87 | -------------------------- 88 | 89 | Bug Fixes 90 | * Fixed problem with new error checks which broke Landsat-7 case 91 | 92 | Version 0.5.2 (2018-12-12) 93 | -------------------------- 94 | 95 | Bug Fixes 96 | * Fix issue with entry points on Windows 97 | 98 | Enhancements 99 | * Ensure temporary files are removed on Windows 100 | 101 | Version 0.5.1 (2018-11-26) 102 | -------------------------- 103 | 104 | Enhancements 105 | * Added better support for Conda packaging on Windows 106 | * Upgraded license to GPL v3 107 | 108 | Version 0.5.0 (2018-10-18) 109 | -------------------------- 110 | 111 | Enhancements 112 | * For Sentinel-2, added support for parallax test to remove false cloud in bright (typically 113 | urban) areas, as per Frantz (2018). Optional, defaults to off. Thanks to Vincent Schut 114 | for assistance in kicking that across the line. 115 | * For Sentinel-2, added command line switch to directly use the SAFE directory as 116 | provided by ESA, to avoid the need to manually stack up the individual bands externally. 117 | * For Sentinel-2, turn off the (cloud prob > 99%) test, as suggested by Zhu (2015). It is 118 | in effect for Landsat. 119 | 120 | 121 | Version 0.4.5 (2017-07-12) 122 | -------------------------- 123 | 124 | Bug Fixes: 125 | * Handling old formats of USGS MTL files 126 | * Fixes for numpy 1.13 127 | 128 | 129 | Version 0.4.4 (2017-04-04) 130 | -------------------------- 131 | 132 | Enhancements 133 | * Added commandline options to set cloud probability threshold, and the two reflectance 134 | thresholds used for the snow mask. 135 | * Call gdal.UseExceptions(), so that when bad things happen, they are more likely 136 | to be reported accurately by the exception message, and reduce confusion for users. 137 | 138 | 139 | Version 0.4.3 (2016-11-26) 140 | -------------------------- 141 | 142 | Bug Fixes: 143 | * Fix 32 bit builds 144 | * Fix help message for fmask_usgsLandsatStacked.py 145 | 146 | Enhancements 147 | * Helper .bat file for Windows to expand wildcards 148 | * Changes to 'nodata' handling to make processing in parallel possible with RIOS 149 | 150 | 151 | Version 0.4.2 (2016-09-01) 152 | -------------------------- 153 | 154 | Bug Fixes 155 | * Fixed fall-back default values for Landsat brightness temperature equation constants, 156 | as required when processing older USGS files which do not have these present in the MTL file. 157 | * For Sentinel-2 only, added a work-around for the alarming random null pixels which 158 | ESA leave in the cirrus band. This avoids leaving corresponding null pixels in the 159 | resulting output masks. 160 | 161 | 162 | Version 0.4.0 (2016-06-10) 163 | -------------------------- 164 | 165 | Bug fixes 166 | * Proper null mask taken from all reflective bands combined, not just the blue band 167 | * Trap seg-faults in valueindexes C code 168 | * Use null value of 32767 for Landsat TOA image 169 | * Cope when Sentinel-2 metadata only has sensor angles for a subset of bands. 170 | 171 | Enhancements 172 | * Landsat angles code is now in a module, with a main program wrapper, consistent 173 | with the rest of the package 174 | * Added :command:`--cloudbufferdistance`, :command:`--shadowbufferdistance` and 175 | :command:`--mincloudsize` options to 176 | main program wrappers (both Landsat and Sentinel-2) to give user control over these 177 | parameters 178 | 179 | 180 | Version 0.3.0 (2016-03-21) 181 | -------------------------- 182 | 183 | Bug fixes 184 | * Added code for estimating per-pixel Landsat sun and sensor angles, to allow proper 185 | shadow tracking, as per original code 186 | * Full use of Sentinel-2 metadata XML, including per-pixel angles grid 187 | 188 | -------------------------------------------------------------------------------- /fmask/__init__.py: -------------------------------------------------------------------------------- 1 | # This file is part of 'python-fmask' - a cloud masking module 2 | # Copyright (C) 2015 Neil Flood 3 | # 4 | # This program is free software; you can redistribute it and/or 5 | # modify it under the terms of the GNU General Public License 6 | # as published by the Free Software Foundation; either version 3 7 | # of the License, or (at your option) any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU General Public License for more details. 13 | # 14 | # You should have received a copy of the GNU General Public License 15 | # along with this program; if not, write to the Free Software 16 | # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 17 | 18 | __version__ = '0.5.10' 19 | -------------------------------------------------------------------------------- /fmask/cmdline/__init__.py: -------------------------------------------------------------------------------- 1 | # This file is part of 'python-fmask' - a cloud masking module 2 | # Copyright (C) 2015 Neil Flood 3 | # 4 | # This program is free software; you can redistribute it and/or 5 | # modify it under the terms of the GNU General Public License 6 | # as published by the Free Software Foundation; either version 3 7 | # of the License, or (at your option) any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU General Public License for more details. 13 | # 14 | # You should have received a copy of the GNU General Public License 15 | # along with this program; if not, write to the Free Software 16 | # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 17 | 18 | -------------------------------------------------------------------------------- /fmask/cmdline/sentinel2Stacked.py: -------------------------------------------------------------------------------- 1 | """ 2 | Script that takes a stacked Sentinel 2 Level 1C image and runs 3 | fmask on it. 4 | """ 5 | # This file is part of 'python-fmask' - a cloud masking module 6 | # Copyright (C) 2015 Neil Flood 7 | # 8 | # This program is free software; you can redistribute it and/or 9 | # modify it under the terms of the GNU General Public License 10 | # as published by the Free Software Foundation; either version 3 11 | # of the License, or (at your option) any later version. 12 | # 13 | # This program is distributed in the hope that it will be useful, 14 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 15 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 | # GNU General Public License for more details. 17 | # 18 | # You should have received a copy of the GNU General Public License 19 | # along with this program; if not, write to the Free Software 20 | # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 21 | from __future__ import print_function, division 22 | 23 | import sys 24 | import os 25 | import argparse 26 | import tempfile 27 | import glob 28 | 29 | from osgeo import gdal 30 | from osgeo_utils import gdal_merge 31 | 32 | from rios import fileinfo 33 | from rios.imagewriter import DEFAULTDRIVERNAME, dfltDriverOptions 34 | 35 | from fmask import config 36 | from fmask import fmaskerrors 37 | from fmask.cmdline import sentinel2makeAnglesImage 38 | from fmask import fmask 39 | from fmask import sen2meta 40 | 41 | # for GDAL. 42 | CMDLINECREATIONOPTIONS = [] 43 | if DEFAULTDRIVERNAME in dfltDriverOptions: 44 | for opt in dfltDriverOptions[DEFAULTDRIVERNAME]: 45 | CMDLINECREATIONOPTIONS.append('-co') 46 | CMDLINECREATIONOPTIONS.append(opt) 47 | 48 | 49 | def getCmdargs(argv=None): 50 | """ 51 | Get command line arguments 52 | 53 | If argv is given, it should be a list of pairs of parameter and arguments like in command line. 54 | See getCmdargs(['-h']) for details on available parameters. 55 | Example: getCmdargs(['--safedir', '<.SAFE directory>', '-o', '']) 56 | 57 | If argv is None or not given, command line sys.args are used, see argparse.parse_args. 58 | """ 59 | parser = argparse.ArgumentParser() 60 | parser.add_argument("--safedir", help=("Name of .SAFE directory, as unzipped from " + 61 | "a standard ESA L1C zip file. Using this option will automatically create intermediate " + 62 | "stacks of the input bands, and so does NOT require --toa or --anglesfile. ")) 63 | parser.add_argument("--granuledir", help=("Name of granule sub-directory within the " + 64 | ".SAFE directory, as unzipped from a standard ESA L1C zip file. This option is an " + 65 | "alternative to --safedir, for use with ESA's old format zipfiles which had multiple " + 66 | "granules in each zipfile. Specify the subdirectory of the single tile, under the " + 67 | "/GRANULE/ directory. " + 68 | "Using this option will automatically create intermediate " + 69 | "stacks of the input bands, and so does NOT require --toa or --anglesfile. ")) 70 | parser.add_argument('-a', '--toa', 71 | help=('Input stack of TOA reflectance (as supplied by ESA). This is obsolete, and is ' + 72 | 'only required if NOT using the --safedir or --granuledir option. ')) 73 | parser.add_argument('-z', '--anglesfile', 74 | help=("Input angles file containing satellite and sun azimuth and zenith. " + 75 | "See fmask_sentinel2makeAnglesImage.py for assistance in creating this. " + 76 | "This option is obsolete, and is only required if NOT using the --safedir " + 77 | "or --granuledir option. ")) 78 | parser.add_argument('-o', '--output', help='Output cloud mask') 79 | parser.add_argument('-v', '--verbose', dest='verbose', default=False, 80 | action='store_true', help='verbose output') 81 | parser.add_argument("--pixsize", default=20, type=int, 82 | help="Output pixel size in metres (default=%(default)s)") 83 | parser.add_argument('-k', '--keepintermediates', 84 | default=False, action='store_true', help='Keep intermediate temporary files (normally deleted)') 85 | parser.add_argument('-e', '--tempdir', 86 | default='.', help="Temp directory to use (default='%(default)s')") 87 | 88 | params = parser.add_argument_group(title="Configurable parameters", description=""" 89 | Changing these parameters will affect the way the algorithm works, and thus the 90 | quality of the final output masks. 91 | """) 92 | params.add_argument("--mincloudsize", type=int, default=0, 93 | help="Mininum cloud size (in pixels) to retain, before any buffering. Default=%(default)s)") 94 | params.add_argument("--cloudbufferdistance", type=float, default=150, 95 | help="Distance (in metres) to buffer final cloud objects (default=%(default)s)") 96 | params.add_argument("--shadowbufferdistance", type=float, default=300, 97 | help="Distance (in metres) to buffer final cloud shadow objects (default=%(default)s)") 98 | defaultCloudProbThresh = 100 * config.FmaskConfig.Eqn17CloudProbThresh 99 | params.add_argument("--cloudprobthreshold", type=float, default=defaultCloudProbThresh, 100 | help=("Cloud probability threshold (percentage) (default=%(default)s). This is "+ 101 | "the constant term at the end of equation 17, given in the paper as 0.2 (i.e. 20%%). "+ 102 | "To reduce commission errors, increase this value, but this will also increase "+ 103 | "omission errors. ")) 104 | dfltNirSnowThresh = config.FmaskConfig.Eqn20NirSnowThresh 105 | params.add_argument("--nirsnowthreshold", default=dfltNirSnowThresh, type=float, 106 | help=("Threshold for NIR reflectance (range [0-1]) for snow detection "+ 107 | "(default=%(default)s). Increase this to reduce snow commission errors")) 108 | dfltGreenSnowThresh = config.FmaskConfig.Eqn20GreenSnowThresh 109 | params.add_argument("--greensnowthreshold", default=dfltGreenSnowThresh, type=float, 110 | help=("Threshold for Green reflectance (range [0-1]) for snow detection "+ 111 | "(default=%(default)s). Increase this to reduce snow commission errors")) 112 | params.add_argument("--parallaxtest", default=False, action="store_true", 113 | help="Turn on the parallax displacement test from Frantz (2018) (default will not use this test)") 114 | 115 | cmdargs = parser.parse_args(argv) 116 | 117 | # Do some sanity checks on what was given 118 | safeDirGiven = (cmdargs.safedir is not None) 119 | granuleDirGiven = (cmdargs.granuledir is not None) 120 | if granuleDirGiven and safeDirGiven: 121 | print("Only give one of --safedir or --granuledir. The --granuledir is only ") 122 | print("required for multi-tile zipfiles in the old ESA format") 123 | sys.exit(1) 124 | stackAnglesGiven = (cmdargs.toa is not None and cmdargs.anglesfile is not None) 125 | multipleInputGiven = (safeDirGiven or granuleDirGiven) and stackAnglesGiven 126 | inputGiven = safeDirGiven or granuleDirGiven or stackAnglesGiven 127 | if cmdargs.output is None or multipleInputGiven or not inputGiven: 128 | parser.print_help() 129 | sys.exit(1) 130 | 131 | return cmdargs 132 | 133 | 134 | def checkAnglesFile(inputAnglesFile, toafile): 135 | """ 136 | Check that the resolution of the input angles file matches that of the input 137 | TOA reflectance file. If not, make a VRT file which will resample it 138 | on-the-fly. Only checks the resolution, assumes that if these match, then everything 139 | else will match too. 140 | 141 | Return the name of the angles file to use. 142 | 143 | """ 144 | toaImgInfo = fileinfo.ImageInfo(toafile) 145 | anglesImgInfo = fileinfo.ImageInfo(inputAnglesFile) 146 | 147 | outputAnglesFile = inputAnglesFile 148 | if (toaImgInfo.xRes != anglesImgInfo.xRes) or (toaImgInfo.yRes != anglesImgInfo.yRes): 149 | (fd, vrtName) = tempfile.mkstemp(prefix='angles', suffix='.vrt') 150 | os.close(fd) 151 | 152 | options = gdal.WarpOptions(format='VRT', xRes=toaImgInfo.xRes, 153 | yRes=toaImgInfo.yRes, outputBounds=[toaImgInfo.xMin, toaImgInfo.yMin, 154 | toaImgInfo.xMax, toaImgInfo.yMax], resampleAlg='near') 155 | gdal.Warp(vrtName, inputAnglesFile, options=options) 156 | 157 | outputAnglesFile = vrtName 158 | 159 | return outputAnglesFile 160 | 161 | 162 | def makeStackAndAngles(cmdargs): 163 | """ 164 | Make an intermediate stack of all the TOA reflectance bands. Also make an image 165 | of the angles. Fill in the names of these in the cmdargs object. 166 | 167 | """ 168 | if cmdargs.granuledir is None and cmdargs.safedir is not None: 169 | cmdargs.granuledir = findGranuleDir(cmdargs.safedir) 170 | 171 | # Make the angles file 172 | (fd, anglesfile) = tempfile.mkstemp(dir=cmdargs.tempdir, prefix="angles_tmp_", 173 | suffix=".img") 174 | os.close(fd) 175 | xmlfile = findGranuleXml(cmdargs.granuledir) 176 | if cmdargs.verbose: 177 | print("Making angles image") 178 | sentinel2makeAnglesImage.makeAngles(xmlfile, anglesfile) 179 | cmdargs.anglesfile = anglesfile 180 | 181 | # Make a stack of the reflectance bands. Not that we do an explicit resample to the 182 | # output pixel size, to avoid picking up the overview layers with the ESA jpg files. 183 | # According to @vincentschut, these are shifted slightly, and should be avoided. 184 | bandList = ['B01', 'B02', 'B03', 'B04', 'B05', 'B06', 'B07', 'B08', 'B8A', 185 | 'B09', 'B10', 'B11', 'B12'] 186 | imgDir = "{}/IMG_DATA".format(cmdargs.granuledir) 187 | resampledBands = [] 188 | for band in bandList: 189 | (fd, tmpBand) = tempfile.mkstemp(dir=cmdargs.tempdir, prefix="tmp_{}_".format(band), 190 | suffix=".vrt") 191 | os.close(fd) 192 | inBandImgList = glob.glob("{}/*_{}.jp2".format(imgDir, band)) 193 | if len(inBandImgList) != 1: 194 | raise fmaskerrors.FmaskFileError("Cannot find input band {}".format(band)) 195 | inBandImg = inBandImgList[0] 196 | 197 | # Now make a resampled copy to the desired pixel size, using the right resample method 198 | resampleMethod = chooseResampleMethod(cmdargs.pixsize, inBandImg) 199 | 200 | options = gdal.WarpOptions(format='VRT', resampleAlg=resampleMethod, 201 | xRes=cmdargs.pixsize, yRes=cmdargs.pixsize) 202 | gdal.Warp(tmpBand, inBandImg, options=options) 203 | 204 | resampledBands.append(tmpBand) 205 | 206 | # Now make a stack of these 207 | if cmdargs.verbose: 208 | print("Making stack of all bands, at {}m pixel size".format(cmdargs.pixsize)) 209 | (fd, tmpStack) = tempfile.mkstemp(dir=cmdargs.tempdir, prefix="tmp_allbands_", 210 | suffix=".img") 211 | os.close(fd) 212 | cmdargs.toa = tmpStack 213 | 214 | # We need to turn off exceptions while using gdal_merge, as it doesn't cope 215 | usingExceptions = gdal.GetUseExceptions() 216 | gdal.DontUseExceptions() 217 | gdal_merge.main(['-q', '-of', DEFAULTDRIVERNAME] + CMDLINECREATIONOPTIONS + 218 | ['-separate', '-o', cmdargs.toa] + resampledBands) 219 | if usingExceptions: 220 | gdal.UseExceptions() 221 | 222 | for fn in resampledBands: 223 | fmask.deleteRaster(fn) 224 | 225 | return resampledBands 226 | 227 | 228 | def chooseResampleMethod(outpixsize, inBandImg): 229 | """ 230 | Choose the right resample method, given the image and the desired output pixel size 231 | """ 232 | imginfo = fileinfo.ImageInfo(inBandImg) 233 | inPixsize = imginfo.xRes 234 | 235 | if outpixsize == inPixsize: 236 | resample = "near" 237 | elif outpixsize > inPixsize: 238 | resample = "average" 239 | else: 240 | resample = "cubic" 241 | 242 | return resample 243 | 244 | 245 | def findGranuleDir(safedir): 246 | """ 247 | Search the given .SAFE directory, and find the main XML file at the GRANULE level. 248 | 249 | Note that this currently only works for the new-format zip files, with one 250 | tile per zipfile. The old ones are being removed from service, so we won't 251 | cope with them. 252 | 253 | """ 254 | granuleDirPattern = "{}/GRANULE/L1C_*".format(safedir) 255 | granuleDirList = glob.glob(granuleDirPattern) 256 | if len(granuleDirList) == 0: 257 | raise fmaskerrors.FmaskFileError("Unable to find GRANULE sub-directory {}".format(granuleDirPattern)) 258 | elif len(granuleDirList) > 1: 259 | dirstring = ','.join(granuleDirList) 260 | msg = "Found multiple GRANULE sub-directories: {}".format(dirstring) 261 | raise fmaskerrors.FmaskFileError(msg) 262 | 263 | granuleDir = granuleDirList[0] 264 | return granuleDir 265 | 266 | 267 | def findGranuleXml(granuleDir): 268 | """ 269 | Find the granule-level XML file, given the granule dir 270 | """ 271 | xmlfile = "{}/MTD_TL.xml".format(granuleDir) 272 | if not os.path.exists(xmlfile): 273 | # Might be old-format zipfile, so search for *.xml 274 | xmlfilePattern = "{}/*.xml".format(granuleDir) 275 | xmlfileList = glob.glob(xmlfilePattern) 276 | if len(xmlfileList) == 1: 277 | xmlfile = xmlfileList[0] 278 | else: 279 | raise fmaskerrors.FmaskFileError("Unable to find XML file {}".format(xmlfile)) 280 | return xmlfile 281 | 282 | 283 | def readTopLevelMeta(cmdargs): 284 | """ 285 | Since ESA introduced the radiometric offsets in version 04.00 286 | of their processing, we now need to read the XML metadata from 287 | the top level of the SAFE directory. 288 | 289 | Return a sen2meta.Sen2ZipfileMeta object. 290 | 291 | """ 292 | safeDir = cmdargs.safedir 293 | if safeDir is None: 294 | safeDir = os.path.join(cmdargs.granuledir, os.pardir, os.pardir) 295 | if not os.path.exists(os.path.join(safeDir, 'GRANULE')): 296 | msg = "Cannot identify the SAFE-level directory, which is now required (since ESA version 04.00)" 297 | raise fmaskerrors.FmaskFileError(msg) 298 | 299 | # If we can find the top-level XML file, then we can check it for 300 | # offset values. First look for the stndard named file 301 | topLevelXML = os.path.join(safeDir, 'MTD_MSIL1C.xml') 302 | if not os.path.exists(topLevelXML): 303 | # We may have an old-format zip file, in which the XML is 304 | # named for the date of acquisition. It should be the only 305 | # .xml file in that directory. 306 | topLevelXML = None 307 | xmlList = [f for f in glob.glob(os.path.join(safeDir, "*.xml")) if "INSPIRE.xml" not in f] 308 | if len(xmlList) == 1: 309 | topLevelXML = xmlList[0] 310 | if topLevelXML is None: 311 | msg = "Unable to find top-level XML file, which is now required" 312 | raise fmaskerrors.FmaskFileError(msg) 313 | 314 | topMeta = sen2meta.Sen2ZipfileMeta(xmlfilename=topLevelXML) 315 | return topMeta 316 | 317 | 318 | def makeRefOffsetDict(topMeta): 319 | """ 320 | Take the given sen2meta.Sen2ZipfileMeta object and convert it 321 | into a dictionary suitable to give to FmaskConfig.setTOARefOffsetDict. 322 | 323 | """ 324 | # This dictionary established a correspondance between the string 325 | # given in sen2meta.nameFromBandId and the internal index values used 326 | # for bands within python-fmask. 327 | # Note that this should include every band used within the Fmask code, 328 | # although not necessarily every Sentinel-2 band. 329 | bandIndexNameDict = {config.BAND_BLUE: "B02", config.BAND_GREEN: "B03", 330 | config.BAND_RED: "B04", config.BAND_NIR: "B08", 331 | config.BAND_SWIR1: "B11", config.BAND_SWIR2: "B12", 332 | config.BAND_CIRRUS: "B10", config.BAND_S2CDI_NIR8A: "B08A", 333 | config.BAND_S2CDI_NIR7: "B07", config.BAND_WATERVAPOUR: "B09"} 334 | 335 | offsetDict = {} 336 | for bandNdx in bandIndexNameDict: 337 | bandNameStr = bandIndexNameDict[bandNdx] 338 | offsetVal = topMeta.offsetValDict[bandNameStr] 339 | offsetDict[bandNdx] = offsetVal 340 | return offsetDict 341 | 342 | 343 | def mainRoutine(argv=None): 344 | """ 345 | Main routine that calls fmask 346 | 347 | If argv is given, it should be a list of pairs of parameter and arguments like in command line. 348 | See mainRoutine(['-h']) for details on available parameters. 349 | Example: mainRoutine(['--safedir', '<.SAFE directory>', '-o', '']) 350 | 351 | If argv is None or not given, command line sys.args are used, see argparse.parse_args. 352 | """ 353 | cmdargs = getCmdargs(argv) 354 | tempStack = False 355 | if cmdargs.safedir is not None or cmdargs.granuledir is not None: 356 | tempStack = True 357 | makeStackAndAngles(cmdargs) 358 | topMeta = readTopLevelMeta(cmdargs) 359 | 360 | anglesfile = checkAnglesFile(cmdargs.anglesfile, cmdargs.toa) 361 | anglesInfo = config.AnglesFileInfo(anglesfile, 3, anglesfile, 2, anglesfile, 1, anglesfile, 0) 362 | 363 | fmaskFilenames = config.FmaskFilenames() 364 | fmaskFilenames.setTOAReflectanceFile(cmdargs.toa) 365 | fmaskFilenames.setOutputCloudMaskFile(cmdargs.output) 366 | 367 | fmaskConfig = config.FmaskConfig(config.FMASK_SENTINEL2) 368 | fmaskConfig.setAnglesInfo(anglesInfo) 369 | fmaskConfig.setKeepIntermediates(cmdargs.keepintermediates) 370 | fmaskConfig.setVerbose(cmdargs.verbose) 371 | fmaskConfig.setTempDir(cmdargs.tempdir) 372 | fmaskConfig.setTOARefScaling(topMeta.scaleVal) 373 | offsetDict = makeRefOffsetDict(topMeta) 374 | fmaskConfig.setTOARefOffsetDict(offsetDict) 375 | fmaskConfig.setMinCloudSize(cmdargs.mincloudsize) 376 | fmaskConfig.setEqn17CloudProbThresh(cmdargs.cloudprobthreshold / 100) # Note conversion from percentage 377 | fmaskConfig.setEqn20NirSnowThresh(cmdargs.nirsnowthreshold) 378 | fmaskConfig.setEqn20GreenSnowThresh(cmdargs.greensnowthreshold) 379 | fmaskConfig.setSen2displacementTest(cmdargs.parallaxtest) 380 | 381 | # Work out a suitable buffer size, in pixels, dependent on the resolution of the input TOA image 382 | toaImgInfo = fileinfo.ImageInfo(cmdargs.toa) 383 | fmaskConfig.setCloudBufferSize(int(cmdargs.cloudbufferdistance / toaImgInfo.xRes)) 384 | fmaskConfig.setShadowBufferSize(int(cmdargs.shadowbufferdistance / toaImgInfo.xRes)) 385 | 386 | fmask.doFmask(fmaskFilenames, fmaskConfig) 387 | 388 | if (anglesfile != cmdargs.anglesfile): 389 | # Must have been a temporary, so remove it 390 | fmask.deleteRaster(anglesfile) 391 | 392 | if tempStack and not cmdargs.keepintermediates: 393 | for fn in [cmdargs.toa, cmdargs.anglesfile]: 394 | if os.path.exists(fn): 395 | fmask.deleteRaster(fn) 396 | 397 | -------------------------------------------------------------------------------- /fmask/cmdline/sentinel2makeAnglesImage.py: -------------------------------------------------------------------------------- 1 | # This file is part of 'python-fmask' - a cloud masking module 2 | # Copyright (C) 2015 Neil Flood 3 | # 4 | # This program is free software; you can redistribute it and/or 5 | # modify it under the terms of the GNU General Public License 6 | # as published by the Free Software Foundation; either version 3 7 | # of the License, or (at your option) any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU General Public License for more details. 13 | # 14 | # You should have received a copy of the GNU General Public License 15 | # along with this program; if not, write to the Free Software 16 | # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 17 | """ 18 | Make a 4-layer image of satellite and sun angles, from the given tile metadata 19 | file. 20 | 21 | The sun angles are exactly as provided in the XML. The satellite angles are more 22 | complicated, because they vary slightly with each band. I have done some limited 23 | testing and concluded that the variation is generally quite small, and so the 24 | layers written here are the per-pixel averages over all bands. The satellite 25 | zenith this appears to vary only a couple of degrees across bands, while the 26 | variation for satellite azimuth is somewhat larger, depending on which bands. I 27 | think that the 60m bands vary a bit more, but not sure. This is also complicated 28 | by the variation across the scan, due to the different view angles of each detector 29 | module. The pushbroom appears to be made up of 12 such modules, each looking 30 | in slightly different directions, and each band within each module looking slightly 31 | differently. Complicated...... sigh...... 32 | 33 | A bit of a description of how the instrument is structured can be found at 34 | https://sentinels.copernicus.eu/web/sentinel/technical-guides/sentinel-2-msi/msi-instrument 35 | 36 | """ 37 | from __future__ import print_function, division 38 | 39 | import sys 40 | import argparse 41 | 42 | import numpy 43 | from osgeo import gdal 44 | from osgeo import osr 45 | 46 | from rios import applier 47 | from rios import calcstats 48 | from rios import cuiprogress 49 | 50 | from fmask import sen2meta 51 | 52 | # This scale value will convert between DN and radians in output image file, 53 | # radians = dn * SCALE_TO_RADIANS 54 | SCALE_TO_RADIANS = 0.01 55 | 56 | 57 | def getCmdargs(): 58 | """ 59 | Get commandline arguments 60 | """ 61 | p = argparse.ArgumentParser() 62 | p.add_argument("-i", "--infile", help="Input sentinel-2 tile metafile") 63 | p.add_argument("-o", "--outfile", help="Output angles image file") 64 | cmdargs = p.parse_args() 65 | if cmdargs.infile is None or cmdargs.outfile is None: 66 | p.print_help() 67 | sys.exit() 68 | 69 | return cmdargs 70 | 71 | 72 | def mainRoutine(): 73 | """ 74 | Main routine 75 | """ 76 | cmdargs = getCmdargs() 77 | 78 | makeAngles(cmdargs.infile, cmdargs.outfile) 79 | 80 | 81 | def makeAngles(infile, outfile): 82 | """ 83 | Callable main routine 84 | """ 85 | info = sen2meta.Sen2TileMeta(filename=infile) 86 | 87 | ds = createOutfile(outfile, info) 88 | nullValDN = 1000 89 | 90 | # Get a sorted list of the Sentinel-2 band names. Note that sometimes this 91 | # is an incomplete list of band names, which appears to be due to a bug in 92 | # earlier versions of ESA's processing software. I suspect it relates to 93 | # Anomaly number 11 in the following page. 94 | # https://sentinel.esa.int/web/sentinel/news/-/article/new-processing-baseline-for-sentinel-2-products 95 | bandNames = sorted(info.viewAzimuthDict.keys()) 96 | 97 | # Mean over all bands 98 | satAzDeg = numpy.array([info.viewAzimuthDict[i] for i in bandNames]) 99 | satAzDegMeanOverBands = satAzDeg.mean(axis=0) 100 | 101 | satZenDeg = numpy.array([info.viewZenithDict[i] for i in bandNames]) 102 | satZenDegMeanOverBands = satZenDeg.mean(axis=0) 103 | 104 | sunAzDeg = info.sunAzimuthGrid 105 | 106 | sunZenDeg = info.sunZenithGrid 107 | 108 | stackDeg = numpy.array([satAzDegMeanOverBands, satZenDegMeanOverBands, sunAzDeg, sunZenDeg]) 109 | stackRadians = numpy.radians(stackDeg) 110 | 111 | stackDN = numpy.round(stackRadians / SCALE_TO_RADIANS) 112 | nullmask = numpy.isnan(stackDeg) 113 | stackDN[nullmask] = nullValDN 114 | stackDN = stackDN.astype(numpy.int16) 115 | 116 | lnames = ['SatelliteAzimuth', 'SatelliteZenith', 'SunAzimuth', 'SunZenith'] 117 | for i in range(ds.RasterCount): 118 | b = ds.GetRasterBand(i + 1) 119 | b.WriteArray(stackDN[i]) 120 | b.SetNoDataValue(nullValDN) 121 | b.SetDescription(lnames[i]) 122 | calcstats.calcStats(ds, ignore=nullValDN, progress=cuiprogress.SilentProgress()) 123 | del ds 124 | 125 | 126 | def createOutfile(filename, info): 127 | """ 128 | Create the empty output image file 129 | """ 130 | drvr = gdal.GetDriverByName(applier.DEFAULTDRIVERNAME) 131 | (nrows, ncols) = info.anglesGridShape 132 | ds = drvr.Create(filename, ncols, nrows, 4, gdal.GDT_Int16, applier.DEFAULTCREATIONOPTIONS) 133 | gt = (info.anglesULXY[0], info.angleGridXres, 0, info.anglesULXY[1], 0.0, -info.angleGridYres) 134 | ds.SetGeoTransform(gt) 135 | 136 | sr = osr.SpatialReference() 137 | sr.ImportFromEPSG(int(info.epsg)) 138 | ds.SetProjection(sr.ExportToWkt()) 139 | return ds 140 | -------------------------------------------------------------------------------- /fmask/cmdline/usgsLandsatMakeAnglesImage.py: -------------------------------------------------------------------------------- 1 | # This file is part of 'python-fmask' - a cloud masking module 2 | # Copyright (C) 2015 Neil Flood 3 | # 4 | # This program is free software; you can redistribute it and/or 5 | # modify it under the terms of the GNU General Public License 6 | # as published by the Free Software Foundation; either version 3 7 | # of the License, or (at your option) any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU General Public License for more details. 13 | # 14 | # You should have received a copy of the GNU General Public License 15 | # along with this program; if not, write to the Free Software 16 | # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 17 | 18 | """ 19 | Generate an output image file with estimates of per-pixel angles for 20 | sun and satellite azimuth and zenith. These are rough estimates, using the generic 21 | characteristics of the Landsat 5 platform, and are not particularly accurate, 22 | but good enough for the current purposes. 23 | In the future, the USGS have plans to distribute a set of parameters which can 24 | be used to directly generate these angles, derived from the actual orbit ephemeris 25 | of the pass. This program is intended for use when these parameters are not 26 | available. Quite possibly, when they do make these available, I will modify this 27 | program to use those parameters if they are present in the MTL file, but fall back 28 | to the old estimates if not. 29 | 30 | The general approach for satellite angles is to estimate the nadir line by running it 31 | down the middle of the image data area. The satellite azimuth is assumed to be 32 | at right angles to this nadir line, which is only roughly correct. For the whisk-broom 33 | sensors on Landsat-5 and Landsat-7, this angles is not 90 degrees, but is affected by 34 | earth rotation and is latitude dependent, while for Landsat-8, the scan line is at 35 | right angles, due to the compensation for earth rotation, but the push-broom is 36 | made up of sub-modules which point in slightly different directions, giving 37 | slightly different satellite azimuths along the scan line. None of these effects 38 | are included in the current estimates. The satellite zenith is estimated based on the 39 | nadir point, the scan-line, and the assumed satellite altitude, and includes the 40 | appropriate allowance for earth curvature. 41 | 42 | Because this works by searching the imagery for the non-null area, and assumes that 43 | this represents a full-swath image, it would not work for a subset of a full image. 44 | 45 | The sun angles are approximated using the algorithm found in the Fortran code with 46 | 6S (Second Simulation of the Satellite Signal in the Solar Spectrum). The subroutine 47 | in question is the POSSOL() routine. I translated the Fortran code into Python for 48 | inclusion here. 49 | 50 | """ 51 | from __future__ import print_function, division 52 | 53 | import sys 54 | import argparse 55 | 56 | from fmask import landsatangles 57 | from fmask import config 58 | 59 | from rios import fileinfo 60 | 61 | 62 | def getCmdargs(): 63 | """ 64 | Get commandline arguments 65 | """ 66 | p = argparse.ArgumentParser() 67 | p.add_argument("-m", "--mtl", help="MTL text file of USGS metadata") 68 | p.add_argument("-t", "--templateimg", 69 | help="Image filename to use as template for output angles image") 70 | p.add_argument("-o", "--outfile", help="Output image file") 71 | cmdargs = p.parse_args() 72 | if (cmdargs.mtl is None or cmdargs.templateimg is None or 73 | cmdargs.outfile is None): 74 | p.print_help() 75 | sys.exit(1) 76 | return cmdargs 77 | 78 | 79 | def mainRoutine(): 80 | """ 81 | Main routine 82 | """ 83 | cmdargs = getCmdargs() 84 | 85 | makeAngles(cmdargs.mtl, cmdargs.templateimg, cmdargs.outfile) 86 | 87 | 88 | def makeAngles(mtlfile, templateimg, outfile): 89 | """ 90 | Callable main routine 91 | """ 92 | mtlInfo = config.readMTLFile(mtlfile) 93 | 94 | imgInfo = fileinfo.ImageInfo(templateimg) 95 | corners = landsatangles.findImgCorners(templateimg, imgInfo) 96 | nadirLine = landsatangles.findNadirLine(corners) 97 | 98 | extentSunAngles = landsatangles.sunAnglesForExtent(imgInfo, mtlInfo) 99 | satAzimuth = landsatangles.satAzLeftRight(nadirLine) 100 | 101 | landsatangles.makeAnglesImage(templateimg, outfile, 102 | nadirLine, extentSunAngles, satAzimuth, imgInfo) 103 | -------------------------------------------------------------------------------- /fmask/cmdline/usgsLandsatSaturationMask.py: -------------------------------------------------------------------------------- 1 | # This file is part of 'python-fmask' - a cloud masking module 2 | # Copyright (C) 2015 Neil Flood 3 | # 4 | # This program is free software; you can redistribute it and/or 5 | # modify it under the terms of the GNU General Public License 6 | # as published by the Free Software Foundation; either version 3 7 | # of the License, or (at your option) any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU General Public License for more details. 13 | # 14 | # You should have received a copy of the GNU General Public License 15 | # along with this program; if not, write to the Free Software 16 | # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 17 | from __future__ import print_function, division 18 | 19 | import sys 20 | import argparse 21 | from fmask import saturationcheck 22 | from fmask import config 23 | 24 | 25 | def getCmdargs(): 26 | """ 27 | Get command line arguments 28 | """ 29 | parser = argparse.ArgumentParser() 30 | parser.add_argument('-i', '--infile', help='Input raw DN radiance image') 31 | parser.add_argument('-m', '--mtl', help='.MTL file') 32 | parser.add_argument('-o', '--output', help='Output saturation mask file') 33 | 34 | cmdargs = parser.parse_args() 35 | 36 | if (cmdargs.infile is None or cmdargs.mtl is None or 37 | cmdargs.output is None): 38 | parser.print_help() 39 | sys.exit() 40 | 41 | return cmdargs 42 | 43 | 44 | def mainRoutine(): 45 | cmdargs = getCmdargs() 46 | 47 | makeSaturationMask(cmdargs.mtl, cmdargs.infile, cmdargs.output) 48 | 49 | 50 | def makeSaturationMask(mtlfile, infile, outfile): 51 | """ 52 | Callable main routine 53 | """ 54 | mtlInfo = config.readMTLFile(mtlfile) 55 | landsat = mtlInfo['SPACECRAFT_ID'][-1] 56 | 57 | if landsat == '4': 58 | sensor = config.FMASK_LANDSAT47 59 | elif landsat == '5': 60 | sensor = config.FMASK_LANDSAT47 61 | elif landsat == '7': 62 | sensor = config.FMASK_LANDSAT47 63 | elif landsat in ('8', '9'): 64 | sensor = config.FMASK_LANDSATOLI 65 | else: 66 | raise SystemExit('Unsupported Landsat sensor') 67 | 68 | # needed so the saturation function knows which 69 | # bands are visible etc. 70 | fmaskConfig = config.FmaskConfig(sensor) 71 | 72 | saturationcheck.makeSaturationMask(fmaskConfig, infile, outfile) 73 | 74 | -------------------------------------------------------------------------------- /fmask/cmdline/usgsLandsatStacked.py: -------------------------------------------------------------------------------- 1 | """ 2 | Script that takes USGS landsat stacked separately for 3 | reflective and thermal and runs the fmask on it. 4 | """ 5 | # This file is part of 'python-fmask' - a cloud masking module 6 | # Copyright (C) 2015 Neil Flood 7 | # 8 | # This program is free software; you can redistribute it and/or 9 | # modify it under the terms of the GNU General Public License 10 | # as published by the Free Software Foundation; either version 3 11 | # of the License, or (at your option) any later version. 12 | # 13 | # This program is distributed in the hope that it will be useful, 14 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 15 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 | # GNU General Public License for more details. 17 | # 18 | # You should have received a copy of the GNU General Public License 19 | # along with this program; if not, write to the Free Software 20 | # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 21 | from __future__ import print_function, division 22 | 23 | import os 24 | import glob 25 | import argparse 26 | import tempfile 27 | from fmask import landsatTOA 28 | from fmask.cmdline import usgsLandsatMakeAnglesImage, usgsLandsatSaturationMask 29 | from fmask import config 30 | from fmask import fmaskerrors 31 | from fmask import fmask 32 | 33 | from osgeo_utils import gdal_merge 34 | from osgeo import gdal 35 | 36 | from rios import fileinfo 37 | from rios.imagewriter import DEFAULTDRIVERNAME, dfltDriverOptions 38 | 39 | # for GDAL. 40 | CMDLINECREATIONOPTIONS = [] 41 | if DEFAULTDRIVERNAME in dfltDriverOptions: 42 | for opt in dfltDriverOptions[DEFAULTDRIVERNAME]: 43 | CMDLINECREATIONOPTIONS.append('-co') 44 | CMDLINECREATIONOPTIONS.append(opt) 45 | 46 | 47 | def getCmdargs(): 48 | """ 49 | Get command line arguments 50 | """ 51 | parser = argparse.ArgumentParser() 52 | parser.add_argument('-t', '--thermal', help='Input stack of thermal bands') 53 | parser.add_argument('-a', '--toa', 54 | help='Input stack of TOA reflectance (see fmask_usgsLandsatTOA.py)') 55 | parser.add_argument('-m', '--mtl', help='Input .MTL file') 56 | parser.add_argument('-s', '--saturation', 57 | help='Input saturation mask (see fmask_usgsLandsatSaturationMask.py)') 58 | parser.add_argument("-z", "--anglesfile", 59 | help="Image of sun and satellite angles (see fmask_usgsLandsatMakeAnglesImage.py)") 60 | parser.add_argument('-o', '--output', dest='output', 61 | help='output cloud mask') 62 | parser.add_argument('-v', '--verbose', default=False, 63 | action='store_true', help='verbose output') 64 | parser.add_argument('-k', '--keepintermediates', dest='keepintermediates', 65 | default=False, action='store_true', help='verbose output') 66 | parser.add_argument('-e', '--tempdir', 67 | default='.', help="Temp directory to use (default=%(default)s)") 68 | parser.add_argument('--scenedir', help='Path to the unzipped USGS Landsat scene. ' + 69 | 'Using this option will automatically create intermediate stacks of the input ' + 70 | 'bands, and so does NOT require --toa, --thermal, --saturation, --mtl or --anglesfile') 71 | 72 | params = parser.add_argument_group(title="Configurable parameters", description=""" 73 | Changing these parameters will affect the way the algorithm works, and thus the 74 | quality of the final output masks. 75 | """) 76 | params.add_argument("--mincloudsize", type=int, default=0, 77 | help="Mininum cloud size (in pixels) to retain, before any buffering. Default=%(default)s)") 78 | params.add_argument("--cloudbufferdistance", type=float, default=150, 79 | help="Distance (in metres) to buffer final cloud objects (default=%(default)s)") 80 | params.add_argument("--shadowbufferdistance", type=float, default=300, 81 | help="Distance (in metres) to buffer final cloud shadow objects (default=%(default)s)") 82 | defaultCloudProbThresh = 100 * config.FmaskConfig.Eqn17CloudProbThresh 83 | params.add_argument("--cloudprobthreshold", type=float, default=defaultCloudProbThresh, 84 | help=("Cloud probability threshold (percentage) (default=%(default)s). This is "+ 85 | "the constant term at the end of equation 17, given in the paper as 0.2 (i.e. 20%%). "+ 86 | "To reduce commission errors, increase this value, but this will also increase "+ 87 | "omission errors. ")) 88 | dfltNirSnowThresh = config.FmaskConfig.Eqn20NirSnowThresh 89 | params.add_argument("--nirsnowthreshold", default=dfltNirSnowThresh, type=float, 90 | help=("Threshold for NIR reflectance (range [0-1]) for snow detection "+ 91 | "(default=%(default)s). Increase this to reduce snow commission errors")) 92 | dfltGreenSnowThresh = config.FmaskConfig.Eqn20GreenSnowThresh 93 | params.add_argument("--greensnowthreshold", default=dfltGreenSnowThresh, type=float, 94 | help=("Threshold for Green reflectance (range [0-1]) for snow detection "+ 95 | "(default=%(default)s). Increase this to reduce snow commission errors")) 96 | 97 | cmdargs = parser.parse_args() 98 | 99 | return cmdargs 100 | 101 | 102 | def makeStacksAndAngles(cmdargs): 103 | """ 104 | Find the name of the MTL file. 105 | Make an intermediate stacks of all the TOA reflectance and thermal bands. 106 | Also make an image of the angles, saturation and TOA reflectance. 107 | Fill in the names of these in the cmdargs object. 108 | 109 | """ 110 | # find MTL file 111 | wldpath = os.path.join(cmdargs.scenedir, '*_MTL.txt') 112 | mtlList = glob.glob(wldpath) 113 | if len(mtlList) != 1: 114 | raise fmaskerrors.FmaskFileError("Cannot find a *_MTL.txt file in specified dir") 115 | 116 | cmdargs.mtl = mtlList[0] 117 | 118 | # we need to find the 'SPACECRAFT_ID' to work out the wildcards to use 119 | mtlInfo = config.readMTLFile(cmdargs.mtl) 120 | landsat = mtlInfo['SPACECRAFT_ID'][-1] 121 | 122 | if landsat == '4' or landsat == '5': 123 | refWildcard = 'L*_B[1,2,3,4,5,7].TIF' 124 | thermalWildcard = 'L*_B6.TIF' 125 | elif landsat == '7': 126 | refWildcard = 'L*_B[1,2,3,4,5,7].TIF' 127 | thermalWildcard = 'L*_B6_VCID_?.TIF' 128 | elif landsat in ('8', '9'): 129 | refWildcard = 'LC*_B[1-7,9].TIF' 130 | thermalWildcard = 'LC*_B1[0,1].TIF' 131 | else: 132 | raise SystemExit('Unsupported Landsat sensor') 133 | 134 | wldpath = os.path.join(cmdargs.scenedir, refWildcard) 135 | refFiles = sorted(glob.glob(wldpath)) 136 | if len(refFiles) == 0: 137 | raise fmaskerrors.FmaskFileError("Cannot find expected reflectance files for sensor") 138 | 139 | wldpath = os.path.join(cmdargs.scenedir, thermalWildcard) 140 | thermalFiles = sorted(glob.glob(wldpath)) 141 | if len(thermalFiles) == 0: 142 | raise fmaskerrors.FmaskFileError("Cannot find expected thermal files for sensor") 143 | 144 | # We need to turn off exceptions while using gdal_merge, as it doesn't cope 145 | usingExceptions = gdal.GetUseExceptions() 146 | 147 | if cmdargs.verbose: 148 | print("Making stack of all reflectance bands") 149 | (fd, tmpRefStack) = tempfile.mkstemp(dir=cmdargs.tempdir, prefix="tmp_allrefbands_", 150 | suffix=".img") 151 | os.close(fd) 152 | 153 | gdal.DontUseExceptions() 154 | gdal_merge.main(['-q', '-of', DEFAULTDRIVERNAME] + 155 | CMDLINECREATIONOPTIONS + ['-separate', '-o', tmpRefStack] + refFiles) 156 | if usingExceptions: 157 | gdal.UseExceptions() 158 | 159 | # stash so we can delete later 160 | cmdargs.refstack = tmpRefStack 161 | 162 | if cmdargs.verbose: 163 | print("Making stack of all thermal bands") 164 | (fd, tmpThermStack) = tempfile.mkstemp(dir=cmdargs.tempdir, prefix="tmp_allthermalbands_", 165 | suffix=".img") 166 | os.close(fd) 167 | 168 | gdal.DontUseExceptions() 169 | gdal_merge.main(['-q', '-of', DEFAULTDRIVERNAME] + 170 | CMDLINECREATIONOPTIONS + ['-separate', '-o', tmpThermStack] + thermalFiles) 171 | if usingExceptions: 172 | gdal.UseExceptions() 173 | 174 | cmdargs.thermal = tmpThermStack 175 | 176 | # now the angles 177 | if cmdargs.verbose: 178 | print("Creating angles file") 179 | (fd, anglesfile) = tempfile.mkstemp(dir=cmdargs.tempdir, prefix="angles_tmp_", 180 | suffix=".img") 181 | os.close(fd) 182 | usgsLandsatMakeAnglesImage.makeAngles(cmdargs.mtl, tmpRefStack, anglesfile) 183 | cmdargs.anglesfile = anglesfile 184 | 185 | # saturation 186 | if cmdargs.verbose: 187 | print("Creating saturation file") 188 | (fd, saturationfile) = tempfile.mkstemp(dir=cmdargs.tempdir, prefix="saturation_tmp_", 189 | suffix=".img") 190 | os.close(fd) 191 | usgsLandsatSaturationMask.makeSaturationMask(cmdargs.mtl, tmpRefStack, saturationfile) 192 | cmdargs.saturation = saturationfile 193 | 194 | # TOA 195 | if cmdargs.verbose: 196 | print("Creating TOA file") 197 | (fs, toafile) = tempfile.mkstemp(dir=cmdargs.tempdir, prefix="toa_tmp_", 198 | suffix=".img") 199 | os.close(fd) 200 | landsatTOA.makeTOAReflectance(tmpRefStack, cmdargs.mtl, anglesfile, toafile) 201 | cmdargs.toa = toafile 202 | 203 | 204 | def mainRoutine(): 205 | """ 206 | Main routine that calls fmask 207 | """ 208 | cmdargs = getCmdargs() 209 | tempStack = False 210 | if cmdargs.scenedir is not None: 211 | tempStack = True 212 | makeStacksAndAngles(cmdargs) 213 | 214 | if (cmdargs.thermal is None or cmdargs.anglesfile is None or 215 | cmdargs.mtl is None is None or cmdargs.output is None or 216 | cmdargs.toa is None): 217 | raise SystemExit('Not all required input parameters supplied') 218 | 219 | # 1040nm thermal band should always be the first (or only) band in a 220 | # stack of Landsat thermal bands 221 | thermalInfo = config.readThermalInfoFromLandsatMTL(cmdargs.mtl) 222 | 223 | anglesfile = cmdargs.anglesfile 224 | anglesInfo = config.AnglesFileInfo(anglesfile, 3, anglesfile, 2, anglesfile, 1, anglesfile, 0) 225 | 226 | mtlInfo = config.readMTLFile(cmdargs.mtl) 227 | landsat = mtlInfo['SPACECRAFT_ID'][-1] 228 | 229 | if landsat == '4': 230 | sensor = config.FMASK_LANDSAT47 231 | elif landsat == '5': 232 | sensor = config.FMASK_LANDSAT47 233 | elif landsat == '7': 234 | sensor = config.FMASK_LANDSAT47 235 | elif landsat in ('8', '9'): 236 | sensor = config.FMASK_LANDSATOLI 237 | else: 238 | raise SystemExit('Unsupported Landsat sensor') 239 | 240 | fmaskFilenames = config.FmaskFilenames() 241 | fmaskFilenames.setTOAReflectanceFile(cmdargs.toa) 242 | fmaskFilenames.setThermalFile(cmdargs.thermal) 243 | fmaskFilenames.setOutputCloudMaskFile(cmdargs.output) 244 | if cmdargs.saturation is not None: 245 | fmaskFilenames.setSaturationMask(cmdargs.saturation) 246 | else: 247 | print('saturation mask not supplied - see fmask_usgsLandsatSaturationMask.py') 248 | 249 | fmaskConfig = config.FmaskConfig(sensor) 250 | fmaskConfig.setThermalInfo(thermalInfo) 251 | fmaskConfig.setAnglesInfo(anglesInfo) 252 | fmaskConfig.setKeepIntermediates(cmdargs.keepintermediates) 253 | fmaskConfig.setVerbose(cmdargs.verbose) 254 | fmaskConfig.setTempDir(cmdargs.tempdir) 255 | fmaskConfig.setMinCloudSize(cmdargs.mincloudsize) 256 | fmaskConfig.setEqn17CloudProbThresh(cmdargs.cloudprobthreshold / 100) # Note conversion from percentage 257 | fmaskConfig.setEqn20NirSnowThresh(cmdargs.nirsnowthreshold) 258 | fmaskConfig.setEqn20GreenSnowThresh(cmdargs.greensnowthreshold) 259 | 260 | # Work out a suitable buffer size, in pixels, dependent on the resolution of the input TOA image 261 | toaImgInfo = fileinfo.ImageInfo(cmdargs.toa) 262 | fmaskConfig.setCloudBufferSize(int(cmdargs.cloudbufferdistance / toaImgInfo.xRes)) 263 | fmaskConfig.setShadowBufferSize(int(cmdargs.shadowbufferdistance / toaImgInfo.xRes)) 264 | 265 | fmask.doFmask(fmaskFilenames, fmaskConfig) 266 | 267 | if tempStack and not cmdargs.keepintermediates: 268 | for fn in [cmdargs.refstack, cmdargs.thermal, cmdargs.anglesfile, 269 | cmdargs.saturation, cmdargs.toa]: 270 | if os.path.exists(fn): 271 | fmask.deleteRaster(fn) 272 | 273 | -------------------------------------------------------------------------------- /fmask/cmdline/usgsLandsatTOA.py: -------------------------------------------------------------------------------- 1 | # This file is part of 'python-fmask' - a cloud masking module 2 | # Copyright (C) 2015 Neil Flood 3 | # 4 | # This program is free software; you can redistribute it and/or 5 | # modify it under the terms of the GNU General Public License 6 | # as published by the Free Software Foundation; either version 3 7 | # of the License, or (at your option) any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU General Public License for more details. 13 | # 14 | # You should have received a copy of the GNU General Public License 15 | # along with this program; if not, write to the Free Software 16 | # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 17 | from __future__ import print_function, division 18 | 19 | import sys 20 | import argparse 21 | from fmask import landsatTOA 22 | 23 | 24 | def getCmdargs(): 25 | """ 26 | Get command line arguments 27 | """ 28 | parser = argparse.ArgumentParser() 29 | parser.add_argument('-i', '--infile', help='Input raw DN radiance image') 30 | parser.add_argument('-m', '--mtl', help='.MTL file') 31 | parser.add_argument("-z", "--anglesfile", 32 | help="Image of sun and satellite angles (see fmask_usgsLandsatMakeAnglesImage.py)") 33 | parser.add_argument('-o', '--output', help='Output TOA reflectance file') 34 | 35 | cmdargs = parser.parse_args() 36 | 37 | if (cmdargs.infile is None or cmdargs.mtl is None or 38 | cmdargs.output is None or cmdargs.anglesfile is None): 39 | parser.print_help() 40 | sys.exit() 41 | return cmdargs 42 | 43 | 44 | def mainRoutine(): 45 | cmdargs = getCmdargs() 46 | 47 | landsatTOA.makeTOAReflectance(cmdargs.infile, cmdargs.mtl, cmdargs.anglesfile, cmdargs.output) 48 | -------------------------------------------------------------------------------- /fmask/config.py: -------------------------------------------------------------------------------- 1 | """ 2 | Configuration classes that define the inputs and parameters 3 | for the fmask function. 4 | """ 5 | # This file is part of 'python-fmask' - a cloud masking module 6 | # Copyright (C) 2015 Neil Flood 7 | # 8 | # This program is free software; you can redistribute it and/or 9 | # modify it under the terms of the GNU General Public License 10 | # as published by the Free Software Foundation; either version 3 11 | # of the License, or (at your option) any later version. 12 | # 13 | # This program is distributed in the hope that it will be useful, 14 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 15 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 | # GNU General Public License for more details. 17 | # 18 | # You should have received a copy of the GNU General Public License 19 | # along with this program; if not, write to the Free Software 20 | # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 21 | from __future__ import print_function, division 22 | 23 | import abc 24 | import numpy 25 | import scipy.constants 26 | 27 | from osgeo import gdal 28 | from rios import applier 29 | from . import fmaskerrors 30 | 31 | gdal.UseExceptions() 32 | 33 | FMASK_LANDSAT47 = 0 34 | "Landsat 4 to 7" 35 | FMASK_LANDSAT8 = 1 36 | "Landsat 8" 37 | FMASK_SENTINEL2 = 2 38 | "Sentinel 2" 39 | FMASK_LANDSATOLI = 3 40 | "Landsat OLI" 41 | 42 | """ 43 | Some constants for the various reflective bands used in fmask. 44 | """ 45 | #: ~475nm 46 | BAND_BLUE = 0 47 | #: ~560nm 48 | BAND_GREEN = 1 49 | #: ~660nm 50 | BAND_RED = 2 51 | #: ~780nm 52 | BAND_NIR = 3 53 | #: ~1360nm 54 | BAND_CIRRUS = 4 # Sentinel2 + Landsat8 only 55 | #: ~1610nm 56 | BAND_SWIR1 = 5 57 | #: ~2200nm 58 | BAND_SWIR2 = 6 59 | 60 | # These bands are used only for the Sentinel-2 Cloud Displacement Index code. They 61 | # are NIR bands, with slightly different look angles, and used as per Frantz et al 2017. 62 | #: ~865nm 63 | BAND_S2CDI_NIR8A = 7 64 | #: ~783nm 65 | BAND_S2CDI_NIR7 = 8 66 | 67 | BAND_WATERVAPOUR = 9 68 | 69 | 70 | class FmaskConfig(object): 71 | """ 72 | Class that contains the configuration parameters of the fmask 73 | run. 74 | """ 75 | # some parameters for fmask operation 76 | keepIntermediates = False 77 | cloudBufferSize = 5 78 | shadowBufferSize = 10 79 | verbose = False 80 | strictFmask = False 81 | tempDir = '.' 82 | TOARefScaling = 10000.0 83 | TOARefDNoffsetDict = None 84 | # Minimum number of pixels in a single cloud (before buffering). A non-zero value 85 | # would allow filtering of very small clouds. 86 | minCloudSize_pixels = 0 87 | 88 | # constants from the paper that could probably be tweaked 89 | # equation numbers are from the original paper. 90 | Eqn1Swir2Thresh = 0.03 91 | Eqn1ThermThresh = 27 92 | Eqn2WhitenessThresh = 0.7 93 | cirrusBandTestThresh = 0.01 94 | Eqn7Swir2Thresh = 0.03 95 | Eqn20ThermThresh = 3.8 96 | Eqn20NirSnowThresh = 0.11 97 | Eqn20GreenSnowThresh = 0.1 98 | cirrusProbRatio = 0.04 99 | Eqn19NIRFillThresh = 0.02 100 | 101 | # Constant term at the end of Equation 17. Zhu's MATLAB code now has this as a configurable 102 | # value, which they recommend as 22.5% (i.e. 0.225) 103 | Eqn17CloudProbThresh = 0.2 104 | 105 | # GDAL driver for final output file 106 | gdalDriverName = applier.DEFAULTDRIVERNAME 107 | 108 | # Do we do the Sentinel-2 Cloud Displacement Test ? 109 | sen2displacementTest = False 110 | sen2cdiWindow = 7 111 | 112 | def __init__(self, sensor): 113 | """ 114 | Pass in the sensor (one of: FMASK_LANDSAT47, FMASK_LANDSAT8 or 115 | FMASK_SENTINEL2) and default of parameters will be set. These 116 | can be overridden using the functions on this object. 117 | """ 118 | self.sensor = sensor 119 | 120 | # Some standard file configurations for different sensors. 121 | # Assumed that panchromatic + thermal bands stored in separate files. 122 | # zero based indexing 123 | if sensor == FMASK_LANDSAT47: 124 | self.bands = {BAND_BLUE: 0, BAND_GREEN: 1, BAND_RED: 2, BAND_NIR: 3, 125 | BAND_SWIR1: 4, BAND_SWIR2: 5} 126 | elif sensor in (FMASK_LANDSAT8, FMASK_LANDSATOLI): 127 | self.bands = {BAND_BLUE: 1, BAND_GREEN: 2, BAND_RED: 3, BAND_NIR: 4, 128 | BAND_SWIR1: 5, BAND_SWIR2: 6, BAND_CIRRUS: 7} 129 | elif sensor == FMASK_SENTINEL2: 130 | # Assumes the input stack has ALL bands, in their numeric order (with 8A after 8) 131 | self.bands = {BAND_BLUE: 1, BAND_GREEN: 2, BAND_RED: 3, BAND_NIR: 7, 132 | BAND_SWIR1: 11, BAND_SWIR2: 12, BAND_WATERVAPOUR: 9, BAND_CIRRUS: 10, 133 | BAND_S2CDI_NIR7: 6, BAND_S2CDI_NIR8A: 8} 134 | else: 135 | msg = 'unrecognised sensor' 136 | raise fmaskerrors.FmaskParameterError(msg) 137 | 138 | # we can't do anything with the thermal yet since 139 | # we need a .mtl file or equivalent to get the gains etc 140 | self.thermalInfo = None 141 | 142 | # same with angles 143 | self.anglesInfo = None 144 | 145 | # obtain the usual extension for the GDAL driver used by RIOS 146 | # so we can create temporary files with this extension. 147 | driver = gdal.GetDriverByName(applier.DEFAULTDRIVERNAME) 148 | if driver is None: 149 | msg = 'Cannot find GDAL driver %s used by RIOS' 150 | msg = msg % applier.DEFAULTDRIVERNAME 151 | raise fmaskerrors.FmaskParameterError(msg) 152 | 153 | ext = driver.GetMetadataItem('DMD_EXTENSION') 154 | if ext is None: 155 | self.defaultExtension = '.tmp' 156 | else: 157 | self.defaultExtension = '.' + ext 158 | 159 | def setReflectiveBand(self, band, index): 160 | """ 161 | Tell fmask which band is in which index in the reflectance 162 | data stack file. band should be one of the BAND_* constants. 163 | index is zero based (ie 0 is first band in the file). 164 | 165 | These are set to default values for each sensor which are 166 | normally correct, but this function can be used to update. 167 | 168 | """ 169 | self.bands[band] = index 170 | 171 | def setThermalInfo(self, info): 172 | """ 173 | Set an instance of ThermalFileInfo. By default this is 174 | None and fmask assumes there is no thermal data available. 175 | 176 | The :func:`fmask.config.readThermalInfoFromLandsatMTL` 177 | function can be used to obtain this from a Landsat .mtl file. 178 | 179 | """ 180 | self.thermalInfo = info 181 | 182 | def setAnglesInfo(self, info): 183 | """ 184 | Set an instance of AnglesInfo. By default this is 185 | None and will need to be set before fmask will run. 186 | 187 | The :func:`fmask.config.readAnglesFromLandsatMTL` 188 | function can be used to obtain this from a Landsat .mtl file. 189 | 190 | """ 191 | self.anglesInfo = info 192 | 193 | def setTOARefScaling(self, scaling): 194 | """ 195 | Set the scaling used in the Top of Atmosphere reflectance 196 | image. The calculation is done as 197 | 198 | ref = (dn + dnOffset) / scaling 199 | 200 | and so is used in conjunction with the offset values 201 | (see setTOARefOffsets). 202 | 203 | The dnOffset was added in 2021 to cope with ESA's absurd 204 | decision to suddenly introduce an offset in their Sentinel-2 205 | TOA reflectance imagery. For Landsat, there is no need for 206 | it ever to be non-zero. 207 | 208 | """ 209 | self.TOARefScaling = scaling 210 | 211 | def setTOARefOffsetDict(self, offsetDict): 212 | """ 213 | Set the reflectance offsets to the given list. 214 | This should contain an offset value for each band used with 215 | the Fmask code. The keys are the named constants in the 216 | config module, BAND_*. 217 | 218 | The offset is added to the corresponding band pixel values 219 | before dividing by the scaling value. 220 | 221 | This facility is made available largely for use with Sentinel-2, 222 | after ESA unilaterally starting using non-zero offsets in their 223 | Level-1C imagery (Nov 2021). However, it can be used 224 | with Landsat if required. 225 | 226 | """ 227 | if len(offsetDict) != len(self.bands): 228 | msg = "Must supply offsets for all bands being used" 229 | raise fmaskerrors.FmaskParameterError(msg) 230 | 231 | self.TOARefDNoffsetDict = offsetDict 232 | 233 | def setKeepIntermediates(self, keepIntermediates): 234 | """ 235 | Set to True to keep the intermediate files created in 236 | processed. This is False by default. 237 | 238 | """ 239 | self.keepIntermediates = keepIntermediates 240 | 241 | def setCloudBufferSize(self, bufferSize): 242 | """ 243 | Extra buffer of this many pixels on cloud layer. Defaults to 5. 244 | 245 | """ 246 | self.cloudBufferSize = bufferSize 247 | 248 | def setShadowBufferSize(self, bufferSize): 249 | """ 250 | Extra buffer of this many pixels on cloud layer. Defaults to 10. 251 | 252 | """ 253 | self.shadowBufferSize = bufferSize 254 | 255 | def setMinCloudSize(self, minCloudSize): 256 | """ 257 | Set the minimum cloud size retained. This minimum is applied before any 258 | buffering of clouds. Size is specified as an area, in pixels. 259 | """ 260 | self.minCloudSize_pixels = minCloudSize 261 | 262 | def setVerbose(self, verbose): 263 | """ 264 | Print informative messages. Defaults to False. 265 | 266 | """ 267 | self.verbose = verbose 268 | 269 | def setStrictFmask(self, strictFmask): 270 | """ 271 | Set whatever options are necessary to run strictly as per Fmask paper 272 | (Zhu & Woodcock). Setting this will override the settings of other 273 | parameters on this object. 274 | 275 | """ 276 | self.strictFmask = strictFmask 277 | 278 | def setTempDir(self, tempDir): 279 | """ 280 | Temporary directory to use. Defaults to '.' (the current directory). 281 | 282 | """ 283 | self.tempDir = tempDir 284 | 285 | def setDefaultExtension(self, extension): 286 | """ 287 | Sets the default extension used by temporary files created by 288 | fmask. Defaults to the extension of the driver that RIOS 289 | is configured to use. 290 | 291 | Note that this should include the '.' - ie '.img'. 292 | 293 | """ 294 | self.defaultExtension = extension 295 | 296 | def setEqn1Swir2Thresh(self, thresh): 297 | """ 298 | Change the threshold used by Equation 1 for the SWIR2 band. 299 | This defaults to 0.03 300 | 301 | """ 302 | self.Eqn1Swir2Thresh = thresh 303 | 304 | def setEqn1ThermThresh(self, thresh): 305 | """ 306 | Change the threshold used by Equation one for BT. 307 | This defaults to 27. 308 | 309 | """ 310 | self.Eqn1ThermThresh = thresh 311 | 312 | def setEqn2WhitenessThresh(self, thresh): 313 | """ 314 | Change the threshold used by Equation 2 to determine 315 | whiteness from visible bands. This defaults to 0.7. 316 | 317 | """ 318 | self.Eqn2WhitenessThresh = thresh 319 | 320 | def setCirrusBandTestThresh(self, thresh): 321 | """ 322 | Change the threshold used by Zhu et al 2015, section 2.2.1 323 | for the cirrus band test. Defaults to 0.01. 324 | 325 | """ 326 | self.cirrusBandTestThresh = thresh 327 | 328 | def setEqn7Swir2Thresh(self, thresh): 329 | """ 330 | Change the threshold used by Equation 7 (water test) 331 | for the Swir2 band. This defaults to 0.03. 332 | 333 | """ 334 | self.Eqn7Swir2Thresh = thresh 335 | 336 | def setEqn17CloudProbThresh(self, thresh): 337 | """ 338 | Change the threshold used by Equation 17. The threshold 339 | given here is the constant term added to the end of the equation 340 | for the land probability threshold. Original paper had this as 0.2, 341 | although Zhu et al's MATLAB code now defaults it to 0.225 (i.e. 22.5%) 342 | 343 | """ 344 | self.Eqn17CloudProbThresh = thresh 345 | 346 | def setEqn20ThermThresh(self, thresh): 347 | """ 348 | Change the threshold used by Equation 20 (snow) 349 | for BT. This defaults to 3.8. 350 | 351 | """ 352 | self.Eqn20ThermThresh = thresh 353 | 354 | def setEqn20NirSnowThresh(self, thresh): 355 | """ 356 | Change the threshold used by Equation 20 (snow) 357 | for NIR reflectance. This defaults to 0.11 358 | 359 | """ 360 | self.Eqn20NirSnowThresh = thresh 361 | 362 | def setEqn20GreenSnowThresh(self, thresh): 363 | """ 364 | Change the threshold used by Equation 20 (snow) 365 | for green reflectance. This defaults to 0.1 366 | 367 | """ 368 | self.Eqn20GreenSnowThresh = thresh 369 | 370 | def setCirrusProbRatio(self, ratio): 371 | """ 372 | Change the ratio used by Zhu et al 2015 Equation 1 373 | to determine the cirrus cloud probability. Defaults 374 | to 0.04. 375 | 376 | """ 377 | self.cirrusProbRatio = ratio 378 | 379 | def setEqn19NIRFillThresh(self, thresh): 380 | """ 381 | Change the threshold used by Equation 19 to determine 382 | potential cloud shadow from the difference between NIR 383 | and flood filled NIR. Defaults to 0.02. 384 | 385 | """ 386 | self.Eqn19NIRFillThresh = thresh 387 | 388 | def setSen2displacementTest(self, useDisplacementTest): 389 | """ 390 | Set whether or not to use the Frantz (2018) parallax displacement test 391 | to remove false clouds. Pass True if the test is desired, False otherwise. 392 | 393 | """ 394 | self.sen2displacementTest = useDisplacementTest 395 | 396 | def setGdalDriverName(self, driverName): 397 | """ 398 | Change the GDAL driver used for writing the final output file. Default 399 | value is taken from the default for the RIOS package, as per $RIOS_DFLT_DRIVER. 400 | """ 401 | self.gdalDriverName = driverName 402 | 403 | 404 | class FmaskFilenames(object): 405 | """ 406 | Class that contains the filenames used in the fmask run. 407 | """ 408 | toaRef = None 409 | thermal = None 410 | saturationMask = None 411 | outputMask = None 412 | 413 | def __init__(self, toaRefFile=None, thermalFile=None, outputMask=None, 414 | saturationMask=None): 415 | self.toaRef = toaRefFile 416 | self.thermal = thermalFile 417 | self.saturationMask = saturationMask 418 | self.outputMask = outputMask 419 | 420 | def setThermalFile(self, thermalFile): 421 | """ 422 | Set the path of the input thermal file. To make use 423 | of this, the :func:`fmask.config.FmaskConfig.setThermalInfo` 424 | function must also be called so that fmask knows how 425 | to use the file. 426 | 427 | This file should be in any GDAL readable format. 428 | 429 | """ 430 | self.thermal = thermalFile 431 | 432 | def setTOAReflectanceFile(self, toaRefFile): 433 | """ 434 | Set the path of the input top of atmosphere (TOA) file. It pays 435 | to check that the default set of bands match what fmask expects in 436 | the :class:`fmask.config.FmaskConfig` class and update if necessary. 437 | 438 | This should have numbers which are reflectance * 1000 439 | 440 | Use the :func:`fmask.landsatTOA.makeTOAReflectance` function to create 441 | this file from raw Landsat radiance (or the fmask_usgsLandsatTOA 442 | command line program supplied with fmask). 443 | 444 | It is assumed that any values that are nulls in the original radiance 445 | image are set to the ignore values in the toaRefFile. 446 | 447 | This file should be in any GDAL readable format. 448 | 449 | """ 450 | self.toaRef = toaRefFile 451 | 452 | def setSaturationMask(self, mask): 453 | """ 454 | Set the mask to use for ignoring saturated pixels. By default 455 | no mask is used and all pixels are assumed to be unsaturated. 456 | This will cause problems for the whiteness test if some pixels 457 | are in fact saturated, but not masked out. 458 | 459 | Use the :func:`fmask.saturation.makeSaturationMask` function to 460 | create this from input radiance data. 461 | 462 | This mask should be 1 for pixels that are saturated, 0 otherwise. 463 | 464 | Note that this is not in the original paper so cannot be considered 465 | 'strict', but if provided is used no matter the strict setting in 466 | :class:`fmask.config.FmaskConfig`. 467 | 468 | This file should be in any GDAL readable format. 469 | 470 | """ 471 | self.saturationMask = mask 472 | 473 | def setOutputCloudMaskFile(self, cloudMask): 474 | """ 475 | Set the output cloud mask path. 476 | 477 | Note that this file will be written in the format 478 | that RIOS is currently configured to use. See the 479 | `RIOS documentation `_ 480 | for more details. Note that the default is HFA (.img) and can 481 | be overridden using environment variables. 482 | 483 | """ 484 | self.outputMask = cloudMask 485 | 486 | 487 | class ThermalFileInfo(object): 488 | """ 489 | Contains parameters for interpreting thermal file. 490 | See :func:`fmask.config.readThermalInfoFromLandsatMTL`. 491 | 492 | """ 493 | thermalBand1040um = None 494 | thermalGain1040um = None 495 | thermalOffset1040um = None 496 | thermalK1_1040um = None 497 | thermalK2_1040um = None 498 | 499 | def __init__(self, thermalBand1040um, thermalGain1040um, 500 | thermalOffset1040um, thermalK1_1040um, thermalK2_1040um): 501 | self.thermalBand1040um = thermalBand1040um 502 | self.thermalGain1040um = thermalGain1040um 503 | self.thermalOffset1040um = thermalOffset1040um 504 | self.thermalK1_1040um = thermalK1_1040um 505 | self.thermalK2_1040um = thermalK2_1040um 506 | 507 | def scaleThermalDNtoC(self, scaledBT): 508 | """ 509 | Use the given params to unscale the thermal, and then 510 | convert it from K to C. Return a single 2-d array of the 511 | temperature in deg C. 512 | """ 513 | KELVIN_ZERO_DEGC = scipy.constants.zero_Celsius 514 | rad = (scaledBT[self.thermalBand1040um].astype(float) * 515 | self.thermalGain1040um + self.thermalOffset1040um) 516 | # see http://www.yale.edu/ceo/Documentation/Landsat_DN_to_Kelvin.pdf 517 | # and https://landsat.usgs.gov/Landsat8_Using_Product.php 518 | rad[rad <= 0] = 0.00001 # to stop errors below 519 | temp = self.thermalK2_1040um / numpy.log(self.thermalK1_1040um / rad + 1.0) 520 | bt = temp - KELVIN_ZERO_DEGC 521 | return bt 522 | 523 | 524 | # Keys within a .mtl file for each band 525 | LANDSAT_RADIANCE_MULT = 'RADIANCE_MULT_BAND_%s' 526 | LANDSAT_RADIANCE_ADD = 'RADIANCE_ADD_BAND_%s' 527 | LANDSAT_K1_CONST = 'K1_CONSTANT_BAND_%s' 528 | LANDSAT_K2_CONST = 'K2_CONSTANT_BAND_%s' 529 | 530 | # Oldest format of MTL file has only min/max values 531 | LANDSAT_LMAX_KEY = 'LMAX_BAND%s' 532 | LANDSAT_LMIN_KEY = 'LMIN_BAND%s' 533 | LANDSAT_QCALMAX_KEY = 'QCALMAX_BAND%s' 534 | LANDSAT_QCALMIN_KEY = 'QCALMIN_BAND%s' 535 | 536 | # band numbers in mtl file for gain and offset for thermal 537 | LANDSAT_TH_BAND_NUM_DICT = {'LANDSAT_4': '6', 538 | 'LANDSAT_5': '6', 539 | 'LANDSAT_7': '6_VCID_1', 540 | 'LANDSAT_8': '10', 541 | 'LANDSAT_9': '10'} 542 | 543 | 544 | # for some reason L4, 5, and 7 don't 545 | # have these numbers in the mtl file, but L8 does 546 | # from http://www.yale.edu/ceo/Documentation/Landsat_DN_to_Kelvin.pdf 547 | LANDSAT_K1_DICT = {'TM': 607.76, 'ETM': 666.09, 'ETM+': 666.09} 548 | LANDSAT_K2_DICT = {'TM': 1260.56, 'ETM': 1282.71, 'ETM+': 1282.71} 549 | 550 | 551 | def readThermalInfoFromLandsatMTL(mtlfile, thermalBand1040um=0): 552 | """ 553 | Returns an instance of ThermalFileInfo given a path to the mtl 554 | file and the index of the thermal band. 555 | 556 | """ 557 | mtlData = readMTLFile(mtlfile) 558 | gain = None 559 | offset = None 560 | k1 = None 561 | k2 = None 562 | if 'SPACECRAFT_ID' in mtlData: 563 | # we can now grab the gain and offset 564 | spaceCraft = mtlData['SPACECRAFT_ID'] 565 | band = LANDSAT_TH_BAND_NUM_DICT[spaceCraft] 566 | 567 | s = LANDSAT_RADIANCE_MULT % band 568 | 569 | oldestMtlFormat = (s not in mtlData) 570 | 571 | if not oldestMtlFormat: 572 | gain = float(mtlData[s]) 573 | s = LANDSAT_RADIANCE_ADD % band 574 | offset = float(mtlData[s]) 575 | else: 576 | # Oldest format MTL file 577 | if spaceCraft == "LANDSAT_7": 578 | band = "61" 579 | lMax = float(mtlData[LANDSAT_LMAX_KEY % band]) 580 | lMin = float(mtlData[LANDSAT_LMIN_KEY % band]) 581 | qcalMax = float(mtlData[LANDSAT_QCALMAX_KEY % band]) 582 | qcalMin = float(mtlData[LANDSAT_QCALMIN_KEY % band]) 583 | gain = (lMax - lMin) / (qcalMax - qcalMin) 584 | offset = lMin - qcalMin * gain 585 | 586 | if 'SENSOR_ID' in mtlData: 587 | # look for k1 and k2 588 | sensor = mtlData['SENSOR_ID'] 589 | s = LANDSAT_K1_CONST % band 590 | if s in mtlData: 591 | k1 = float(mtlData[s]) 592 | else: 593 | # drop back to our own values if not in file 594 | k1 = LANDSAT_K1_DICT[sensor] 595 | 596 | s = LANDSAT_K2_CONST % band 597 | if s in mtlData: 598 | k2 = float(mtlData[s]) 599 | else: 600 | # drop back to our own values if not in file 601 | k2 = LANDSAT_K2_DICT[sensor] 602 | 603 | if gain is not None and offset is not None and k1 is not None and k2 is not None: 604 | thermalInfo = ThermalFileInfo(thermalBand1040um, gain, 605 | offset, k1, k2) 606 | else: 607 | msg = 'Cannot find SPACECRAFT_ID/SENSOR_ID in MTL file' 608 | raise fmaskerrors.FmaskFileError(msg) 609 | 610 | return thermalInfo 611 | 612 | 613 | class AnglesInfo(object): 614 | """ 615 | Abstract base class that Contains view and solar angle 616 | information for file (in radians). 617 | 618 | """ 619 | __metaclass__ = abc.ABCMeta 620 | 621 | def prepareForQuerying(self): 622 | """ 623 | Called when fmask is about to query this object for angles. 624 | Derived class should do any reading of files into memory required here. 625 | """ 626 | 627 | def releaseMemory(self): 628 | """ 629 | Called when fmask has finished querying this object. 630 | Can release any allocated memory. 631 | """ 632 | 633 | @abc.abstractmethod 634 | def getSolarZenithAngle(self, indices): 635 | """ 636 | Return the average solar zenith angle for the given indices 637 | """ 638 | 639 | @abc.abstractmethod 640 | def getSolarAzimuthAngle(self, indices): 641 | """ 642 | Return the average solar azimuth angle for the given indices 643 | """ 644 | 645 | @abc.abstractmethod 646 | def getViewZenithAngle(self, indices): 647 | """ 648 | Return the average view zenith angle for the given indices 649 | """ 650 | 651 | @abc.abstractmethod 652 | def getViewAzimuthAngle(self, indices): 653 | """ 654 | Return the average view azimuth angle for the given indices 655 | """ 656 | 657 | @abc.abstractmethod 658 | def setScaleToRadians(self, scale): 659 | """ 660 | Set scaling factor to get radians from angles image values. 661 | """ 662 | 663 | 664 | class AnglesFileInfo(AnglesInfo): 665 | """ 666 | An implementation of AnglesInfo that reads the information from 667 | GDAL supported files. 668 | """ 669 | def __init__(self, solarZenithFilename, solarZenithBand, solarAzimuthFilename, 670 | solarAzimuthBand, viewZenithFilename, viewZenithBand, 671 | viewAzimuthFilename, viewAzimuthBand): 672 | """ 673 | Initialises the object with the names and band numbers of the angles. 674 | band numbers should be 0 based - ie first band is 0. 675 | """ 676 | self.solarZenithFilename = solarZenithFilename 677 | self.solarZenithBand = solarZenithBand 678 | self.solarAzimuthFilename = solarAzimuthFilename 679 | self.solarAzimuthBand = solarAzimuthBand 680 | self.viewZenithFilename = viewZenithFilename 681 | self.viewZenithBand = viewZenithBand 682 | self.viewAzimuthFilename = viewAzimuthFilename 683 | self.viewAzimuthBand = viewAzimuthBand 684 | # these will contain the actual image data once read 685 | # by prepareForQuerying() 686 | self.solarZenithData = None 687 | self.solarAzimuthData = None 688 | self.viewZenithData = None 689 | self.viewAzimuthData = None 690 | 691 | # This default value matches the file produced by fmask_usgsLandsatMakeAnglesImage 692 | self.scaleToRadians = 0.01 693 | 694 | @staticmethod 695 | def readData(filename, bandNum): 696 | ds = gdal.Open(filename) 697 | band = ds.GetRasterBand(bandNum + 1) 698 | data = band.ReadAsArray() 699 | del ds 700 | return data 701 | 702 | def prepareForQuerying(self): 703 | """ 704 | Called when fmask is about to query this object for angles. 705 | """ 706 | self.solarZenithData = self.readData(self.solarZenithFilename, 707 | self.solarZenithBand) 708 | self.solarAzimuthData = self.readData(self.solarAzimuthFilename, 709 | self.solarAzimuthBand) 710 | self.viewZenithData = self.readData(self.viewZenithFilename, 711 | self.viewZenithBand) 712 | self.viewAzimuthData = self.readData(self.viewAzimuthFilename, 713 | self.viewAzimuthBand) 714 | 715 | def releaseMemory(self): 716 | """ 717 | Called when fmask has finished querying this object. 718 | """ 719 | del self.solarZenithData 720 | del self.solarAzimuthData 721 | del self.viewZenithData 722 | del self.viewAzimuthData 723 | 724 | def getSolarZenithAngle(self, indices): 725 | """ 726 | Return the average solar zenith angle for the given indices 727 | """ 728 | return self.solarZenithData[indices].mean() * self.scaleToRadians 729 | 730 | def getSolarAzimuthAngle(self, indices): 731 | """ 732 | Return the average solar azimuth angle for the given indices 733 | """ 734 | return self.solarAzimuthData[indices].mean() * self.scaleToRadians 735 | 736 | def getViewZenithAngle(self, indices): 737 | """ 738 | Return the average view zenith angle for the given indices 739 | """ 740 | return self.viewZenithData[indices].mean() * self.scaleToRadians 741 | 742 | def getViewAzimuthAngle(self, indices): 743 | """ 744 | Return the average view azimuth angle for the given indices 745 | """ 746 | return self.viewAzimuthData[indices].mean() * self.scaleToRadians 747 | 748 | def setScaleToRadians(self, scale): 749 | """ 750 | Set scaling factor to get radians from angles image values. 751 | """ 752 | self.scaleToRadians = scale 753 | 754 | 755 | class AngleConstantInfo(AnglesInfo): 756 | """ 757 | An implementation of AnglesInfo that uses constant 758 | angles accross the scene. 759 | """ 760 | def __init__(self, solarZenithAngle, solarAzimuthAngle, viewZenithAngle, 761 | viewAzimuthAngle): 762 | self.solarZenithAngle = solarZenithAngle 763 | self.solarAzimuthAngle = solarAzimuthAngle 764 | self.viewZenithAngle = viewZenithAngle 765 | self.viewAzimuthAngle = viewAzimuthAngle 766 | 767 | def getSolarZenithAngle(self, indices): 768 | """ 769 | Return the solar zenith angle 770 | """ 771 | return self.solarZenithAngle 772 | 773 | def getSolarAzimuthAngle(self, indices): 774 | """ 775 | Return the solar azimuth angle 776 | """ 777 | return self.solarAzimuthAngle 778 | 779 | def getViewZenithAngle(self, indices): 780 | """ 781 | Return the view zenith angle 782 | """ 783 | return self.viewZenithAngle 784 | 785 | def getViewAzimuthAngle(self, indices): 786 | """ 787 | Return the view azimuth angle 788 | """ 789 | return self.viewAzimuthAngle 790 | 791 | 792 | def readMTLFile(mtl): 793 | """ 794 | Very simple .mtl file reader that just creates a dictionary 795 | of key and values and returns it 796 | """ 797 | dict = {} 798 | for line in open(mtl): 799 | arr = line.split('=') 800 | if len(arr) == 2: 801 | (key, value) = arr 802 | dict[key.strip()] = value.replace('"', '').strip() 803 | 804 | # For the older format of the MTL file, a few fields had different names. So, we fake the 805 | # new names, so that the rest of the code can just use those. 806 | if 'ACQUISITION_DATE' in dict: 807 | dict['DATE_ACQUIRED'] = dict['ACQUISITION_DATE'] 808 | if 'SCENE_CENTER_SCAN_TIME' in dict: 809 | dict['SCENE_CENTER_TIME'] = dict['SCENE_CENTER_SCAN_TIME'] 810 | 811 | # Oldest format has spacecraft ID string formatted differently, so reformat it. 812 | spaceCraft = dict['SPACECRAFT_ID'] 813 | if spaceCraft.startswith('Landsat') and '_' not in spaceCraft: 814 | satNum = spaceCraft[-1] 815 | dict['SPACECRAFT_ID'] = "LANDSAT_" + satNum 816 | 817 | return dict 818 | 819 | 820 | def readAnglesFromLandsatMTL(mtlfile): 821 | """ 822 | Given the path to a Landsat USGS .MTL file, read the angles 823 | out and return an instance of AngleConstantInfo. 824 | 825 | This is no longer supported, and this routine now raises an exception. 826 | 827 | """ 828 | msg = ("The simplified option of assuming constant angles across the whole image is "+ 829 | "no longer supported. You must use per-pixel angles. ") 830 | raise fmaskerrors.FmaskNotSupportedError(msg) 831 | -------------------------------------------------------------------------------- /fmask/fillminima.py: -------------------------------------------------------------------------------- 1 | """ 2 | Module to implement filling of local minima in a raster surface. 3 | 4 | The algorithm is from 5 | Soille, P., and Gratin, C. (1994). An efficient algorithm for drainage network 6 | extraction on DEMs. J. Visual Communication and Image Representation. 7 | 5(2). 181-189. 8 | 9 | The algorithm is intended for hydrological processing of a DEM, but is used by the 10 | Fmask cloud shadow algorithm as part of its process for finding local minima which 11 | represent potential shadow objects. 12 | 13 | """ 14 | # This file is part of 'python-fmask' - a cloud masking module 15 | # Copyright (C) 2015 Neil Flood 16 | # 17 | # This program is free software; you can redistribute it and/or 18 | # modify it under the terms of the GNU General Public License 19 | # as published by the Free Software Foundation; either version 3 20 | # of the License, or (at your option) any later version. 21 | # 22 | # This program is distributed in the hope that it will be useful, 23 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 24 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 25 | # GNU General Public License for more details. 26 | # 27 | # You should have received a copy of the GNU General Public License 28 | # along with this program; if not, write to the Free Software 29 | # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 30 | 31 | import os 32 | import numpy 33 | from scipy.ndimage import grey_dilation 34 | 35 | # Fail slightly less drastically when running from ReadTheDocs 36 | if os.getenv('READTHEDOCS', default='False') != 'True': 37 | from . import _fillminima 38 | 39 | 40 | def fillMinima(img, nullval, boundaryval): 41 | """ 42 | Fill all local minima in the input img. The input 43 | array should be a numpy 2-d array. This function returns 44 | an array of the same shape and datatype, with the same contents, but 45 | with local minima filled using the reconstruction-by-erosion algorithm. 46 | 47 | """ 48 | (nrows, ncols) = img.shape 49 | dtype = img.dtype 50 | nullmask = (img == nullval) 51 | nonNullmask = numpy.logical_not(nullmask) 52 | (hMax, hMin) = (int(img[nonNullmask].max()), int(img[nonNullmask].min())) 53 | boundaryval = max(boundaryval, hMin) 54 | img2 = numpy.zeros((nrows, ncols), dtype=dtype) 55 | img2.fill(hMax) 56 | 57 | if nullmask.sum() > 0: 58 | nullmaskDilated = grey_dilation(nullmask, size=(3, 3)) 59 | innerBoundary = nullmaskDilated ^ nullmask 60 | (boundaryRows, boundaryCols) = numpy.where(innerBoundary) 61 | else: 62 | img2[0, :] = img[0, :] 63 | img2[-1, :] = img[-1, :] 64 | img2[:, 0] = img[:, 0] 65 | img2[:, -1] = img[:, -1] 66 | (boundaryRows, boundaryCols) = numpy.where(img2!=hMax) 67 | 68 | # on some systems (32 bit only?) numpy.where returns int32 69 | # rather than int64. Convert so we don't have to handle both in C. 70 | boundaryRows = boundaryRows.astype(numpy.int64) 71 | boundaryCols = boundaryCols.astype(numpy.int64) 72 | 73 | _fillminima.fillMinima(img, img2, hMin, hMax, nullmask, boundaryval, 74 | boundaryRows, boundaryCols) 75 | 76 | img2[nullmask] = nullval 77 | 78 | return img2 79 | -------------------------------------------------------------------------------- /fmask/fmaskerrors.py: -------------------------------------------------------------------------------- 1 | """ 2 | Exceptions used within fmask 3 | """ 4 | 5 | # This file is part of 'python-fmask' - a cloud masking module 6 | # Copyright (C) 2015 Neil Flood 7 | # 8 | # This program is free software; you can redistribute it and/or 9 | # modify it under the terms of the GNU General Public License 10 | # as published by the Free Software Foundation; either version 3 11 | # of the License, or (at your option) any later version. 12 | # 13 | # This program is distributed in the hope that it will be useful, 14 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 15 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 | # GNU General Public License for more details. 17 | # 18 | # You should have received a copy of the GNU General Public License 19 | # along with this program; if not, write to the Free Software 20 | # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 21 | 22 | 23 | class FmaskException(Exception): 24 | "An exception rasied by Fmask" 25 | 26 | 27 | class FmaskParameterError(FmaskException): 28 | "Something is wrong with a parameter" 29 | 30 | 31 | class FmaskFileError(FmaskException): 32 | "Data in file is incorrect" 33 | 34 | 35 | class FmaskNotSupportedError(FmaskException): 36 | "Requested operation is not supported" 37 | 38 | 39 | class FmaskInstallationError(FmaskException): 40 | "Problem with installation of Fmask or a required package" 41 | 42 | 43 | class Sen2MetaError(Exception): 44 | "Error with Sentinel-2 metadata" 45 | -------------------------------------------------------------------------------- /fmask/landsatTOA.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ 3 | Module that handles convertion of scaled radiance (DN) 4 | values from USGS to Top of Atmosphere (TOA) reflectance (\*1000). 5 | """ 6 | # This file is part of 'python-fmask' - a cloud masking module 7 | # Copyright (C) 2015 Neil Flood 8 | # 9 | # This program is free software; you can redistribute it and/or 10 | # modify it under the terms of the GNU General Public License 11 | # as published by the Free Software Foundation; either version 3 12 | # of the License, or (at your option) any later version. 13 | # 14 | # This program is distributed in the hope that it will be useful, 15 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 16 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 17 | # GNU General Public License for more details. 18 | # 19 | # You should have received a copy of the GNU General Public License 20 | # along with this program; if not, write to the Free Software 21 | # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 22 | from __future__ import print_function, division 23 | 24 | import numpy 25 | from osgeo import gdal 26 | from rios import applier, cuiprogress, fileinfo 27 | from . import config 28 | 29 | gdal.UseExceptions() 30 | 31 | # Derived by Pete Bunting from 6S 32 | LANDSAT8_ESUN = [1876.61, 1970.03, 1848.9, 1571.3, 967.66, 245.73, 82.03, 361.72] 33 | # From Chander, G., Markham, B.L., Helder, D.L. (2008) 34 | # Summary of current radiometric calibration coefficients for Landsat MSS, TM, ETM+, and EO-1 ALI sensors 35 | # http://landsathandbook.gsfc.nasa.gov/pdfs/Landsat_Calibration_Summary_RSE.pdf 36 | LANDSAT4_ESUN = [1983.0, 1795.0, 1539.0, 1028.0, 219.8, 83.49] 37 | LANDSAT5_ESUN = [1983.0, 1796.0, 1536.0, 1031.0, 220.0, 83.44] 38 | LANDSAT7_ESUN = [1997.0, 1812.0, 1533.0, 1039.0, 230.8, 84.90] 39 | # Derived by RSC, using Landsat-9 RSR & Kurucz solar spectrum 40 | LANDSAT9_ESUN = [1881.97, 1972.61, 1851.75, 1572.03, 969.37, 246.18, 82.11, 360.48] 41 | 42 | ESUN_LOOKUP = {'LANDSAT_4': LANDSAT4_ESUN, 43 | 'LANDSAT_5': LANDSAT5_ESUN, 44 | 'LANDSAT_7': LANDSAT7_ESUN, 45 | 'LANDSAT_8': LANDSAT8_ESUN, 46 | 'LANDSAT_9': LANDSAT9_ESUN} 47 | 48 | RADIANCE_MULT = 'RADIANCE_MULT_BAND_%d' 49 | RADIANCE_ADD = 'RADIANCE_ADD_BAND_%d' 50 | 51 | LMAX_KEY = 'LMAX_BAND%d' 52 | LMIN_KEY = 'LMIN_BAND%d' 53 | QCALMAX_KEY = 'QCALMAX_BAND%d' 54 | QCALMIN_KEY = 'QCALMIN_BAND%d' 55 | 56 | 57 | # band numbers in mtl file for gain and offset for reflective 58 | BAND_NUM_DICT = {'LANDSAT_4': (1, 2, 3, 4, 5, 7), 59 | 'LANDSAT_5': (1, 2, 3, 4, 5, 7), 60 | 'LANDSAT_7': (1, 2, 3, 4, 5, 7), 61 | 'LANDSAT_8': (1, 2, 3, 4, 5, 6, 7, 9), 62 | 'LANDSAT_9': (1, 2, 3, 4, 5, 6, 7, 9)} 63 | 64 | 65 | def readGainsOffsets(mtlInfo): 66 | """ 67 | Read the gains and offsets out of the .MTL file 68 | """ 69 | spaceCraft = mtlInfo['SPACECRAFT_ID'] 70 | nbands = len(BAND_NUM_DICT[spaceCraft]) 71 | 72 | gains = numpy.zeros(nbands) 73 | offsets = numpy.zeros(nbands) 74 | 75 | if (RADIANCE_MULT % 1) in mtlInfo: 76 | # Newer format for MTL file 77 | for idx, band in enumerate(BAND_NUM_DICT[spaceCraft]): 78 | s = RADIANCE_MULT % band 79 | gain = float(mtlInfo[s]) 80 | gains[idx] = gain 81 | 82 | s = RADIANCE_ADD % band 83 | offset = float(mtlInfo[s]) 84 | offsets[idx] = offset 85 | else: 86 | # Old format, calculate gain and offset from min/max values 87 | for (idx, band) in enumerate(BAND_NUM_DICT[spaceCraft]): 88 | lMax = float(mtlInfo[LMAX_KEY % band]) 89 | lMin = float(mtlInfo[LMIN_KEY % band]) 90 | qcalMax = float(mtlInfo[QCALMAX_KEY % band]) 91 | qcalMin = float(mtlInfo[QCALMIN_KEY % band]) 92 | 93 | gain = (lMax - lMin) / (qcalMax - qcalMin) 94 | offset = lMin - qcalMin * gain 95 | 96 | gains[idx] = gain 97 | offsets[idx] = offset 98 | 99 | return gains, offsets 100 | 101 | 102 | def earthSunDistance(date): 103 | """ 104 | Given a date in YYYYMMDD will compute the earth sun distance in astronomical units 105 | """ 106 | import datetime 107 | year = int(date[:4]) 108 | month = int(date[4:6]) 109 | day = int(date[6:]) 110 | d1 = datetime.datetime(year, month, day) 111 | d2 = datetime.datetime(year, 1, 1) # first day of year 112 | deltaT = d1 - d2 113 | jday = deltaT.days + 1 # julian day of year. 114 | ds = (1.0 - 0.01673 * numpy.cos(0.9856 * (jday - 4) * numpy.pi / 180.0)) 115 | return ds 116 | 117 | 118 | def riosTOA(info, inputs, outputs, otherinputs): 119 | """ 120 | Called from RIOS 121 | """ 122 | nbands = inputs.infile.shape[0] 123 | 124 | infile = inputs.infile.astype(numpy.float32) 125 | inIgnore = otherinputs.inNull 126 | if inIgnore is None: 127 | inIgnore = 0 128 | 129 | cosSunZen = numpy.cos(inputs.angles[3] * otherinputs.anglesToRadians) 130 | 131 | nullMask = (inputs.infile == inIgnore).any(axis=0) 132 | 133 | toaRefList = [] 134 | for band in range(nbands): 135 | rtoa = infile[band] * otherinputs.gains[band] + otherinputs.offsets[band] 136 | 137 | p = numpy.pi * rtoa * otherinputs.earthSunDistanceSq / (otherinputs.esun[band] * cosSunZen) 138 | # clip to a sensible range 139 | numpy.clip(p, 0.0, 2.0, out=p) 140 | 141 | toaRefList.append(p) 142 | 143 | out = numpy.array(toaRefList) * 10000.0 144 | # convert to int16 145 | outputs.outfile = out.astype(numpy.int16) 146 | # Mask out where input is null 147 | for i in range(len(outputs.outfile)): 148 | outputs.outfile[i][nullMask] = otherinputs.outNull 149 | 150 | 151 | def makeTOAReflectance(infile, mtlFile, anglesfile, outfile): 152 | """ 153 | Main routine - does the calculation 154 | 155 | The eqn for TOA reflectance, p, is 156 | p = pi * L * d^2 / E * cos(theta) 157 | 158 | d = earthSunDistance(date) 159 | L = image pixel (radiance) 160 | E = exoatmospheric irradiance for the band, and 161 | theta = solar zenith angle. 162 | 163 | Assumes infile is radiance values in DN from USGS. 164 | mtlFile is the .mtl file. 165 | outfile will be created in the default format that RIOS 166 | is configured to use and will be top of atmosphere 167 | reflectance values *10000. Also assumes that the 168 | angles image file is scaled as radians*100, and has layers for 169 | satAzimuth, satZenith, sunAzimuth, sunZenith, in that order. 170 | 171 | """ 172 | mtlInfo = config.readMTLFile(mtlFile) 173 | spaceCraft = mtlInfo['SPACECRAFT_ID'] 174 | date = mtlInfo['DATE_ACQUIRED'] 175 | date = date.replace('-', '') 176 | 177 | inputs = applier.FilenameAssociations() 178 | inputs.infile = infile 179 | inputs.angles = anglesfile 180 | 181 | outputs = applier.FilenameAssociations() 182 | outputs.outfile = outfile 183 | 184 | otherinputs = applier.OtherInputs() 185 | otherinputs.earthSunDistance = earthSunDistance(date) 186 | otherinputs.earthSunDistanceSq = otherinputs.earthSunDistance * otherinputs.earthSunDistance 187 | otherinputs.esun = ESUN_LOOKUP[spaceCraft] 188 | gains, offsets = readGainsOffsets(mtlInfo) 189 | otherinputs.gains = gains 190 | otherinputs.offsets = offsets 191 | otherinputs.anglesToRadians = 0.01 192 | otherinputs.outNull = 32767 193 | imginfo = fileinfo.ImageInfo(infile) 194 | otherinputs.inNull = imginfo.nodataval[0] 195 | 196 | controls = applier.ApplierControls() 197 | controls.progress = cuiprogress.GDALProgressBar() 198 | controls.setStatsIgnore(otherinputs.outNull) 199 | controls.setCalcStats(False) 200 | 201 | applier.apply(riosTOA, inputs, outputs, otherinputs, controls=controls) 202 | 203 | # Explicitly set the null value in the output 204 | ds = gdal.Open(outfile, gdal.GA_Update) 205 | for i in range(ds.RasterCount): 206 | ds.GetRasterBand(i + 1).SetNoDataValue(otherinputs.outNull) 207 | -------------------------------------------------------------------------------- /fmask/landsatangles.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # This file is part of 'python-fmask' - a cloud masking module 3 | # Copyright (C) 2015 Neil Flood 4 | # 5 | # This program is free software; you can redistribute it and/or 6 | # modify it under the terms of the GNU General Public License 7 | # as published by the Free Software Foundation; either version 3 8 | # of the License, or (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with this program; if not, write to the Free Software 17 | # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 18 | """ 19 | Functions relating to estimating the per-pixel sun and satellite angles for 20 | a given Landsat image. These are rough estimates, using the generic 21 | characteristics of the Landsat 5 platform, and are not particularly accurate, 22 | but good enough for the current purposes. 23 | 24 | Historically, the USGS have not supplied satellite zenith/azimuth angles, and have only 25 | supplied scene-centre values for sun zenith/azimuth angles. Since the satellite 26 | view geometry is important in correctly tracking a shadow when matching shadows 27 | to their respective clouds, the Fmask algorithm requires good estimates of all these 28 | angles. The routines contained here are used to derive per-pixel estimates of 29 | these angles. 30 | 31 | As of mid-2016, the USGS are planning to supply sufficient information to calculate 32 | these angles directly from orbit ephemeris data. When that comes about, it seems likely 33 | that the need for the routines here will diminish, but any data downloaded from USGS 34 | prior to then will still require this approach, as the associated angle metadata will 35 | not be present. 36 | 37 | The core Fmask code in this package is adaptable enough to be configured for either 38 | approach. 39 | 40 | The general approach for satellite angles is to estimate the nadir line by running it 41 | down the middle of the image data area. The satellite azimuth is assumed to be 42 | at right angles to this nadir line, which is only roughly correct. For the whisk-broom 43 | sensors on Landsat-5 and Landsat-7, this angle is not 90 degrees, but is affected by 44 | earth rotation and is latitude dependent. For Landsat-8, the scan line is at 45 | right angles, due to the compensation for earth rotation, but the push-broom is 46 | made up of sub-modules which point in slightly different directions, giving 47 | slightly different satellite azimuths along the scan line. None of these effects 48 | are included in the current estimates. The satellite zenith is estimated based on the 49 | nadir point, the scan-line, and the assumed satellite altitude, and includes the 50 | appropriate allowance for earth curvature. 51 | 52 | Because this works by searching the imagery for the non-null area, and assumes that 53 | this represents a full-swath image, it would not work for a subset of a full image. 54 | 55 | The sun angles are approximated using the algorithm found in the Fortran code with 56 | 6S (Second Simulation of the Satellite Signal in the Solar Spectrum). The subroutine 57 | in question is the POSSOL() routine. I translated the Fortran code into Python for 58 | inclusion here. 59 | 60 | """ 61 | from __future__ import print_function, division 62 | 63 | import datetime 64 | 65 | import numpy 66 | from osgeo import osr 67 | 68 | from rios import applier 69 | from rios import fileinfo 70 | 71 | osr.UseExceptions() 72 | 73 | 74 | def findImgCorners(img, imgInfo): 75 | """ 76 | Find the corners of the data within the given template image 77 | Return a numpy array of (x, y) coordinates. The array has 2 columns, for X and Y. 78 | Each row is a corner, in the order: 79 | 80 | top-left, top-right, bottom-left, bottom-right. 81 | 82 | Uses RIOS to pass through the image searching for non-null data, 83 | and find the extremes. Assumes we are working with a full-swathe Landsat 84 | image. 85 | 86 | Each list element is a numpy array of (x, y) 87 | 88 | """ 89 | infiles = applier.FilenameAssociations() 90 | outfiles = applier.FilenameAssociations() 91 | otherargs = applier.OtherInputs() 92 | 93 | infiles.img = img 94 | otherargs.tl = None 95 | otherargs.tr = None 96 | otherargs.bl = None 97 | otherargs.br = None 98 | otherargs.nullVal = imgInfo.nodataval[0] 99 | if otherargs.nullVal is None: 100 | otherargs.nullVal = 0 101 | 102 | applier.apply(findCorners, infiles, outfiles, otherargs) 103 | 104 | corners = numpy.array([ 105 | otherargs.tl, 106 | otherargs.tr, 107 | otherargs.bl, 108 | otherargs.br, 109 | ]) 110 | return corners 111 | 112 | 113 | def findCorners(info, inputs, outputs, otherargs): 114 | """ 115 | Called from RIOS 116 | 117 | Checks non-null area of image block. Finds extremes, records coords 118 | of extremes against those already in otherargs. 119 | 120 | Note that the logic is very specific to the orientation of the usual Landsat 121 | descending pass imagery. The same logic should not be applied to swathes 122 | oriented in other, e.g. for other satellites. 123 | 124 | """ 125 | (xblock, yblock) = info.getBlockCoordArrays() 126 | 127 | nonnull = (inputs.img != otherargs.nullVal).all(axis=0) 128 | 129 | xNonnull = xblock[nonnull] 130 | yNonnull = yblock[nonnull] 131 | 132 | if len(xNonnull) > 0: 133 | topNdx = numpy.argmax(yNonnull) 134 | topXY = (xNonnull[topNdx], yNonnull[topNdx]) 135 | leftNdx = numpy.argmin(xNonnull) 136 | leftXY = (xNonnull[leftNdx], yNonnull[leftNdx]) 137 | bottomNdx = numpy.argmin(yNonnull) 138 | bottomXY = (xNonnull[bottomNdx], yNonnull[bottomNdx]) 139 | rightNdx = numpy.argmax(xNonnull) 140 | rightXY = (xNonnull[rightNdx], yNonnull[rightNdx]) 141 | 142 | # If these are more extreme than those already in otherargs, replace them 143 | if otherargs.tl is None or topXY[1] > otherargs.tl[1]: 144 | otherargs.tl = topXY 145 | if otherargs.tr is None or rightXY[0] > otherargs.tr[0]: 146 | otherargs.tr = rightXY 147 | if otherargs.bl is None or leftXY[0] < otherargs.bl[0]: 148 | otherargs.bl = leftXY 149 | if otherargs.br is None or bottomXY[1] < otherargs.br[1]: 150 | otherargs.br = bottomXY 151 | 152 | 153 | def findNadirLine(corners): 154 | """ 155 | Return the equation of the nadir line, from the given corners of the swathe. 156 | Returns a numpy array of [b, m], for the equation 157 | 158 | y = mx + b 159 | 160 | giving the y coordinate of the nadir as a function of the x coordinate. 161 | 162 | """ 163 | # Find the top and bottom mid-points. 164 | topMid = (corners[0] + corners[1]) / 2.0 165 | bottomMid = (corners[2] + corners[3]) / 2.0 166 | 167 | slope = (topMid[1] - bottomMid[1]) / (topMid[0] - bottomMid[0]) 168 | intercept = topMid[1] - slope * topMid[0] 169 | 170 | coeffs = numpy.array([intercept, slope]) 171 | return coeffs 172 | 173 | 174 | def satAzLeftRight(nadirLine): 175 | """ 176 | Calculate the satellite azimuth for the left and right sides of the nadir line. 177 | Assume that the satellite azimuth vector is at right angles to the nadir line 178 | (which is not really true, but reasonably close), and that there are only 179 | two possibilities, as a pixel is either to the left or to the right of the nadir 180 | line. 181 | 182 | Return a numpy array of [satAzLeft, satAzRight], with angles in radians, 183 | in the range [-pi, pi] 184 | 185 | """ 186 | slope = nadirLine[1] 187 | # Slope of a line perpendicular to the nadir 188 | slopePerp = -1 / slope 189 | 190 | # Azimuth for pixels to the left of the line 191 | azimuthLeft = numpy.pi / 2.0 - numpy.arctan(slopePerp) 192 | # Azimuth for pixels to the right is directly opposite 193 | azimuthRight = azimuthLeft - numpy.pi 194 | 195 | return numpy.array([azimuthLeft, azimuthRight]) 196 | 197 | 198 | def localRadius(latitude): 199 | """ 200 | Calculate a local radius of curvature, for the given geodetic latitude. 201 | This approximates the earth curvature at this latitude. The given 202 | latitude is in degrees. This is probably overkill, given some of the other 203 | approximations I am making.... 204 | 205 | """ 206 | latRadians = numpy.radians(latitude) 207 | 208 | # Earth axis lengths 209 | a = osr.SRS_WGS84_SEMIMAJOR 210 | invFlat = osr.SRS_WGS84_INVFLATTENING 211 | f = 1 / invFlat 212 | eSqd = 2 * f - f**2 213 | 214 | # Radius of curvature 215 | R = a / numpy.sqrt(1 - eSqd * numpy.sin(latRadians)**2) 216 | return R 217 | 218 | 219 | def sunAnglesForExtent(imgInfo, mtlInfo): 220 | """ 221 | Return array of sun azimuth and zenith for each of the corners of the image 222 | extent. Note that this is the raster extent, not the corners of the swathe. 223 | 224 | The algorithm used here has been copied from the 6S possol() subroutine. The 225 | Fortran code I copied it from was .... up to the usual standard in 6S. So, the 226 | notation is not always clear. 227 | 228 | """ 229 | cornerLatLong = imgInfo.getCorners(outEPSG=4326) 230 | (ul_long, ul_lat, ur_long, ur_lat, lr_long, lr_lat, ll_long, ll_lat) = cornerLatLong 231 | pts = numpy.array([ 232 | [ul_long, ul_lat], 233 | [ur_long, ur_lat], 234 | [ll_long, ll_lat], 235 | [lr_long, lr_lat] 236 | ]) 237 | longDeg = pts[:, 0] 238 | latDeg = pts[:, 1] 239 | 240 | # Date/time in UTC 241 | dateStr = mtlInfo['DATE_ACQUIRED'] 242 | timeStr = mtlInfo['SCENE_CENTER_TIME'].replace('Z', '') 243 | ymd = [int(i) for i in dateStr.split('-')] 244 | dateObj = datetime.date(ymd[0], ymd[1], ymd[2]) 245 | julianDay = (dateObj - datetime.date(ymd[0], 1, 1)).days + 1 246 | juldayYearEnd = (datetime.date(ymd[0], 12, 31) - datetime.date(ymd[0], 1, 1)).days + 1 247 | # Julian day as a proportion of the year 248 | jdp = julianDay / juldayYearEnd 249 | # Hour in UTC 250 | hms = [float(x) for x in timeStr.split(':')] 251 | hourGMT = hms[0] + hms[1] / 60.0 + hms[2] / 3600.0 252 | 253 | (sunAz, sunZen) = sunAnglesForPoints(latDeg, longDeg, hourGMT, jdp) 254 | 255 | sunAngles = numpy.vstack((sunAz, sunZen)).T 256 | return sunAngles 257 | 258 | 259 | def sunAnglesForPoints(latDeg, longDeg, hourGMT, jdp): 260 | """ 261 | Calculate sun azimuth and zenith for the given location(s), for the given 262 | time of year. jdp is the julian day as a proportion, ranging from 0 to 1, where 263 | Jan 1 is 1.0/365 and Dec 31 is 1.0. 264 | Location is given in latitude/longitude, in degrees, and can be arrays to 265 | calculate for multiple locations. hourGMT is a decimal hour number giving the time 266 | of day in GMT (i.e. UTC). 267 | 268 | Return a tuple of (sunAz, sunZen). If latDeg and longDeg are arrays, then returned 269 | values will be arrays of the same shape. 270 | 271 | """ 272 | latRad = numpy.radians(latDeg) 273 | # Express jdp in radians 274 | jdpr = jdp * 2 * numpy.pi 275 | 276 | # Now work out the solar position. This is copied from the 6S code, but 277 | # is also documented in the 6S manual. The notation 278 | a = numpy.array([0.000075, 0.001868, 0.032077, 0.014615, 0.040849]) 279 | meanSolarTime = hourGMT + longDeg / 15.0 280 | localSolarDiff = (a[0] + a[1] * numpy.cos(jdpr) - a[2] * numpy.sin(jdpr) - 281 | a[3] * numpy.cos(2 * jdpr) - a[4] * numpy.sin(2 * jdpr)) * 12 * 60 / numpy.pi 282 | trueSolarTime = meanSolarTime + localSolarDiff / 60 - 12.0 283 | # Hour as an angle 284 | ah = trueSolarTime * numpy.radians(15) 285 | 286 | b = numpy.array([0.006918, 0.399912, 0.070257, 0.006758, 0.000907, 0.002697, 0.001480]) 287 | delta = (b[0] - b[1] * numpy.cos(jdpr) + b[2] * numpy.sin(jdpr) - 288 | b[3] * numpy.cos(2. * jdpr) + b[4] * numpy.sin(2. * jdpr) - 289 | b[5] * numpy.cos(3. * jdpr) + b[6] * numpy.sin(3. * jdpr)) 290 | 291 | cosSunZen = (numpy.sin(latRad) * numpy.sin(delta) + 292 | numpy.cos(latRad) * numpy.cos(delta) * numpy.cos(ah)) 293 | sunZen = numpy.arccos(cosSunZen) 294 | 295 | # sun azimuth from south, turning west (yeah, I know, weird, isn't it....) 296 | sinSunAzSW = numpy.cos(delta) * numpy.sin(ah) / numpy.sin(sunZen) 297 | sinSunAzSW = sinSunAzSW.clip(-1.0, 1.0) 298 | 299 | # This next bit seems to be to get the azimuth in the correct quadrant. I do 300 | # not really understand it. 301 | cosSunAzSW = (-numpy.cos(latRad) * numpy.sin(delta) + 302 | numpy.sin(latRad) * numpy.cos(delta) * numpy.cos(ah)) / numpy.sin(sunZen) 303 | sunAzSW = numpy.arcsin(sinSunAzSW) 304 | sunAzSW = numpy.where(cosSunAzSW <= 0, numpy.pi - sunAzSW, sunAzSW) 305 | sunAzSW = numpy.where((cosSunAzSW > 0) & (sinSunAzSW <= 0), 2 * numpy.pi + sunAzSW, sunAzSW) 306 | 307 | # Now convert to azimuth from north, turning east, as is usual convention 308 | sunAz = sunAzSW + numpy.pi 309 | # Keep within [0, 2pi] range 310 | sunAz = numpy.where(sunAz > 2 * numpy.pi, sunAz - 2 * numpy.pi, sunAz) 311 | 312 | return (sunAz, sunZen) 313 | 314 | 315 | def makeAnglesImage(templateimg, outfile, nadirLine, extentSunAngles, satAzimuth, imgInfo): 316 | """ 317 | Make a single output image file of the sun and satellite angles for every 318 | pixel in the template image. 319 | 320 | """ 321 | imgInfo = fileinfo.ImageInfo(templateimg) 322 | 323 | infiles = applier.FilenameAssociations() 324 | outfiles = applier.FilenameAssociations() 325 | otherargs = applier.OtherInputs() 326 | controls = applier.ApplierControls() 327 | 328 | infiles.img = templateimg 329 | outfiles.angles = outfile 330 | 331 | (ctrLat, ctrLong) = getCtrLatLong(imgInfo) 332 | otherargs.R = localRadius(ctrLat) 333 | otherargs.nadirLine = nadirLine 334 | otherargs.xMin = imgInfo.xMin 335 | otherargs.xMax = imgInfo.xMax 336 | otherargs.yMin = imgInfo.yMin 337 | otherargs.yMax = imgInfo.yMax 338 | otherargs.extentSunAngles = extentSunAngles 339 | otherargs.satAltitude = 705000 # Landsat nominal altitude in metres 340 | otherargs.satAzimuth = satAzimuth 341 | otherargs.radianScale = 100 # Store pixel values as (radians * radianScale) 342 | controls.setStatsIgnore(500) 343 | 344 | applier.apply(makeAngles, infiles, outfiles, otherargs, controls=controls) 345 | 346 | 347 | def makeAngles(info, inputs, outputs, otherargs): 348 | """ 349 | Called from RIOS 350 | 351 | Make 4-layer sun and satellite angles for the image block 352 | 353 | """ 354 | (xblock, yblock) = info.getBlockCoordArrays() 355 | # Nadir line coefficients of y=mx+b 356 | (b, m) = otherargs.nadirLine 357 | 358 | # Distance of each pixel from the nadir line 359 | dist = numpy.absolute((m * xblock - yblock + b) / numpy.sqrt(m**2 + 1)) 360 | 361 | # Zenith angle assuming a flat earth 362 | satZenith = numpy.arctan(dist / otherargs.satAltitude) 363 | 364 | # Adjust satZenith for earth curvature. This is a very simple approximation, but 365 | # the adjustment is less than one degree anyway, so this is accurate enough. 366 | curveAngle = numpy.arctan(dist / otherargs.R) 367 | satZenith += curveAngle 368 | 369 | # Work out whether we are left or right of the nadir line 370 | isLeft = (yblock - (m * xblock + b)) > 0 371 | (satAzimuthLeft, satAzimuthRight) = otherargs.satAzimuth 372 | satAzimuth = numpy.where(isLeft, satAzimuthLeft, satAzimuthRight) 373 | 374 | # Interpolate the sun angles from those calculated at the corners of the whole raster extent 375 | (xMin, xMax, yMin, yMax) = (otherargs.xMin, otherargs.xMax, otherargs.yMin, otherargs.yMax) 376 | sunAzimuth = bilinearInterp(xMin, xMax, yMin, yMax, otherargs.extentSunAngles[:, 0], xblock, yblock) 377 | sunZenith = bilinearInterp(xMin, xMax, yMin, yMax, otherargs.extentSunAngles[:, 1], xblock, yblock) 378 | 379 | angleStack = numpy.array([satAzimuth, satZenith, sunAzimuth, sunZenith]) 380 | angleStackDN = angleStack * otherargs.radianScale 381 | 382 | outputs.angles = numpy.round(angleStackDN).astype(numpy.int16) 383 | 384 | 385 | def bilinearInterp(xMin, xMax, yMin, yMax, cornerVals, x, y): 386 | """ 387 | Evaluate the given value on a grid of (x, y) points. The exact value is given 388 | on a set of corner points (top-left, top-right, bottom-left, bottom-right). 389 | The corner locations are implied from xMin, xMax, yMin, yMax. 390 | 391 | """ 392 | p = (y - yMin) / (yMax - yMin) 393 | q = (x - xMin) / (xMax - xMin) 394 | 395 | # Give the known corner values some simple names 396 | (tl, tr, bl, br) = cornerVals 397 | 398 | # Calculate the interpolated values 399 | vals = tr * p * q + tl * p * (1 - q) + br * (1 - p) * q + bl * (1 - p) * (1 - q) 400 | return vals 401 | 402 | 403 | def getCtrLatLong(imgInfo): 404 | """ 405 | Return the lat/long of the centre of the image as 406 | (ctrLat, ctrLong) 407 | 408 | """ 409 | cornerLatLong = imgInfo.getCorners(outEPSG=4326) 410 | (ul_long, ul_lat, ur_long, ur_lat, lr_long, lr_lat, ll_long, ll_lat) = cornerLatLong 411 | ctrLat = numpy.array([ul_lat, ur_lat, lr_lat, ll_lat]).mean() 412 | ctrLong = numpy.array([ul_long, ur_long, lr_long, ll_long]).mean() 413 | return (ctrLat, ctrLong) 414 | -------------------------------------------------------------------------------- /fmask/saturationcheck.py: -------------------------------------------------------------------------------- 1 | """ 2 | Module for doing checks of visible band and reporting 3 | and saturation. Note that this works off the original 4 | radiance file, not the TOA reflectance. 5 | """ 6 | 7 | # This file is part of 'python-fmask' - a cloud masking module 8 | # Copyright (C) 2015 Neil Flood 9 | # 10 | # This program is free software; you can redistribute it and/or 11 | # modify it under the terms of the GNU General Public License 12 | # as published by the Free Software Foundation; either version 3 13 | # of the License, or (at your option) any later version. 14 | # 15 | # This program is distributed in the hope that it will be useful, 16 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 17 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 18 | # GNU General Public License for more details. 19 | # 20 | # You should have received a copy of the GNU General Public License 21 | # along with this program; if not, write to the Free Software 22 | # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 23 | from __future__ import print_function, division 24 | 25 | import numpy 26 | from rios import applier, cuiprogress 27 | from . import config 28 | 29 | 30 | def makeSaturationMask(fmaskConfig, radiancefile, outMask): 31 | """ 32 | Checks the radianceFile and creates a mask with 33 | 1's where there is saturation in one of more visible bands. 34 | 0 otherwise. 35 | 36 | The fmaskConfig parameter should be an instance of 37 | :class:`fmask.config.FmaskConfig`. This is used to determine 38 | which bands are visible. 39 | 40 | This mask is advisible since the whiteness test Eqn 2. 41 | and Equation 6 are affected by saturated pixels and 42 | may determine a pixel is not cloud when it is. 43 | 44 | The format of outMask will be the current 45 | RIOS default format. 46 | 47 | It is assumed that the input radianceFile has values in the 48 | range 0-255 and saturated pixels are set to 255. 49 | 50 | """ 51 | inputs = applier.FilenameAssociations() 52 | inputs.radiance = radiancefile 53 | 54 | outputs = applier.FilenameAssociations() 55 | outputs.mask = outMask 56 | 57 | otherargs = applier.OtherInputs() 58 | otherargs.radianceBands = fmaskConfig.bands 59 | 60 | controls = applier.ApplierControls() 61 | controls.progress = cuiprogress.GDALProgressBar() 62 | 63 | applier.apply(riosSaturationMask, inputs, outputs, 64 | otherargs, controls=controls) 65 | 66 | 67 | def riosSaturationMask(info, inputs, outputs, otherargs): 68 | """ 69 | Called from RIOS. Does the actual saturation test. Currently assumes that 70 | only 8-bit radiance inputs can be saturated, but if this turns out 71 | not to be true, we can come back to this. 72 | 73 | """ 74 | if inputs.radiance.dtype == numpy.uint8: 75 | blue = otherargs.radianceBands[config.BAND_BLUE] 76 | green = otherargs.radianceBands[config.BAND_GREEN] 77 | red = otherargs.radianceBands[config.BAND_RED] 78 | 79 | satMaskList = [] 80 | for band in [blue, green, red]: 81 | satMaskList.append(inputs.radiance[band] == 255) 82 | 83 | outputs.mask = numpy.array(satMaskList).astype(numpy.uint8) 84 | else: 85 | # Assume that anything larger than 8-bit is immune to saturation 86 | outShape = (3, ) + inputs.radiance[0].shape 87 | outputs.mask = numpy.zeros(outShape, dtype=numpy.uint8) 88 | 89 | -------------------------------------------------------------------------------- /fmask/sen2meta.py: -------------------------------------------------------------------------------- 1 | """ 2 | Classes for handling the various metadata files which come with Sentinel-2. 3 | 4 | Currently only has a class for the tile-based metadata file. 5 | 6 | """ 7 | # This file is part of 'python-fmask' - a cloud masking module 8 | # Copyright (C) 2015 Neil Flood 9 | # 10 | # This program is free software; you can redistribute it and/or 11 | # modify it under the terms of the GNU General Public License 12 | # as published by the Free Software Foundation; either version 3 13 | # of the License, or (at your option) any later version. 14 | # 15 | # This program is distributed in the hope that it will be useful, 16 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 17 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 18 | # GNU General Public License for more details. 19 | # 20 | # You should have received a copy of the GNU General Public License 21 | # along with this program; if not, write to the Free Software 22 | # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 23 | 24 | from __future__ import print_function, division 25 | 26 | import datetime 27 | from xml.etree import ElementTree 28 | 29 | import numpy 30 | from osgeo import osr 31 | 32 | from . import fmaskerrors 33 | 34 | osr.UseExceptions() 35 | 36 | 37 | class Sen2TileMeta(object): 38 | """ 39 | Metadata for a single 100km tile 40 | """ 41 | def __init__(self, filename=None): 42 | """ 43 | Constructor takes a filename for the XML file of tile-based metadata. 44 | 45 | """ 46 | f = open(filename) 47 | 48 | root = ElementTree.fromstring(f.read()) 49 | # Stoopid XML namespace prefix 50 | nsPrefix = root.tag[:root.tag.index('}') + 1] 51 | nsDict = {'n1': nsPrefix[1:-1]} 52 | 53 | generalInfoNode = root.find('n1:General_Info', nsDict) 54 | # N.B. I am still not entirely convinced that this SENSING_TIME is really 55 | # the acquisition time, but the documentation is rubbish. 56 | sensingTimeNode = generalInfoNode.find('SENSING_TIME') 57 | sensingTimeStr = sensingTimeNode.text.strip() 58 | self.datetime = datetime.datetime.strptime(sensingTimeStr, "%Y-%m-%dT%H:%M:%S.%fZ") 59 | tileIdNode = generalInfoNode.find('TILE_ID') 60 | tileIdFullStr = tileIdNode.text.strip() 61 | self.tileId = tileIdFullStr.split('_')[-2] 62 | self.satId = tileIdFullStr[:3] 63 | self.procLevel = tileIdFullStr[13:16] # Not sure whether to use absolute pos or split by '_'.... 64 | 65 | geomInfoNode = root.find('n1:Geometric_Info', nsDict) 66 | geocodingNode = geomInfoNode.find('Tile_Geocoding') 67 | epsgNode = geocodingNode.find('HORIZONTAL_CS_CODE') 68 | self.epsg = epsgNode.text.split(':')[1] 69 | 70 | # Dimensions of images at different resolutions. 71 | self.dimsByRes = {} 72 | sizeNodeList = geocodingNode.findall('Size') 73 | for sizeNode in sizeNodeList: 74 | res = sizeNode.attrib['resolution'] 75 | nrows = int(sizeNode.find('NROWS').text) 76 | ncols = int(sizeNode.find('NCOLS').text) 77 | self.dimsByRes[res] = (nrows, ncols) 78 | 79 | # Upper-left corners of images at different resolutions. As far as I can 80 | # work out, these coords appear to be the upper left corner of the upper left 81 | # pixel, i.e. equivalent to GDAL's convention. This also means that they 82 | # are the same for the different resolutions, which is nice. 83 | self.ulxyByRes = {} 84 | posNodeList = geocodingNode.findall('Geoposition') 85 | for posNode in posNodeList: 86 | res = posNode.attrib['resolution'] 87 | ulx = float(posNode.find('ULX').text) 88 | uly = float(posNode.find('ULY').text) 89 | self.ulxyByRes[res] = (ulx, uly) 90 | 91 | # Sun and satellite angles. 92 | tileAnglesNode = geomInfoNode.find('Tile_Angles') 93 | sunZenithNode = tileAnglesNode.find('Sun_Angles_Grid').find('Zenith') 94 | self.angleGridXres = float(sunZenithNode.find('COL_STEP').text) 95 | self.angleGridYres = float(sunZenithNode.find('ROW_STEP').text) 96 | self.sunZenithGrid = self.makeValueArray(sunZenithNode.find('Values_List')) 97 | sunAzimuthNode = tileAnglesNode.find('Sun_Angles_Grid').find('Azimuth') 98 | self.sunAzimuthGrid = self.makeValueArray(sunAzimuthNode.find('Values_List')) 99 | self.anglesGridShape = self.sunAzimuthGrid.shape 100 | 101 | # Now build up the viewing angle per grid cell, from the separate layers 102 | # given for each detector for each band. Initially I am going to keep 103 | # the bands separate, just to see how that looks. 104 | # The names of things in the XML suggest that these are view angles, 105 | # but the numbers suggest that they are angles as seen from the pixel's 106 | # frame of reference on the ground, i.e. they are in fact what we ultimately want. 107 | viewingAngleNodeList = tileAnglesNode.findall('Viewing_Incidence_Angles_Grids') 108 | self.viewZenithDict = self.buildViewAngleArr(viewingAngleNodeList, 'Zenith') 109 | self.viewAzimuthDict = self.buildViewAngleArr(viewingAngleNodeList, 'Azimuth') 110 | 111 | # Make a guess at the coordinates of the angle grids. These are not given 112 | # explicitly in the XML, and don't line up exactly with the other grids, so I am 113 | # making a rough estimate. Because the angles don't change rapidly across these 114 | # distances, it is not important if I am a bit wrong (although it would be nice 115 | # to be exactly correct!). 116 | (ulx, uly) = self.ulxyByRes["10"] 117 | self.anglesULXY = (ulx - self.angleGridXres / 2.0, uly + self.angleGridYres / 2.0) 118 | 119 | @staticmethod 120 | def makeValueArray(valuesListNode): 121 | """ 122 | Take a node from the XML, and return an array of the values contained 123 | within it. This will be a 2-d numpy array of float32 values (should I pass the dtype in??) 124 | 125 | """ 126 | valuesList = valuesListNode.findall('VALUES') 127 | vals = [] 128 | for valNode in valuesList: 129 | text = valNode.text 130 | vals.append([numpy.float32(x) for x in text.strip().split()]) 131 | return numpy.array(vals) 132 | 133 | def buildViewAngleArr(self, viewingAngleNodeList, angleName): 134 | """ 135 | Build up the named viewing angle array from the various detector strips given as 136 | separate arrays. I don't really understand this, and may need to re-write it once 137 | I have worked it out...... 138 | 139 | The angleName is one of 'Zenith' or 'Azimuth'. 140 | Returns a dictionary of 2-d arrays, keyed by the bandId string. 141 | """ 142 | angleArrDict = {} 143 | for viewingAngleNode in viewingAngleNodeList: 144 | bandId = viewingAngleNode.attrib['bandId'] 145 | angleNode = viewingAngleNode.find(angleName) 146 | angleArr = self.makeValueArray(angleNode.find('Values_List')) 147 | if bandId not in angleArrDict: 148 | angleArrDict[bandId] = angleArr 149 | else: 150 | mask = (~numpy.isnan(angleArr)) 151 | angleArrDict[bandId][mask] = angleArr[mask] 152 | return angleArrDict 153 | 154 | def getUTMzone(self): 155 | """ 156 | Return the UTM zone of the tile, as an integer 157 | """ 158 | if not (self.epsg.startswith("327") or self.epsg.startswith("326")): 159 | raise fmaskerrors.Sen2MetaError("Cannot determine UTM zone from EPSG:{}".format(self.epsg)) 160 | return int(self.epsg[3:]) 161 | 162 | def getCtrXY(self): 163 | """ 164 | Return the (X, Y) coordinates of the scene centre (in image projection, generally UTM) 165 | """ 166 | (nrows, ncols) = self.dimsByRes['10'] 167 | (ctrRow, ctrCol) = (nrows // 2, ncols // 2) 168 | (ulx, uly) = self.ulxyByRes['10'] 169 | (ctrX, ctrY) = (ulx + ctrCol * 10, uly - ctrRow * 10) 170 | return (ctrX, ctrY) 171 | 172 | def getCtrLongLat(self): 173 | """ 174 | Return the (longitude, latitude) of the scene centre 175 | """ 176 | (ctrX, ctrY) = self.getCtrXY() 177 | srUTM = osr.SpatialReference() 178 | srUTM.ImportFromEPSG(int(self.epsg)) 179 | srLL = osr.SpatialReference() 180 | srLL.ImportFromEPSG(4326) 181 | if hasattr(srLL, 'SetAxisMappingStrategy'): 182 | # We are in GDAL >= 3, so guard against axis swapping 183 | srLL.SetAxisMappingStrategy(osr.OAMS_TRADITIONAL_GIS_ORDER) 184 | srUTM.SetAxisMappingStrategy(osr.OAMS_TRADITIONAL_GIS_ORDER) 185 | tr = osr.CoordinateTransformation(srUTM, srLL) 186 | (longitude, latitude, z) = tr.TransformPoint(ctrX, ctrY) 187 | return (longitude, latitude) 188 | 189 | 190 | # ESA use stoopid index numbers in the XML, known as bandId. This list turns 191 | # them into a band name. Note that these names will also sort into the 192 | # same band order as their index numbers, because we have used 'B08A' 193 | # instead of B8A. 194 | nameFromBandId = ['B01', 'B02', 'B03', 'B04', 'B05', 'B06', 'B07', 'B08', 195 | 'B08A', 'B09', 'B10', 'B11', 'B12'] 196 | 197 | 198 | class Sen2ZipfileMeta(object): 199 | """ 200 | Metadata from the top-level XML file. 201 | 202 | Only loading the few things we need from this file, most of it 203 | is ignored. 204 | 205 | """ 206 | def __init__(self, xmlfilename=None): 207 | xmlStr = open(xmlfilename).read() 208 | root = ElementTree.fromstring(xmlStr) 209 | nsPrefix = root.tag[:root.tag.index('}') + 1] 210 | nsDict = {'n1': nsPrefix[1:-1]} 211 | 212 | generalInfoNode = root.find('n1:General_Info', nsDict) 213 | prodImgCharactNode = generalInfoNode.find('Product_Image_Characteristics', nsDict) 214 | scaleValNode = prodImgCharactNode.find('QUANTIFICATION_VALUE', nsDict) 215 | self.scaleVal = float(scaleValNode.text) 216 | # Plough through the bizarrely organised special values list 217 | specialValuesList = prodImgCharactNode.findall('Special_Values', nsDict) 218 | for node in specialValuesList: 219 | nameNode = node.find('SPECIAL_VALUE_TEXT', nsDict) 220 | valNode = node.find('SPECIAL_VALUE_INDEX', nsDict) 221 | name = nameNode.text 222 | val = int(valNode.text) 223 | if name == "NODATA": 224 | self.nodataVal = val 225 | elif name == "SATURATED": 226 | self.saturatedVal = val 227 | 228 | self.offsetValDict = {} 229 | offsetNodeList = generalInfoNode.findall('Product_Image_Characteristics/Radiometric_Offset_List/RADIO_ADD_OFFSET', nsDict) 230 | if len(offsetNodeList) == 0: 231 | for k in nameFromBandId: 232 | self.offsetValDict[k] = 0 233 | else: 234 | for node in offsetNodeList: 235 | bandId = int(node.attrib['band_id']) 236 | bandName = nameFromBandId[bandId] 237 | offsetVal = int(node.text) 238 | self.offsetValDict[bandName] = offsetVal 239 | 240 | baselineVersionNode = generalInfoNode.find('Product_Info/PROCESSING_BASELINE', nsDict) 241 | self.baselineVersion = baselineVersionNode.text 242 | -------------------------------------------------------------------------------- /fmask/valueindexes.py: -------------------------------------------------------------------------------- 1 | # This file is part of 'python-fmask' - a cloud masking module 2 | # Copyright (C) 2015 Neil Flood 3 | # 4 | # This program is free software; you can redistribute it and/or 5 | # modify it under the terms of the GNU General Public License 6 | # as published by the Free Software Foundation; either version 3 7 | # of the License, or (at your option) any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU General Public License for more details. 13 | # 14 | # You should have received a copy of the GNU General Public License 15 | # along with this program; if not, write to the Free Software 16 | # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 17 | from __future__ import print_function, division 18 | 19 | import os 20 | import numpy 21 | 22 | # Fail slightly less drastically when running from ReadTheDocs 23 | if os.getenv('READTHEDOCS', default='False') != 'True': 24 | from . import _valueindexes 25 | 26 | 27 | class ValueIndexesError(Exception): 28 | pass 29 | 30 | 31 | class NonIntTypeError(ValueIndexesError): 32 | pass 33 | 34 | 35 | class RangeError(ValueIndexesError): 36 | pass 37 | 38 | 39 | class ValueIndexes(object): 40 | """ 41 | An object which contains the indexes for every value in a given array. 42 | This class is intended to mimic the reverse_indices clause in IDL, 43 | only nicer. 44 | 45 | Takes an array, works out what unique values exist in this array. Provides 46 | a method which will return all the indexes into the original array for 47 | a given value. 48 | 49 | The array must be of an integer-like type. Floating point arrays will 50 | not work. If one needs to look at ranges of values within a float array, 51 | it is possible to use numpy.digitize() to create an integer array 52 | corresponding to a set of bins, and then use ValueIndexes with that. 53 | 54 | Example usage, for a given array a:: 55 | 56 | valIndexes = ValueIndexes(a) 57 | for val in valIndexes.values: 58 | ndx = valIndexes.getIndexes(val) 59 | # Do something with all the indexes 60 | 61 | 62 | This is a much faster and more efficient alternative to something like:: 63 | 64 | values = numpy.unique(a) 65 | for val in values: 66 | mask = (a == val) 67 | # Do something with the mask 68 | 69 | The point is that when a is large, and/or the number of possible values 70 | is large, this becomes very slow, and can take up lots of memory. Each 71 | loop iteration involves searching through a again, looking for a different 72 | value. This class provides a much more efficient way of doing the same 73 | thing, requiring only one pass through a. When a is small, or when the 74 | number of possible values is very small, it probably makes little difference. 75 | 76 | If one or more null values are given to the constructor, these will not 77 | be counted, and will not be available to the getIndexes() method. This 78 | makes it more memory-efficient, so it doesn't store indexes of a whole 79 | lot of nulls. 80 | 81 | A ValueIndexes object has the following attributes: 82 | 83 | * **values** Array of all values indexed 84 | * **counts** Array of counts for each value 85 | * **nDims** Number of dimensions of original array 86 | * **indexes** Packed array of indexes 87 | * **start** Starting points in indexes array for each value 88 | * **end** End points in indexes for each value 89 | * **valLU** Lookup table for each value, to find it in the values array without explicitly searching. 90 | * **nullVals** Array of the null values requested. 91 | 92 | Limitations: 93 | The array index values are handled using unsigned 32bit int values, so 94 | it won't work if the data array is larger than 4Gb. I don't think it would 95 | fail very gracefully, either. 96 | 97 | """ 98 | def __init__(self, a, nullVals=[]): 99 | """ 100 | Creates a ValueIndexes object for the given array a. 101 | A sequence of null values can be given, and these will not be included 102 | in the results, so that indexes for these cannot be determined. 103 | 104 | """ 105 | if not numpy.issubdtype(a.dtype, numpy.integer): 106 | raise NonIntTypeError("ValueIndexes only works on integer-like types. Array is %s"%a.dtype) 107 | 108 | if numpy.isscalar(nullVals): 109 | self.nullVals = [nullVals] 110 | else: 111 | self.nullVals = nullVals 112 | 113 | # Get counts of all values in a 114 | minval = a.min() 115 | maxval = a.max() 116 | (counts, binEdges) = numpy.histogram(a, range=(minval, maxval + 1), 117 | bins=(maxval - minval + 1)) 118 | 119 | # Mask counts for any requested null values. 120 | maskedCounts = counts.copy() 121 | for val in self.nullVals: 122 | maskedCounts[binEdges[:-1]==val] = 0 123 | self.values = binEdges[:-1][maskedCounts>0].astype(a.dtype) 124 | self.counts = maskedCounts[maskedCounts>0] 125 | 126 | # Allocate space to store all indexes 127 | totalCounts = self.counts.sum() 128 | self.nDims = a.ndim 129 | self.indexes = numpy.zeros((totalCounts, a.ndim), dtype=numpy.uint32) 130 | self.end = self.counts.cumsum() 131 | self.start = self.end - self.counts 132 | 133 | if len(self.values) > 0: 134 | # A lookup table to make searching for a value very fast. 135 | valrange = numpy.array([self.values.min(), self.values.max()]) 136 | numLookups = valrange[1] - valrange[0] + 1 137 | maxUint32 = 2**32 - 1 138 | if numLookups > maxUint32: 139 | raise RangeError("Range of different values is too great for uint32") 140 | self.valLU = numpy.zeros(numLookups, dtype=numpy.uint32) 141 | self.valLU.fill(maxUint32) # A value to indicate "not found", must match _valndxFunc below 142 | self.valLU[self.values - self.values[0]] = range(len(self.values)) 143 | 144 | # For use within C. For each value, the current index 145 | # into the indexes array. A given element is incremented whenever it finds 146 | # a new element of that value. 147 | currentIndex = self.start.copy().astype(numpy.uint32) 148 | 149 | _valueindexes.valndxFunc(a, self.indexes, valrange[0], valrange[1], 150 | self.valLU, currentIndex) 151 | 152 | def getIndexes(self, val): 153 | """ 154 | Return a set of indexes into the original array, for which the 155 | value in the array is equal to val. 156 | 157 | """ 158 | # Find where this value is listed. 159 | valNdx = (self.values == val).nonzero()[0] 160 | 161 | # If this value is not actually in those listed, then we 162 | # must return empty indexes 163 | if len(valNdx) == 0: 164 | start = 0 165 | end = 0 166 | else: 167 | # The index into counts, etc. for this value. 168 | valNdx = valNdx[0] 169 | start = self.start[valNdx] 170 | end = self.end[valNdx] 171 | 172 | # Create a tuple of index arrays, one for each index of the original array. 173 | ndx = () 174 | for i in range(self.nDims): 175 | ndx += (self.indexes[start:end, i], ) 176 | return ndx 177 | -------------------------------------------------------------------------------- /fmask/zerocheck.py: -------------------------------------------------------------------------------- 1 | """ 2 | Utility function to determine if a particular band in a 3 | file is all zeros. This is useful for Landsat8 since 4 | although the file and metadata exist for thermal, it may 5 | be all zeroes and hence unusable form cloud masking. 6 | """ 7 | 8 | # This file is part of 'python-fmask' - a cloud masking module 9 | # Copyright (C) 2015 Neil Flood 10 | # 11 | # This program is free software; you can redistribute it and/or 12 | # modify it under the terms of the GNU General Public License 13 | # as published by the Free Software Foundation; either version 3 14 | # of the License, or (at your option) any later version. 15 | # 16 | # This program is distributed in the hope that it will be useful, 17 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 18 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 19 | # GNU General Public License for more details. 20 | # 21 | # You should have received a copy of the GNU General Public License 22 | # along with this program; if not, write to the Free Software 23 | # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 24 | from __future__ import print_function, division 25 | 26 | from rios import applier 27 | from rios import fileinfo 28 | 29 | 30 | def isBandAllZeroes(filename, band=0): 31 | """ 32 | Checks the specified band within a file to see if it is all zeroes. 33 | band should be 0 based. 34 | 35 | Returns True if all zeroes, False otherwise. 36 | 37 | This function firstly checks the stats if present and uses this 38 | and assumes that they are correct. 39 | 40 | If they are not present, then the band is read and the values determined. 41 | 42 | """ 43 | 44 | info = fileinfo.ImageFileStats(filename) 45 | maxVal = info[band].max 46 | if maxVal is not None: 47 | # we have valid stats 48 | return maxVal == 0 49 | 50 | # otherwise read the thing with rios 51 | infiles = applier.FilenameAssociations() 52 | infiles.input = filename 53 | 54 | outfiles = applier.FilenameAssociations() 55 | 56 | otherArgs = applier.OtherInputs() 57 | otherArgs.nonZeroFound = False 58 | otherArgs.band = band 59 | 60 | applier.apply(riosAllZeroes, infiles, outfiles, otherArgs) 61 | 62 | return not otherArgs.nonZeroFound 63 | 64 | 65 | def riosAllZeroes(info, inputs, outputs, otherArgs): 66 | """ 67 | Called from RIOS. Does the actual work. 68 | 69 | """ 70 | if inputs.input[otherArgs.band].max() > 0: 71 | otherArgs.nonZeroFound = True 72 | 73 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | # Note that we do not explicitly list the requirements. This is because the 2 | # main one is GDAL (via rios), which is not managed on PyPI (for good reasons), and 3 | # therefore cannot be satisfactorily managed with pip. If we list it as a 4 | # requirement, this mainly just creates confusion. GDAL must be installed 5 | # by other means. For the same reason, this package is itself not available 6 | # from PyPI. 7 | # The actual requirements are rios (which requires GDAL) and scipy. 8 | # 9 | # Installation requires pip>=23.0. 10 | # 11 | 12 | [build-system] 13 | requires = ["setuptools>=61.0", "wheel", "numpy"] 14 | build-backend = "setuptools.build_meta" 15 | 16 | [project] 17 | name = "python-fmask" 18 | dynamic = ["version"] 19 | authors = [ 20 | {name = "Sam Gillingham"}, 21 | {name = "Neil Flood"} 22 | ] 23 | description = "Implement the Fmask cloud masking algorithm (Zhu, Wang & Woodcock 2015)" 24 | readme = "README.md" 25 | license = {file = "LICENSE.txt"} 26 | 27 | [project.scripts] 28 | fmask_sentinel2makeAnglesImage = "fmask.cmdline.sentinel2makeAnglesImage:mainRoutine" 29 | fmask_sentinel2Stacked = "fmask.cmdline.sentinel2Stacked:mainRoutine" 30 | fmask_usgsLandsatMakeAnglesImage = "fmask.cmdline.usgsLandsatMakeAnglesImage:mainRoutine" 31 | fmask_usgsLandsatSaturationMask = "fmask.cmdline.usgsLandsatSaturationMask:mainRoutine" 32 | fmask_usgsLandsatStacked = "fmask.cmdline.usgsLandsatStacked:mainRoutine" 33 | fmask_usgsLandsatTOA = "fmask.cmdline.usgsLandsatTOA:mainRoutine" 34 | 35 | [project.urls] 36 | Repository = "https://github.com/ubarsc/python-fmask.git" 37 | Homepage = "https://www.pythonfmask.org" 38 | 39 | [tool.setuptools.dynamic] 40 | version = {attr = "fmask.__version__"} 41 | 42 | [tool.setuptools.packages.find] 43 | namespaces = false # Excludes subdirectories with no __init__.py 44 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # This file is part of 'python-fmask' - a cloud masking module 2 | # Copyright (C) 2015 Neil Flood 3 | # 4 | # This program is free software; you can redistribute it and/or 5 | # modify it under the terms of the GNU General Public License 6 | # as published by the Free Software Foundation; either version 3 7 | # of the License, or (at your option) any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU General Public License for more details. 13 | # 14 | # You should have received a copy of the GNU General Public License 15 | # along with this program; if not, write to the Free Software 16 | # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 17 | 18 | """ 19 | This setup.py now only covers how to compile the C extension modules. All 20 | other information has been moved into pyprojects.toml. 21 | """ 22 | 23 | import sys 24 | 25 | from setuptools import setup, Extension 26 | 27 | 28 | # So I can import fmask itself, which will allow access to fmask.__version__ 29 | # to support the dynamic version attribute specified in pyproject.toml 30 | sys.path.append(".") 31 | 32 | try: 33 | from numpy import get_include as numpy_get_include 34 | withExtensions = True 35 | except ImportError: 36 | # I suspect that this can no longer happen. We had to add numpy as a 37 | # build-time dependency inside the pyproject.toml file, which I think 38 | # means that it will always be present during any build process, even 39 | # on ReadTheDocs or similar. 40 | withExtensions = False 41 | 42 | # Trigger the most up-to-date numpy compile-time deprecation warnings. 43 | # See https://numpy.org/doc/stable/reference/c-api/deprecations.html 44 | # for a not-very-clear explanation. 45 | # The messages are not visible during the install process (i.e. using pip), 46 | # but ARE visible when building a wheel file (e.g. when using "python -m build") 47 | NUMPY_DEPR_WARN = ('NPY_NO_DEPRECATED_API', 'NPY_2_0_API_VERSION') 48 | 49 | if withExtensions: 50 | # This is for a normal build 51 | fillminimaC = Extension(name='fmask._fillminima', 52 | define_macros=[NUMPY_DEPR_WARN], 53 | sources=['c_src/fillminima.c'], 54 | include_dirs=[numpy_get_include()]) 55 | valueIndexesC = Extension(name='fmask._valueindexes', 56 | define_macros=[NUMPY_DEPR_WARN], 57 | sources=['c_src/valueindexes.c'], 58 | include_dirs=[numpy_get_include()]) 59 | extensionsList = [fillminimaC, valueIndexesC] 60 | else: 61 | # This would be for a ReadTheDocs build. As noted above, this probably never 62 | # happens anymore, since the addition of pyproject.toml 63 | extensionsList = [] 64 | 65 | # do the setup 66 | setup(ext_modules=extensionsList) 67 | 68 | --------------------------------------------------------------------------------