├── requirements.txt ├── PyFMask ├── __init__.py └── fmask.py ├── .gitignore ├── setup.py ├── README.md └── LICENSE.md /requirements.txt: -------------------------------------------------------------------------------- 1 | pygdal 2 | numexpr 3 | scikit-image 4 | numpy 5 | scipy 6 | -------------------------------------------------------------------------------- /PyFMask/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | 5 | __version__ = "3.2.0" 6 | 7 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .tox/ 2 | tmp_in.txt 3 | *tmp.in* 4 | *.pyc 5 | .redcar 6 | build/* 7 | dist/* 8 | doc/build/* 9 | .ropeproject 10 | *.ipynb 11 | .coverage 12 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # module: setup.py 4 | 5 | 6 | from __future__ import division 7 | from __future__ import print_function 8 | from __future__ import absolute_import 9 | 10 | from setuptools import setup, find_packages 11 | 12 | import PyFMask as package 13 | 14 | setup( 15 | name=package.__name__, 16 | version=package.__version__, 17 | packages=find_packages(), 18 | install_requires=[ 19 | 'numpy', 20 | 'scipy', 21 | 'numexpr', 22 | 'scikit-image', 23 | 'pygdal', 24 | ], 25 | entry_points=""" 26 | [console_scripts] 27 | fmask.py=PyFMask.fmask:main 28 | """, 29 | ) 30 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # PyFMask 2 | **There are major issues in the code, and it should not be used in its current state. I suggest you investigate https://bitbucket.org/chchrsc/python-fmask instead** 3 | 4 | 5 | Python version of the FMask Landsat Cloud Masking code 6 | 7 | ## Requirements 8 | Install the Python requirements listed in requirements.txt, and also ensure that you have GDAL installed (with Python bindings) 9 | 10 | ## License 11 | This code is licensed under the GPL v2, see the LICENSE.md file for more details. 12 | 13 | The code was taken from https://github.com/ceholden/config_fmask and modified slightly (with most modifications contributed back to this project). That code, in turn, was taken from a near-literal port of the original Matlab FMask code by Mitchell Wheeler and Josh Sixsmith 14 | 15 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | GNU GENERAL PUBLIC LICENSE 2 | Version 2, June 1991 3 | 4 | Copyright (C) 1989, 1991 Free Software Foundation, Inc., 5 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 6 | Everyone is permitted to copy and distribute verbatim copies 7 | of this license document, but changing it is not allowed. 8 | 9 | Preamble 10 | 11 | The licenses for most software are designed to take away your 12 | freedom to share and change it. By contrast, the GNU General Public 13 | License is intended to guarantee your freedom to share and change free 14 | software--to make sure the software is free for all its users. This 15 | General Public License applies to most of the Free Software 16 | Foundation's software and to any other program whose authors commit to 17 | using it. (Some other Free Software Foundation software is covered by 18 | the GNU Lesser General Public License instead.) You can apply it to 19 | your programs, too. 20 | 21 | When we speak of free software, we are referring to freedom, not 22 | price. Our General Public Licenses are designed to make sure that you 23 | have the freedom to distribute copies of free software (and charge for 24 | this service if you wish), that you receive source code or can get it 25 | if you want it, that you can change the software or use pieces of it 26 | in new free programs; and that you know you can do these things. 27 | 28 | To protect your rights, we need to make restrictions that forbid 29 | anyone to deny you these rights or to ask you to surrender the rights. 30 | These restrictions translate to certain responsibilities for you if you 31 | distribute copies of the software, or if you modify it. 32 | 33 | For example, if you distribute copies of such a program, whether 34 | gratis or for a fee, you must give the recipients all the rights that 35 | you have. You must make sure that they, too, receive or can get the 36 | source code. And you must show them these terms so they know their 37 | rights. 38 | 39 | We protect your rights with two steps: (1) copyright the software, and 40 | (2) offer you this license which gives you legal permission to copy, 41 | distribute and/or modify the software. 42 | 43 | Also, for each author's protection and ours, we want to make certain 44 | that everyone understands that there is no warranty for this free 45 | software. If the software is modified by someone else and passed on, we 46 | want its recipients to know that what they have is not the original, so 47 | that any problems introduced by others will not reflect on the original 48 | authors' reputations. 49 | 50 | Finally, any free program is threatened constantly by software 51 | patents. We wish to avoid the danger that redistributors of a free 52 | program will individually obtain patent licenses, in effect making the 53 | program proprietary. To prevent this, we have made it clear that any 54 | patent must be licensed for everyone's free use or not licensed at all. 55 | 56 | The precise terms and conditions for copying, distribution and 57 | modification follow. 58 | 59 | GNU GENERAL PUBLIC LICENSE 60 | TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 61 | 62 | 0. This License applies to any program or other work which contains 63 | a notice placed by the copyright holder saying it may be distributed 64 | under the terms of this General Public License. The "Program", below, 65 | refers to any such program or work, and a "work based on the Program" 66 | means either the Program or any derivative work under copyright law: 67 | that is to say, a work containing the Program or a portion of it, 68 | either verbatim or with modifications and/or translated into another 69 | language. (Hereinafter, translation is included without limitation in 70 | the term "modification".) Each licensee is addressed as "you". 71 | 72 | Activities other than copying, distribution and modification are not 73 | covered by this License; they are outside its scope. The act of 74 | running the Program is not restricted, and the output from the Program 75 | is covered only if its contents constitute a work based on the 76 | Program (independent of having been made by running the Program). 77 | Whether that is true depends on what the Program does. 78 | 79 | 1. You may copy and distribute verbatim copies of the Program's 80 | source code as you receive it, in any medium, provided that you 81 | conspicuously and appropriately publish on each copy an appropriate 82 | copyright notice and disclaimer of warranty; keep intact all the 83 | notices that refer to this License and to the absence of any warranty; 84 | and give any other recipients of the Program a copy of this License 85 | along with the Program. 86 | 87 | You may charge a fee for the physical act of transferring a copy, and 88 | you may at your option offer warranty protection in exchange for a fee. 89 | 90 | 2. You may modify your copy or copies of the Program or any portion 91 | of it, thus forming a work based on the Program, and copy and 92 | distribute such modifications or work under the terms of Section 1 93 | above, provided that you also meet all of these conditions: 94 | 95 | a) You must cause the modified files to carry prominent notices 96 | stating that you changed the files and the date of any change. 97 | 98 | b) You must cause any work that you distribute or publish, that in 99 | whole or in part contains or is derived from the Program or any 100 | part thereof, to be licensed as a whole at no charge to all third 101 | parties under the terms of this License. 102 | 103 | c) If the modified program normally reads commands interactively 104 | when run, you must cause it, when started running for such 105 | interactive use in the most ordinary way, to print or display an 106 | announcement including an appropriate copyright notice and a 107 | notice that there is no warranty (or else, saying that you provide 108 | a warranty) and that users may redistribute the program under 109 | these conditions, and telling the user how to view a copy of this 110 | License. (Exception: if the Program itself is interactive but 111 | does not normally print such an announcement, your work based on 112 | the Program is not required to print an announcement.) 113 | 114 | These requirements apply to the modified work as a whole. If 115 | identifiable sections of that work are not derived from the Program, 116 | and can be reasonably considered independent and separate works in 117 | themselves, then this License, and its terms, do not apply to those 118 | sections when you distribute them as separate works. But when you 119 | distribute the same sections as part of a whole which is a work based 120 | on the Program, the distribution of the whole must be on the terms of 121 | this License, whose permissions for other licensees extend to the 122 | entire whole, and thus to each and every part regardless of who wrote it. 123 | 124 | Thus, it is not the intent of this section to claim rights or contest 125 | your rights to work written entirely by you; rather, the intent is to 126 | exercise the right to control the distribution of derivative or 127 | collective works based on the Program. 128 | 129 | In addition, mere aggregation of another work not based on the Program 130 | with the Program (or with a work based on the Program) on a volume of 131 | a storage or distribution medium does not bring the other work under 132 | the scope of this License. 133 | 134 | 3. You may copy and distribute the Program (or a work based on it, 135 | under Section 2) in object code or executable form under the terms of 136 | Sections 1 and 2 above provided that you also do one of the following: 137 | 138 | a) Accompany it with the complete corresponding machine-readable 139 | source code, which must be distributed under the terms of Sections 140 | 1 and 2 above on a medium customarily used for software interchange; or, 141 | 142 | b) Accompany it with a written offer, valid for at least three 143 | years, to give any third party, for a charge no more than your 144 | cost of physically performing source distribution, a complete 145 | machine-readable copy of the corresponding source code, to be 146 | distributed under the terms of Sections 1 and 2 above on a medium 147 | customarily used for software interchange; or, 148 | 149 | c) Accompany it with the information you received as to the offer 150 | to distribute corresponding source code. (This alternative is 151 | allowed only for noncommercial distribution and only if you 152 | received the program in object code or executable form with such 153 | an offer, in accord with Subsection b above.) 154 | 155 | The source code for a work means the preferred form of the work for 156 | making modifications to it. For an executable work, complete source 157 | code means all the source code for all modules it contains, plus any 158 | associated interface definition files, plus the scripts used to 159 | control compilation and installation of the executable. However, as a 160 | special exception, the source code distributed need not include 161 | anything that is normally distributed (in either source or binary 162 | form) with the major components (compiler, kernel, and so on) of the 163 | operating system on which the executable runs, unless that component 164 | itself accompanies the executable. 165 | 166 | If distribution of executable or object code is made by offering 167 | access to copy from a designated place, then offering equivalent 168 | access to copy the source code from the same place counts as 169 | distribution of the source code, even though third parties are not 170 | compelled to copy the source along with the object code. 171 | 172 | 4. You may not copy, modify, sublicense, or distribute the Program 173 | except as expressly provided under this License. Any attempt 174 | otherwise to copy, modify, sublicense or distribute the Program is 175 | void, and will automatically terminate your rights under this License. 176 | However, parties who have received copies, or rights, from you under 177 | this License will not have their licenses terminated so long as such 178 | parties remain in full compliance. 179 | 180 | 5. You are not required to accept this License, since you have not 181 | signed it. However, nothing else grants you permission to modify or 182 | distribute the Program or its derivative works. These actions are 183 | prohibited by law if you do not accept this License. Therefore, by 184 | modifying or distributing the Program (or any work based on the 185 | Program), you indicate your acceptance of this License to do so, and 186 | all its terms and conditions for copying, distributing or modifying 187 | the Program or works based on it. 188 | 189 | 6. Each time you redistribute the Program (or any work based on the 190 | Program), the recipient automatically receives a license from the 191 | original licensor to copy, distribute or modify the Program subject to 192 | these terms and conditions. You may not impose any further 193 | restrictions on the recipients' exercise of the rights granted herein. 194 | You are not responsible for enforcing compliance by third parties to 195 | this License. 196 | 197 | 7. If, as a consequence of a court judgment or allegation of patent 198 | infringement or for any other reason (not limited to patent issues), 199 | conditions are imposed on you (whether by court order, agreement or 200 | otherwise) that contradict the conditions of this License, they do not 201 | excuse you from the conditions of this License. If you cannot 202 | distribute so as to satisfy simultaneously your obligations under this 203 | License and any other pertinent obligations, then as a consequence you 204 | may not distribute the Program at all. For example, if a patent 205 | license would not permit royalty-free redistribution of the Program by 206 | all those who receive copies directly or indirectly through you, then 207 | the only way you could satisfy both it and this License would be to 208 | refrain entirely from distribution of the Program. 209 | 210 | If any portion of this section is held invalid or unenforceable under 211 | any particular circumstance, the balance of the section is intended to 212 | apply and the section as a whole is intended to apply in other 213 | circumstances. 214 | 215 | It is not the purpose of this section to induce you to infringe any 216 | patents or other property right claims or to contest validity of any 217 | such claims; this section has the sole purpose of protecting the 218 | integrity of the free software distribution system, which is 219 | implemented by public license practices. Many people have made 220 | generous contributions to the wide range of software distributed 221 | through that system in reliance on consistent application of that 222 | system; it is up to the author/donor to decide if he or she is willing 223 | to distribute software through any other system and a licensee cannot 224 | impose that choice. 225 | 226 | This section is intended to make thoroughly clear what is believed to 227 | be a consequence of the rest of this License. 228 | 229 | 8. If the distribution and/or use of the Program is restricted in 230 | certain countries either by patents or by copyrighted interfaces, the 231 | original copyright holder who places the Program under this License 232 | may add an explicit geographical distribution limitation excluding 233 | those countries, so that distribution is permitted only in or among 234 | countries not thus excluded. In such case, this License incorporates 235 | the limitation as if written in the body of this License. 236 | 237 | 9. The Free Software Foundation may publish revised and/or new versions 238 | of the General Public License from time to time. Such new versions will 239 | be similar in spirit to the present version, but may differ in detail to 240 | address new problems or concerns. 241 | 242 | Each version is given a distinguishing version number. If the Program 243 | specifies a version number of this License which applies to it and "any 244 | later version", you have the option of following the terms and conditions 245 | either of that version or of any later version published by the Free 246 | Software Foundation. If the Program does not specify a version number of 247 | this License, you may choose any version ever published by the Free Software 248 | Foundation. 249 | 250 | 10. If you wish to incorporate parts of the Program into other free 251 | programs whose distribution conditions are different, write to the author 252 | to ask for permission. For software which is copyrighted by the Free 253 | Software Foundation, write to the Free Software Foundation; we sometimes 254 | make exceptions for this. Our decision will be guided by the two goals 255 | of preserving the free status of all derivatives of our free software and 256 | of promoting the sharing and reuse of software generally. 257 | 258 | NO WARRANTY 259 | 260 | 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY 261 | FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN 262 | OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES 263 | PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED 264 | OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 265 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS 266 | TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE 267 | PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, 268 | REPAIR OR CORRECTION. 269 | 270 | 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 271 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR 272 | REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, 273 | INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING 274 | OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED 275 | TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY 276 | YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER 277 | PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE 278 | POSSIBILITY OF SUCH DAMAGES. 279 | 280 | END OF TERMS AND CONDITIONS 281 | 282 | How to Apply These Terms to Your New Programs 283 | 284 | If you develop a new program, and you want it to be of the greatest 285 | possible use to the public, the best way to achieve this is to make it 286 | free software which everyone can redistribute and change under these terms. 287 | 288 | To do so, attach the following notices to the program. It is safest 289 | to attach them to the start of each source file to most effectively 290 | convey the exclusion of warranty; and each file should have at least 291 | the "copyright" line and a pointer to where the full notice is found. 292 | 293 | 294 | Copyright (C) 295 | 296 | This program is free software; you can redistribute it and/or modify 297 | it under the terms of the GNU General Public License as published by 298 | the Free Software Foundation; either version 2 of the License, or 299 | (at your option) any later version. 300 | 301 | This program is distributed in the hope that it will be useful, 302 | but WITHOUT ANY WARRANTY; without even the implied warranty of 303 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 304 | GNU General Public License for more details. 305 | 306 | You should have received a copy of the GNU General Public License along 307 | with this program; if not, write to the Free Software Foundation, Inc., 308 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 309 | 310 | Also add information on how to contact you by electronic and paper mail. 311 | 312 | If the program is interactive, make it output a short notice like this 313 | when it starts in an interactive mode: 314 | 315 | Gnomovision version 69, Copyright (C) year name of author 316 | Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 317 | This is free software, and you are welcome to redistribute it 318 | under certain conditions; type `show c' for details. 319 | 320 | The hypothetical commands `show w' and `show c' should show the appropriate 321 | parts of the General Public License. Of course, the commands you use may 322 | be called something other than `show w' and `show c'; they could even be 323 | mouse-clicks or menu items--whatever suits your program. 324 | 325 | You should also get your employer (if you work as a programmer) or your 326 | school, if any, to sign a "copyright disclaimer" for the program, if 327 | necessary. Here is a sample; alter the names: 328 | 329 | Yoyodyne, Inc., hereby disclaims all copyright interest in the program 330 | `Gnomovision' (which makes passes at compilers) written by James Hacker. 331 | 332 | , 1 April 1989 333 | Ty Coon, President of Vice 334 | 335 | This General Public License does not permit incorporating your program into 336 | proprietary programs. If your program is a subroutine library, you may 337 | consider it more useful to permit linking proprietary applications with the 338 | library. If this is what you want to do, use the GNU Lesser General 339 | Public License instead of this License. 340 | -------------------------------------------------------------------------------- /PyFMask/fmask.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # coding=utf-8 4 | # A near literal port of the FMask matlab code to python using numpy/scipy and 5 | # itk. - As a result of the literal port, original comments should still be 6 | # intact - as is the original code structure. 7 | 8 | # Transcription History 9 | # 1.6.3 transcription Mitchell Wheeler 10 | # 3.0 transcription Josh Sixsmith 11 | # + added skimage library 12 | # + better handling for imagery of different resolutions 13 | 14 | import sys 15 | import gc 16 | import math 17 | import logging 18 | import datetime 19 | import os.path 20 | from glob import glob 21 | import argparse 22 | import numpy 23 | import numexpr 24 | import scipy.stats 25 | import scipy.signal 26 | import scipy.ndimage.morphology 27 | from osgeo import gdal 28 | import skimage 29 | from skimage import morphology 30 | from skimage import measure 31 | from skimage import segmentation 32 | 33 | skimage_version = [int(n) for n in skimage.__version__.split('.') if n != ''] 34 | 35 | logger = logging.getLogger('root.' + __name__) 36 | 37 | # Sun earth distance look up table 38 | sun_earth_distance = {1: 0.98331, 2: 0.98330, 3: 0.98330, 4: 0.98330, 5: 0.98330, 6: 0.98332, 7: 0.98333, 8: 0.98335, 9: 0.98338, 10: 0.98341, 11: 0.98345, 12: 0.98349, 13: 0.98354, 14: 0.98359, 15: 0.98365, 16: 0.98371, 17: 0.98378, 18: 0.98385, 19: 0.98393, 20: 0.98401, 21: 0.98410, 22: 0.98419, 23: 0.98428, 24: 0.98439, 25: 0.98449, 26: 0.98460, 27: 0.98472, 28: 0.98484, 29: 0.98496, 30: 0.98509, 31: 0.98523, 32: 0.98536, 33: 0.98551, 34: 0.98565, 35: 0.98580, 36: 0.98596, 37: 0.98612, 38: 0.98628, 39: 0.98645, 40: 0.98662, 41: 0.98680, 42: 0.98698, 43: 0.98717, 44: 0.98735, 45: 0.98755, 46: 0.98774, 47: 0.98794, 48: 0.98814, 49: 0.98835, 50: 0.98856, 51: 0.98877, 52: 0.98899, 53: 0.98921, 54: 0.98944, 55: 0.98966, 56: 0.98989, 57: 0.99012, 58: 0.99036, 59: 0.99060, 60: 0.99084, 61: 0.99108, 62: 0.99133, 63: 0.99158, 64: 0.99183, 65: 0.99208, 66: 0.99234, 67: 0.99260, 68: 0.99286, 69: 0.99312, 70: 0.99339, 71: 0.99365, 72: 0.99392, 73: 0.99419, 74: 0.99446, 75: 0.99474, 76: 0.99501, 77: 0.99529, 78: 0.99556, 79: 0.99584, 80: 0.99612, 81: 0.99640, 82: 0.99669, 83: 0.99697, 84: 0.99725, 85: 0.99754, 86: 0.99782, 87: 0.99811, 88: 0.99840, 89: 0.99868, 90: 0.99897, 91: 0.99926, 92: 0.99954, 93: 0.99983, 94: 1.00012, 95: 1.00041, 96: 1.00069, 97: 1.00098, 98: 1.00127, 99: 1.00155, 100: 1.00184, 101: 1.00212, 102: 1.00240, 103: 1.00269, 104: 1.00297, 105: 1.00325, 106: 1.00353, 107: 1.00381, 108: 1.00409, 109: 1.00437, 110: 1.00464, 111: 1.00492, 112: 1.00519, 113: 1.00546, 114: 1.00573, 115: 1.00600, 116: 1.00626, 117: 1.00653, 118: 1.00679, 119: 1.00705, 120: 1.00731, 121: 1.00756, 122: 1.00781, 123: 1.00806, 124: 1.00831, 125: 1.00856, 126: 1.00880, 127: 1.00904, 128: 1.00928, 129: 1.00952, 130: 1.00975, 131: 1.00998, 132: 1.01020, 133: 1.01043, 134: 1.01065, 135: 1.01087, 136: 1.01108, 137: 1.01129, 138: 1.01150, 139: 1.01170, 140: 1.01191, 141: 1.01210, 142: 1.01230, 143: 1.01249, 144: 1.01267, 145: 1.01286, 146: 1.01304, 147: 1.01321, 148: 1.01338, 149: 1.01355, 150: 1.01371, 151: 1.01387, 152: 1.01403, 153: 1.01418, 154: 1.01433, 155: 1.01447, 156: 1.01461, 157: 1.01475, 158: 1.01488, 159: 1.01500, 160: 1.01513, 161: 1.01524, 162: 1.01536, 163: 1.01547, 164: 1.01557, 165: 1.01567, 166: 1.01577, 167: 1.01586, 168: 1.01595, 169: 1.01603, 170: 1.01610, 171: 1.01618, 172: 1.01625, 173: 1.01631, 174: 1.01637, 175: 1.01642, 176: 1.01647, 177: 1.01652, 178: 1.01656, 179: 1.01659, 180: 1.01662, 181: 1.01665, 182: 1.01667, 183: 1.01668, 184: 1.01670, 185: 1.01670, 186: 1.01670, 39 | 187: 1.01670, 188: 1.01669, 189: 1.01668, 190: 1.01666, 191: 1.01664, 192: 1.01661, 193: 1.01658, 194: 1.01655, 195: 1.01650, 196: 1.01646, 197: 1.01641, 198: 1.01635, 199: 1.01629, 200: 1.01623, 201: 1.01616, 202: 1.01609, 203: 1.01601, 204: 1.01592, 205: 1.01584, 206: 1.01575, 207: 1.01565, 208: 1.01555, 209: 1.01544, 210: 1.01533, 211: 1.01522, 212: 1.01510, 213: 1.01497, 214: 1.01485, 215: 1.01471, 216: 1.01458, 217: 1.01444, 218: 1.01429, 219: 1.01414, 220: 1.01399, 221: 1.01383, 222: 1.01367, 223: 1.01351, 224: 1.01334, 225: 1.01317, 226: 1.01299, 227: 1.01281, 228: 1.01263, 229: 1.01244, 230: 1.01225, 231: 1.01205, 232: 1.01186, 233: 1.01165, 234: 1.01145, 235: 1.01124, 236: 1.01103, 237: 1.01081, 238: 1.01060, 239: 1.01037, 240: 1.01015, 241: 1.00992, 242: 1.00969, 243: 1.00946, 244: 1.00922, 245: 1.00898, 246: 1.00874, 247: 1.00850, 248: 1.00825, 249: 1.00800, 250: 1.00775, 251: 1.00750, 252: 1.00724, 253: 1.00698, 254: 1.00672, 255: 1.00646, 256: 1.00620, 257: 1.00593, 258: 1.00566, 259: 1.00539, 260: 1.00512, 261: 1.00485, 262: 1.00457, 263: 1.00430, 264: 1.00402, 265: 1.00374, 266: 1.00346, 267: 1.00318, 268: 1.00290, 269: 1.00262, 270: 1.00234, 271: 1.00205, 272: 1.00177, 273: 1.00148, 274: 1.00119, 275: 1.00091, 276: 1.00062, 277: 1.00033, 278: 1.00005, 279: 0.99976, 280: 0.99947, 281: 0.99918, 282: 0.99890, 283: 0.99861, 284: 0.99832, 285: 0.99804, 286: 0.99775, 287: 0.99747, 288: 0.99718, 289: 0.99690, 290: 0.99662, 291: 0.99634, 292: 0.99605, 293: 0.99577, 294: 0.99550, 295: 0.99522, 296: 0.99494, 297: 0.99467, 298: 0.99440, 299: 0.99412, 300: 0.99385, 301: 0.99359, 302: 0.99332, 303: 0.99306, 304: 0.99279, 305: 0.99253, 306: 0.99228, 307: 0.99202, 308: 0.99177, 309: 0.99152, 310: 0.99127, 311: 0.99102, 312: 0.99078, 313: 0.99054, 314: 0.99030, 315: 0.99007, 316: 0.98983, 317: 0.98961, 318: 0.98938, 319: 0.98916, 320: 0.98894, 321: 0.98872, 322: 0.98851, 323: 0.98830, 324: 0.98809, 325: 0.98789, 326: 0.98769, 327: 0.98750, 328: 0.98731, 329: 0.98712, 330: 0.98694, 331: 0.98676, 332: 0.98658, 333: 0.98641, 334: 0.98624, 335: 0.98608, 336: 0.98592, 337: 0.98577, 338: 0.98562, 339: 0.98547, 340: 0.98533, 341: 0.98519, 342: 0.98506, 343: 0.98493, 344: 0.98481, 345: 0.98469, 346: 0.98457, 347: 0.98446, 348: 0.98436, 349: 0.98426, 350: 0.98416, 351: 0.98407, 352: 0.98399, 353: 0.98391, 354: 0.98383, 355: 0.98376, 356: 0.98370, 357: 0.98363, 358: 0.98358, 359: 0.98353, 360: 0.98348, 361: 0.98344, 362: 0.98340, 363: 0.98337, 364: 0.98335, 365: 0.98333, 366: 0.98331} 40 | 41 | # Replacement for original dir() function in this module. 42 | # Renamed to avoid name collision with builtin. 43 | 44 | 45 | def match_file(dir_path, pattern): 46 | res = glob(os.path.join(dir_path, pattern)) 47 | 48 | if len(res) == 0: 49 | return None 50 | else: 51 | return res[0] 52 | 53 | 54 | def im_info(filename): 55 | """ A function to retrieve the geotransform and projection details using 56 | GDAL. The original MATLAB code handles it differently, we'll just implement 57 | something unique here and let GDAL automatically handle the read/write of 58 | the projection info. Less messy than the MATLAB code which assumes that 59 | everything is in UTM. Some products may come as Lat/Lon geographic 60 | projections. """ 61 | 62 | img = gdal.Open(filename) 63 | geoT = img.GetGeoTransform() 64 | prj = img.GetProjection() 65 | size = (img.RasterYSize, img.RasterXSize) 66 | ul_coord = (geoT[3], geoT[0]) 67 | return (geoT, prj, size, ul_coord) 68 | 69 | 70 | def imread(filename, resample=False, samples=None, lines=None): 71 | img = gdal.Open(filename) 72 | band = img.GetRasterBand(1) 73 | if resample: 74 | driver = gdal.GetDriverByName('MEM') 75 | outds = driver.Create("", samples, lines, 1, band.DataType) 76 | outds.SetGeoTransform(img.GetGeoTransform()) 77 | outds.SetProjection(img.GetProjection()) 78 | gdal.ReprojectImage(img, outds) 79 | return outds.ReadAsArray() 80 | else: 81 | return band.ReadAsArray() 82 | 83 | 84 | def imfill_skimage(img): 85 | """ Replicates the imfill function available within MATLAB. Based on the 86 | example provided in 87 | http://scikit-image.org/docs/dev/auto_examples/plot_holes_and_peaks.html#example-plot-holes-and-peaks-py. 88 | 89 | """ 90 | 91 | seed = img.copy() 92 | 93 | # Define seed points and the start points for the erosion process. 94 | seed[1:-1, 1:-1] = img.max() 95 | 96 | # Define the mask; Probably unneeded. 97 | mask = img 98 | 99 | # Fill the holes 100 | filled = morphology.reconstruction(seed, mask, method='erosion') 101 | 102 | return filled 103 | 104 | 105 | def lndhdrread(filename): 106 | """ 107 | Load Landsat scene MTL file metadata. 108 | 109 | :param filename: 110 | A string containing the full path the scene's MTL file. 111 | """ 112 | # Read in Landsat TM/ETM+ MTL header for Fmask 113 | # [Lmax,Lmin,Qcalmax,Qcalmin,ijdim_ref,ijdim_thm,reso_ref,reso_thm,ul,zen,azi,zc,Lnum,doy]=lndhdrread(filename) 114 | # Where: 115 | # Inputs: 116 | # filename='L*MTL.txt' 117 | # Ouenuts: 118 | # 1) Lmax = Max radiances 119 | # 2) Lmin = Min radiances 120 | # 3) Qcalmax = Max calibrated DNs 121 | # 4) Qcalmin = Min calibrated DNs 122 | # 5) ijdim_ref = [nrows,ncols] # dimension of optical bands 123 | # 6) ijdim_ref = [nrows,ncols] # dimension of thermal band 124 | # 7) reo_ref = 28/30 # resolution of optical bands 125 | # 8) reo_thm = 60/120 # resolution of thermal band 126 | # 9) ul = [upperleft_mapx upperleft_mapy] 127 | # 10) zen = solar zenith angle (degrees) 128 | # 11) azi = solar azimuth angle (degrees) 129 | # 12) zc = Zone Number 130 | # 13) Lnum = 4,5,or 7 Landsat sensor number 131 | # 14) doy = day of year (1,2,3,...,356) 132 | # 133 | ## 134 | # open and read hdr file 135 | data = {} 136 | fl = open(filename, 'r') 137 | file_lines = fl.readlines() 138 | for line in file_lines: 139 | values = line.split(' = ') 140 | if len(values) != 2: 141 | continue 142 | 143 | data[values[0].strip()] = values[1].strip().strip('"') 144 | 145 | fl.close() 146 | 147 | # Identify Landsat Number (Lnum = 4, 5 or 7) 148 | LID = data['SPACECRAFT_ID'] 149 | Lnum = int(LID[len(LID) - 1]) 150 | 151 | if ((Lnum >= 4) & (Lnum <= 7)): 152 | # LS8 variables only. The original MATLAB function returns all 153 | # variables. 154 | Refmax = None 155 | # LS8 variables only. The original MATLAB function returns all 156 | # variables. 157 | Refmin = None 158 | # Test for New/Old MTL file 159 | if not ('LANDSAT_SCENE_ID' in data.keys()): 160 | # Process Old version of MTL file 161 | 162 | # read in LMAX 163 | Lmax_B1 = numpy.float32(data['LMAX_BAND1']) 164 | Lmax_B2 = numpy.float32(data['LMAX_BAND2']) 165 | Lmax_B3 = numpy.float32(data['LMAX_BAND3']) 166 | Lmax_B4 = numpy.float32(data['LMAX_BAND4']) 167 | Lmax_B5 = numpy.float32(data['LMAX_BAND5']) 168 | if Lnum == 7: 169 | Lmax_B6 = numpy.float32(data['LMAX_BAND61']) 170 | else: 171 | Lmax_B6 = numpy.float32(data['LMAX_BAND6']) 172 | 173 | Lmax_B7 = numpy.float32(data['LMAX_BAND7']) 174 | Lmax = (Lmax_B1, Lmax_B2, Lmax_B3, 175 | Lmax_B4, Lmax_B5, Lmax_B6, Lmax_B7) 176 | 177 | # Read in LMIN 178 | Lmin_B1 = numpy.float32(data['LMIN_BAND1']) 179 | Lmin_B2 = numpy.float32(data['LMIN_BAND2']) 180 | Lmin_B3 = numpy.float32(data['LMIN_BAND3']) 181 | Lmin_B4 = numpy.float32(data['LMIN_BAND4']) 182 | Lmin_B5 = numpy.float32(data['LMIN_BAND5']) 183 | if Lnum == 7: 184 | Lmin_B6 = numpy.float32(data['LMIN_BAND61']) 185 | else: 186 | Lmin_B6 = numpy.float32(data['LMIN_BAND6']) 187 | 188 | Lmin_B7 = numpy.float32(data['LMIN_BAND7']) 189 | Lmin = (Lmin_B1, Lmin_B2, Lmin_B3, 190 | Lmin_B4, Lmin_B5, Lmin_B6, Lmin_B7) 191 | 192 | # Read in QCALMAX 193 | Qcalmax_B1 = numpy.float32(data['QCALMAX_BAND1']) 194 | Qcalmax_B2 = numpy.float32(data['QCALMAX_BAND2']) 195 | Qcalmax_B3 = numpy.float32(data['QCALMAX_BAND3']) 196 | Qcalmax_B4 = numpy.float32(data['QCALMAX_BAND4']) 197 | Qcalmax_B5 = numpy.float32(data['QCALMAX_BAND5']) 198 | if Lnum == 7: 199 | Qcalmax_B6 = numpy.float32(data['QCALMAX_BAND61']) 200 | else: 201 | Qcalmax_B6 = numpy.float32(data['QCALMAX_BAND6']) 202 | 203 | Qcalmax_B7 = numpy.float32(data['QCALMAX_BAND7']) 204 | Qcalmax = (Qcalmax_B1, Qcalmax_B2, Qcalmax_B3, 205 | Qcalmax_B4, Qcalmax_B5, Qcalmax_B6, Qcalmax_B7) 206 | 207 | # Read in QCALMIN 208 | Qcalmin_B1 = numpy.float32(data['QCALMIN_BAND1']) 209 | Qcalmin_B2 = numpy.float32(data['QCALMIN_BAND2']) 210 | Qcalmin_B3 = numpy.float32(data['QCALMIN_BAND3']) 211 | Qcalmin_B4 = numpy.float32(data['QCALMIN_BAND4']) 212 | Qcalmin_B5 = numpy.float32(data['QCALMIN_BAND5']) 213 | if Lnum == 7: 214 | Qcalmin_B6 = numpy.float32(data['QCALMIN_BAND61']) 215 | else: 216 | Qcalmin_B6 = numpy.float32(data['QCALMIN_BAND6']) 217 | 218 | Qcalmin_B7 = numpy.float32(data['QCALMIN_BAND7']) 219 | Qcalmin = (Qcalmin_B1, Qcalmin_B2, Qcalmin_B3, 220 | Qcalmin_B4, Qcalmin_B5, Qcalmin_B6, Qcalmin_B7) 221 | 222 | # Read in nrows & ncols of optical bands 223 | Sample_ref = int(data['PRODUCT_SAMPLES_REF']) 224 | Line_ref = int(data['PRODUCT_LINES_REF']) 225 | # record ijdimension of optical bands 226 | ijdim_ref = (Line_ref, Sample_ref) 227 | 228 | Sample_thm = int(data['PRODUCT_SAMPLES_THM']) 229 | Line_thm = int(data['PRODUCT_LINES_THM']) 230 | # record thermal band dimensions (i,j) 231 | ijdim_thm = (Line_thm, Sample_thm) 232 | 233 | # Read in resolution of optical and thermal bands 234 | reso_ref = numpy.float32(data['GRID_CELL_SIZE_REF']) 235 | reso_thm = numpy.float32(data['GRID_CELL_SIZE_THM']) 236 | 237 | # Read in UTM Zone Number 238 | zc = numpy.float32(data['ZONE_NUMBER']) 239 | # Read in Solar Azimuth & Elevation angle (degrees) 240 | azi = numpy.float32(data['SUN_AZIMUTH']) 241 | zen = 90 - numpy.float32(data['SUN_ELEVATION']) 242 | # Read in upperleft mapx,y 243 | ulx = numpy.float32(data['PRODUCT_UL_CORNER_MAPX']) 244 | uly = numpy.float32(data['PRODUCT_UL_CORNER_MAPY']) 245 | ul = (ulx, uly) 246 | # Read in date of year 247 | char_doy = data['DATEHOUR_CONTACT_PERIOD'] 248 | doy = int(char_doy[2:5]) 249 | else: 250 | # Process New version of MTL file 251 | 252 | # read in LMAX 253 | Lmax_B1 = numpy.float32(data['RADIANCE_MAXIMUM_BAND_1']) 254 | Lmax_B2 = numpy.float32(data['RADIANCE_MAXIMUM_BAND_2']) 255 | Lmax_B3 = numpy.float32(data['RADIANCE_MAXIMUM_BAND_3']) 256 | Lmax_B4 = numpy.float32(data['RADIANCE_MAXIMUM_BAND_4']) 257 | Lmax_B5 = numpy.float32(data['RADIANCE_MAXIMUM_BAND_5']) 258 | if Lnum == 7: 259 | Lmax_B6 = numpy.float32(data['RADIANCE_MAXIMUM_BAND_6_VCID_1']) 260 | else: 261 | Lmax_B6 = numpy.float32(data['RADIANCE_MAXIMUM_BAND_6']) 262 | 263 | Lmax_B7 = numpy.float32(data['RADIANCE_MAXIMUM_BAND_7']) 264 | Lmax = (Lmax_B1, Lmax_B2, Lmax_B3, 265 | Lmax_B4, Lmax_B5, Lmax_B6, Lmax_B7) 266 | 267 | # Read in LMIN 268 | Lmin_B1 = numpy.float32(data['RADIANCE_MINIMUM_BAND_1']) 269 | Lmin_B2 = numpy.float32(data['RADIANCE_MINIMUM_BAND_2']) 270 | Lmin_B3 = numpy.float32(data['RADIANCE_MINIMUM_BAND_3']) 271 | Lmin_B4 = numpy.float32(data['RADIANCE_MINIMUM_BAND_4']) 272 | Lmin_B5 = numpy.float32(data['RADIANCE_MINIMUM_BAND_5']) 273 | if Lnum == 7: 274 | Lmin_B6 = numpy.float32(data['RADIANCE_MINIMUM_BAND_6_VCID_1']) 275 | else: 276 | Lmin_B6 = numpy.float32(data['RADIANCE_MINIMUM_BAND_6']) 277 | 278 | Lmin_B7 = numpy.float32(data['RADIANCE_MINIMUM_BAND_7']) 279 | Lmin = (Lmin_B1, Lmin_B2, Lmin_B3, 280 | Lmin_B4, Lmin_B5, Lmin_B6, Lmin_B7) 281 | 282 | # Read in QCALMAX 283 | Qcalmax_B1 = numpy.float32(data['QUANTIZE_CAL_MAX_BAND_1']) 284 | Qcalmax_B2 = numpy.float32(data['QUANTIZE_CAL_MAX_BAND_2']) 285 | Qcalmax_B3 = numpy.float32(data['QUANTIZE_CAL_MAX_BAND_3']) 286 | Qcalmax_B4 = numpy.float32(data['QUANTIZE_CAL_MAX_BAND_4']) 287 | Qcalmax_B5 = numpy.float32(data['QUANTIZE_CAL_MAX_BAND_5']) 288 | if Lnum == 7: 289 | Qcalmax_B6 = numpy.float32( 290 | data['QUANTIZE_CAL_MAX_BAND_6_VCID_1']) 291 | else: 292 | Qcalmax_B6 = numpy.float32(data['QUANTIZE_CAL_MAX_BAND_6']) 293 | 294 | Qcalmax_B7 = numpy.float32(data['QUANTIZE_CAL_MAX_BAND_7']) 295 | Qcalmax = (Qcalmax_B1, Qcalmax_B2, Qcalmax_B3, 296 | Qcalmax_B4, Qcalmax_B5, Qcalmax_B6, Qcalmax_B7) 297 | 298 | # Read in QCALMIN 299 | Qcalmin_B1 = numpy.float32(data['QUANTIZE_CAL_MIN_BAND_1']) 300 | Qcalmin_B2 = numpy.float32(data['QUANTIZE_CAL_MIN_BAND_2']) 301 | Qcalmin_B3 = numpy.float32(data['QUANTIZE_CAL_MIN_BAND_3']) 302 | Qcalmin_B4 = numpy.float32(data['QUANTIZE_CAL_MIN_BAND_4']) 303 | Qcalmin_B5 = numpy.float32(data['QUANTIZE_CAL_MIN_BAND_5']) 304 | if Lnum == 7: 305 | Qcalmin_B6 = numpy.float32( 306 | data['QUANTIZE_CAL_MIN_BAND_6_VCID_1']) 307 | else: 308 | Qcalmin_B6 = numpy.float32(data['QUANTIZE_CAL_MIN_BAND_6']) 309 | 310 | Qcalmin_B7 = numpy.float32(data['QUANTIZE_CAL_MIN_BAND_7']) 311 | Qcalmin = (Qcalmin_B1, Qcalmin_B2, Qcalmin_B3, 312 | Qcalmin_B4, Qcalmin_B5, Qcalmin_B6, Qcalmin_B7) 313 | 314 | # Read in nrows & ncols of optical bands 315 | Sample_ref = int(data['REFLECTIVE_SAMPLES']) 316 | Line_ref = int(data['REFLECTIVE_LINES']) 317 | # record ijdimension of optical bands 318 | ijdim_ref = (Line_ref, Sample_ref) 319 | 320 | Sample_thm = int(data['THERMAL_SAMPLES']) 321 | Line_thm = int(data['THERMAL_LINES']) 322 | # record thermal band dimensions (i,j) 323 | ijdim_thm = (Line_thm, Sample_thm) 324 | 325 | # Read in resolution of optical and thermal bands 326 | reso_ref = numpy.float32(data['GRID_CELL_SIZE_REFLECTIVE']) 327 | reso_thm = numpy.float32(data['GRID_CELL_SIZE_THERMAL']) 328 | 329 | # Read in UTM Zone Number 330 | zc = numpy.float32(data['UTM_ZONE']) 331 | # Read in Solar Azimuth & Elevation angle (degrees) 332 | azi = numpy.float32(data['SUN_AZIMUTH']) 333 | zen = 90 - numpy.float32(data['SUN_ELEVATION']) 334 | # Read in upperleft mapx,y 335 | ulx = numpy.float32(data['CORNER_UL_PROJECTION_X_PRODUCT']) 336 | uly = numpy.float32(data['CORNER_UL_PROJECTION_Y_PRODUCT']) 337 | ul = (ulx, uly) 338 | # Read in date of year 339 | char_doy = data['LANDSAT_SCENE_ID'] 340 | # This may need to change to 14:16. TODO Test this! 341 | doy = int(char_doy[13:16]) 342 | 343 | elif (Lnum == 8): 344 | # Retrieve LS8 info 345 | Lmax_B2 = numpy.float32(data['RADIANCE_MAXIMUM_BAND_2']) 346 | Lmax_B3 = numpy.float32(data['RADIANCE_MAXIMUM_BAND_3']) 347 | Lmax_B4 = numpy.float32(data['RADIANCE_MAXIMUM_BAND_4']) 348 | Lmax_B5 = numpy.float32(data['RADIANCE_MAXIMUM_BAND_5']) 349 | Lmax_B6 = numpy.float32(data['RADIANCE_MAXIMUM_BAND_6']) 350 | Lmax_B7 = numpy.float32(data['RADIANCE_MAXIMUM_BAND_7']) 351 | Lmax_B9 = numpy.float32(data['RADIANCE_MAXIMUM_BAND_9']) 352 | Lmax_B10 = numpy.float32(data['RADIANCE_MAXIMUM_BAND_10']) 353 | 354 | Lmax = (Lmax_B2, Lmax_B3, Lmax_B4, Lmax_B5, 355 | Lmax_B6, Lmax_B7, Lmax_B9, Lmax_B10) 356 | 357 | # Read in LMIN 358 | Lmin_B2 = numpy.float32(data['RADIANCE_MINIMUM_BAND_2']) 359 | Lmin_B3 = numpy.float32(data['RADIANCE_MINIMUM_BAND_3']) 360 | Lmin_B4 = numpy.float32(data['RADIANCE_MINIMUM_BAND_4']) 361 | Lmin_B5 = numpy.float32(data['RADIANCE_MINIMUM_BAND_5']) 362 | Lmin_B6 = numpy.float32(data['RADIANCE_MINIMUM_BAND_6']) 363 | Lmin_B7 = numpy.float32(data['RADIANCE_MINIMUM_BAND_7']) 364 | Lmin_B9 = numpy.float32(data['RADIANCE_MINIMUM_BAND_9']) 365 | Lmin_B10 = numpy.float32(data['RADIANCE_MINIMUM_BAND_10']) 366 | 367 | Lmin = (Lmin_B2, Lmin_B3, Lmin_B4, Lmin_B5, 368 | Lmin_B6, Lmin_B7, Lmin_B9, Lmin_B10) 369 | 370 | # Read in QCALMAX 371 | Qcalmax_B2 = numpy.float32(data['QUANTIZE_CAL_MAX_BAND_2']) 372 | Qcalmax_B3 = numpy.float32(data['QUANTIZE_CAL_MAX_BAND_3']) 373 | Qcalmax_B4 = numpy.float32(data['QUANTIZE_CAL_MAX_BAND_4']) 374 | Qcalmax_B5 = numpy.float32(data['QUANTIZE_CAL_MAX_BAND_5']) 375 | Qcalmax_B6 = numpy.float32(data['QUANTIZE_CAL_MAX_BAND_6']) 376 | Qcalmax_B7 = numpy.float32(data['QUANTIZE_CAL_MAX_BAND_7']) 377 | Qcalmax_B9 = numpy.float32(data['QUANTIZE_CAL_MAX_BAND_9']) 378 | Qcalmax_B10 = numpy.float32(data['QUANTIZE_CAL_MAX_BAND_10']) 379 | 380 | Qcalmax = (Qcalmax_B2, Qcalmax_B3, Qcalmax_B4, Qcalmax_B5, 381 | Qcalmax_B6, Qcalmax_B7, Qcalmax_B9, Qcalmax_B10) 382 | 383 | # Read in QCALMIN 384 | Qcalmin_B2 = numpy.float32(data['QUANTIZE_CAL_MIN_BAND_2']) 385 | Qcalmin_B3 = numpy.float32(data['QUANTIZE_CAL_MIN_BAND_3']) 386 | Qcalmin_B4 = numpy.float32(data['QUANTIZE_CAL_MIN_BAND_4']) 387 | Qcalmin_B5 = numpy.float32(data['QUANTIZE_CAL_MIN_BAND_5']) 388 | Qcalmin_B6 = numpy.float32(data['QUANTIZE_CAL_MIN_BAND_6']) 389 | Qcalmin_B7 = numpy.float32(data['QUANTIZE_CAL_MIN_BAND_7']) 390 | Qcalmin_B9 = numpy.float32(data['QUANTIZE_CAL_MIN_BAND_9']) 391 | Qcalmin_B10 = numpy.float32(data['QUANTIZE_CAL_MIN_BAND_10']) 392 | 393 | Qcalmin = (Qcalmin_B2, Qcalmin_B3, Qcalmin_B4, Qcalmin_B5, 394 | Qcalmin_B6, Qcalmin_B7, Qcalmin_B9, Qcalmin_B10) 395 | 396 | # Read in Refmax 397 | Refmax_B2 = numpy.float32(data['REFLECTANCE_MAXIMUM_BAND_2']) 398 | Refmax_B3 = numpy.float32(data['REFLECTANCE_MAXIMUM_BAND_3']) 399 | Refmax_B4 = numpy.float32(data['REFLECTANCE_MAXIMUM_BAND_4']) 400 | Refmax_B5 = numpy.float32(data['REFLECTANCE_MAXIMUM_BAND_5']) 401 | Refmax_B6 = numpy.float32(data['REFLECTANCE_MAXIMUM_BAND_6']) 402 | Refmax_B7 = numpy.float32(data['REFLECTANCE_MAXIMUM_BAND_7']) 403 | Refmax_B9 = numpy.float32(data['REFLECTANCE_MAXIMUM_BAND_9']) 404 | 405 | Refmax = (Refmax_B2, Refmax_B3, Refmax_B4, 406 | Refmax_B5, Refmax_B6, Refmax_B7, Refmax_B9) 407 | 408 | # Read in Refmin 409 | Refmin_B2 = numpy.float32(data['REFLECTANCE_MINIMUM_BAND_2']) 410 | Refmin_B3 = numpy.float32(data['REFLECTANCE_MINIMUM_BAND_3']) 411 | Refmin_B4 = numpy.float32(data['REFLECTANCE_MINIMUM_BAND_4']) 412 | Refmin_B5 = numpy.float32(data['REFLECTANCE_MINIMUM_BAND_5']) 413 | Refmin_B6 = numpy.float32(data['REFLECTANCE_MINIMUM_BAND_6']) 414 | Refmin_B7 = numpy.float32(data['REFLECTANCE_MINIMUM_BAND_7']) 415 | Refmin_B9 = numpy.float32(data['REFLECTANCE_MINIMUM_BAND_9']) 416 | 417 | Refmin = (Refmin_B2, Refmin_B3, Refmin_B4, 418 | Refmin_B5, Refmin_B6, Refmin_B7, Refmin_B9) 419 | 420 | # Read in nrows & ncols of optical bands 421 | Sample_ref = int(data['REFLECTIVE_SAMPLES']) 422 | Line_ref = int(data['REFLECTIVE_LINES']) 423 | # record ijdimension of optical bands 424 | ijdim_ref = (Line_ref, Sample_ref) 425 | 426 | Sample_thm = int(data['THERMAL_SAMPLES']) 427 | Line_thm = int(data['THERMAL_LINES']) 428 | # record thermal band dimensions (i,j) 429 | ijdim_thm = (Line_thm, Sample_thm) 430 | 431 | # Read in resolution of optical and thermal bands 432 | reso_ref = numpy.float32(data['GRID_CELL_SIZE_REFLECTIVE']) 433 | reso_thm = numpy.float32(data['GRID_CELL_SIZE_THERMAL']) 434 | 435 | # Read in UTM Zone Number 436 | zc = numpy.float32(data['UTM_ZONE']) 437 | # Read in Solar Azimuth & Elevation angle (degrees) 438 | azi = numpy.float32(data['SUN_AZIMUTH']) 439 | zen = 90 - numpy.float32(data['SUN_ELEVATION']) 440 | # Read in upperleft mapx,y 441 | ulx = numpy.float32(data['CORNER_UL_PROJECTION_X_PRODUCT']) 442 | uly = numpy.float32(data['CORNER_UL_PROJECTION_Y_PRODUCT']) 443 | ul = (ulx, uly) 444 | # Read in date of year 445 | char_doy = data['LANDSAT_SCENE_ID'] 446 | doy = int(char_doy[13:16]) 447 | else: 448 | raise Exception('This sensor is not Landsat 4, 5, 7, or 8!') 449 | 450 | if ((doy < 1) or (doy > 366)): 451 | raise ValueError( 452 | 'Invalid Day of Year metadata value - expected (1,366) got %s' % doy) 453 | 454 | # The new version returns Lmax,Lmin,Qcalmax,Qcalmin,Refmax,Refmin,ijdim_ref,ijdim_thm,reso_ref,reso_thm,ul,zen,azi,zc,Lnum,doy 455 | # return 456 | # (Lmax,Lmin,Qcalmax,Qcalmin,ijdim_ref,ijdim_thm,reso_ref,reso_thm,ul,zen,azi,zc,Lnum,doy) 457 | return (Lmax, Lmin, Qcalmax, Qcalmin, 458 | Refmax, Refmin, 459 | ijdim_ref, ijdim_thm, 460 | reso_ref, reso_thm, 461 | ul, 462 | zen, azi, 463 | zc, 464 | Lnum, 465 | doy) 466 | 467 | 468 | def nd2toarbt(filename, images=None): 469 | """ Load metadata from MTL file & calculate reflectance values for scene 470 | bands. 471 | 472 | :param filename: A string containing the file path of the MTL file for the 473 | landsat scene. 474 | 475 | :param images: A numpy.ndarray of pre-calculated reflectance values for each 476 | landsat band, to be used instead of calculating our own. 477 | """ 478 | Lmax, Lmin, Qcalmax, Qcalmin, Refmax, Refmin, ijdim_ref, ijdim_thm, reso_ref, reso_thm, ul, zen, azi, zc, Lnum, doy = lndhdrread( 479 | filename) 480 | 481 | base = os.path.dirname(filename) 482 | 483 | # LPGS Upper left corner alignment (see Landsat handbook for detail) 484 | # Changed from (ul[0]-15,ul[1]+15), GA products are 25m, this should also 485 | # allow for other resolutions as well 486 | ul = (ul[0] - float(reso_ref) / 2, ul[1] + float(reso_ref) / 2) 487 | resolu = (reso_ref, reso_ref) 488 | 489 | if ((Lnum >= 4) & (Lnum <= 7)): 490 | 491 | # Band6 492 | if Lnum == 7: 493 | n_B6 = match_file(base, '*B6*1.*') 494 | else: 495 | n_B6 = match_file(base, '*B6.*') 496 | 497 | # Check that the thermal band resolution matches the reflectance bands. 498 | ref_lines, ref_samples = ijdim_ref 499 | thm_lines, thm_samples = ijdim_thm 500 | if ((thm_lines != ref_lines) | (thm_samples != ref_samples)): 501 | im_B6 = imread(n_B6, resample=True, samples=ref_samples, 502 | lines=ref_lines).astype(numpy.float32) 503 | else: 504 | im_B6 = imread(n_B6).astype(numpy.float32) 505 | 506 | # convert Band6 from radiance to BT 507 | # fprintf('From Band 6 Radiance to Brightness Temperature') 508 | # see G. Chander et al. RSE 113 (2009) 893-903 509 | K1_L4 = 671.62 510 | K2_L4 = 1284.30 511 | K1_L5 = 607.76 512 | K2_L5 = 1260.56 513 | K1_L7 = 666.09 514 | K2_L7 = 1282.71 515 | 516 | if Lnum == 7: 517 | K1 = K1_L7 518 | K2 = K2_L7 519 | elif Lnum == 5: 520 | K1 = K1_L5 521 | K2 = K2_L5 522 | elif Lnum == 4: 523 | K1 = K1_L4 524 | K2 = K2_L4 525 | 526 | if images is not None: 527 | im_B1 = images[0, :, :].astype(numpy.float32) 528 | im_B2 = images[1, :, :].astype(numpy.float32) 529 | im_B3 = images[2, :, :].astype(numpy.float32) 530 | im_B4 = images[3, :, :].astype(numpy.float32) 531 | im_B5 = images[4, :, :].astype(numpy.float32) 532 | im_B7 = images[6, :, :].astype(numpy.float32) 533 | del images 534 | 535 | # find pixels that are saturated in the visible bands 536 | B1Satu = im_B1 == 255.0 537 | B2Satu = im_B2 == 255.0 538 | B3Satu = im_B3 == 255.0 539 | 540 | # only processing pixesl where all bands have values (id_mssing) 541 | id_missing = numexpr.evaluate( 542 | "(im_B1 == 0.0) | (im_B2 == 0.0) | (im_B3 == 0.0) | (im_B4 == 0.0) |(im_B5 == 0.0) | (im_B6 == 0.0) | (im_B7 == 0.0)") 543 | 544 | else: 545 | # Band1 546 | n_B1 = match_file(base, '*B1.*') 547 | im_B1 = imread(n_B1).astype(numpy.float32) 548 | # Band2 549 | n_B2 = match_file(base, '*B2.*') 550 | im_B2 = imread(n_B2).astype(numpy.float32) 551 | # Band3 552 | n_B3 = match_file(base, '*B3.*') 553 | im_B3 = imread(n_B3).astype(numpy.float32) 554 | # Band4 555 | n_B4 = match_file(base, '*B4.*') 556 | im_B4 = imread(n_B4).astype(numpy.float32) 557 | # Band5 558 | n_B5 = match_file(base, '*B5.*') 559 | im_B5 = imread(n_B5).astype(numpy.float32) 560 | # Band7 561 | n_B7 = match_file(base, '*B7.*') 562 | im_B7 = imread(n_B7).astype(numpy.float32) 563 | 564 | # Retrieve the projection and geotransform info from the blue band 565 | # (B1 LS 4,5,7) 566 | geoT, prj, sz, ul_coord = im_info(n_B1) 567 | 568 | # find pixels that are saturated in the visible bands 569 | B1Satu = im_B1 == 255.0 570 | B2Satu = im_B2 == 255.0 571 | B3Satu = im_B3 == 255.0 572 | 573 | # only processing pixesl where all bands have values (id_mssing) 574 | id_missing = numexpr.evaluate( 575 | "(im_B1 == 0.0) | (im_B2 == 0.0) | (im_B3 == 0.0) | (im_B4 == 0.0) | (im_B5 == 0.0) | (im_B6 == 0.0) | (im_B7 == 0.0)") 576 | 577 | # ND to radiance first 578 | im_B1 = numexpr.evaluate("((Lma - Lmi) / (Qma - Qmi)) * (im_B1 - Qmi) + Lmi", { 579 | 'Lma': Lmax[0], 'Lmi': Lmin[0], 'Qma': Qcalmax[0], 'Qmi': Qcalmin[0]}, locals()) 580 | im_B2 = numexpr.evaluate("((Lma - Lmi) / (Qma - Qmi)) * (im_B2 - Qmi) + Lmi", { 581 | 'Lma': Lmax[1], 'Lmi': Lmin[1], 'Qma': Qcalmax[1], 'Qmi': Qcalmin[1]}, locals()) 582 | im_B3 = numexpr.evaluate("((Lma - Lmi) / (Qma - Qmi)) * (im_B3 - Qmi) + Lmi", { 583 | 'Lma': Lmax[2], 'Lmi': Lmin[2], 'Qma': Qcalmax[2], 'Qmi': Qcalmin[2]}, locals()) 584 | im_B4 = numexpr.evaluate("((Lma - Lmi) / (Qma - Qmi)) * (im_B4 - Qmi) + Lmi", { 585 | 'Lma': Lmax[3], 'Lmi': Lmin[3], 'Qma': Qcalmax[3], 'Qmi': Qcalmin[3]}, locals()) 586 | im_B5 = numexpr.evaluate("((Lma - Lmi) / (Qma - Qmi)) * (im_B5 - Qmi) + Lmi", { 587 | 'Lma': Lmax[4], 'Lmi': Lmin[4], 'Qma': Qcalmax[4], 'Qmi': Qcalmin[4]}, locals()) 588 | im_B6 = numexpr.evaluate("((Lma - Lmi) / (Qma - Qmi)) * (im_B6 - Qmi) + Lmi", { 589 | 'Lma': Lmax[5], 'Lmi': Lmin[5], 'Qma': Qcalmax[5], 'Qmi': Qcalmin[5]}, locals()) 590 | im_B7 = numexpr.evaluate("((Lma - Lmi) / (Qma - Qmi)) * (im_B7 - Qmi) + Lmi", { 591 | 'Lma': Lmax[6], 'Lmi': Lmin[6], 'Qma': Qcalmax[6], 'Qmi': Qcalmin[6]}, locals()) 592 | 593 | # radiance to TOA reflectances 594 | # fprintf('From Radiances to TOA ref') 595 | # # Solar Spectral Irradiances from LEDAPS 596 | # esun_L7=[1969.000, 1840.000, 1551.000, 1044.000, 225.700, -1.0, 82.07] 597 | # esun_L5=[1957.0, 1826.0, 1554.0, 1036.0, 215.0, -1.0, 80.67] 598 | # esun_L4=[1957.0, 1825.0, 1557.0, 1033.0, 214.9, -1.0, 80.72] 599 | 600 | # see G. Chander et al. RSE 113 (2009) 893-903 601 | esun_L7 = [1997.000, 1812.000, 1533.000, 602 | 1039.000, 230.800, -1.0, 84.90] 603 | esun_L5 = [1983.0, 1796.0, 1536.0, 1031.0, 220.0, -1.0, 83.44] 604 | esun_L4 = [1983.0, 1795.0, 1539.0, 1028.0, 219.8, -1.0, 83.49] 605 | 606 | if Lnum == 7: 607 | ESUN = esun_L7 608 | elif Lnum == 5: 609 | ESUN = esun_L5 610 | elif Lnum == 4: 611 | ESUN = esun_L4 612 | 613 | # # Interpolate earth-sun distance with day of year from LEDAPS 614 | # dsun_table_doy = [1,15,32,46,60,74,91,106,121,135,152,166,182,196,213,227,242,258,274,288,305,319,335,349,366] 615 | # dsun_table_dis= [0.9832,0.9836,0.9853,0.9878,0.9909,0.9945,0.9993,1.0033,1.0076,1.0109,1.0140,1.0158,1.0167, 616 | # 1.0165,1.0149,1.0128,1.0092,1.0057,1.0011,0.9972,0.9925,0.9892,0.9860,0.9843,0.9833] 617 | # 618 | # for i=1:length(dsun_table_doy)-1 619 | # if doy >=dsun_table_doy(i) and doy <=dsun_table_doy(i+1) 620 | # break 621 | # end 622 | # end 623 | # 624 | # dsun_doy=dsun_table_dis(i)+ 625 | # (dsun_table_dis(i+1)-dsun_table_dis(i))*(doy-dsun_table_doy(i))/(dsun_table_doy(i+1)-dsun_table_doy(i)) 626 | 627 | # earth-sun distance see G. Chander et al. RSE 113 (2009) 893-903 628 | dsun_doy = sun_earth_distance[doy] 629 | 630 | # compute TOA reflectances 631 | # converted from degrees to radiance 632 | s_zen = math.radians(zen) 633 | stack = { 634 | 'a': numpy.float32(10000.0 * math.pi), 635 | 'b': numpy.float32(dsun_doy * dsun_doy), 636 | 'c': numpy.float32(math.cos(s_zen)) 637 | } 638 | 639 | im_B1 = numexpr.evaluate("a * im_B1 * b / (sun * c)", dict( 640 | stack.items() + {'sun': numpy.float32(ESUN[0])}.items()), locals()) 641 | im_B2 = numexpr.evaluate("a * im_B2 * b / (sun * c)", dict( 642 | stack.items() + {'sun': numpy.float32(ESUN[1])}.items()), locals()) 643 | im_B3 = numexpr.evaluate("a * im_B3 * b / (sun * c)", dict( 644 | stack.items() + {'sun': numpy.float32(ESUN[2])}.items()), locals()) 645 | im_B4 = numexpr.evaluate("a * im_B4 * b / (sun * c)", dict( 646 | stack.items() + {'sun': numpy.float32(ESUN[3])}.items()), locals()) 647 | im_B5 = numexpr.evaluate("a * im_B5 * b / (sun * c)", dict( 648 | stack.items() + {'sun': numpy.float32(ESUN[4])}.items()), locals()) 649 | im_B7 = numexpr.evaluate("a * im_B7 * b / (sun * c)", dict( 650 | stack.items() + {'sun': numpy.float32(ESUN[6])}.items()), locals()) 651 | 652 | # convert from Kelvin to Celcius with 0.01 scale_facor 653 | im_B6 = numexpr.evaluate("a * ((K2 / log((K1 / im_B6) + one)) - b)", {'a': numpy.float32( 654 | 100), 'b': numpy.float32(273.15), 'one': numpy.float32(1.0)}, locals()) 655 | 656 | # get data ready for Fmask 657 | im_B1[id_missing] = -9999 658 | im_B2[id_missing] = -9999 659 | im_B3[id_missing] = -9999 660 | im_B4[id_missing] = -9999 661 | im_B5[id_missing] = -9999 662 | im_B6[id_missing] = -9999 663 | im_B7[id_missing] = -9999 664 | del id_missing 665 | 666 | images = numpy.array( 667 | [im_B1, im_B2, im_B3, im_B4, im_B5, im_B7], 'float32') 668 | del im_B1, im_B2, im_B3, im_B4, im_B5, im_B7 669 | 670 | # We'll modify the return argument for the Python implementation (geoT,prj) are added to the list 671 | # return 672 | # [im_B6,images,ijdim_ref,ul,zen,azi,zc,B1Satu,B2Satu,B3Satu,resolu,geoT,prj] 673 | return [im_B6, images, sz, ul_coord, zen, azi, zc, 674 | B1Satu, B2Satu, B3Satu, resolu, geoT, prj] 675 | 676 | elif (Lnum == 8): 677 | n_B10 = match_file(base, '*B10.*') 678 | # Check that the thermal band resolution matches the reflectance bands. 679 | ref_lines, ref_samples = ijdim_ref 680 | thm_lines, thm_samples = ijdim_thm 681 | 682 | if ((thm_lines != ref_lines) | (thm_samples != ref_samples)): 683 | im_B10 = imread(n_B10, resample=True, samples=ref_samples, 684 | lines=ref_lines).astype(numpy.float32) 685 | else: 686 | im_B10 = imread(n_B10).astype(numpy.float32) 687 | 688 | # Band2 689 | n_B2 = match_file(base, '*B2.*') 690 | im_B2 = imread(n_B2).astype(numpy.float32) 691 | # Band3 692 | n_B3 = match_file(base, '*B3.*') 693 | im_B3 = imread(n_B3).astype(numpy.float32) 694 | # Band4 695 | n_B4 = match_file(base, '*B4.*') 696 | im_B4 = imread(n_B4).astype(numpy.float32) 697 | # Band5 698 | n_B5 = match_file(base, '*B5.*') 699 | im_B5 = imread(n_B5).astype(numpy.float32) 700 | # Band6 701 | n_B6 = match_file(base, '*B6.*') 702 | im_B6 = imread(n_B6).astype(numpy.float32) 703 | # Band7 704 | n_B7 = match_file(base, '*B7.*') 705 | im_B7 = imread(n_B7).astype(numpy.float32) 706 | # Band9 707 | n_B9 = match_file(base, '*B9.*') 708 | im_B9 = imread(n_B9).astype(numpy.float32) 709 | 710 | # Retrieve the projection and geotransform info from the blue band (B2 711 | # in LS8) 712 | geoT, prj, sz, ul_coord = im_info(n_B2) 713 | 714 | # only processing pixesl where all bands have values (id_mssing) 715 | id_missing = numexpr.evaluate( 716 | "(im_B2 == 0.0) | (im_B3 == 0.0) | (im_B4 == 0.0) | (im_B5 == 0.0) | (im_B6 == 0.0) | (im_B7 == 0.0) | (im_B9 == 0.0) | (im_B10 == 0.0)") 717 | 718 | # find pixels that are saturated in the visible bands 719 | B1Satu = im_B2 == 65535.0 720 | B2Satu = im_B3 == 65535.0 721 | B3Satu = im_B4 == 65535.0 722 | 723 | # DN to TOA reflectance with 0.0001 scale_factor 724 | # This formulae is similar to that used for LS 4,5,7. But is different to that given by 725 | # https://landsat.usgs.gov/Landsat8_Using_Product.php : Noted JS 726 | # 2013/11/28 727 | logger.info('From DNs to TOA ref & BT') 728 | im_B2 = numexpr.evaluate("((Rma - Rmi) / (Qma - Qmi)) * (im_B2 - Qmi) + Rmi", { 729 | 'Rma': Refmax[0], 'Rmi': Refmin[0], 'Qma': Qcalmax[0], 'Qmi': Qcalmin[0]}, locals()) 730 | im_B3 = numexpr.evaluate("((Rma - Rmi) / (Qma - Qmi)) * (im_B3 - Qmi) + Rmi", { 731 | 'Rma': Refmax[1], 'Rmi': Refmin[1], 'Qma': Qcalmax[1], 'Qmi': Qcalmin[1]}, locals()) 732 | im_B4 = numexpr.evaluate("((Rma - Rmi) / (Qma - Qmi)) * (im_B4 - Qmi) + Rmi", { 733 | 'Rma': Refmax[2], 'Rmi': Refmin[2], 'Qma': Qcalmax[2], 'Qmi': Qcalmin[2]}, locals()) 734 | im_B5 = numexpr.evaluate("((Rma - Rmi) / (Qma - Qmi)) * (im_B5 - Qmi) + Rmi", { 735 | 'Rma': Refmax[3], 'Rmi': Refmin[3], 'Qma': Qcalmax[3], 'Qmi': Qcalmin[3]}, locals()) 736 | im_B6 = numexpr.evaluate("((Rma - Rmi) / (Qma - Qmi)) * (im_B6 - Qmi) + Rmi", { 737 | 'Rma': Refmax[4], 'Rmi': Refmin[4], 'Qma': Qcalmax[4], 'Qmi': Qcalmin[4]}, locals()) 738 | im_B7 = numexpr.evaluate("((Rma - Rmi) / (Qma - Qmi)) * (im_B7 - Qmi) + Rmi", { 739 | 'Rma': Refmax[5], 'Rmi': Refmin[5], 'Qma': Qcalmax[5], 'Qmi': Qcalmin[5]}, locals()) 740 | im_B9 = numexpr.evaluate("((Rma - Rmi) / (Qma - Qmi)) * (im_B9 - Qmi) + Rmi", { 741 | 'Rma': Refmax[6], 'Rmi': Refmin[6], 'Qma': Qcalmax[6], 'Qmi': Qcalmin[6]}, locals()) 742 | im_B10 = numexpr.evaluate("((Lma - Lmi) / (Qma - Qmi)) * (im_B10 - Qmi) + Lmi", { 743 | 'Lma': Lmax[7], 'Lmi': Lmin[7], 'Qma': Qcalmax[7], 'Qmi': Qcalmin[7]}, locals()) 744 | 745 | s_zen = numpy.deg2rad(zen) 746 | im_B2 = numexpr.evaluate("10000 * im_B2 / cos(s_zen)") 747 | im_B3 = numexpr.evaluate("10000 * im_B3 / cos(s_zen)") 748 | im_B4 = numexpr.evaluate("10000 * im_B4 / cos(s_zen)") 749 | im_B5 = numexpr.evaluate("10000 * im_B5 / cos(s_zen)") 750 | im_B6 = numexpr.evaluate("10000 * im_B6 / cos(s_zen)") 751 | im_B7 = numexpr.evaluate("10000 * im_B7 / cos(s_zen)") 752 | im_B9 = numexpr.evaluate("10000 * im_B9 / cos(s_zen)") 753 | 754 | # convert Band6 from radiance to BT 755 | # fprintf('From Band 6 Radiance to Brightness Temperature'); 756 | K1_B10 = numpy.float32(774.89) 757 | K2_B10 = numpy.float32(1321.08) 758 | one = numpy.float32(1) 759 | 760 | im_B10 = numexpr.evaluate("K2_B10 / log((K1_B10 / im_B10) + one)") 761 | 762 | # convert from Kelvin to Celcius with 0.01 scale_factor 763 | K = numpy.float32(273.15) 764 | im_B10 = numexpr.evaluate("100 * (im_B10 - K)") 765 | 766 | # get data ready for Fmask 767 | im_B2[id_missing] = -9999 768 | im_B3[id_missing] = -9999 769 | im_B4[id_missing] = -9999 770 | im_B5[id_missing] = -9999 771 | im_B6[id_missing] = -9999 772 | im_B7[id_missing] = -9999 773 | im_B9[id_missing] = -9999 774 | im_B10[id_missing] = -9999 775 | del id_missing 776 | 777 | images = numpy.array( 778 | [im_B2, im_B3, im_B4, im_B5, im_B6, im_B7, im_B9], 'float32') 779 | del im_B2, im_B3, im_B4, im_B5, im_B6, im_B7, im_B9 780 | 781 | # We'll modify the return argument for the Python implementation (geoT,prj) are added to the list 782 | # return 783 | # [im_B10,images,ijdim_ref,ul,zen,azi,zc,B1Satu,B2Satu,B3Satu,resolu,geoT,prj] 784 | return [im_B10, images, sz, ul_coord, zen, azi, zc, 785 | B1Satu, B2Satu, B3Satu, resolu, geoT, prj] 786 | 787 | else: 788 | raise Exception('This sensor is not Landsat 4, 5, 7, or 8!') 789 | 790 | 791 | def plcloud(filename, cldprob=22.5, num_Lst=None, images=None, 792 | shadow_prob=False, mask=None): 793 | """ 794 | Calculates a cloud mask for a landsat 5/7 scene. 795 | 796 | :param filename: 797 | A string containing the file path of the landsat scene MTL file. 798 | 799 | :param cldprob: 800 | The cloud probability for the scene (defaults to 22.5%). 801 | 802 | :param num_Lst: 803 | The Landsat satellite number. 804 | 805 | :param images: 806 | A numpy.ndarray of pre-loaded, scaled, and corrected bands. 807 | 808 | :param log_filename: 809 | A string containing the file path of the output log produced by FMask. 810 | 811 | :param shadow_prob: 812 | A flag indicating if the shadow probability should be 813 | calculated or not (required by FMask cloud shadow). Type Bool. 814 | 815 | :return: 816 | Tuple (zen,azi,ptm, temperature band (celcius*100),t_templ,t_temph, 817 | water mask, snow mask, cloud mask, shadow probability,dim, 818 | ul,resolu,zc). 819 | """ 820 | Temp, data, dim, ul, zen, azi, zc, satu_B1, satu_B2, satu_B3, resolu, geoT, prj = nd2toarbt( 821 | filename, images) 822 | 823 | if num_Lst < 8: # Landsat 4~7 824 | Thin_prob = 0 # there is no contribution from the new bands 825 | else: 826 | Thin_prob = numexpr.evaluate( 827 | "cirrus / 400", {'cirrus': data[-1]}, locals()) 828 | 829 | Cloud = numpy.zeros(dim, 'uint8') # cloud mask 830 | Snow = numpy.zeros(dim, 'uint8') # Snow mask 831 | WT = numpy.zeros(dim, 'uint8') # Water msk 832 | 833 | # process only the overlap area 834 | if mask is None: 835 | mask = Temp > -9999 836 | else: 837 | mask = mask.astype('bool') 838 | 839 | Shadow = numpy.zeros(dim, 'uint8') # shadow mask 840 | 841 | data1 = data[0, :, :] 842 | data2 = data[1, :, :] 843 | data3 = data[2, :, :] 844 | data4 = data[3, :, :] 845 | data5 = data[4, :, :] 846 | data6 = data[5, :, :] 847 | 848 | NDVI = numexpr.evaluate("(data4 - data3) / (data4 + data3)") 849 | NDSI = numexpr.evaluate("(data2 - data5) / (data2 + data5)") 850 | 851 | NDVI[numexpr.evaluate("(data4 + data3) == 0")] = 0.01 852 | NDSI[numexpr.evaluate("(data2 + data5) == 0")] = 0.01 853 | 854 | # saturation in the three visible bands 855 | satu_Bv = numexpr.evaluate("(satu_B1 | satu_B2 | satu_B3)") 856 | del satu_B1 857 | # Basic cloud test 858 | idplcd = numexpr.evaluate( 859 | "(NDSI < 0.8) & (NDVI < 0.8) & (data6 > 300) & (Temp < 2700)") 860 | 861 | # Snow test 862 | # It takes every snow pixels including snow pixel under thin clouds or icy 863 | # clouds 864 | Snow[numexpr.evaluate( 865 | "(NDSI > 0.15) & (Temp < 1000) & (data4 > 1100) & (data2 > 1000)")] = 1 866 | #Snow[mask == 0] = 255 867 | # Water test 868 | # Zhe's water test (works over thin cloud) 869 | WT[numexpr.evaluate( 870 | "((NDVI < 0.01) & (data4 < 1100)) | ((NDVI < 0.1) & (NDVI > 0) & (data4 < 500))")] = 1 871 | WT[mask == 0] = 255 872 | # ################################################ Whiteness test 873 | # visible bands flatness (sum(abs)/mean < 0.6 => brigt and dark cloud ) 874 | visimean = numexpr.evaluate("(data1 + data2 + data3) / 3 ") 875 | whiteness = numexpr.evaluate( 876 | "(abs(data1 - visimean) + abs(data2 - visimean)+ abs(data3 - visimean)) / visimean") 877 | del visimean 878 | 879 | # update idplcd 880 | whiteness[satu_Bv] = 0 # If one visible is saturated whiteness == 0 881 | idplcd &= whiteness < 0.7 882 | 883 | # Haze test 884 | HOT = numexpr.evaluate("data1 - 0.5 * data3 - 800") # Haze test 885 | idplcd &= numexpr.evaluate("(HOT > 0) | satu_Bv") 886 | del HOT # need to find thick warm cloud 887 | 888 | # Ratio4/5>0.75 cloud test 889 | idplcd &= numexpr.evaluate("(data4 / data5) > 0.75") 890 | 891 | # Cirrus tests from Landsat 8 892 | idplcd |= numexpr.evaluate("Thin_prob > 0.25") 893 | 894 | ####################################constants########################## 895 | l_pt = 0.175 # low percent 896 | h_pt = 1 - l_pt # high percent 897 | # (temperature & snow test ) 898 | # test whether use thermal or not 899 | idclr = numexpr.evaluate("(idplcd == False) & (mask == 1)") 900 | ptm = 100 * idclr.sum() / mask.sum() # percent of del pixel 901 | idlnd = numexpr.evaluate("idclr & (WT == False)") 902 | idwt = numexpr.evaluate("idclr & (WT == True)") # &data(:,:,6)<=300; 903 | lndptm = 100 * idlnd.sum() / mask.sum() 904 | 905 | logger.debug('idlnd: %s', idlnd) 906 | logger.debug('idlnd.sum(): %s', idlnd.sum()) 907 | logger.debug('lndptm: %s', lndptm) 908 | sys.stdout.flush() 909 | 910 | if ptm <= 0.1: # no thermal test => meanless for snow detection (0~1) 911 | # fprintf('Clear pixel NOT exist in this scene (del prct = 912 | # #.2f)',ptm) 913 | Cloud[idplcd] = 1 # all cld 914 | 915 | # mask out the non-contiguous pixels 916 | Cloud[~(mask)] = 0 917 | 918 | # # improving by majority filtering 919 | # Cloud=bwmorph(Cloud,'majority')# exclude <5/9 920 | #Cloud = scipy.signal.convolve2d(Cloud,numpy.ones((3,3),Cloud.dtype.name))[1:-1,1:-1] 921 | #Cloud = (Cloud > 4).astype('uint8') 922 | # Applying twice, makes a cleaner result 923 | #Cloud = scipy.signal.convolve2d(Cloud,numpy.ones((3,3),Cloud.dtype.name))[1:-1,1:-1] 924 | #Cloud = Cloud > 4 925 | 926 | Shadow[Cloud == 0] = 1 927 | Temp = -1 928 | t_templ = -1 929 | t_temph = -1 930 | else: 931 | # fprintf('Clear pixel EXIST in this scene (del prct = #.2f)',ptm) 932 | # (temperature test ) 933 | if lndptm >= 0.1: 934 | F_temp = Temp[idlnd] # get land temperature 935 | # fprintf('Land temperature') 936 | else: 937 | F_temp = Temp[idclr] # get del temperature 938 | # fprintf('Clear temperature') 939 | 940 | # Get cloud prob over water 941 | # temperature test (over water) 942 | # F_wtemp = Temp[numexpr.evaluate("(WT == 1) & (data6 <= 300)")] # get 943 | # del water temperature 944 | F_wtemp = Temp[idwt] 945 | if len(F_wtemp) == 0: 946 | t_wtemp = 0 947 | else: 948 | t_wtemp = scipy.stats.scoreatpercentile(F_wtemp, 100 * h_pt) 949 | wTemp_prob = numexpr.evaluate('(t_wtemp - Temp) / 400') 950 | wTemp_prob[numexpr.evaluate('wTemp_prob < 0')] = 0 951 | 952 | # Brightness test (over water) 953 | t_bright = 1100 954 | Brightness_prob = data5 / t_bright 955 | Brightness_prob[Brightness_prob > 1] = 1 956 | Brightness_prob[Brightness_prob < 0] = 0 957 | 958 | # Final prob mask (water) 959 | # cloud over water probability 960 | wfinal_prob = numexpr.evaluate( 961 | '100 * wTemp_prob * Brightness_prob + 100 * Thin_prob') 962 | wclr_max = scipy.stats.scoreatpercentile( 963 | wfinal_prob[idwt], 100 * h_pt) + cldprob # dynamic threshold (land) 964 | # wclr_max=50;% fixed threshold (water) 965 | 966 | # release memory 967 | del wTemp_prob 968 | del Brightness_prob 969 | 970 | # Temperature test 971 | t_buffer = 4 * 100 972 | if len(F_temp) != 0: 973 | # 0.175 percentile background temperature (low) 974 | t_templ = scipy.stats.scoreatpercentile(F_temp, 100 * l_pt) 975 | # 0.825 percentile background temperature (high) 976 | t_temph = scipy.stats.scoreatpercentile(F_temp, 100 * h_pt) 977 | else: 978 | t_templ = 0 979 | t_temph = 0 980 | 981 | t_tempL = t_templ - t_buffer 982 | t_tempH = t_temph + t_buffer 983 | Temp_l = t_tempH - t_tempL 984 | Temp_prob = (t_tempH - Temp) / Temp_l 985 | # Temperature can have prob > 1 986 | Temp_prob[Temp_prob < 0] = 0 987 | # Temp_prob(Temp_prob > 1) = 1 988 | 989 | NDSI[numexpr.evaluate('satu_B2 & (NDSI < 0)')] = 0 990 | NDVI[numexpr.evaluate('satu_B3 & (NDVI > 0)')] = 0 991 | 992 | Vari_prob = 1 - \ 993 | numpy.maximum(numpy.maximum(numpy.absolute( 994 | NDSI), numpy.absolute(NDVI)), whiteness) 995 | 996 | # release memory 997 | del satu_B2 998 | del satu_B3 999 | del NDSI 1000 | del NDVI 1001 | del whiteness 1002 | 1003 | # Final prob mask (land) 1004 | final_prob = 100 * Temp_prob * Vari_prob + 100 * \ 1005 | Thin_prob # cloud over land probability 1006 | clr_max = scipy.stats.scoreatpercentile( 1007 | final_prob[idlnd], 100 * h_pt) + cldprob # dynamic threshold (land) 1008 | 1009 | # release memory 1010 | del Vari_prob 1011 | del Temp_prob 1012 | del Thin_prob 1013 | 1014 | logger.debug('cldprob: %s', cldprob) 1015 | logger.debug('clr_max: %s', clr_max) 1016 | logger.debug('t_templ: %s', t_templ) 1017 | sys.stdout.flush() 1018 | 1019 | # fprintf('pcloud probability threshold (land) = .2f#',clr_max) 1020 | # cloud over land : (idplcd & (final_prob > clr_max) & (WT == 0)) 1021 | # thin cloud over water : (idplcd & (wfinal_prob > wclr_max) & (WT == 1)) 1022 | # high prob cloud (land) : (final_prob > 99.0) & (WT == 0) 1023 | # extremly cold cloud : (Temp < t_templ - 3500) 1024 | id_final_cld = numexpr.evaluate( 1025 | '(idplcd & (final_prob > clr_max) & (WT == 0)) | (idplcd & (wfinal_prob > wclr_max) & (WT == 1)) | (Temp < t_templ - 3500)') 1026 | 1027 | # Star with potential cloud mask 1028 | # # potential cloud mask 1029 | Cloud[id_final_cld] = 1 1030 | 1031 | # release memory 1032 | del final_prob 1033 | del wfinal_prob 1034 | del id_final_cld 1035 | 1036 | # Start with potential cloud shadow mask 1037 | if shadow_prob: 1038 | 1039 | # band 4 flood fill 1040 | nir = data4.astype('float32') 1041 | # estimating background (land) Band 4 ref 1042 | backg_B4 = scipy.stats.scoreatpercentile(nir[idlnd], 100.0 * l_pt) 1043 | nir[mask == 0] = backg_B4 1044 | # fill in regional minimum Band 4 ref 1045 | nir = imfill_skimage(nir) 1046 | nir = nir - data4 1047 | 1048 | # band 5 flood fill 1049 | swir = data5 1050 | # estimating background (land) Band 4 ref 1051 | backg_B5 = scipy.stats.scoreatpercentile(swir[idlnd], 100.0 * l_pt) 1052 | swir[mask == 0] = backg_B5 1053 | # fill in regional minimum Band 5 ref 1054 | swir = imfill_skimage(swir) 1055 | swir = swir - data5 1056 | 1057 | # compute shadow probability 1058 | shadow_prob = numpy.minimum(nir, swir) 1059 | # release remory 1060 | del nir 1061 | del swir 1062 | 1063 | Shadow[shadow_prob > 200] = 1 1064 | # release remory 1065 | del shadow_prob 1066 | 1067 | # Cloud[idplcd==True]=1 # all cld 1068 | 1069 | #*************************************************************************************# 1070 | #*************************************************************************************# 1071 | #*************************************************************************************# 1072 | #************The following code may be removed as the new code has changed************# 1073 | # Mask the non-contiguous pixels 1074 | #Cloud[~(mask)] = 0 1075 | 1076 | # improving by majority filtering 1077 | # ERROR: not aware of a similar filter in scipy (though one may very well exist) 1078 | # Doing convolution of all surrounding pixels & filtering those > 4, which should in theory have the same result 1079 | #Cloud = scipy.signal.convolve2d(Cloud, numpy.ones((3,3), Cloud.dtype.name))[1:-1,1:-1] 1080 | #Cloud = (Cloud > 4).astype('uint8') 1081 | # Applying twice, makes a cleaner result 1082 | #Cloud = scipy.signal.convolve2d(Cloud, numpy.ones((3,3), Cloud.dtype.name))[1:-1,1:-1] 1083 | # Cloud=(Cloud>4).astype('uint8') 1084 | # 3rd, still some single pixels at tile edges 1085 | # Cloud=scipy.signal.convolve2d(Cloud,numpy.ones((3,3),Cloud.dtype.name))[1:-1,1:-1] 1086 | #Cloud = Cloud > 4 1087 | #************The above code may be removed as the new code has changed****************# 1088 | #*************************************************************************************# 1089 | #*************************************************************************************# 1090 | #*************************************************************************************# 1091 | 1092 | del data 1093 | images = None 1094 | gc.collect() 1095 | 1096 | # refine Water mask - Zhe's water mask (no confusion water/cloud) 1097 | WT[numexpr.evaluate("(WT == 1) & (Cloud == 0)")] = 1 1098 | # bwmorph changed Cloud to Binary 1099 | Cloud = Cloud.astype('uint8') 1100 | Cloud[mask == 0] = 255 1101 | Shadow[mask == 0] = 255 1102 | 1103 | gc.collect() 1104 | cloud_mask = (Cloud == 1) & mask 1105 | 1106 | if ptm > 0.1: 1107 | cloud_temp = Temp[cloud_mask] 1108 | 1109 | logger.info("Snow Percent: %f" % 1110 | ((float(Snow[mask].sum()) / float(mask.sum())) * 100.0)) 1111 | logger.info("Cloud Mean: %f C" % (numpy.mean(cloud_temp) / 100.0)) 1112 | 1113 | if numpy.sum(cloud_mask) > 0: 1114 | cloud_stddev = numpy.std( 1115 | cloud_temp, dtype='float64', ddof=1) / 100.0 1116 | pct_upper = numpy.percentile(cloud_temp, 97.5) / 100.0 1117 | pct_lower = numpy.percentile(cloud_temp, 83.5) / 100.0 1118 | pct_upper_max = numpy.percentile(cloud_temp, 98.75) / 100.0 1119 | 1120 | logger.debug("Standard Deviation: %f C" % cloud_stddev) 1121 | logger.debug("97.5 percentile: %f C" % pct_upper) 1122 | logger.debug("83.5 percentile: %f C" % pct_lower) 1123 | logger.debug("98.75 percentile: %f C" % pct_upper_max) 1124 | 1125 | cloud_skew = 0.0 # TODO 1126 | 1127 | logger.info("Final Cloud Layer Percent: %f" % 1128 | ((float(Cloud[cloud_mask].sum()) / float(mask.sum())) * 100.0)) 1129 | 1130 | logger.info("Completed processing FMASK cloud cover...") 1131 | 1132 | # We'll modify the return argument for the Python implementation 1133 | # (geoT,prj) are added to the list 1134 | return (zen, azi, ptm, Temp, t_templ, t_temph, 1135 | WT, Snow, Cloud, Shadow, dim, ul, resolu, zc, geoT, prj) 1136 | 1137 | 1138 | def plcloud_warm(toa_bt, cldprob=22.5, num_Lst=None, 1139 | shadow_prob=False, mask=None): 1140 | """ 1141 | Calculates a cloud mask for a landsat 5/7 scene. 1142 | 1143 | :param filename: 1144 | A string containing the file path of the landsat scene MTL file. 1145 | 1146 | :param cldprob: 1147 | The cloud probability for the scene (defaults to 22.5%). 1148 | 1149 | :param num_Lst: 1150 | The Landsat satellite number. 1151 | 1152 | :param images: 1153 | A numpy.ndarray of pre-loaded, scaled, and corrected bands. 1154 | 1155 | :param log_filename: 1156 | A string containing the file path of the output log produced by FMask. 1157 | 1158 | :param shadow_prob: 1159 | A flag indicating if the shadow probability should be 1160 | calculated or not (required by FMask cloud shadow). Type Bool. 1161 | 1162 | :return: 1163 | Tuple (zen,azi,ptm, temperature band (celcius*100),t_templ,t_temph, 1164 | water mask, snow mask, cloud mask , shadow probability,dim, 1165 | ul,resolu,zc). 1166 | """ 1167 | Temp, data, \ 1168 | dim, ul, zen, azi, zc, \ 1169 | satu_B1, satu_B2, satu_B3, \ 1170 | resolu, geoT, prj = toa_bt 1171 | 1172 | if num_Lst < 8: # Landsat 4~7 1173 | Thin_prob = 0 # there is no contribution from the new bands 1174 | else: 1175 | Thin_prob = numexpr.evaluate( 1176 | "cirrus / 400", {'cirrus': data[-1]}, locals()) 1177 | 1178 | Cloud = numpy.zeros(dim, 'uint8') # cloud mask 1179 | Snow = numpy.zeros(dim, 'uint8') # Snow mask 1180 | WT = numpy.zeros(dim, 'uint8') # Water msk 1181 | 1182 | # process only the overlap area 1183 | if mask is None: 1184 | mask = Temp > -9999 1185 | else: 1186 | mask = mask.astype('bool') 1187 | 1188 | Shadow = numpy.zeros(dim, 'uint8') # shadow mask 1189 | 1190 | data1 = data[0, :, :] 1191 | data2 = data[1, :, :] 1192 | data3 = data[2, :, :] 1193 | data4 = data[3, :, :] 1194 | data5 = data[4, :, :] 1195 | data6 = data[5, :, :] 1196 | 1197 | NDVI = numexpr.evaluate("(data4 - data3) / (data4 + data3)") 1198 | NDSI = numexpr.evaluate("(data2 - data5) / (data2 + data5)") 1199 | 1200 | NDVI[numexpr.evaluate("(data4 + data3) == 0")] = 0.01 1201 | NDSI[numexpr.evaluate("(data2 + data5) == 0")] = 0.01 1202 | 1203 | # saturation in the three visible bands 1204 | satu_Bv = numexpr.evaluate("(satu_B1 | satu_B2 | satu_B3)") 1205 | del satu_B1 1206 | # Basic cloud test 1207 | idplcd = numexpr.evaluate( 1208 | "(NDSI < 0.8) & (NDVI < 0.8) & (data6 > 300) & (Temp < 2700)") 1209 | 1210 | # Snow test 1211 | # It takes every snow pixels including snow pixel under thin clouds or icy 1212 | # clouds 1213 | Snow[numexpr.evaluate( 1214 | "(NDSI > 0.15) & (Temp < 1000) & (data4 > 1100) & (data2 > 1000)")] = 1 1215 | #Snow[mask == 0] = 255 1216 | # Water test 1217 | # Zhe's water test (works over thin cloud) 1218 | WT[numexpr.evaluate( 1219 | "((NDVI < 0.01) & (data4 < 1100)) | ((NDVI < 0.1) & (NDVI > 0) & (data4 < 500))")] = 1 1220 | WT[mask == 0] = 255 1221 | # ################################################ Whiteness test 1222 | # visible bands flatness (sum(abs)/mean < 0.6 => brigt and dark cloud ) 1223 | visimean = numexpr.evaluate("(data1 + data2 + data3) / 3 ") 1224 | whiteness = numexpr.evaluate( 1225 | "(abs(data1 - visimean) + abs(data2 - visimean)+ abs(data3 - visimean)) / visimean") 1226 | del visimean 1227 | 1228 | # update idplcd 1229 | whiteness[satu_Bv] = 0 # If one visible is saturated whiteness == 0 1230 | idplcd &= whiteness < 0.7 1231 | 1232 | # Haze test 1233 | HOT = numexpr.evaluate("data1 - 0.5 * data3 - 800") # Haze test 1234 | idplcd &= numexpr.evaluate("(HOT > 0) | satu_Bv") 1235 | del HOT # need to find thick warm cloud 1236 | 1237 | # Ratio4/5>0.75 cloud test 1238 | idplcd &= numexpr.evaluate("(data4 / data5) > 0.75") 1239 | 1240 | # Cirrus tests from Landsat 8 1241 | idplcd |= numexpr.evaluate("Thin_prob > 0.25") 1242 | 1243 | ####################################constants########################## 1244 | l_pt = 0.175 # low percent 1245 | h_pt = 1 - l_pt # high percent 1246 | # (temperature & snow test ) 1247 | # test whether use thermal or not 1248 | idclr = numexpr.evaluate("(idplcd == False) & (mask == 1)") 1249 | ptm = 100 * idclr.sum() / mask.sum() # percent of del pixel 1250 | idlnd = numexpr.evaluate("idclr & (WT == False)") 1251 | idwt = numexpr.evaluate("idclr & (WT == True)") # &data(:,:,6)<=300; 1252 | lndptm = 100 * idlnd.sum() / mask.sum() 1253 | 1254 | logger.debug('idlnd: %s', idlnd) 1255 | logger.debug('idlnd.sum(): %s', idlnd.sum()) 1256 | logger.debug('lndptm: %s', lndptm) 1257 | sys.stdout.flush() 1258 | 1259 | if ptm <= 0.1: # no thermal test => meanless for snow detection (0~1) 1260 | # fprintf('Clear pixel NOT exist in this scene (del prct = 1261 | # #.2f)',ptm) 1262 | Cloud[idplcd] = 1 # all cld 1263 | 1264 | # mask out the non-contiguous pixels 1265 | Cloud[~(mask)] = 0 1266 | 1267 | # # improving by majority filtering 1268 | # Cloud=bwmorph(Cloud,'majority')# exclude <5/9 1269 | #Cloud = scipy.signal.convolve2d(Cloud,numpy.ones((3,3),Cloud.dtype.name))[1:-1,1:-1] 1270 | #Cloud = (Cloud > 4).astype('uint8') 1271 | # Applying twice, makes a cleaner result 1272 | #Cloud = scipy.signal.convolve2d(Cloud,numpy.ones((3,3),Cloud.dtype.name))[1:-1,1:-1] 1273 | #Cloud = Cloud > 4 1274 | 1275 | Shadow[Cloud == 0] = 1 1276 | Temp = -1 1277 | t_templ = -1 1278 | t_temph = -1 1279 | else: 1280 | # fprintf('Clear pixel EXIST in this scene (del prct = #.2f)',ptm) 1281 | # (temperature test ) 1282 | if lndptm >= 0.1: 1283 | F_temp = Temp[idlnd] # get land temperature 1284 | # fprintf('Land temperature') 1285 | else: 1286 | F_temp = Temp[idclr] # get del temperature 1287 | # fprintf('Clear temperature') 1288 | 1289 | # Get cloud prob over water 1290 | # temperature test (over water) 1291 | # F_wtemp = Temp[numexpr.evaluate("(WT == 1) & (data6 <= 300)")] # get 1292 | # del water temperature 1293 | F_wtemp = Temp[idwt] 1294 | if len(F_wtemp) == 0: 1295 | t_wtemp = 0 1296 | else: 1297 | t_wtemp = scipy.stats.scoreatpercentile(F_wtemp, 100 * h_pt) 1298 | wTemp_prob = numexpr.evaluate('(t_wtemp - Temp) / 400') 1299 | wTemp_prob[numexpr.evaluate('wTemp_prob < 0')] = 0 1300 | 1301 | # Brightness test (over water) 1302 | t_bright = 1100 1303 | Brightness_prob = data5 / t_bright 1304 | Brightness_prob[Brightness_prob > 1] = 1 1305 | Brightness_prob[Brightness_prob < 0] = 0 1306 | 1307 | # Final prob mask (water) 1308 | # cloud over water probability 1309 | wfinal_prob = numexpr.evaluate( 1310 | '100 * wTemp_prob * Brightness_prob + 100 * Thin_prob') 1311 | wclr_max = scipy.stats.scoreatpercentile( 1312 | wfinal_prob[idwt], 100 * h_pt) + cldprob # dynamic threshold (land) 1313 | # wclr_max=50;% fixed threshold (water) 1314 | 1315 | # release memory 1316 | del wTemp_prob 1317 | del Brightness_prob 1318 | 1319 | # Temperature test 1320 | t_buffer = 4 * 100 1321 | if len(F_temp) != 0: 1322 | # 0.175 percentile background temperature (low) 1323 | t_templ = scipy.stats.scoreatpercentile(F_temp, 100 * l_pt) 1324 | # 0.825 percentile background temperature (high) 1325 | t_temph = scipy.stats.scoreatpercentile(F_temp, 100 * h_pt) 1326 | else: 1327 | t_templ = 0 1328 | t_temph = 0 1329 | 1330 | t_tempL = t_templ - t_buffer 1331 | t_tempH = t_temph + t_buffer 1332 | Temp_l = t_tempH - t_tempL 1333 | Temp_prob = (t_tempH - Temp) / Temp_l 1334 | # Temperature can have prob > 1 1335 | Temp_prob[Temp_prob < 0] = 0 1336 | # Temp_prob(Temp_prob > 1) = 1 1337 | 1338 | NDSI[numexpr.evaluate('satu_B2 & (NDSI < 0)')] = 0 1339 | NDVI[numexpr.evaluate('satu_B3 & (NDVI > 0)')] = 0 1340 | 1341 | Vari_prob = 1 - \ 1342 | numpy.maximum(numpy.maximum(numpy.absolute( 1343 | NDSI), numpy.absolute(NDVI)), whiteness) 1344 | 1345 | # release memory 1346 | del satu_B2 1347 | del satu_B3 1348 | del NDSI 1349 | del NDVI 1350 | del whiteness 1351 | 1352 | # Final prob mask (land) 1353 | final_prob = 100 * Temp_prob * Vari_prob + 100 * \ 1354 | Thin_prob # cloud over land probability 1355 | clr_max = scipy.stats.scoreatpercentile( 1356 | final_prob[idlnd], 100 * h_pt) + cldprob # dynamic threshold (land) 1357 | 1358 | # release memory 1359 | del Vari_prob 1360 | del Temp_prob 1361 | del Thin_prob 1362 | 1363 | logger.debug('cldprob: %s', cldprob) 1364 | logger.debug('clr_max: %s', clr_max) 1365 | logger.debug('t_templ: %s', t_templ) 1366 | sys.stdout.flush() 1367 | 1368 | # fprintf('pcloud probability threshold (land) = .2f#',clr_max) 1369 | # cloud over land : (idplcd & (final_prob > clr_max) & (WT == 0)) 1370 | # thin cloud over water : (idplcd & (wfinal_prob > wclr_max) & (WT == 1)) 1371 | # high prob cloud (land) : (final_prob > 99.0) & (WT == 0) 1372 | # extremly cold cloud : (Temp < t_templ - 3500) 1373 | id_final_cld = numexpr.evaluate( 1374 | '(idplcd & (final_prob > clr_max) & (WT == 0)) | (idplcd & (wfinal_prob > wclr_max) & (WT == 1)) | (Temp < t_templ - 3500)') 1375 | 1376 | # Star with potential cloud mask 1377 | # # potential cloud mask 1378 | Cloud[id_final_cld] = 1 1379 | 1380 | # release memory 1381 | del final_prob 1382 | del wfinal_prob 1383 | del id_final_cld 1384 | 1385 | # Start with potential cloud shadow mask 1386 | if shadow_prob: 1387 | # band 4 flood fill 1388 | nir = data4.astype('float32') 1389 | # estimating background (land) Band 4 ref 1390 | backg_B4 = scipy.stats.scoreatpercentile(nir[idlnd], 100.0 * l_pt) 1391 | nir[mask == 0] = backg_B4 1392 | # fill in regional minimum Band 4 ref 1393 | nir = imfill_skimage(nir) 1394 | nir = nir - data4 1395 | 1396 | # band 5 flood fill 1397 | swir = data5 1398 | # estimating background (land) Band 4 ref 1399 | backg_B5 = scipy.stats.scoreatpercentile(swir[idlnd], 100.0 * l_pt) 1400 | swir[mask == 0] = backg_B5 1401 | # fill in regional minimum Band 5 ref 1402 | swir = imfill_skimage(swir) 1403 | swir = swir - data5 1404 | 1405 | # compute shadow probability 1406 | shadow_prob = numpy.minimum(nir, swir) 1407 | # release remory 1408 | del nir 1409 | del swir 1410 | 1411 | Shadow[shadow_prob > 200] = 1 1412 | # release remory 1413 | del shadow_prob 1414 | 1415 | # Cloud[idplcd==True]=1 # all cld 1416 | 1417 | #*************************************************************************************# 1418 | #*************************************************************************************# 1419 | #*************************************************************************************# 1420 | #************The following code may be removed as the new code has changed************# 1421 | # Mask the non-contiguous pixels 1422 | #Cloud[~(mask)] = 0 1423 | 1424 | # improving by majority filtering 1425 | # ERROR: not aware of a similar filter in scipy (though one may very well exist) 1426 | # Doing convolution of all surrounding pixels & filtering those > 4, which should in theory have the same result 1427 | #Cloud = scipy.signal.convolve2d(Cloud, numpy.ones((3,3), Cloud.dtype.name))[1:-1,1:-1] 1428 | #Cloud = (Cloud > 4).astype('uint8') 1429 | # Applying twice, makes a cleaner result 1430 | #Cloud = scipy.signal.convolve2d(Cloud, numpy.ones((3,3), Cloud.dtype.name))[1:-1,1:-1] 1431 | # Cloud=(Cloud>4).astype('uint8') 1432 | # 3rd, still some single pixels at tile edges 1433 | # Cloud=scipy.signal.convolve2d(Cloud,numpy.ones((3,3),Cloud.dtype.name))[1:-1,1:-1] 1434 | #Cloud = Cloud > 4 1435 | #************The above code may be removed as the new code has changed****************# 1436 | #*************************************************************************************# 1437 | #*************************************************************************************# 1438 | #*************************************************************************************# 1439 | 1440 | del data 1441 | images = None 1442 | gc.collect() 1443 | 1444 | # refine Water mask - Zhe's water mask (no confusion water/cloud) 1445 | WT[numexpr.evaluate("(WT == 1) & (Cloud == 0)")] = 1 1446 | # bwmorph changed Cloud to Binary 1447 | Cloud = Cloud.astype('uint8') 1448 | Cloud[mask == 0] = 255 1449 | Shadow[mask == 0] = 255 1450 | 1451 | gc.collect() 1452 | cloud_mask = (Cloud == 1) & mask 1453 | 1454 | if ptm > 0.1: 1455 | cloud_temp = Temp[cloud_mask] 1456 | 1457 | logger.debug("Snow Percent: %f" % 1458 | ((float(Snow[mask].sum()) / float(mask.sum())) * 100.0)) 1459 | logger.debug("Cloud Mean: %f C" % (numpy.mean(cloud_temp) / 100.0)) 1460 | 1461 | if numpy.sum(cloud_mask) > 0: 1462 | cloud_stddev = numpy.std( 1463 | cloud_temp, dtype='float64', ddof=1) / 100.0 1464 | pct_upper = numpy.percentile(cloud_temp, 97.5) / 100.0 1465 | pct_lower = numpy.percentile(cloud_temp, 83.5) / 100.0 1466 | pct_upper_max = numpy.percentile(cloud_temp, 98.75) / 100.0 1467 | 1468 | logger.debug("Standard Deviation: %f C" % cloud_stddev) 1469 | logger.debug("97.5 percentile: %f C" % pct_upper) 1470 | logger.debug("83.5 percentile: %f C" % pct_lower) 1471 | logger.debug("98.75 percentile: %f C" % pct_upper_max) 1472 | 1473 | logger.info("Final Cloud Layer Percent: %f" % 1474 | ((float(Cloud[cloud_mask].sum()) / float(mask.sum())) * 100.0)) 1475 | 1476 | logger.info("Completed processing FMASK cloud cover...") 1477 | 1478 | # We'll modify the return argument for the Python implementation 1479 | # (geoT,prj) are added to the list 1480 | return (zen, azi, ptm, Temp, t_templ, t_temph, WT, Snow, Cloud, Shadow, dim, ul, resolu, zc, geoT, prj) 1481 | 1482 | 1483 | def fcssm(Sun_zen, Sun_azi, ptm, Temp, t_templ, t_temph, Water, Snow, plcim, 1484 | plsim, ijDim, resolu, ZC, cldpix, sdpix, snpix): 1485 | """ Calculates the cloud shadow 1486 | mask for a scene, given solar geometry information, the thermal band for the 1487 | scene & a cloud mask. 1488 | 1489 | :param Sun_zen: 1490 | Solar Elevation angle (degrees). 1491 | 1492 | :param Sun_azi: 1493 | Solar Azimuth angle (degrees). 1494 | 1495 | :param ptm: 1496 | Percentage of deleted pixels. 1497 | 1498 | :param Temp: 1499 | A numpy.ndarray containing the temperature band for the landsat scene (Celcius*100). 1500 | :param t_templ: 1501 | 0.175 percentile background temperature (low). 1502 | 1503 | :param t_temph: 1504 | 0.825 percentile background temperature (high). 1505 | 1506 | :param Water: 1507 | A numpy.ndarray of type Bool containing the water mask calculated by FMask. 1508 | 1509 | :param Snow: 1510 | A numpy.ndarray of type Bool containing the snow mask calculated by FMask. 1511 | 1512 | :param plcim: 1513 | A numpy.ndarray of type Bool containing the cloud mask calculated by FMask. 1514 | 1515 | :param plsim: 1516 | A numpy.ndarray of type Bool containing the cloud shadow mask calculated by FMask. 1517 | 1518 | :param ijDim: 1519 | A tuple containing the resolution of the scene bands (height, width). 1520 | 1521 | :param resolu: 1522 | A tuple (number, numpber). 1523 | 1524 | :param ZC: 1525 | The UTM Zone Number of the scene. 1526 | 1527 | :param cldpix: 1528 | A number for the cloud mask dilation (in pixels). 1529 | 1530 | :param sdpix: 1531 | A number for the cloud shadow mask dilation (in pixels) 1532 | """ 1533 | # Function for Cloud, cloud Shadow, and Snow Masking 1.6.3sav 1534 | # History of revisions: 1535 | # cloud shadow do not have to overlap with potential cloud shadow layer (Zhe Zhu 04/24/2011) 1536 | # exclude small cloud object <= 25 pixels (zhe Zhu 3/07/2011) 1537 | # dilate shadow again (3 pixels as default) (Zhe Zhu 12/23/2010); 1538 | # similarity < 0.95 (Zhe Zhu 11/06/2010) 1539 | # boosts data by >5/9 (Zhe Zhu 12/08/2009) 1540 | # use temperature to narrow iteration height (Zhe Zhu 12/09/2009) 1541 | # fixed bug for height (Zhe Zhu 12/09/2009) 1542 | # cloud DEM by thermal in cloud and shadow match (Zhe Zhu 1/03/2009) 1543 | 1544 | # solar elevation angle 1545 | Sun_ele = 90.0 - Sun_zen 1546 | sun_ele_rad = math.radians(Sun_ele) 1547 | # solar azimuth anngle 1548 | Sun_tazi = Sun_azi - 90.0 1549 | sun_tazi_rad = math.radians(Sun_tazi) 1550 | # assume resolu.x=resolu.y 1551 | sub_size = resolu[0] 1552 | win_height = ijDim[0] 1553 | win_width = ijDim[1] 1554 | 1555 | # potential cloud & shadow layer 1556 | cloud_test = numpy.zeros(ijDim, 'uint8') 1557 | shadow_test = numpy.zeros(ijDim, 'uint8') 1558 | # matched cloud & shadow layer 1559 | shadow_cal = numpy.zeros(ijDim, 'uint8') 1560 | cloud_cal = numpy.zeros(ijDim, 'uint8') 1561 | # cloud_height=zeros(ijDim)# cloud relative height (m) 1562 | # boundary layer 1563 | boundary_test = numpy.zeros(ijDim, 'uint8') 1564 | # final cloud, shadow and snow mask 1565 | cs_final = numpy.zeros(ijDim, 'uint8') 1566 | 1567 | # get potential mask values 1568 | shadow_test[plsim == 1] = 1 # plshadow layer 1569 | del plsim # empty memory 1570 | 1571 | boundary_test[plcim < 255] = 1 # boundary layer 1572 | cloud_test[plcim == 1] = 1 # plcloud layer 1573 | del plcim # empty memory 1574 | 1575 | # revised percent of cloud on the scene after plcloud 1576 | revised_ptm = numpy.sum(cloud_test) / numpy.sum(boundary_test) 1577 | # no t test => more than 98 # clouds and partly cloud over land 1578 | # => no match => rest are definite shadows 1579 | 1580 | # cloud covers more than 90# of the scene 1581 | # => no match => rest are definite shadows 1582 | # fprintf('Cloud and cloud shadow matching ...') 1583 | 1584 | if ptm <= 0.1 or revised_ptm >= 0.90: 1585 | # fprintf('No Shadow Match due to too much cloud (>90 percent)') 1586 | cloud_cal[cloud_test == True] = 1 1587 | shadow_cal[cloud_test == False] = 1 1588 | similar_num = -1 1589 | # height_num=-1 1590 | 1591 | else: 1592 | # fprintf('Shadow Match in processing') 1593 | 1594 | # define constants 1595 | Tsimilar = 0.30 1596 | Tbuffer = 0.95 # threshold for matching buffering 1597 | max_similar = 0.95 # max similarity threshold 1598 | num_cldoj = 3 # minimum matched cloud object (pixels) 1599 | num_pix = 3 # number of inward pixes (90m) for cloud base temperature 1600 | 1601 | # enviromental lapse rate 6.5 degrees/km 1602 | # dry adiabatic lapse rate 9.8 degrees/km 1603 | rate_elapse = 6.5 # degrees/km 1604 | rate_dlapse = 9.8 # degrees/km 1605 | 1606 | # fprintf('Set cloud similarity = #.3f',Tsimilar) 1607 | # fprintf('Set matching buffer = #.3f',Tbuffer) 1608 | # fprintf('Shadow match for cloud object >= #d pixels',num_cldoj) 1609 | 1610 | i_step = 2 * sub_size * math.tan(sun_ele_rad) # move 2 pixel at a time 1611 | 1612 | # get moving direction 1613 | (rows, cols) = numpy.nonzero(boundary_test) 1614 | (y_ul, num) = (rows.min(), rows.argmin()) 1615 | x_ul = cols[num] 1616 | 1617 | (y_lr, num) = (rows.max(), rows.argmax()) 1618 | x_lr = cols[num] 1619 | 1620 | (x_ll, num) = (cols.min(), cols.argmin()) 1621 | y_ll = rows[num] 1622 | 1623 | (x_ur, num) = (cols.max(), cols.argmax()) 1624 | y_ur = rows[num] 1625 | 1626 | # get view angle geometry 1627 | # print x_ul, y_ul, x_ur, y_ur, x_ll, y_ll, x_lr, y_lr 1628 | (A, B, C, omiga_par, omiga_per) = viewgeo(float(x_ul), float(y_ul), float( 1629 | x_ur), float(y_ur), float(x_ll), float(y_ll), float(x_lr), float(y_lr)) 1630 | 1631 | # Segmentate each cloud 1632 | # fprintf('Cloud segmentation & matching') 1633 | (segm_cloud_init, segm_cloud_init_features) = scipy.ndimage.measurements.label( 1634 | cloud_test, scipy.ndimage.morphology.generate_binary_structure(2, 2)) 1635 | 1636 | # filter out cloud object < than num_cldoj pixels 1637 | morphology.remove_small_objects( 1638 | segm_cloud_init, num_cldoj, in_place=True) 1639 | if skimage_version[1] >= 10: 1640 | segm_cloud, fw, inv = segmentation.relabel_sequential( 1641 | segm_cloud_init) 1642 | else: 1643 | segm_cloud, fw, inv = segmentation.relabel_from_one( 1644 | segm_cloud_init) 1645 | num = numpy.max(segm_cloud) 1646 | 1647 | # NOTE: properties is deprecated as of version 0.9 and all properties 1648 | # are computed. Currently using version 0.8.2. If this version or a 1649 | # later versionproves too slow, I'll implement another method. JS 1650 | # 16/12/2013 The properties is taking approx 3min, I can cut that down 1651 | # to just a few seconds using another method, but will leave for the 1652 | # time being. 1653 | if skimage_version[1] > 9: 1654 | s = measure.regionprops(segm_cloud) 1655 | else: 1656 | s = measure.regionprops(segm_cloud, properties=[ 1657 | 'Area', 'Coordinates']) 1658 | 1659 | # Use iteration to get the optimal move distance 1660 | # Calulate the moving cloud shadow 1661 | 1662 | # height_num=zeros(num) # cloud relative height (m) 1663 | similar_num = numpy.zeros(num) # cloud shadow match similarity (m) 1664 | 1665 | # Newer method of looping through the cloud objects/segments JS 1666 | # 16/12/2013 1667 | for cloud_type in s: 1668 | 1669 | cld_area = cloud_type['Area'] 1670 | cld_label = cloud_type['Label'] 1671 | 1672 | num_pixels = cld_area 1673 | 1674 | # Have re-formatted the arrays to be Python style memory ordering 1675 | # (2,num_pixels) JS 16/12/2013 moving cloud xys 1676 | XY_type = numpy.zeros((2, num_pixels), dtype='uint32') 1677 | 1678 | # record the max threshold moving cloud xys 1679 | tmp_XY_type = numpy.zeros((2, num_pixels), dtype='uint32') 1680 | 1681 | # corrected for view angle xys 1682 | # Leave as float for the time being 1683 | tmp_xys = numpy.zeros((2, num_pixels)) 1684 | 1685 | # record this original ids 1686 | orin_cid = (cloud_type['Coordinates'][:, 0], 1687 | cloud_type['Coordinates'][:, 1]) 1688 | 1689 | # Temperature of the cloud object 1690 | temp_obj = Temp[orin_cid] 1691 | 1692 | # assume object is round r_obj is radium of object 1693 | r_obj = math.sqrt(cld_area / math.pi) 1694 | 1695 | # number of inward pixes for correct temperature 1696 | # num_pix=8 1697 | pct_obj = math.pow(r_obj - num_pix, 2) / math.pow(r_obj, 2) 1698 | # pct of edge pixel should be less than 1 1699 | pct_obj = numpy.minimum(pct_obj, 1) 1700 | t_obj = scipy.stats.mstats.mquantiles(temp_obj, pct_obj) 1701 | 1702 | # put the edge of the cloud the same value as t_obj 1703 | temp_obj[temp_obj > t_obj] = t_obj 1704 | 1705 | # wet adiabatic lapse rate 6.5 degrees/km 1706 | # dry adiabatic lapse rate 9.8 degrees/km 1707 | # rate_wlapse=6.5# degrees/km 1708 | # rate_dlapse=9.8# degrees/km 1709 | 1710 | Max_cl_height = 12000 # Max cloud base height (m) 1711 | Min_cl_height = 200 # Min cloud base height (m) 1712 | 1713 | # refine cloud height range (m) 1714 | Min_cl_height = max(Min_cl_height, 10 * 1715 | (t_templ - 400 - t_obj) / rate_dlapse) 1716 | Max_cl_height = min(Max_cl_height, 10 * (t_temph + 400 - t_obj)) 1717 | 1718 | # initialize height and similarity info 1719 | record_h = 0.0 1720 | record_thresh = 0.0 1721 | 1722 | # iterate in height (m) 1723 | for base_h in numpy.arange(Min_cl_height, Max_cl_height, i_step): 1724 | # Get the true postion of the cloud 1725 | # calculate cloud DEM with initial base height 1726 | h = (10 * (t_obj - temp_obj) / rate_elapse + base_h) 1727 | tmp_xys[1, :], tmp_xys[0, :] = mat_truecloud(orin_cid[1], orin_cid[ 1728 | 0], h, A, B, C, omiga_par, omiga_per) # Function is returned as (x_new,y_new) 1729 | 1730 | # shadow moved distance (pixel) 1731 | # i_xy=h*cos(sun_tazi_rad)/(sub_size*math.tan(sun_ele_rad)) 1732 | i_xy = h / (sub_size * math.tan(sun_ele_rad)) 1733 | 1734 | if Sun_azi < 180: 1735 | XY_type[1, :] = numpy.round( 1736 | tmp_xys[1, :] - i_xy * math.cos(sun_tazi_rad)) # X is for j,1 1737 | XY_type[0, :] = numpy.round( 1738 | tmp_xys[0, :] - i_xy * math.sin(sun_tazi_rad)) # Y is for i,0 1739 | else: 1740 | XY_type[1, :] = numpy.round( 1741 | tmp_xys[1, :] + i_xy * math.cos(sun_tazi_rad)) # X is for j,1 1742 | XY_type[0, :] = numpy.round( 1743 | tmp_xys[0, :] + i_xy * math.sin(sun_tazi_rad)) # Y is for i,0 1744 | 1745 | tmp_j = XY_type[1, :] # col 1746 | tmp_i = XY_type[0, :] # row 1747 | 1748 | # the id that is out of the image 1749 | out_id = (tmp_i < 0) | (tmp_i >= win_height) | ( 1750 | tmp_j < 0) | (tmp_j >= win_width) 1751 | out_all = numpy.sum(out_id) 1752 | 1753 | tmp_ii = tmp_i[out_id == 0] 1754 | tmp_jj = tmp_j[out_id == 0] 1755 | 1756 | tmp_id = [tmp_ii, tmp_jj] 1757 | 1758 | # the id that is matched (exclude original cloud) 1759 | match_id = numexpr.evaluate("(b_test == 0) | ((seg != label) & ((cld_test > 0) | (shad_test == 1)))", {'b_test': boundary_test[ 1760 | tmp_id], 'seg': segm_cloud[tmp_id], 'label': cld_label, 'cld_test': cloud_test[tmp_id], 'shad_test': shadow_test[tmp_id]}) 1761 | matched_all = numpy.sum(match_id) + out_all 1762 | 1763 | # the id that is the total pixel (exclude original cloud) 1764 | total_id = segm_cloud[tmp_id] != cld_label 1765 | total_all = numpy.sum(total_id) + out_all 1766 | 1767 | thresh_match = numpy.float32(matched_all) / total_all 1768 | if (thresh_match >= (Tbuffer * record_thresh)) and (base_h < (Max_cl_height - i_step)) and (record_thresh < 0.95): 1769 | if thresh_match > record_thresh: 1770 | record_thresh = thresh_match 1771 | record_h = h 1772 | 1773 | elif record_thresh > Tsimilar: 1774 | # -1 to account for the zero based index used by Python (MATLAB is 1 one based). 1775 | similar_num[cld_label - 1] = record_thresh 1776 | i_vir = record_h / (sub_size * math.tan(sun_ele_rad)) 1777 | # height_num=record_h 1778 | 1779 | if Sun_azi < 180: 1780 | tmp_XY_type[1, :] = numpy.round( 1781 | tmp_xys[1, :] - i_vir * math.cos(sun_tazi_rad)) # X is for col j,2 1782 | tmp_XY_type[0, :] = numpy.round( 1783 | tmp_xys[0, :] - i_vir * math.sin(sun_tazi_rad)) # Y is for row i,1 1784 | else: 1785 | tmp_XY_type[1, :] = numpy.round( 1786 | tmp_xys[1, :] + i_vir * math.cos(sun_tazi_rad)) # X is for col j,2 1787 | tmp_XY_type[0, :] = numpy.round( 1788 | tmp_xys[0, :] + i_vir * math.sin(sun_tazi_rad)) # Y is for row i,1 1789 | 1790 | tmp_scol = tmp_XY_type[1, :] 1791 | tmp_srow = tmp_XY_type[0, :] 1792 | 1793 | # put data within range 1794 | tmp_srow[tmp_srow < 0] = 0 1795 | tmp_srow[tmp_srow >= win_height] = win_height - 1 1796 | tmp_scol[tmp_scol < 0] = 0 1797 | tmp_scol[tmp_scol >= win_width] = win_width - 1 1798 | 1799 | # sub2ind(ijDim,tmp_srow,tmp_scol) 1800 | tmp_sid = [tmp_srow, tmp_scol] 1801 | # give shadow_cal=1 1802 | shadow_cal[tmp_sid] = 1 1803 | # record matched cloud 1804 | # cloud_cal(orin_cid)=1 1805 | # cloud_height[orin_cid]=record_h 1806 | break 1807 | else: 1808 | record_thresh = 0.0 1809 | 1810 | # # dilate each cloud and shadow object by 3 and 6 pixel outward in 8 connect directions 1811 | # cldpix=3 # number of pixels to be dilated for cloud 1812 | # sdpix=3 # number of pixels to be dilated for shadow 1813 | # fprintf('Dilate #d pixels for cloud & #d pixels for shadow 1814 | # objects',cldpix,sdpix) 1815 | 1816 | # NOTE: An alternative for a structuring element would be to use the iterations parameter of binary_dilation() 1817 | # The number of iterations is equal to the number of dilations if using 1818 | # a 3x3 structuring element. JS 16/12/2013 1819 | SEc = 2 * cldpix + 1 1820 | SEc = numpy.ones((SEc, SEc), 'uint8') 1821 | SEs = 2 * sdpix + 1 1822 | SEs = numpy.ones((SEs, SEs), 'uint8') 1823 | SEsn = 2 * snpix + 1 1824 | SEsn = numpy.ones((SEsn, SEsn), 'uint8') 1825 | 1826 | # dilate shadow first 1827 | # NOTE: The original transcription returned the inverse, i.e. 1828 | # cloud_shadow = 0 rather than 1. We'll try inverting it outside this 1829 | # function in order to preserve the original return values of Fmask 1830 | shadow_cal = scipy.ndimage.morphology.binary_dilation( 1831 | shadow_cal, structure=SEs) 1832 | 1833 | # # find shadow within plshadow 1834 | # shadow_cal(shadow_test~=1)=0 1835 | # # dilate shadow again with the more accurate cloud shadow 1836 | # shadow_cal=imdilate(shadow_cal,SEs) 1837 | 1838 | segm_cloud_tmp = numexpr.evaluate("segm_cloud != 0") 1839 | # NOTE: The original transcription returned the inverse, i.e. cloud = 0 1840 | # rather than 1. We'll try inverting it outside this function in order 1841 | # to preserve the original return values of Fmask 1842 | cloud_cal = scipy.ndimage.morphology.binary_dilation( 1843 | segm_cloud_tmp, structure=SEc) 1844 | 1845 | Snow = scipy.ndimage.morphology.binary_dilation(Snow, structure=SEsn) 1846 | 1847 | cs_final[Water == 1] = 1 1848 | # mask from plcloud 1849 | # step 1 snow or unknow 1850 | cs_final[Snow == 1] = 3 # snow 1851 | # step 2 shadow above snow and everyting 1852 | cs_final[shadow_cal == 1] = 2 # shadow 1853 | # step 3 cloud above all 1854 | cs_final[cloud_cal == 1] = 4 # cloud 1855 | cs_final[boundary_test == 0] = 255 1856 | 1857 | # record cloud and cloud shadow percent 1858 | tmpcs = ((cs_final == 1) | (cs_final == 3)).astype('uint8') 1859 | cspt = 100.0 * (tmpcs.sum() / boundary_test.sum()) 1860 | 1861 | return (similar_num, cspt, shadow_cal, cs_final) 1862 | 1863 | # viewgeo function 1864 | 1865 | 1866 | def viewgeo(x_ul, y_ul, x_ur, y_ur, x_ll, y_ll, x_lr, y_lr): 1867 | # imput "x",j 1868 | # imput "y",i 1869 | # imput cloud height "h" 1870 | 1871 | x_u = (x_ul + x_ur) / 2 1872 | x_l = (x_ll + x_lr) / 2 1873 | y_u = (y_ul + y_ur) / 2 1874 | y_l = (y_ll + y_lr) / 2 1875 | 1876 | # get k of the upper left and right points 1877 | K_ulr = (y_ul - y_ur) / max(1, (x_ul - x_ur)) 1878 | # get k of the lower left and right points 1879 | K_llr = (y_ll - y_lr) / max(1, (x_ll - x_lr)) 1880 | K_aver = (K_ulr + K_llr) / 2 1881 | omiga_par = math.atan(K_aver) # get the angle of parallel lines k (in pi) 1882 | 1883 | # AX(j)+BY(i)+C=0 1884 | A = y_u - y_l 1885 | B = x_l - x_u 1886 | C = y_l * x_u - x_l * y_u 1887 | 1888 | # get the angle which is perpendicular to the trace line 1889 | omiga_per = math.atan(B / A) 1890 | return (A, B, C, omiga_par, omiga_per) 1891 | 1892 | # mat_truecloud function 1893 | 1894 | 1895 | def mat_truecloud(x, y, h, A, B, C, omiga_par, omiga_per): 1896 | # imput "x",j col 1897 | # imput "y",i row 1898 | # imput cloud height "h" 1899 | H = 705000 # average Landsat 7 height (m) 1900 | # from the cetral perpendicular (unit: pixel) 1901 | dist = (A * x + B * y + C) / math.sqrt(A * A + B * B) 1902 | dist_par = dist / math.cos(omiga_per - omiga_par) 1903 | dist_move = dist_par * h / H # cloud move distance (m) 1904 | delt_x = dist_move * math.cos(omiga_par) 1905 | delt_y = dist_move * math.sin(omiga_par) 1906 | 1907 | x_new = x + delt_x # new x, j 1908 | y_new = y + delt_y # new y, i 1909 | 1910 | return (x_new, y_new) 1911 | 1912 | 1913 | def run_FMask(mtl, outdir=None, cldprob=22.5, cldpix=3, sdpix=3, snpix=3): 1914 | # Check that the MTL file exists 1915 | assert os.path.exists(mtl), "Invalid filename: %s" % mtl 1916 | 1917 | if outdir is None: 1918 | outdir = os.path.dirname(mtl) 1919 | 1920 | if not os.path.exists(outdir): 1921 | os.mkdir(outdir) 1922 | 1923 | # Check that the output directory exists 1924 | assert os.path.exists(outdir), "Directory doesn't exist: %s" % outdir 1925 | 1926 | # Create the output filenames 1927 | log_fname = os.path.join(outdir, 'FMASK_LOGFILE.txt') 1928 | cloud_fname = os.path.join(outdir, 'fmask_cloud') 1929 | cloud_shadow_fname = os.path.join(outdir, 'fmask_shadow') 1930 | final_mask_fname = os.path.join(outdir, 'fmask_final_mask') 1931 | fmask_fname = os.path.join(outdir, 'fmask') 1932 | 1933 | # Open the MTL file. 1934 | # The original MATLAB code opens the file twice to 1935 | # retrieve the Landsat number. It would be better to open it once and 1936 | # restructure the function parameters. 1937 | data = {} 1938 | fl = open(mtl, 'r') 1939 | file_lines = fl.readlines() 1940 | for line in file_lines: 1941 | values = line.split(' = ') 1942 | if len(values) != 2: 1943 | continue 1944 | 1945 | data[values[0].strip()] = values[1].strip().strip('"') 1946 | 1947 | fl.close() 1948 | 1949 | # Identify Landsat Number (Lnum = 4, 5 or 7) 1950 | LID = data['SPACECRAFT_ID'] 1951 | Lnum = int(LID[len(LID) - 1]) 1952 | 1953 | st = datetime.datetime.now() 1954 | zen, azi, ptm, Temp, t_templ, t_temph, WT, Snow, Cloud, Shadow, dim, ul, resolu, zc, geoT, prj = plcloud( 1955 | mtl, cldprob, num_Lst=Lnum, shadow_prob=True) 1956 | et = datetime.datetime.now() 1957 | logger.info('time taken for plcloud function: %s', str(et - st)) 1958 | st = datetime.datetime.now() 1959 | similar_num, cspt, shadow_cal, cs_final = fcssm( 1960 | zen, azi, ptm, Temp, t_templ, t_temph, WT, Snow, Cloud, Shadow, dim, resolu, zc, cldpix, sdpix, snpix) 1961 | et = datetime.datetime.now() 1962 | logger.info('time taken for fcssm function: %s', str(et - st)) 1963 | 1964 | c = gdal.GetDriverByName('ENVI').Create(cloud_fname, Cloud.shape[ 1965 | 1], Cloud.shape[0], 1, gdal.GDT_Byte) 1966 | c.SetGeoTransform(geoT) 1967 | c.SetProjection(prj) 1968 | c.GetRasterBand(1).WriteArray(Cloud * 255) 1969 | c = None 1970 | 1971 | c = gdal.GetDriverByName('ENVI').Create(cloud_shadow_fname, shadow_cal.shape[ 1972 | 1], shadow_cal.shape[0], 1, gdal.GDT_Byte) 1973 | c.SetGeoTransform(geoT) 1974 | c.SetProjection(prj) 1975 | c.GetRasterBand(1).WriteArray(shadow_cal * 255) 1976 | c = None 1977 | 1978 | c = gdal.GetDriverByName('ENVI').Create(fmask_fname, cs_final.shape[ 1979 | 1], cs_final.shape[0], 1, gdal.GDT_Byte) 1980 | c.SetGeoTransform(geoT) 1981 | c.SetProjection(prj) 1982 | c.GetRasterBand(1).WriteArray(cs_final) 1983 | c = None 1984 | 1985 | final_mask = ((cs_final != 0)).astype('uint8') 1986 | c = gdal.GetDriverByName('ENVI').Create(final_mask_fname, final_mask.shape[ 1987 | 1], final_mask.shape[0], 1, gdal.GDT_Byte) 1988 | c.SetGeoTransform(geoT) 1989 | c.SetProjection(prj) 1990 | c.GetRasterBand(1).WriteArray(final_mask * 255) 1991 | c = None 1992 | # TODO: Save water/snow masks? 1993 | 1994 | 1995 | def main(): 1996 | parser = argparse.ArgumentParser( 1997 | description='Computes the Fmask algorithm. Cloud, cloud shadow and Fmask combined (contains thecloud, cloud shadow and snow masks in a single array) are output to disk.') 1998 | parser.add_argument('--mtl', required=True, 1999 | help='The full file path to the Landsat MTL file.') 2000 | parser.add_argument('--cldprob', type=float, default=22.5, 2001 | help='The cloud probability for the scene. Default is 22.5 percent.') 2002 | parser.add_argument('--cldpix', type=int, default=3, 2003 | help='The number of pixels to be dilated for the cloud mask. Default is 3.') 2004 | parser.add_argument('--sdpix', type=int, default=3, 2005 | help='The number of pixels to be dilated for the cloud shadow mask. Default is 3.') 2006 | parser.add_argument('--snpix', type=int, default=3, 2007 | help='The number of pixels to be dilated for the snow mask. Default is 3.') 2008 | parser.add_argument('--outdir', required=True, 2009 | help='The full file path of the output directory that will contain the Fmask results.') 2010 | 2011 | parsed_args = parser.parse_args() 2012 | mtl = parsed_args.mtl 2013 | cldprob = parsed_args.cldprob 2014 | cldpix = parsed_args.cldpix 2015 | sdpix = parsed_args.sdpix 2016 | snpix = parsed_args.snpix 2017 | outdir = parsed_args.outdir 2018 | 2019 | logger.setLevel(logging.INFO) 2020 | logging.basicConfig() 2021 | run_FMask(mtl, outdir, cldprob, cldpix, sdpix, snpix) 2022 | 2023 | 2024 | if __name__ == '__main__': 2025 | main() 2026 | --------------------------------------------------------------------------------