├── README.md ├── run_llr_template.js ├── colab_template.ipynb ├── LICENSE ├── readme_js.md ├── landsatlinkr_workshop_template.ipynb ├── landsatlinkr.js └── landsatlinkr.py /README.md: -------------------------------------------------------------------------------- 1 | # ee-LandsatLinkr 2 | 3 | The aim of `landsatlinkr` is to make it easy to run the LandTrendr algorithm with Landsat MSS, TM, ETM, and OLI data together in Earth Engine. It assembles image collections across the five satellites that carried the MSS sensor, filters those images for quality, calculates TOA reflectance, and calculates the MSScvm cloud mask. It also allows the user to manually exclude MSS images with scan line issues or other noise. These MSS images are converted to pseudo-TM by building a relationship between MSS and TM coincident images. These converted MSS to TM images are then grouped with TM and OLI images and run through LandTrendr to track changes over time. 4 | 5 | ## Notes 6 | 7 | The most recent version of ee-LandsatLinkr is written for the Earth Engine Python API and executed using a Colab notebook. 8 | 9 | As of 2022-10-25 the code is still changing frequently, so check back for updates. 10 | 11 | Right now the best place to start is with a presentation and notebook by Annie Taylor and Justin Braaten given during a presentation at [Geo for Good '22](https://earthoutreachonair.withgoogle.com/events/geoforgood22#) 12 | 13 | - [Presentation](https://earthoutreachonair.withgoogle.com/events/geoforgood22/watch?talk=day3-track1-talk1) 14 | - [Slides](https://docs.google.com/presentation/d/1lBL2omCxVDRqVqu7wRoVYAHC-IgAXL-7tIHJB6ZEX6k/edit#slide=id.g941972ef10_0_0) 15 | - [Worksheet](https://docs.google.com/document/d/1hhJhamfjfrhlPkQbd2WrYgpliilzeq9nnQj2MRPndCw/edit) 16 | - [Notebook](https://github.com/gee-community/ee-LandsatLinkr/blob/main/landsatlinkr_workshop_template.ipynb) 17 | -------------------------------------------------------------------------------- /run_llr_template.js: -------------------------------------------------------------------------------- 1 | // ############################################################################# 2 | // ############################################################################# 3 | // ############################################################################# 4 | 5 | /** 6 | * 1. View WRS-1 granules - figure out what WRS-1 granule to process 7 | * -- Make a processing dir: https://gist.github.com/jdbcode/36f5a04329d5d85c43c0408176c51e6d 8 | * 2. Create MSS WRS-1 reference image - for MSS WRS1 to MSS WRS2 harmonization 9 | * 3. View WRS-1 collection - identify bad MSS images 10 | * 4. Prepare MSS WRS-1 images 11 | * 5. Get TM-to-MSS correction coefficients 12 | * 6. Export MSS-to-TM corrected images 13 | * 7. Inspect the full time series collection - explore time series via animation and inspector tool to check for noise 14 | * 8. Run LandTrendr and display the fitted collection on the map 15 | * 9. Display the year and magnitude of the greatest disturbance during the time series 16 | */ 17 | 18 | var LLR_STEP = 1; 19 | 20 | // ############################################################################# 21 | 22 | var PROJ_PATH = 'users/braaten/LandTrendr'; // Must be the same path used to create the asset folder - cannot contain / at end - check for this in the code. 23 | var WRS_1_GRANULE = '049030'; 24 | var CRS = 'EPSG:3857'; 25 | 26 | var DOY_RANGE = [160, 254]; 27 | var MAX_CLOUD = 50; 28 | var MAX_GEOM_RMSE = 0.5; 29 | 30 | var EXCLUDE_IDS = [ 31 | 'LM10490301972210AAA05', 32 | 'LM20490301975167AAA02', 33 | 'LM20490301975185AAA02', 34 | 'LM20490301976216GDS01', 35 | 'LM20490301976252GDS01', 36 | 'LM30490301978196GDS03', 37 | 'LM20490301978241AAA02', 38 | 'LM20490301979200AAA05', 39 | 'LM20490301981171AAA03' 40 | ]; 41 | 42 | // ############################################################################# 43 | // ############################################################################# 44 | // ############################################################################# 45 | 46 | var params = { 47 | maxRmseVerify: MAX_GEOM_RMSE, 48 | maxCloudCover: MAX_CLOUD, 49 | doyRange: DOY_RANGE, 50 | wrs1: WRS_1_GRANULE, 51 | crs: CRS, 52 | excludeIds: EXCLUDE_IDS, 53 | baseDir: PROJ_PATH + '/' + WRS_1_GRANULE 54 | }; 55 | 56 | var llr = require('users/jstnbraaten/modules:landsatlinkr/landsatlinkr.js'); 57 | switch (LLR_STEP) { 58 | case 1: 59 | llr.wrs1GranuleSelector(); 60 | break; 61 | case 2: 62 | llr.exportMssRefImg(params); 63 | break; 64 | case 3: 65 | llr.viewWrs1Col(params); 66 | break; 67 | case 4: 68 | llr.processMssWrs1Imgs(params); 69 | break; 70 | case 5: 71 | llr.exportMss2TmCoefCol(params); 72 | break; 73 | case 6: 74 | llr.exportFinalCorrectedMssCol(params); 75 | break; 76 | case 7: 77 | var col = llr.getColForLandTrendrFromAsset(params); 78 | llr.displayCollection(col); 79 | llr.animateCollection(col); 80 | break; 81 | case 8: 82 | var lt = llr.runLandTrendrMss2Tm(params); 83 | //llr.displayFittedCollection() not built yet 84 | break; 85 | case 9: 86 | var lt = llr.runLandTrendrMss2Tm(params); 87 | llr.displayGreatestDisturbance(lt, params); 88 | break; 89 | } 90 | -------------------------------------------------------------------------------- /colab_template.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "source": [ 6 | "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/gee-community/ee-LandsatLinkr/blob/main/colab_template.ipynb)" 7 | ], 8 | "metadata": { 9 | "id": "ElDwLGzPc-VS" 10 | } 11 | }, 12 | { 13 | "cell_type": "markdown", 14 | "metadata": { 15 | "id": "G-uiqaRAOo_Q" 16 | }, 17 | "source": [ 18 | "# Environment setup\n", 19 | "\n" 20 | ] 21 | }, 22 | { 23 | "cell_type": "markdown", 24 | "metadata": { 25 | "id": "gzkiZpJPOXwm" 26 | }, 27 | "source": [ 28 | "Import the Earth Engine API and authenticate to the service" 29 | ] 30 | }, 31 | { 32 | "cell_type": "code", 33 | "execution_count": null, 34 | "metadata": { 35 | "id": "uPG-SZDcOC7W" 36 | }, 37 | "outputs": [], 38 | "source": [ 39 | "import ee\n", 40 | "ee.Authenticate()\n", 41 | "ee.Initialize()" 42 | ] 43 | }, 44 | { 45 | "cell_type": "markdown", 46 | "metadata": { 47 | "id": "1htUCSlsOajh" 48 | }, 49 | "source": [ 50 | "Get LandsatLinkr library." 51 | ] 52 | }, 53 | { 54 | "cell_type": "code", 55 | "execution_count": null, 56 | "metadata": { 57 | "id": "oSXgOmkYGcQO" 58 | }, 59 | "outputs": [], 60 | "source": [ 61 | "!rm -r /content/ee-LandsatLinkr\n", 62 | "!git clone https://github.com/gee-community/ee-LandsatLinkr --quiet\n", 63 | "import sys\n", 64 | "sys.path.append(\"/content/ee-LandsatLinkr\")\n", 65 | "from landsatlinkr import *" 66 | ] 67 | }, 68 | { 69 | "cell_type": "markdown", 70 | "metadata": { 71 | "id": "P4fR8xWSKj8Z" 72 | }, 73 | "source": [ 74 | "# Identity MSS WRS-1 tile ID(s)\n", 75 | "\n", 76 | "\n", 77 | "1. Visit: https://code.earthengine.google.com/978fb6b2d523007918d77f2b35048c4a \n", 78 | "2. In the map, zoom to your study area and select its location to reveal the WRS-1 tile IDs at that location (this may take a minute or two to load)\n", 79 | "3. Copy the WRS-1 tile ID from the pop-up window. If your study area overlaps with two or more tile footprints, you should note all of the WRS-1 tile IDs and process as many tiles that intersect the study region one at a time – the results can be composited later. For now, select one WRS-1 tile ID to begin with (you can only process one tile ID at a time).\n", 80 | "4. Paste this WRS-1 tile ID into your run script as the WRS_1_GRANULE variable:" 81 | ] 82 | }, 83 | { 84 | "cell_type": "code", 85 | "execution_count": null, 86 | "metadata": { 87 | "id": "vPcdzuQdH8p0" 88 | }, 89 | "outputs": [], 90 | "source": [ 91 | "PROJECT_DIR = 'projects/ee-braaten/assets/llr' \n", 92 | "WRS_1_GRANULE = '050028' #(saint helens) # '041029' (west yellowstone) # '049030' (western oregon)" 93 | ] 94 | }, 95 | { 96 | "cell_type": "markdown", 97 | "metadata": { 98 | "id": "hnsSQ7REP0eW" 99 | }, 100 | "source": [ 101 | "Create a project directory for this MSS WRS-1 ID\n", 102 | "\n", 103 | "TODO: does not warn you if it failed" 104 | ] 105 | }, 106 | { 107 | "cell_type": "code", 108 | "execution_count": null, 109 | "metadata": { 110 | "id": "6BysCvTPOWyN" 111 | }, 112 | "outputs": [], 113 | "source": [ 114 | "createProjectDir(PROJECT_DIR, WRS_1_GRANULE)" 115 | ] 116 | }, 117 | { 118 | "cell_type": "markdown", 119 | "metadata": { 120 | "id": "C2fOKd_VQz8v" 121 | }, 122 | "source": [ 123 | "Set variables TODO: should allow people to select the end year, right now I think it is hard-coded to 2021" 124 | ] 125 | }, 126 | { 127 | "cell_type": "code", 128 | "execution_count": null, 129 | "metadata": { 130 | "id": "i-xS5u88QxyL" 131 | }, 132 | "outputs": [], 133 | "source": [ 134 | "CRS = 'EPSG:3857'\n", 135 | "DOY_RANGE = [160, 254]\n", 136 | "MAX_CLOUD = 50\n", 137 | "MAX_GEOM_RMSE = 0.5\n", 138 | "\n", 139 | "LT_PARAMS = {\n", 140 | " 'ftvBands': ['tcb', 'tcg', 'tcw'], #['red', 'green', 'blue'], # , 'tcb', 'tcg', 'tcw'\n", 141 | " 'maxSegments': 10,\n", 142 | " 'spikeThreshold': 0.7,\n", 143 | " 'vertexCountOvershoot': None,\n", 144 | " 'preventOneYearRecovery': None,\n", 145 | " 'recoveryThreshold': 0.5,\n", 146 | " 'pvalThreshold': None,\n", 147 | " 'bestModelProportion': None,\n", 148 | " 'minObservationsNeeded': None,\n", 149 | " 'scale': 30\n", 150 | "}\n", 151 | "\n", 152 | "params = {\n", 153 | " 'maxRmseVerify': MAX_GEOM_RMSE,\n", 154 | " 'maxCloudCover': MAX_CLOUD,\n", 155 | " 'doyRange': DOY_RANGE,\n", 156 | " 'wrs1': WRS_1_GRANULE,\n", 157 | " 'crs': CRS,\n", 158 | " 'excludeIds': [],\n", 159 | " 'baseDir': PROJECT_DIR + '/' + WRS_1_GRANULE,\n", 160 | " 'projectDir': PROJECT_DIR,\n", 161 | " 'ltParams': LT_PARAMS\n", 162 | "}" 163 | ] 164 | }, 165 | { 166 | "cell_type": "markdown", 167 | "metadata": { 168 | "id": "UyFooaFAhOye" 169 | }, 170 | "source": [ 171 | "Preview MSS WRS-1 images, note bad images and add them to the `EXCLUDE_IDS` list. Run until all bad images are removed." 172 | ] 173 | }, 174 | { 175 | "cell_type": "code", 176 | "execution_count": null, 177 | "metadata": { 178 | "id": "nzCpb0k7g_Jy" 179 | }, 180 | "outputs": [], 181 | "source": [ 182 | "viewWrs1Col(params)" 183 | ] 184 | }, 185 | { 186 | "cell_type": "code", 187 | "execution_count": null, 188 | "metadata": { 189 | "id": "Y9zEeJqnlQSC" 190 | }, 191 | "outputs": [], 192 | "source": [ 193 | "# EXCLUDE_IDS = [\n", 194 | "# 'LM10410291974209GDS03',\n", 195 | "# 'LM20410291975231GDS02',\n", 196 | "# 'LM20410291975249AAA04',\n", 197 | "# 'LM20410291976172GDS04',\n", 198 | "# 'LM10410291976181AAA02',\n", 199 | "# 'LM20410291976190GDS02',\n", 200 | "# 'LM10410291976253AAA05',\n", 201 | "# 'LM20410291977220GDS04',\n", 202 | "# 'LM20410291978179AAA02',\n", 203 | "# 'LM30410291978188AAA02',\n", 204 | "# 'LM20410291978197AAA02',\n", 205 | "# 'LM20410291978215AAA02',\n", 206 | "# 'LM20410291978233AAA02',\n", 207 | "# 'LM30410291978242AAA02',\n", 208 | "# 'LM20410291978251AAA02',\n", 209 | "# 'LM20410291979174XXX01',\n", 210 | "# 'LM20410291979192AAA05',\n", 211 | "# 'LM30410291979201AAA08',\n", 212 | "# 'LM20410291980169AAA08',\n", 213 | "# 'LM20410291980241AAA05',\n", 214 | "# 'LM20410291981199AAA03'\n", 215 | "# ]\n", 216 | "\n", 217 | "EXCLUDE_IDS = ['LM30500281982248AAA03']\n", 218 | "params['excludeIds'] = EXCLUDE_IDS" 219 | ] 220 | }, 221 | { 222 | "cell_type": "markdown", 223 | "metadata": { 224 | "id": "35iL-UTkSs8L" 225 | }, 226 | "source": [ 227 | "Make an MSS reference image. Took about 15-25 minutes" 228 | ] 229 | }, 230 | { 231 | "cell_type": "code", 232 | "execution_count": null, 233 | "metadata": { 234 | "id": "5VZqecuOSsSl" 235 | }, 236 | "outputs": [], 237 | "source": [ 238 | "mssRefTask = exportMssRefImg(params)\n", 239 | "monitorTaskStatus(mssRefTask)" 240 | ] 241 | }, 242 | { 243 | "cell_type": "markdown", 244 | "metadata": { 245 | "id": "uR7I7uwftW0m" 246 | }, 247 | "source": [ 248 | "Prepare MSS images TODO: print a message about what this step is doing, similar to above" 249 | ] 250 | }, 251 | { 252 | "cell_type": "code", 253 | "execution_count": null, 254 | "metadata": { 255 | "id": "V-0lbJNxtZJA" 256 | }, 257 | "outputs": [], 258 | "source": [ 259 | "mssWrs1ToWrs2Task = processMssWrs1Imgs(params)\n", 260 | "monitorTaskStatus(mssWrs1ToWrs2Task)" 261 | ] 262 | }, 263 | { 264 | "cell_type": "markdown", 265 | "metadata": { 266 | "id": "Kpr75NIp-aEY" 267 | }, 268 | "source": [ 269 | "Develop a relationship between MSS and TM images." 270 | ] 271 | }, 272 | { 273 | "cell_type": "code", 274 | "execution_count": null, 275 | "metadata": { 276 | "id": "fUOHSHEXQYat" 277 | }, 278 | "outputs": [], 279 | "source": [ 280 | "mss2TmInfoTask = exportMss2TmCoefCol(params)\n", 281 | "monitorTaskStatus(mss2TmInfoTask)" 282 | ] 283 | }, 284 | { 285 | "cell_type": "markdown", 286 | "metadata": { 287 | "id": "AEMG1HHfFlgh" 288 | }, 289 | "source": [ 290 | "Make MSS imagery match TM." 291 | ] 292 | }, 293 | { 294 | "cell_type": "code", 295 | "execution_count": null, 296 | "metadata": { 297 | "id": "0trxpDSHR3eE" 298 | }, 299 | "outputs": [], 300 | "source": [ 301 | "mss2TmTask = exportFinalCorrectedMssCol(params)\n", 302 | "monitorTaskStatus(mss2TmTask)" 303 | ] 304 | }, 305 | { 306 | "cell_type": "markdown", 307 | "metadata": { 308 | "id": "XMnhczvubntn" 309 | }, 310 | "source": [ 311 | "Run LandTrendr.\n", 312 | "\n" 313 | ] 314 | }, 315 | { 316 | "cell_type": "code", 317 | "execution_count": null, 318 | "metadata": { 319 | "colab": { 320 | "background_save": true 321 | }, 322 | "id": "BkVUN94ebp1h" 323 | }, 324 | "outputs": [], 325 | "source": [ 326 | "ltTask = exportLt(params)\n", 327 | "monitorTaskStatus(ltTask)" 328 | ] 329 | } 330 | ], 331 | "metadata": { 332 | "colab": { 333 | "name": "landsatlinkr_run_template.ipynb", 334 | "provenance": [] 335 | }, 336 | "kernelspec": { 337 | "display_name": "Python 3", 338 | "name": "python3" 339 | }, 340 | "language_info": { 341 | "name": "python" 342 | } 343 | }, 344 | "nbformat": 4, 345 | "nbformat_minor": 0 346 | } 347 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /readme_js.md: -------------------------------------------------------------------------------- 1 | # ee-LandsatLinkr 2 | 3 | The aim of `landsatlinkr` is to make it easy to run the LandTrendr algorithm with Landsat MSS, TM, ETM, and OLI data together in Earth Engine. It assembles image collections across the five satellites that carried the MSS sensor, filters those images for quality, calculates TOA reflectance, and calculates the MSScvm cloud mask. It also allows the user to manually exclude MSS images with scan line issues or other noise. These MSS images are converted to pseudo-TM by building a relationship between MSS and TM coincident images. These converted MSS to TM images are then grouped with TM and OLI images and run through LandTrendr to track changes over time. 4 | 5 | ## Module import 6 | 7 | Include the following line at the top of every script to import the landsatlinkr library. This line is already included in the template script that you will work from in this workflow. 8 | 9 | ```js 10 | var llr = require('users/jstnbraaten/modules:landsatlinkr/landsatlinkr.js'); 11 | ``` 12 | 13 | ## Pre-Processing Steps 14 | 15 | ### Prep Step 1. Create a script folder 16 | Create a script folder or repository to hold your landsat linkr scripts. You will ultimately create a different script for each WRS-1 tile footprint that you work with, so it is helpful to have one folder where all of your landsat linkr scripts are stored. Yours might look like this: `users/user_name/LLR_Projects`. 17 | 18 | ### Prep Step 2. Copy the landsat linkr template script into your folder 19 | Add a copy of the run_llr script to your folder (there are two ways to do this): 20 | - Open the run_llr script at this [link](https://code.earthengine.google.com/e926ef1c0fc1001d46af363ef34559bb?noload=true) and save a copy to your landsat linkr project folder 21 | 22 | OR 23 | - Add the [landsat linkr repository](https://code.earthengine.google.com/?accept_repo=users/jstnbraaten/modules) to your EE account and save a copy of run_llr_template.js to your landsat linkr project folder 24 | 25 | ### Script Step 1. Identity MSS WRS-1 tile ID(s) 26 | Determine the WRS-1 tile ID for your study area. WRS-1 tile IDs refer to the image footprints of the MSS sensor. To do this, open your copy of the run script (see above). We'll be editing this run script to run the Landsat Linkr workflow for your study area. 27 | 28 | 1. In the run script, set LLR_STEP to 1 and run the script: 29 | ```js 30 | var LLR_STEP = 1; 31 | ``` 32 | 2. In the map, zoom to your study area and select its location to reveal the WRS-1 tile IDs at that location (this may take a minute or two to load) 33 | 3. Copy the WRS-1 tile ID from the pop-up window. If your study area overlaps with two or more tile footprints, you should note all of the WRS-1 tile IDs and process as many tiles that intersect the study region one at a time – the results can be composited later. For now, select one WRS-1 tile ID to begin with (you can only process one tile ID at a time). 34 | 4. Paste this WRS-1 tile ID into your run script as the WRS_1_GRANULE variable: 35 | ```js 36 | var WRS_1_GRANULE = '049030'; 37 | ``` 38 | 39 | ### Prep Step 3. Create Asset Folders 40 | The Landsat Linkr workflow currently exports many assets to your Earth Engine account to store the intermediate and final processing results. In this step, we'll create the asset folders that are required for these exports. To make this easy, we have provided a python notebook/script that creates the appropriate folders in your Earth Engine Assets tab. 41 | 42 | 1. Open the [LandsatLinkr Asset Manager Script](https://gist.github.com/jdbcode/36f5a04329d5d85c43c0408176c51e6d) and click Open in Colab to open the script as a python notebook 43 | 2. Run the first code block to authenticate to the EE Python API (you'll be asked to open a link with your EE account and then to copy/paste the access code) 44 | 3. Set `wrs-granule-1` to your study area's WRS-1 tile ID (same as the WRS_1_GRANULE above) 45 | 4. Set `project_dir` to 'users/your_EE_username/LandTrendr' 46 | 5. Run the second, third, and fourth code blocks (Set up project dirs and Create project dirs, the rest of the code isn't needed) 47 | 6. Check your Assets tab to see that the following two folders were created: 48 | * users/your_EE_username/LandTrendr/your_WRS_tile_ID/WRS1_to_TM 49 | * users/your_EE_username/LandTrendr/your_WRS_tile_ID/WRS1_to_WRS2 50 | 51 | 52 | ## Running Landsat Linkr 53 | Now that you have your script and asset folders set up, we can get started with the Landsat Linkr workflow. 54 | 55 | ### Script Step 2. MSS WRS-1 reference image 56 | Create an MSS WRS-1 reference image for MSS WRS1 to MSS WRS2 harmonization. This reference image will be used for spectral normalization, as some of the sensors are inconsistent. This step will create a image 'ref' in your project's asset folder, and should take about 15 minutes. 57 | 58 | 1. Set the PROJ_PATH variable to the asset folder that you created. 59 | ```js 60 | var PROJ_PATH = 'users/your_EE_username/LandTrendr'; 61 | // Must be the same path used to create the asset folder - cannot contain / at end. 62 | ``` 63 | 2. Confirm that the WRS_1_GRANULE variable is set to the WRS-1 tile ID that you identified above 64 | ```js 65 | var WRS_1_GRANULE = '046033'; 66 | ``` 67 | 68 | 3. Set the CRS variable to the geographic projection that is most relevant to your study area. If you plan on calculating areas of disturbance (or other area calculations), use a projection that preserves area such as Albers equal area conic. If there are other datasets that you'll be using in combination with this Landsat analysis, consider using the CRS of those datasets. Otherwise, you can stick with the default CRS of the Earth Engine API, which is Web Mercator (EPSG:3857). 69 | ```js 70 | var CRS = 'EPSG:3857'; 71 | ``` 72 | 73 | 4. Set DOY_RANGE to indicate the days of the year that you would like to include in your analysis. Here are some examples. 74 | To include the entire year: 75 | ```js 76 | var DOY_RANGE = [1, 365]; 77 | ``` 78 | To include the months of August and September (these vary by 1 in a leap year): 79 | ```js 80 | var DOY_RANGE = [213, 273]; 81 | ``` 82 | 5. Set MAX_CLOUD to the maximum allowable percentage of cloudiness for the included images (the default is 50%). Keep in mind that a cloud mask is applied in preparing the final image collection, so some cloudiness may be acceptable during the initial filter. 83 | ``` js 84 | var MAX_CLOUD = 50; 85 | ``` 86 | 6. Set MAX_GEOM_RMSE to the maximum allowable spatial offset in units of a pixel (the default is one half of a pixel, or 0.5). 87 | ```js 88 | var MAX_GEOM_RMSE = 0.5; 89 | ``` 90 | 7. Set LLR_STEP to 2 and run the script. 91 | ```js 92 | var LLR_STEP = 2; 93 | ``` 94 | 8. Once it appears in your Tasks Tab, run the task `MSS-reference-image` – don't change the export settings. This should take about 15 minutes. Once the task is completed and you see an image called 'ref' in your project's asset folder, you can move onto the next step. 95 | 96 | ### Script Step 3. Preview and filter out bad MSS images 97 | This step will print thumbnails of all of the available MSS images based on your parameters so that you can assess image quality and remove the bad images from your image collection. This step will result in an updated `EXCLUDE_IDs` variable, and can take between 10 to 30 minutes (depending on the number of images available). 98 | 99 | 1. Set LLR_STEP to 3 and run the script. 100 | ```js 101 | var LLR_STEP = 3; 102 | ``` 103 | 104 | 2. Once the image thumbnails have loaded in the Console Tab, you'll need to inspect them for issues so that you can exclude bad images from your final collection. A bad image is one with: 105 | - lots of discolored lines 106 | - lines that are shifted 107 | - thin clouds and haze 108 | - very bright or very dark colorization compared to the others 109 | 110 | Landsat Linkr can fix (minimize/reduce to tolerable errors): 111 | - a few discolored lines 112 | - thick clouds 113 | 114 | When you find a bad image, copy it's image ID (printed just above the thumbnail) and paste it as an element in the list variable `EXCLUDE_IDS`. Image IDs should be listed as strings (in quotes) and separated by a comma. The template script may have some image IDs pre-populated – be sure to delete these. 115 | 116 | ```js 117 | var EXCLUDE_IDS = [ 118 | 'LM20460331975128AAA04', 119 | ... // more image IDs 120 | 'LM10460331975227GDS03' 121 | ]; 122 | ``` 123 | 124 | If there are one or two good images for a year near the target date, it is recommended that you discard any images from the same year that are only marginally good (images with more clouds, thin clouds, a few discolored lines, etc). If there are no really good images for a given year, you can include all of the marginally good ones (Landsat Linkr attempts to mask discolored lines and clouds and uses an intra-annual medoid reduction to identify the best pixel). 125 | 126 | This step can take some time, particularly if you have included most or all of each year (see DOY_RANGE above). Once you have previewed all of the MSS images and filtered out the bad ones (by copy/pasting those image IDs to the EXCLUDE_IDs variable), you can move onto the next step. 127 | 128 | ### Script Step 4. Prepare WRS-1 images 129 | This step will correct each included MSS image to the reference image `ref` that you created earlier. After running this step, all of the selected MSS imagery will be harmonized within the MSS time series. This step exports annual composite images that are based on the best pixel for each year, and adds these annual composites to the WRS1 to WRS2 asset folder that you created earlier. This step takes about one hour to run. 130 | 131 | 1. Set LLR_STEP to 4 and run the script. 132 | ```js 133 | var LLR_STEP = 4; 134 | ``` 135 | 136 | 2. This step will create 11 tasks in the EE Task tab, one for each year of analysis. Once they appear, run each task using the default settings. This should take about an hour. 137 | 138 | 3. Once the tasks are completed and you see 11 images named for each year in the `WRS1_to_WRS2` asset folder, you can move onto the next step. You can add these asset images to the map if you would like to check them. 139 | 140 | ### Script Step 5. Correlate the MSS to coincident TM images 141 | This step builds the relationship between MSS and TM images by finding MSS and TM coincident images (Landsat 4 had both sensors) and using this overlap to build a robust linear regression to link the two sensors. The resulting table of correlation coefficients (exported as an asset) contains the slope, intercept, and RMSE for each image pair within your tile (and for each band). This step requires two tasks and takes about 45 minutes to run. 142 | 143 | 1. Set LLR_STEP to 5 and run the script. 144 | ```js 145 | var LLR_STEP = 5; 146 | ``` 147 | 148 | 2. This step will create two tasks in the EE Task tab, one for to generate correlation coefficients `mss2TmCoefCol` and one to generate an offset image `medianOffset`. Once they appear, run each task using the default settings. This should take about 45 minutes. 149 | 150 | 3. Once the tasks are completed and you see two new assets in your project's asset folder (mss2TmCoefCol and mss_offset), you can move onto the next step. 151 | 152 | ### Script Step 6. Harmonize MSS images to TM and export the corrected images 153 | This step harmonizes the WRS-1 MSS images to the TM time series using the coefficients calculated in the previous step. Note that Landsat Linkr doesn't harmonize the WRS-2 Landsat 4 and 5 images at all because there are coincident TM images which are preferred (except for 1993 when there are few TM images). This step should take a little over an hour to run. 154 | 155 | 1. Set LLR_STEP to 6 and run the script. 156 | ```js 157 | var LLR_STEP = 6; 158 | ``` 159 | 160 | 2. This step will create 11 tasks in the EE Task tab, one for each year of analysis. Once they appear, run each task using the default settings. This should take about an hour or longer. 161 | 162 | 3. Once the tasks are completed and you see 11 images named for each year in the `WRS1_to_TM` asset folder, you can move onto the next step. 163 | 164 | ### Script Step 7. Inspect time series 165 | In the assets folder, we have a harmonized MSS collection that is ready to input into the Landtrendr algorithm in EE (add link to the EE guide here). This step collects all of the Landsat imagery from later years (ETM and OLI sensors), masks out cloudy pixels and takes the annual medoid composite. 166 | 167 | 1. Set LLR_STEP to 7 and run the script. This creates the entire Landsat image collection and saves it as a new variable `col`. 168 | ```js 169 | var LLR_STEP = 7; 170 | ``` 171 | 172 | 2. Add the image collection to the map using the example code below and run the script. 173 | ```js 174 | Map.addLayer(col, {}, 'Landsat Collection'); 175 | ``` 176 | 177 | 3. Use the inspector to view the time series at a given point. You can check the time series chart in the Inspector window, making sure there is not a step function at or around 1984. 178 | 179 | 4. If you discover unexpected issues with the data for certain years or see any of the issues detailed below (coming soon), return to Script Step 3 and try to identify the image(s) causing the problem and add their IDs to the list of images to exclude of the relevant year(s). If you find images to newly exclude, you will need to re-run Script Steps 4 through 7. Prior to re-running these steps, you'll need to delete the existing image assets for this WRS-1 tile ID. This [LandsatLinkr Asset Manager Script](https://gist.github.com/jdbcode/36f5a04329d5d85c43c0408176c51e6d) (also linked earlier) has some code to automate deletion from the appropriate asset folders; follow the steps below. 180 | 181 | 1. Open the [LandsatLinkr Asset Manager Script](https://gist.github.com/jdbcode/36f5a04329d5d85c43c0408176c51e6d) and click Open in Colab to open the script as a python notebook 182 | 2. Run the first code block to authenticate to the EE Python API (you'll be asked to open a link with your EE account and then to copy/paste the access code) 183 | 3. Set `wrs-granule-1` to your study area's WRS-1 tile ID (same as the WRS_1_GRANULE variable in your EE script) 184 | 4. Set `project_dir` to 'users/your_EE_username/LandTrendr' 185 | 5. Run the second and third code blocks 186 | 6. Run the code blocks titled 'Remove MSS WRS-1 to WRS-2 images' and 'Remove MSS WRS-1 to TM images' 187 | 7. Check your Assets tab to see that the following folders are empty: 188 | * users/your_EE_username/LandTrendr/your_WRS_tile_ID/WRS1_to_TM 189 | * users/your_EE_username/LandTrendr/your_WRS_tile_ID/WRS1_to_WRS2 190 | * users/your_EE_username/LandTrendr/your_WRS_tile_ID/ (empty except for the 'ref' image and the folders listed above) 191 | 192 | ### Script Step 8. Run Landtrendr on the entire time series 193 | 194 | 1. Set LLR_STEP to 8 and run the script. This runs the LandTrendr algorithm on the entire Landsat image collection and saves it as a new variable `lt`. 195 | ```js 196 | var LLR_STEP = 8; 197 | ``` 198 | 199 | 2. This will add the LandTrendr result to the map. 200 | 201 | ### Script Step 9. Display greatest disturbance map 202 | 203 | 1. Set LLR_STEP to 9 and run the script. 204 | ```js 205 | var LLR_STEP = 9; 206 | ``` 207 | 2. This runs the LandTrendr algorithm on the entire Landsat image collection and saves it as a new variable `lt`. It then runs displayGreatestDisturbance() on this result to produce two layers: 208 | - Magnitude of greatest disturbance (change in NDVI scaled by 100) 209 | - Year of greatest disturbance (rainbow color ramp: 1972 in purple, 2020 in red) 210 | -------------------------------------------------------------------------------- /landsatlinkr_workshop_template.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "id": "ElDwLGzPc-VS" 7 | }, 8 | "source": [ 9 | "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/gee-community/ee-LandsatLinkr/blob/main/landsatlinkr_workshop_template.ipynb)" 10 | ] 11 | }, 12 | { 13 | "cell_type": "markdown", 14 | "metadata": { 15 | "id": "G-uiqaRAOo_Q" 16 | }, 17 | "source": [ 18 | "## **1. Environment Setup**\n", 19 | "\n", 20 | "---" 21 | ] 22 | }, 23 | { 24 | "cell_type": "markdown", 25 | "metadata": { 26 | "id": "gzkiZpJPOXwm" 27 | }, 28 | "source": [ 29 | "Import the Earth Engine API and authenticate to your EE account \n", 30 | "This step requires persistence :)\n", 31 | "1. Run the block below, select Run Anyway if a pop up appears, then open the link that is generated\n", 32 | "2. Choose a Cloud Project to use with this notebook. If you don’t yet have a cloud-enabled project, select Choose Project > Create a New Cloud Project > and then give it a good name (this is permanent). You may have to then accept the Google Cloud Terms of Service. Open the link that is provided and select Agree and Continue on that page.\n", 33 | "3. Click 'Generate Token' (keep read-only scopes checked off)\n", 34 | "4. Choose your EE account\n", 35 | "5. Click Continue (not 'Back to Safety')\n", 36 | "6. Check both of the checkboxes and select 'Continue'\n", 37 | "7. Copy the Authorization Code at the bottom of the page and paste it in the code block below\n", 38 | "\n", 39 | "*If you're already authenticated, you will see 'Successfully saved authorization token' instead of a box to paste the code into*\n", 40 | "\n" 41 | ] 42 | }, 43 | { 44 | "cell_type": "code", 45 | "execution_count": null, 46 | "metadata": { 47 | "id": "uPG-SZDcOC7W" 48 | }, 49 | "outputs": [], 50 | "source": [ 51 | "import ee\n", 52 | "ee.Authenticate()\n", 53 | "ee.Initialize()" 54 | ] 55 | }, 56 | { 57 | "cell_type": "markdown", 58 | "metadata": { 59 | "id": "1htUCSlsOajh" 60 | }, 61 | "source": [ 62 | "Import the LandsatLinkr library from GitHub. This allows us to call the imported functions directly. " 63 | ] 64 | }, 65 | { 66 | "cell_type": "code", 67 | "execution_count": null, 68 | "metadata": { 69 | "id": "oSXgOmkYGcQO" 70 | }, 71 | "outputs": [], 72 | "source": [ 73 | "!rm -f -r /content/ee-LandsatLinkr\n", 74 | "!git clone https://github.com/gee-community/ee-LandsatLinkr --quiet\n", 75 | "import sys\n", 76 | "sys.path.append(\"/content/ee-LandsatLinkr\")\n", 77 | "from landsatlinkr import *" 78 | ] 79 | }, 80 | { 81 | "cell_type": "markdown", 82 | "metadata": { 83 | "id": "P4fR8xWSKj8Z" 84 | }, 85 | "source": [ 86 | "## **2. Identity MSS WRS-1 Tile ID(s)**\n", 87 | "\n", 88 | "---\n", 89 | "\n", 90 | "First, we have to determine which WRS-1 tile we’ll be working with. WRS-1 tile IDs refer to the image footprints of the MSS sensor.\n", 91 | "\n", 92 | "1. Open this EE script: https://code.earthengine.google.com/463f6aef9f6db3c248068bc0c6d9a530?hideCode=true \n", 93 | "2. In the map, zoom to your study area and select its location to reveal the WRS-1 tile IDs at that location (this may take a minute or two to load)\n", 94 | "3. Copy the WRS-1 tile ID from the pop-up window and paste this WRS-1 tile ID into your run script as the **```WRS_1_GRANULE```** variable below\n", 95 | "\n", 96 | "*If your study area overlaps with two or more tile footprints, you should note all of the WRS-1 tile IDs and process as many tiles that intersect the study region one at a time – the results can be composited later. For now, select one WRS-1 tile ID to begin with (you can only process one tile ID at a time).*" 97 | ] 98 | }, 99 | { 100 | "cell_type": "code", 101 | "execution_count": null, 102 | "metadata": { 103 | "id": "vPcdzuQdH8p0" 104 | }, 105 | "outputs": [], 106 | "source": [ 107 | "WRS_1_GRANULE = '047034'" 108 | ] 109 | }, 110 | { 111 | "cell_type": "markdown", 112 | "metadata": { 113 | "id": "hnsSQ7REP0eW" 114 | }, 115 | "source": [ 116 | "## **3. Create Asset Folders**\n", 117 | "\n", 118 | "---\n", 119 | "\n", 120 | "The LandsatLinkr workflow currently exports assets to your Earth Engine account to store the intermediate and final processing results. LandsatLinkr therefore requires a specific folder structure to run and the function **`createProjectDir`** will create this structure once you have updated the **`PROJECT_DIR`** variable to indicate your assets folder. \n", 121 | "\n", 122 | "\n", 123 | "1. Create an asset folder called ‘LandsatLinkr’ to organize all of your LLR projects in one place (each tile ID will generate a new subfolder). Head to the Assets tab of the [EE Code Editor](https://code.earthengine.google.com/) to make this folder in your cloud project assets.\n", 124 | "\n", 125 | "> You all now have at least one [cloud-enabled project](https://developers.google.com/earth-engine/cloud), so you’ll see two types of assets in your EE account: cloud project assets and legacy assets. Create your LandsatLinkr folder in your cloud project assets. The path to your new folder should look like this with your cloud project name replacing the placeholder: \n", 126 | "- `projects/[cloud_project]/assets/LandsatLinkr`\n", 127 | "\n", 128 | "2. Update the **`PROJECT_DIR`** variable to indicate **your** assets folder and run the code block below." 129 | ] 130 | }, 131 | { 132 | "cell_type": "code", 133 | "execution_count": null, 134 | "metadata": { 135 | "id": "6BysCvTPOWyN" 136 | }, 137 | "outputs": [], 138 | "source": [ 139 | "# Cloud Project Asset Folder\n", 140 | "# PROJECT_DIR = 'projects/[cloud_project]/assets/LandsatLinkr'\n", 141 | "\n", 142 | "PROJECT_DIR = 'projects/ee-annalisertaylor/assets/LandsatLinkr'\n", 143 | "createProjectDir(PROJECT_DIR, WRS_1_GRANULE)" 144 | ] 145 | }, 146 | { 147 | "cell_type": "markdown", 148 | "metadata": { 149 | "id": "C2fOKd_VQz8v" 150 | }, 151 | "source": [ 152 | "## **4.1 Set Imagery Parameters**\n", 153 | "\n", 154 | "---\n", 155 | "\n", 156 | "These parameters determine which Landsat images are included in our analysis.\n", 157 | "* **CRS** controls the geographic projection of the outputs.\n", 158 | "* **DOY_RANGE** filters the imagery by certain days of the year, 1 through 365. The final output of LLR is annual composites, so this variable controls which parts of the year are included in the composites. \n", 159 | "* **MAX_CLOUD** (0 to 100) controls the maximum allowable percentage of cloudiness for the included images.\n", 160 | "* **MAX_GEOM_RMSE** controls the maximum allowable spatial offset in units of a pixel." 161 | ] 162 | }, 163 | { 164 | "cell_type": "code", 165 | "execution_count": null, 166 | "metadata": { 167 | "id": "i-xS5u88QxyL" 168 | }, 169 | "outputs": [], 170 | "source": [ 171 | "CRS = 'EPSG:3857'\n", 172 | "DOY_RANGE = [160, 254]\n", 173 | "MAX_CLOUD = 50\n", 174 | "MAX_GEOM_RMSE = 0.5\n", 175 | "\n", 176 | "# All of the parameters needed to run LandTrendr (final step)\n", 177 | "LT_params = {\n", 178 | " # the bands you select here are the only ones you can then visualize as RGB\n", 179 | " # all choices: 'blue', 'green', 'red', 'nir', 'swir1', 'swir2', 'ndvi',\n", 180 | " # 'tcb' (tasseled cap brightness), 'tcg' (tasseled cap greenness),\n", 181 | " # 'tcw' (tasseled cap wetness), and 'tca’ (tasseled cap angle).\n", 182 | " 'ftvBands': ['red', 'green', 'blue'], \n", 183 | " 'maxSegments': 10,\n", 184 | " 'spikeThreshold': 0.7,\n", 185 | " 'vertexCountOvershoot': None,\n", 186 | " 'preventOneYearRecovery': None,\n", 187 | " 'recoveryThreshold': 0.5,\n", 188 | " 'pvalThreshold': None,\n", 189 | " 'bestModelProportion': None,\n", 190 | " 'minObservationsNeeded': None,\n", 191 | " 'scale': 30\n", 192 | "}\n", 193 | "\n", 194 | "# All of the parameters needed to run LandsatLinkr\n", 195 | "LLR_params = {\n", 196 | " 'maxRmseVerify': MAX_GEOM_RMSE,\n", 197 | " 'maxCloudCover': MAX_CLOUD,\n", 198 | " 'doyRange': DOY_RANGE,\n", 199 | " 'wrs1': WRS_1_GRANULE,\n", 200 | " 'crs': CRS,\n", 201 | " 'excludeIds': [],\n", 202 | " 'baseDir': PROJECT_DIR + '/' + WRS_1_GRANULE,\n", 203 | " 'projectDir': PROJECT_DIR,\n", 204 | " 'ltParams': LT_params,\n", 205 | " 'correctOffset': True,\n", 206 | " 'mssResample': 'nearest',\n", 207 | " 'mssScale': 30\n", 208 | "}" 209 | ] 210 | }, 211 | { 212 | "cell_type": "markdown", 213 | "metadata": { 214 | "id": "UyFooaFAhOye" 215 | }, 216 | "source": [ 217 | "## **4.2 Preview and Filter MSS Images**\n", 218 | "\n", 219 | "---\n", 220 | "\n", 221 | "\n", 222 | "This step will print thumbnails of all of the available MSS images based on your parameters so that you can assess image quality and remove the bad images from your image collection. This step will result in an updated **`EXCLUDE_IDs`** variable, and can take between 10 to 30 minutes depending on the number of images available.\n", 223 | "\n", 224 | "Preview MSS WRS-1 images, note bad images and add them to the **`EXCLUDE_IDS`** list. Run until all bad images are removed." 225 | ] 226 | }, 227 | { 228 | "cell_type": "code", 229 | "execution_count": null, 230 | "metadata": { 231 | "id": "nzCpb0k7g_Jy" 232 | }, 233 | "outputs": [], 234 | "source": [ 235 | "# Uncomment these two lines to re-run with your bad image IDs removed\n", 236 | "# EXCLUDE_IDS = []\n", 237 | "# LLR_params['excludeIds'] = EXCLUDE_IDS\n", 238 | "\n", 239 | "viewWrs1Col(LLR_params)" 240 | ] 241 | }, 242 | { 243 | "cell_type": "code", 244 | "execution_count": null, 245 | "metadata": { 246 | "id": "Y9zEeJqnlQSC" 247 | }, 248 | "outputs": [], 249 | "source": [ 250 | "EXCLUDE_IDS = [\n", 251 | " 'LM10470341972208GDS03',\n", 252 | " 'LM10470341972244AAA02',\n", 253 | " 'LM10470341974161AAA02',\n", 254 | " 'LM10470341974197AAA04',\n", 255 | " 'LM20470341975201GDS03',\n", 256 | " 'LM10470341976205AAA05',\n", 257 | " 'LM10470341976223GDS03',\n", 258 | " 'LM20470341976250GDS03',\n", 259 | " 'LM20470341977190GDS04',\n", 260 | " 'LM30470341978194GDS03',\n", 261 | " 'LM20470341977226AAA05',\n", 262 | " 'LM20470341977244GDS03',\n", 263 | " 'LM20470341978185AAA02',\n", 264 | " 'LM20470341978203AAA02',\n", 265 | " 'LM30470341978212GDS03',\n", 266 | " 'LM20470341978221AAA02',\n", 267 | " 'LM20470341978239AAA02',\n", 268 | " 'LM30470341979207AAA02',\n", 269 | " 'LM20470341979216XXX01',\n", 270 | " 'LM20470341980211AAA10',\n", 271 | " 'LM30470341980220AAA03',\n", 272 | " 'LM20470341980229AAA06',\n", 273 | " 'LM20470341981205AAA03',\n", 274 | " 'LM20470341981223AAA03',\n", 275 | " 'LM30470341982173AAA08',\n", 276 | " 'LM30470341982227AAA03'\n", 277 | "]\n", 278 | "\n", 279 | "LLR_params['excludeIds'] = EXCLUDE_IDS" 280 | ] 281 | }, 282 | { 283 | "cell_type": "markdown", 284 | "metadata": { 285 | "id": "35iL-UTkSs8L" 286 | }, 287 | "source": [ 288 | "## **5. Prepare the MSS Imagery**\n", 289 | "---\n", 290 | "\n", 291 | "**First**, we’ll create an MSS reference image. This reference image will be used for spectral normalization, as some of the sensors are inconsistent. This step will run a task that creates an image named 'ref' in your project's asset folder. This should take roughly 10 to 15 minutes. Once the task is completed (the word 'COMPLETED' will print below) and you see an image called 'ref' in your project's asset folder, you can move onto the next step.\n" 292 | ] 293 | }, 294 | { 295 | "cell_type": "code", 296 | "execution_count": null, 297 | "metadata": { 298 | "colab": { 299 | "background_save": true 300 | }, 301 | "id": "5VZqecuOSsSl" 302 | }, 303 | "outputs": [], 304 | "source": [ 305 | "mssRefTask = exportMssRefImg(LLR_params)\n", 306 | "monitorTaskStatus(mssRefTask)" 307 | ] 308 | }, 309 | { 310 | "cell_type": "markdown", 311 | "metadata": { 312 | "id": "uR7I7uwftW0m" 313 | }, 314 | "source": [ 315 | "**Next**, we’ll correct each included MSS image to that reference image **`ref`**. After running the block below, all of the selected MSS imagery will be harmonized within the MSS time series. This function runs a task to export a stack of annual composite images that are based on the mediod for each year, which is saved as **`MSS_WRS1_to_WRS2_stack`** in the asset folder that was created earlier. This step can take between 10 and 25 minutes. Once the task is completed, you can move onto the next step.\n" 316 | ] 317 | }, 318 | { 319 | "cell_type": "code", 320 | "execution_count": null, 321 | "metadata": { 322 | "id": "V-0lbJNxtZJA" 323 | }, 324 | "outputs": [], 325 | "source": [ 326 | "mssWrs1ToWrs2Task = processMssWrs1Imgs(LLR_params)\n", 327 | "monitorTaskStatus(mssWrs1ToWrs2Task)" 328 | ] 329 | }, 330 | { 331 | "cell_type": "markdown", 332 | "metadata": { 333 | "id": "Kpr75NIp-aEY" 334 | }, 335 | "source": [ 336 | "## **6. Harmonize MSS Images to TM**\n", 337 | "\n", 338 | "---\n", 339 | "\n", 340 | "**First**, the function **`exportMss2TmCoefCol`** aggregates a stratified random sample of pixels from coincident (collected simultaneously) MSS-TM image pairs. The resulting table (exported as an asset) is used to build a Random Forest (RF) regression model linking MSS imagery to TM imagery. This function will run a task that generates this table (**`mss_to_tm_coef_fc`**) and can take 10 to 20 minutes. Once this is completed you can move onto the next step.\n" 341 | ] 342 | }, 343 | { 344 | "cell_type": "code", 345 | "execution_count": null, 346 | "metadata": { 347 | "id": "fUOHSHEXQYat" 348 | }, 349 | "outputs": [], 350 | "source": [ 351 | "mss2TmInfoTask = exportMss2TmCoefCol(LLR_params)\n", 352 | "monitorTaskStatus(mss2TmInfoTask)" 353 | ] 354 | }, 355 | { 356 | "cell_type": "markdown", 357 | "metadata": { 358 | "id": "Mu6iqAo-y9Au" 359 | }, 360 | "source": [ 361 | "**Next**, the function **`exportMssOffset`** will use coincident MSS and TM WRS-2 images to calculate a per-pixel offset image. This ensures that the MSS imagery more closely match the TM values at each pixel. This step will run one task (**`MSS_offset`**) that takes 15 to 30 minutes." 362 | ] 363 | }, 364 | { 365 | "cell_type": "code", 366 | "execution_count": null, 367 | "metadata": { 368 | "id": "LMBaLJngzHwp" 369 | }, 370 | "outputs": [], 371 | "source": [ 372 | "mssOffsetTask = exportMssOffset(LLR_params)\n", 373 | "monitorTaskStatus(mssOffsetTask)" 374 | ] 375 | }, 376 | { 377 | "cell_type": "markdown", 378 | "metadata": { 379 | "id": "AEMG1HHfFlgh" 380 | }, 381 | "source": [ 382 | "**Finally**, the function **`exportFinalCorrectedMssCol`** will harmonize the WRS-1 MSS images to the TM time series using the Random Forest regression model calculated in a previous step. This step will run one task (**`WRS1_to_TM_stack`**) that takes 15 to 45 minutes.\n", 383 | "* Note that LandsatLinkr doesn't harmonize the WRS-2 (MSS) Landsat 4 and 5 images at all because there are coincident TM images which are preferred and used instead (except for 1983 when there are few TM images).\n", 384 | "\n", 385 | "Once the task is completed, you can move onto the next step.\n", 386 | "\n", 387 | "**We now have an MSS collection that is harmonized to TM!**" 388 | ] 389 | }, 390 | { 391 | "cell_type": "code", 392 | "execution_count": null, 393 | "metadata": { 394 | "id": "0trxpDSHR3eE" 395 | }, 396 | "outputs": [], 397 | "source": [ 398 | "mss2TmTask = exportFinalCorrectedMssCol(LLR_params)\n", 399 | "monitorTaskStatus(mss2TmTask)" 400 | ] 401 | }, 402 | { 403 | "cell_type": "markdown", 404 | "metadata": { 405 | "id": "XMnhczvubntn" 406 | }, 407 | "source": [ 408 | "## **7. Run LandTrendr on the 50-year Time Series**\n", 409 | "\n", 410 | "---\n", 411 | "\n", 412 | "This step collects all of the Landsat imagery from later years (TM, ETM+, OLI, and OLI-2 sensors), filters them by the same days of year (**`DOY_RANGE`**), masks out cloudy pixels, and creates annual medoid composite images. This is combined with the MSS collection you exported in the previous step for a total of 50 years!\n", 413 | "\n", 414 | "It then runs the [LandTrendr](https://emapr.github.io/LT-GEE/index.html) algorithm to distill 50+ years of spectral information into periods of relative stability and disturbance or change. This temporal segmentation information, such as the years and magnitude of disturbance, is captured in the output image, **`landtrendr`**, which is saved to your assets folder. This step can take approximately 2 hours to run.\n", 415 | "\n", 416 | "For more information on how to interpret and work with this output in Earth Engine, check out this [guide](https://emapr.github.io/LT-GEE/lt-gee-outputs.html) or this [workshop](https://youtu.be/gsfHNzbmY10). \n" 417 | ] 418 | }, 419 | { 420 | "cell_type": "code", 421 | "execution_count": null, 422 | "metadata": { 423 | "id": "BkVUN94ebp1h" 424 | }, 425 | "outputs": [], 426 | "source": [ 427 | "ltTask = exportLt(LLR_params)\n", 428 | "monitorTaskStatus(ltTask)" 429 | ] 430 | }, 431 | { 432 | "cell_type": "markdown", 433 | "metadata": { 434 | "id": "kqmbgye8WD9t" 435 | }, 436 | "source": [ 437 | "## **8. Explore 50 Years of Landsat in Earth Engine**\n", 438 | "\n", 439 | "---\n", 440 | "\n", 441 | "Now that we’ve run all of our exports, we’ll move into the EE Code Editor to visualize and interactively inspect the results. Open [this example script](https://code.earthengine.google.com/?scriptPath=users%2Fannalisertaylor%2FG4G%3ALLR_Workshop_100622.js) in Earth Engine (and login if prompted). You may choose to save a copy to your account and name it ‘LLR_Workshop’ or something similar. Once your **`landtrendr`** asset has exported, change the main parameters in the EE script to investigate your data! " 442 | ] 443 | } 444 | ], 445 | "metadata": { 446 | "colab": { 447 | "collapsed_sections": [], 448 | "provenance": [], 449 | "toc_visible": true 450 | }, 451 | "kernelspec": { 452 | "display_name": "Python 3", 453 | "name": "python3" 454 | }, 455 | "language_info": { 456 | "name": "python" 457 | } 458 | }, 459 | "nbformat": 4, 460 | "nbformat_minor": 0 461 | } 462 | -------------------------------------------------------------------------------- /landsatlinkr.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @license 3 | * Copyright 2020 Justin Braaten 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | 19 | 20 | // Ideas for correcting sensors: https://ieeexplore.ieee.org/abstract/document/9093966 21 | 22 | var msslib = require('users/jstnbraaten/modules:msslib/msslib.js'); 23 | var ltgee = require('users/emaprlab/public:Modules/LandTrendr.js'); 24 | var animation = require('users/gena/packages:animation'); 25 | 26 | 27 | 28 | /** 29 | * Returns a filtered TM WRS-2 T1 surface reflectance image collection. 30 | * 31 | * @param {ee.Geometry | ee.Feature} aoi An ee.Filter to filter TM image collection. 32 | * 33 | * @return {ee.ImageCollection} An MSS WRS-2 image collection filtered by 34 | * bounds and quality. 35 | */ 36 | function getTmWrs2Col(aoi){ 37 | var tm4 = ee.ImageCollection("LANDSAT/LT04/C01/T1_SR") 38 | .filterBounds(aoi); 39 | var tm5 = ee.ImageCollection("LANDSAT/LT05/C01/T1_SR") 40 | .filterBounds(aoi); 41 | return tm4.merge(tm5); 42 | } 43 | exports.getTmWrs2Col = getTmWrs2Col; 44 | 45 | 46 | 47 | 48 | /** 49 | * Add unique path, row, orbit ID as image property for joining TM and MSS collections. 50 | * 51 | * @param {ee.Image} tmWrs2Col A TM image collection. 52 | * @param {ee.Image} mssWrs2Col A MSS image collection. 53 | * 54 | * @return {ee.ImageCollection} An image collection ____WAH_____. 55 | */ 56 | function coincidentTmMssCol(tmWrs2Col, mssWrs2Col){ 57 | var filter = ee.Filter.equals({leftField: 'imgID', rightField: 'imgID'}); 58 | var join = ee.Join.saveFirst('coincidentTmMss'); 59 | return ee.ImageCollection(join.apply(tmWrs2Col, mssWrs2Col, filter)); 60 | } 61 | exports.coincidentTmMssCol = coincidentTmMssCol; 62 | 63 | 64 | 65 | 66 | /** 67 | * Add unique path, row, orbit ID as image property for joining TM and MSS collections. 68 | * 69 | * @param {ee.Image} img A TM or MSS image. 70 | * 71 | * @return {ee.ImageCollection} A copy of the input image with an 'imgID' 72 | * property added to the image describing the unique path, row, orbit. 73 | */ 74 | function addTmToMssJoinId(img){ 75 | //return col.map(function(img) { 76 | var date = ee.Image(img).date(); 77 | var year = ee.Algorithms.String(date.get('year')); 78 | var doy = ee.Algorithms.String(date.getRelative('day', 'year')); 79 | var path = ee.Algorithms.String(img.getNumber('WRS_PATH').toInt()); 80 | var row = ee.Algorithms.String(img.getNumber('WRS_ROW').toInt()); 81 | var yearDoy = year.cat(doy).cat(path).cat(row); 82 | return img.set({'imgID': yearDoy, 83 | 'path': path, 84 | 'row': row 85 | }); 86 | //}); 87 | } 88 | exports.addTmToMssJoinId = addTmToMssJoinId; 89 | 90 | 91 | 92 | 93 | /** 94 | * Returns the footprint of an image as a ee.Geometry.Polygon. 95 | * 96 | * @param {ee.Image} img The image to get the footprint for. 97 | * 98 | * @return {ee.Geometry.Polygon} The ee.Geometry.Polygon representation of 99 | * an image's footprint. 100 | */ 101 | function getFootprint(img){ 102 | return ee.Geometry.Polygon(ee.Geometry(img.get('system:footprint')).coordinates())} 103 | exports.getFootprint = getFootprint; 104 | 105 | 106 | 107 | /** 108 | * Generates an ee.Filter for filtering MSS and TM image collection by 109 | * intersection with a given geometry. 110 | * 111 | * @param {ee.Geometry | ee.Feature} aoi Area of interest to filter collection to. 112 | * Include images less than given value. 113 | * 114 | * @return {ee.Filter} A filter to be passed as an argument to the .filter() 115 | * ee.ImageCollection method. 116 | */ 117 | function filterBounds(aoi) { 118 | return ee.Filter.bounds(aoi); 119 | } 120 | exports.filterBounds = filterBounds; 121 | 122 | 123 | /** 124 | * Returns a cloud and cloud shadow mask from CFmask. 125 | * @param {ee.Image} img Landsat SR image. 126 | * @return {ee.Image} A 0/1 mask image to be used with .updateMask(). 127 | */ 128 | function getCfmask(img) { 129 | var cloudShadowBitMask = 1 << 3; 130 | var cloudsBitMask = 1 << 5; 131 | var qa = img.select('pixel_qa'); 132 | var mask = qa.bitwiseAnd(cloudShadowBitMask) 133 | .eq(0) 134 | .and(qa.bitwiseAnd(cloudsBitMask).eq(0)); 135 | return mask; 136 | } 137 | exports.getCfmask = getCfmask; 138 | 139 | 140 | function applyCfmask(img) { 141 | var mask = getCfmask(img); 142 | return img.updateMask(mask); 143 | } 144 | exports.getCfmask = getCfmask; 145 | 146 | 147 | 148 | // ############################################################################# 149 | // ### Process steps ### 150 | // ############################################################################# 151 | 152 | 153 | /** 154 | * Display the series of WRS-1 images for a given WRS-1 granule. 155 | */ 156 | function viewWrs1Col(params) { 157 | print('Displaying WRS-1 images to the console'); 158 | var granuleGeom = msslib.getWrs1GranuleGeom(params.wrs1); 159 | params.aoi = ee.Geometry(granuleGeom.get('centroid')); 160 | params.wrs = '1'; 161 | var mssDnCol = msslib.getCol(params) 162 | .filter(ee.Filter.eq('pr', params.wrs1)); 163 | msslib.viewThumbnails(mssDnCol); 164 | } 165 | exports.viewWrs1Col = viewWrs1Col; 166 | 167 | /** 168 | * Display the WRS-1 grid to the map. 169 | */ 170 | function wrs1GranuleSelector() { 171 | var wrs1Granules = ee.FeatureCollection('users/jstnbraaten/wrs/wrs1_descending_land'); 172 | Map.addLayer(wrs1Granules, {color: 'grey'}, null, null, 0.5); 173 | 174 | var message = ui.Label({value: 'Click granules to print ID. Wait patiently after clicking. Repeat as needed.', 175 | style: {position: 'top-center'}}); 176 | var holder = ui.Panel({style: {width: '170px', height: '220px', position: 'top-left'}}); 177 | var label = ui.Label({value: 'WRS-1 Granule ID(s):'}); 178 | var ids = ui.Panel({style: {backgroundColor: '#DCDCDC'}}); 179 | holder.add(label); 180 | holder.add(ids); 181 | 182 | Map.add(message); 183 | Map.add(holder); 184 | Map.style().set('cursor', 'crosshair'); 185 | Map.onClick(function(e) { 186 | ids.clear(); 187 | var nLayers = Map.layers().length(); 188 | for(var i=0; i < nLayers-1; i++) { 189 | Map.layers().remove(Map.layers().get(1)); 190 | } 191 | 192 | var point = ee.Geometry.Point(e.lon, e.lat); 193 | var joinFilter = ee.Filter.intersects({leftField: '.geo', rightField: '.geo', maxError: 500}); 194 | var join = ee.Join.simple(); 195 | var intersectingFeatures = join.apply(wrs1Granules, ee.FeatureCollection(point), joinFilter); 196 | 197 | intersectingFeatures.toList(intersectingFeatures.size()).evaluate(function(fList) { 198 | var colors = ['red', 'blue', 'green', 'yellow', 'orange', 'pink', 'purple']; 199 | for(var i in fList) { 200 | var f = ee.Feature(fList[i]); 201 | var outline = ee.Image().byte().paint({ 202 | featureCollection: ee.FeatureCollection(f), 203 | color: 1, 204 | width: 3 205 | }); 206 | var pr = f.get('PR').getInfo(); 207 | var title = pr + ' ' + colors[i]; 208 | Map.addLayer(outline, {palette: colors[i]}, title); 209 | ids.add(ui.Label({value: pr, style: {color: colors[i], backgroundColor: '#DCDCDC'}})); 210 | } 211 | }); 212 | }); 213 | } 214 | exports.wrs1GranuleSelector = wrs1GranuleSelector; 215 | 216 | 217 | 218 | // ############################################################################# 219 | // ### Reference prep ### 220 | // ############################################################################# 221 | 222 | /** 223 | * calculate the medoid of a collection. 224 | * 225 | */ 226 | function getMedoid(col, bands) { 227 | col = col.select(bands); 228 | var median = col.median(); 229 | var difFromMedian = col.map(function(img) { 230 | var dif = ee.Image(img).subtract(median).pow(ee.Image.constant(2)); 231 | return dif.reduce(ee.Reducer.sum()) 232 | .addBands(img); 233 | }); 234 | var bandNames = difFromMedian.first().bandNames(); 235 | var len = bandNames.length(); 236 | var bandsPos = ee.List.sequence(1, len.subtract(1)); 237 | var bandNamesSub = bandNames.slice(1); 238 | return difFromMedian.reduce(ee.Reducer.min(len)).select(bandsPos, bandNamesSub); 239 | } 240 | exports.getMedoid = getMedoid; 241 | 242 | function getRefImg(params) { 243 | var granuleGeoms = msslib.getWrs1GranuleGeom(params.wrs1); 244 | var centroid = ee.Geometry(granuleGeoms.get('centroid')); 245 | var bounds = ee.Geometry(granuleGeoms.get('bounds')); 246 | 247 | var refCol = msslib.getCol({ 248 | aoi: bounds, 249 | wrs: '2', 250 | yearRange: [1983, 1987], // Use five early years - want good coverage, but near to MSS WRS-1 window. 251 | doyRange: params.doyRange, 252 | }).map(addTmToMssJoinId); 253 | 254 | var tmCol = getTmWrs2Col(bounds) 255 | .filterDate('1983-01-01', '1988-01-01') 256 | .map(addTmToMssJoinId); 257 | 258 | var coincident = coincidentTmMssCol(refCol, tmCol) 259 | .map(function(img) { 260 | var mask = getCfmask(ee.Image(img.get('coincidentTmMss'))); 261 | var imgToa = msslib.calcToa(img); 262 | return imgToa.updateMask(mask); 263 | }); 264 | 265 | return msslib.addTc(msslib.addNdvi(getMedoid(coincident, ['green', 'red', 'red_edge', 'nir']))) 266 | .select(['green', 'red', 'nir', 'ndvi', 'tcb', 'tcg', 'tca']) // TODO: scale these data to get them to int16 267 | .set('bounds', bounds); 268 | } 269 | exports.getRefImg = getRefImg; 270 | 271 | 272 | function exportMssRefImg(params) { 273 | print('Preparing reference image export task, please wait'); 274 | var refImg = getRefImg(params); 275 | Export.image.toAsset({ 276 | image: refImg, 277 | description: 'MSS-reference-image', 278 | assetId: params.baseDir + '/ref', 279 | region: ee.Geometry(refImg.get('bounds')), 280 | scale: 60, 281 | crs: params.crs, 282 | maxPixels: 1e13 283 | }); 284 | } 285 | exports.exportMssRefImg = exportMssRefImg; 286 | 287 | 288 | 289 | 290 | 291 | 292 | // ############################################################################# 293 | 294 | 295 | 296 | 297 | /** 298 | * Returns an example Tm image. 299 | * 300 | * @return {ee.Image} Example TM image. 301 | */ 302 | function exampleTmImg() { 303 | return ee.Image('LANDSAT/LT05/C01/T1_SR/LT05_045029_19840728'); 304 | } 305 | exports.exampleTmImg = exampleTmImg; 306 | 307 | 308 | 309 | 310 | 311 | 312 | // Example AOIs 313 | var wrs2045029 = ee.Geometry.Point([-121.454, 44.47]); 314 | exports.wrs2045029 = wrs2045029; 315 | 316 | 317 | 318 | 319 | // ############################################################################# 320 | // ### Process MSS WRS-1 images ### 321 | // ############################################################################# 322 | 323 | 324 | // Create a new image that is the concatenation of three images: a constant, 325 | // the SWIR1 band, and the SWIR2 band. 326 | 327 | /** 328 | * Have found that the best regression is robustLinear on entire image 329 | * with scale set as 60. 330 | * Also tried: robustLinear, scale 300 331 | * linear, scale 300 332 | * stratified sample based on image segmentation k-means - worst - could be poor sampling 333 | * Yet to try using all bands to predicit a given band - multiple regression 334 | * Yest to try using different threshold and scale for masking dif to ref in `correctMssImg` 335 | */ 336 | function calcRegression(xImg, yImg, xBand, yBand, aoi, scale) { 337 | var constant = ee.Image(1); 338 | var xVar = xImg.select(xBand); 339 | var yVar = yImg.select(yBand); 340 | var imgRegress = ee.Image.cat(constant, xVar, yVar); 341 | 342 | var linearRegression = imgRegress.reduceRegion({ 343 | reducer: ee.Reducer.robustLinearRegression({ 344 | numX: 2, 345 | numY: 1 346 | }), 347 | geometry: aoi, 348 | scale: scale, 349 | maxPixels: 1e13 350 | }); 351 | 352 | var coefList = ee.Array(linearRegression.get('coefficients')).toList(); 353 | var intercept = ee.List(coefList.get(0)).get(0); 354 | var slope = ee.List(coefList.get(1)).get(0); 355 | var rmse = ee.Array(linearRegression.get('residuals')).toList().get(0); 356 | return ee.Dictionary({slope: slope, intercept: intercept, rmse: rmse}); 357 | } 358 | exports.calcRegression = calcRegression; 359 | 360 | // Function to apply correction to reference image. 361 | function applyCoef(img, band, coef) { 362 | coef = ee.Dictionary(coef); 363 | return img.select(band) 364 | .multiply(ee.Image.constant(coef.getNumber('slope'))) 365 | .add(ee.Image.constant(coef.getNumber('intercept'))); 366 | } 367 | exports.applyCoef = applyCoef; 368 | 369 | 370 | function getSampleImg(img, ref, band) { 371 | var dif = img.select(band) 372 | .subtract(ref.select(band)).rename('dif'); 373 | 374 | var difThresh = dif.reduceRegion({ 375 | reducer: ee.Reducer.percentile({ 376 | percentiles: [40, 60], 377 | maxRaw: 1000000, 378 | maxBuckets: 1000000, 379 | minBucketWidth: 0.00000000001 380 | }), 381 | geometry: img.geometry(), 382 | scale: 60, 383 | maxPixels: 1e13 384 | }); 385 | 386 | var mask = dif.gt(difThresh.getNumber('dif_p40')) 387 | .and(dif.lt(difThresh.getNumber('dif_p60'))); 388 | 389 | return img.updateMask(mask); 390 | } 391 | exports.getSampleImg = getSampleImg; 392 | 393 | 394 | // Function to make normalization function. 395 | function correctMssImg(img) { 396 | var ref = ee.Image(img.get('ref_img')); 397 | var granuleGeoms = msslib.getWrs1GranuleGeom(img.getString('pr')); 398 | var granule = ee.Feature(granuleGeoms.get('granule')).geometry(); 399 | 400 | // // ** Use three bands for mask 401 | // var allMask = getSampleImg(img, ref, 'green').mask().multiply( 402 | // getSampleImg(img, ref, 'red').mask()).multiply( 403 | // getSampleImg(img, ref, 'nir').mask()); 404 | // var sampleImg = img.updateMask(allMask); 405 | 406 | // var greenCoef = calcRegression(sampleImg, ref, 'green', 'green', granule, 60); 407 | // var redCoef = calcRegression(sampleImg, ref, 'red', 'red', granule, 60); 408 | // var nirCoef = calcRegression(sampleImg, ref, 'nir', 'nir', granule, 60); 409 | // var ndviCoef = calcRegression(sampleImg, ref, 'ndvi', 'ndvi', granule, 60); 410 | // var tcbCoef = calcRegression(sampleImg, ref, 'tcb', 'tcb', granule, 60); 411 | // var tcgCoef = calcRegression(sampleImg, ref, 'tcg', 'tcg', granule, 60); 412 | // var tcaCoef = calcRegression(sampleImg, ref, 'tca', 'tca', granule, 60); 413 | // // ** Use three bands for mask 414 | 415 | var greenCoef = calcRegression(getSampleImg(img, ref, 'green'), ref, 'green', 'green', granule, 60); 416 | var redCoef = calcRegression(getSampleImg(img, ref, 'red'), ref, 'red', 'red', granule, 60); 417 | var nirCoef = calcRegression(getSampleImg(img, ref, 'nir'), ref, 'nir', 'nir', granule, 60); 418 | var ndviCoef = calcRegression(getSampleImg(img, ref, 'ndvi'), ref, 'ndvi', 'ndvi', granule, 60); 419 | var tcbCoef = calcRegression(getSampleImg(img, ref, 'tcb'), ref, 'tcb', 'tcb', granule, 60); 420 | var tcgCoef = calcRegression(getSampleImg(img, ref, 'tcg'), ref, 'tcg', 'tcg', granule, 60); 421 | var tcaCoef = calcRegression(getSampleImg(img, ref, 'tca'), ref, 'tca', 'tca', granule, 60); 422 | 423 | return ee.Image(ee.Image.cat( 424 | applyCoef(img, 'green', greenCoef).toFloat(), 425 | applyCoef(img, 'red', redCoef).toFloat(), 426 | applyCoef(img, 'nir', nirCoef).toFloat(), 427 | applyCoef(img, 'ndvi', nirCoef).toFloat(), 428 | applyCoef(img, 'tcb', tcbCoef).toFloat(), 429 | applyCoef(img, 'tcg', tcgCoef).toFloat(), 430 | applyCoef(img, 'tca', tcaCoef).toFloat()) 431 | .rename(['green', 'red', 'nir', 'ndvi', 'tcb', 'tcg', 'tca']) 432 | .copyProperties(img, img.propertyNames())); 433 | } 434 | 435 | 436 | 437 | function prepMss(img) { 438 | var toa = msslib.calcToa(img); 439 | var toaAddBands = msslib.addTc(msslib.addNdvi(toa)); 440 | var toaAddBandsMask = msslib.applyQaMask(toaAddBands); 441 | return msslib.applyMsscvm(toaAddBandsMask); 442 | } 443 | exports.prepMss = prepMss; 444 | 445 | function processMssWrs1Img(img) { 446 | var toaAddBandsMsscvmMask = prepMss(img); 447 | var corrected = correctMssImg(toaAddBandsMsscvmMask); 448 | return corrected; 449 | } 450 | exports.processMssWrs1Img = processMssWrs1Img; 451 | 452 | function processMssWrs1Imgs(params) { 453 | print('Preparing MSS WRS-1 image processing tasks, please wait'); 454 | var granuleGeom = msslib.getWrs1GranuleGeom(params.wrs1); 455 | params.aoi = ee.Geometry(granuleGeom.get('centroid')); 456 | params.wrs = '1'; 457 | var mssCol = msslib.getCol(params) 458 | .filter(ee.Filter.eq('pr', params.wrs1)) 459 | .map(function(img) { 460 | return img.set('ref_img', ee.Image(params.baseDir + '/ref')); 461 | }); 462 | //print(mssCol); 463 | //var years = ee.List(mssCol.aggregate_array('year').distinct()).sort().getInfo(); // TODO: make this so that all year are written out - include dummies - the script that reads them in assumes that all years exist 464 | 465 | var dummy = ee.Image([0, 0, 0, 0, 0, 0, 0, 0]).selfMask().toShort() 466 | .rename(['blue', 'green', 'red', 'nir', 'ndvi', 'tcb', 'tcg', 'tca']); 467 | var outImg; 468 | for(var y = 1972; y <= 1982; y++) { 469 | var yrCol = mssCol.filter(ee.Filter.eq('year', y)); 470 | if(yrCol.size().getInfo() === 0) { // Try to use ee.Algorithms.If - so that the browser does not hang. 471 | outImg = dummy.set({ 472 | dummy: true, 473 | year: y, 474 | 'system:time_start': ee.Date.fromYMD(y, 1, 1) 475 | }); 476 | } else { 477 | yrCol = yrCol.map(processMssWrs1Img); 478 | outImg = getMedoid(yrCol, ['green', 'red', 'nir', 'ndvi', 'tcb', 'tcg', 'tca']) 479 | .set({ 480 | dummy: false, 481 | year: y, 482 | 'system:time_start': ee.Date.fromYMD(y, 1, 1) 483 | }); 484 | } 485 | 486 | Export.image.toAsset({ 487 | image: outImg, 488 | description: y.toString(), 489 | assetId: params.baseDir + '/WRS1_to_WRS2/' + y.toString(), 490 | region: ee.Feature(granuleGeom.get('granule')).geometry(), 491 | scale: 60, 492 | crs: params.crs 493 | }); 494 | } 495 | } 496 | exports.processMssWrs1Imgs = processMssWrs1Imgs; 497 | 498 | 499 | function correctMssWrs2(params) { // NOTE: this is just grabbing 1983 for now. 500 | var aoi = ee.Feature( 501 | msslib.getWrs1GranuleGeom(params.wrs1).get('granule')).geometry(); 502 | var mssCol = msslib.getCol({ 503 | aoi: aoi, 504 | wrs: '2', 505 | doyRange: params.doyRange 506 | }).filterDate('1983-01-01', '1984-01-01') 507 | .map(prepMss); 508 | 509 | return correctMssImgToMedianTm(mssCol, params); 510 | } 511 | exports.correctMssWrs2 = correctMssWrs2; 512 | 513 | // ############################################################################# 514 | // ### Process TM images ### 515 | // ############################################################################# 516 | 517 | 518 | // Function to get and rename bands of interest from OLI. 519 | function renameOli(img) { 520 | return img.select( 521 | ['B2', 'B3', 'B4', 'B5', 'B6', 'B7', 'pixel_qa'], 522 | ['blue', 'green', 'red', 'nir', 'swir1', 'swir2', 'pixel_qa']); 523 | } 524 | 525 | // Function to get and rename bands of interest from ETM+. 526 | function renameTm(img) { 527 | return img.select( 528 | ['B1', 'B2', 'B3', 'B4', 'B5', 'B7', 'pixel_qa'], 529 | ['blue', 'green', 'red', 'nir', 'swir1', 'swir2', 'pixel_qa']); 530 | } 531 | 532 | function tmAddIndices(img) { 533 | var b = ee.Image(img).select(['blue', 'green', 'red', 'nir', 'swir1', 'swir2']); 534 | var brt_coeffs = ee.Image.constant([0.2043, 0.4158, 0.5524, 0.5741, 0.3124, 0.2303]); 535 | var grn_coeffs = ee.Image.constant([-0.1603, -0.2819, -0.4934, 0.7940, -0.0002, -0.1446]); 536 | var brightness = b.multiply(brt_coeffs).reduce(ee.Reducer.sum()).round().toShort(); 537 | var greenness = b.multiply(grn_coeffs).reduce(ee.Reducer.sum()).round().toShort(); 538 | var angle = (greenness.divide(brightness)).atan().multiply(180 / Math.PI).multiply(100).round().toShort(); 539 | var ndvi = img.normalizedDifference(['nir', 'red']).rename('ndvi').multiply(1000).round().toShort(); 540 | var tc = ee.Image.cat(ndvi, brightness, greenness, angle).rename(['ndvi', 'tcb', 'tcg', 'tca']); 541 | return img.addBands(tc); 542 | } 543 | 544 | function gatherTmCol(params) { 545 | var granuleGeom = msslib.getWrs1GranuleGeom(params.wrs1); 546 | var aoi = ee.Feature(granuleGeom.get('granule')).geometry(); 547 | var dateFilter = ee.Filter.calendarRange(params.doyRange[0], params.doyRange[1], 'day_of_year'); 548 | var startDate = ee.Date.fromYMD(params.yearRange[0], 1, 1); 549 | var endDate = startDate.advance(1, 'year'); 550 | var oliCol = ee.ImageCollection('LANDSAT/LC08/C01/T1_SR') 551 | .filterBounds(aoi).filterDate(startDate, endDate).filter(dateFilter).map(prepOli); 552 | var etmCol = ee.ImageCollection('LANDSAT/LE07/C01/T1_SR') 553 | .filterBounds(aoi).filterDate(startDate, endDate).filter(dateFilter).map(prepTm); 554 | var tm5Col = ee.ImageCollection('LANDSAT/LT05/C01/T1_SR') 555 | .filterBounds(aoi).filterDate(startDate, endDate).filter(dateFilter).map(prepTm); 556 | var tm4Col = ee.ImageCollection('LANDSAT/LT04/C01/T1_SR') 557 | .filterBounds(aoi).filterDate(startDate, endDate).filter(dateFilter).map(prepTm); 558 | return tm4Col.merge(tm5Col).merge(etmCol).merge(oliCol); 559 | } 560 | exports.gatherTmCol = gatherTmCol; 561 | 562 | // Define function to prepare OLI images. 563 | function prepOli(img) { 564 | var orig = img; 565 | img = renameOli(img); 566 | img = tmAddIndices(img); 567 | img = applyCfmask(img); 568 | return ee.Image(img.copyProperties(orig, orig.propertyNames())); 569 | } 570 | 571 | // Define function to prepare ETM+ images. 572 | function prepTm(img) { 573 | var orig = img; 574 | img = renameTm(img); 575 | img = tmAddIndices(img); 576 | img = applyCfmask(img); 577 | return ee.Image(img.copyProperties(orig, orig.propertyNames())); 578 | } 579 | exports.prepTm = prepTm; 580 | 581 | 582 | 583 | function getCoincidentTmMssCol(params) { 584 | var aoi = ee.Feature( 585 | msslib.getWrs1GranuleGeom(params.wrs1).get('granule')).geometry(); 586 | var mssCol = msslib.getCol({ 587 | aoi: aoi, 588 | wrs: '2', 589 | doyRange: params.doyRange, 590 | excludeIds: params.excludeIds 591 | }) 592 | .map(addTmToMssJoinId); 593 | 594 | var tmCol = getTmWrs2Col(aoi).map(addTmToMssJoinId); 595 | var coincident = coincidentTmMssCol(mssCol, tmCol); 596 | return coincident; 597 | } 598 | exports.getCoincidentTmMssCol = getCoincidentTmMssCol; 599 | 600 | // Function to make normalization function. 601 | function getMss2TmCoefCol(img) { 602 | var sampleMask = ee.Image.random().gt(0.90); 603 | var xImg = msslib.addTc(msslib.addNdvi(msslib.calcToa(img))); 604 | var xImgSamp = xImg.updateMask(sampleMask); 605 | var yImg = prepTm(ee.Image(xImg.get('coincidentTmMss'))); 606 | var granule= ee.Feature(ee.FeatureCollection('users/jstnbraaten/wrs/wrs2_descending_land') 607 | .filter(ee.Filter.eq('PR', xImg.getString('pr'))).first()).geometry(); 608 | 609 | var blueCoef = calcRegression(xImgSamp, yImg, 'green', 'blue', granule, 150); // TODO: this 300 is maybe not ideal - could try a small image sample like 10% or 5% or 1%. 610 | var greenCoef = calcRegression(xImgSamp, yImg, 'green', 'green', granule, 150); 611 | var redCoef = calcRegression(xImgSamp, yImg, 'red', 'red', granule, 150); 612 | var nirCoef = calcRegression(xImgSamp, yImg, 'nir', 'nir', granule, 150); 613 | var ndviCoef = calcRegression(xImg, yImg, 'ndvi', 'ndvi', granule, 150); 614 | var tcbCoef = calcRegression(xImg, yImg, 'tcb', 'tcb', granule, 150); 615 | var tcgCoef = calcRegression(xImg, yImg, 'tcg', 'tcg', granule, 150); 616 | var tcaCoef = calcRegression(xImg, yImg, 'tca', 'tca', granule, 150); 617 | 618 | var coef = { 619 | blue_coef: blueCoef, 620 | green_coef: greenCoef, 621 | red_coef: redCoef, 622 | nir_coef: nirCoef, 623 | ndvi_coef: ndviCoef, 624 | tcb_coef: tcbCoef, 625 | tcg_coef: tcgCoef, 626 | tca_coef: tcaCoef, 627 | }; 628 | 629 | xImg = xImg.set('mss_2_tm_coef', coef); 630 | var xImgCor = _correctMssImg(xImg); 631 | 632 | return yImg.select(xImgCor.bandNames()).subtract(xImgCor.select(xImgCor.bandNames())) 633 | .copyProperties(xImgCor, xImgCor.propertyNames()); 634 | } 635 | exports.getMss2TmCoefCol = getMss2TmCoefCol; 636 | 637 | 638 | function exportTm2mssCoefCol(params) { 639 | var col = getCoincidentTmMssCol(params); 640 | var coefFc = col.map(getTm2mssCoefCol); 641 | Export.table.toAsset({ 642 | collection: coefFc, 643 | description: 'tm2MssCoefCol', 644 | assetId: params.baseDir + '/tm2MssCoefCol' 645 | }); 646 | } 647 | exports.exportTm2mssCoefCol = exportTm2mssCoefCol; 648 | 649 | function exportMss2TmCoefCol(params) { 650 | var col = getCoincidentTmMssCol(params); 651 | var mssOffsetCol = col.map(getMss2TmCoefCol); 652 | var coefFc = mssOffsetCol.map(function(img) { 653 | var coefs = ee.Dictionary(img.get('mss_2_tm_coef')); 654 | var blueCoef = ee.Dictionary(coefs.get('blue_coef')); 655 | var greenCoef = ee.Dictionary(coefs.get('green_coef')); 656 | var redCoef = ee.Dictionary(coefs.get('red_coef')); 657 | var nirCoef = ee.Dictionary(coefs.get('nir_coef')); 658 | var ndviCoef = ee.Dictionary(coefs.get('ndvi_coef')); 659 | var tcbCoef = ee.Dictionary(coefs.get('tcb_coef')); 660 | var tcgCoef = ee.Dictionary(coefs.get('tcg_coef')); 661 | var tcaCoef = ee.Dictionary(coefs.get('tca_coef')); 662 | 663 | var coef = { 664 | 'blue_slope': blueCoef.getNumber('slope'), 665 | 'blue_intercept': blueCoef.getNumber('intercept'), 666 | 'blue_rmse': blueCoef.getNumber('rmse'), 667 | 'green_slope': greenCoef.getNumber('slope'), 668 | 'green_intercept': greenCoef.getNumber('intercept'), 669 | 'green_rmse': greenCoef.getNumber('rmse'), 670 | 'red_slope': redCoef.getNumber('slope'), 671 | 'red_intercept': redCoef.getNumber('intercept'), 672 | 'red_rmse': redCoef.getNumber('rmse'), 673 | 'nir_slope': nirCoef.getNumber('slope'), 674 | 'nir_intercept': nirCoef.getNumber('intercept'), 675 | 'nir_rmse': nirCoef.getNumber('rmse'), 676 | 'ndvi_slope': ndviCoef.getNumber('slope'), 677 | 'ndvi_intercept': ndviCoef.getNumber('intercept'), 678 | 'ndvi_rmse': ndviCoef.getNumber('rmse'), 679 | 'tcb_slope': tcbCoef.getNumber('slope'), 680 | 'tcb_intercept': tcbCoef.getNumber('intercept'), 681 | 'tcb_rmse': tcbCoef.getNumber('rmse'), 682 | 'tcg_slope': tcgCoef.getNumber('slope'), 683 | 'tcg_intercept': tcgCoef.getNumber('intercept'), 684 | 'tcg_rmse': tcgCoef.getNumber('rmse'), 685 | 'tca_slope': tcaCoef.getNumber('slope'), 686 | 'tca_intercept': tcaCoef.getNumber('intercept'), 687 | 'tca_rmse': tcaCoef.getNumber('rmse'), 688 | }; 689 | 690 | return ee.Feature(ee.Geometry.Point(0, 0)).set(coef) 691 | .copyProperties(img, ['imgID', 'year', 'path', 'row', 'pr']); 692 | }); 693 | 694 | var medianOffset = mssOffsetCol.median().round().toShort(); 695 | var granuleGeom = msslib.getWrs1GranuleGeom(params.wrs1); 696 | Export.image.toAsset({ 697 | image: medianOffset, 698 | description: 'medianOffset', 699 | assetId: params.baseDir + '/mss_offset', 700 | region: ee.Feature(granuleGeom.get('granule')).geometry(), 701 | scale: 60, 702 | crs: params.crs 703 | }); 704 | 705 | Export.table.toAsset({ 706 | collection: coefFc, 707 | description: 'mss2TmCoefCol', 708 | assetId: params.baseDir + '/mss2TmCoefCol' 709 | }); 710 | } 711 | exports.exportMss2TmCoefCol = exportMss2TmCoefCol; 712 | 713 | 714 | function _getMedianCoef(table, coef) { 715 | return ee.List(table.aggregate_array(coef)) 716 | .reduce(ee.Reducer.median()); 717 | } 718 | 719 | function getMedianCoef(table) { 720 | return ee.Dictionary({ 721 | blue_coef: { 722 | slope: _getMedianCoef(table, 'blue_slope'), 723 | intercept: _getMedianCoef(table, 'blue_intercept') 724 | }, 725 | green_coef: { 726 | slope: _getMedianCoef(table, 'green_slope'), 727 | intercept: _getMedianCoef(table, 'green_intercept') 728 | }, 729 | red_coef: { 730 | slope: _getMedianCoef(table, 'red_slope'), 731 | intercept: _getMedianCoef(table, 'red_intercept') 732 | }, 733 | nir_coef: { 734 | slope: _getMedianCoef(table, 'nir_slope'), 735 | intercept: _getMedianCoef(table, 'nir_intercept') 736 | }, 737 | ndvi_coef: { 738 | slope: _getMedianCoef(table, 'ndvi_slope'), 739 | intercept: _getMedianCoef(table, 'ndvi_intercept') 740 | }, 741 | tcb_coef: { 742 | slope: _getMedianCoef(table, 'tcb_slope'), 743 | intercept: _getMedianCoef(table, 'tcb_intercept') 744 | }, 745 | tcg_coef: { 746 | slope: _getMedianCoef(table, 'tcg_slope'), 747 | intercept: _getMedianCoef(table, 'tcg_intercept') 748 | }, 749 | tca_coef: { 750 | slope: _getMedianCoef(table, 'tca_slope'), 751 | intercept: _getMedianCoef(table, 'tca_intercept') 752 | } 753 | }); 754 | } 755 | exports.getMedianCoef = getMedianCoef; 756 | 757 | function _correctMssImg(img) { 758 | var coefs = ee.Dictionary(img.get('mss_2_tm_coef')); 759 | return ee.Image(ee.Image.cat( 760 | applyCoef(img, 'green', ee.Dictionary(coefs.get('blue_coef'))).toFloat(), 761 | applyCoef(img, 'green', ee.Dictionary(coefs.get('green_coef'))).toFloat(), 762 | applyCoef(img, 'red', ee.Dictionary(coefs.get('red_coef'))).toFloat(), 763 | applyCoef(img, 'nir', ee.Dictionary(coefs.get('nir_coef'))).toFloat(), 764 | applyCoef(img, 'ndvi', ee.Dictionary(coefs.get('ndvi_coef'))).toFloat(), 765 | applyCoef(img, 'tcb', ee.Dictionary(coefs.get('tcb_coef'))).toFloat(), 766 | applyCoef(img, 'tcg', ee.Dictionary(coefs.get('tcg_coef'))).toFloat(), 767 | applyCoef(img, 'tca', ee.Dictionary(coefs.get('tca_coef'))).toFloat()) 768 | .rename(['blue', 'green', 'red', 'nir', 'ndvi', 'tcb', 'tcg', 'tca']) 769 | .copyProperties(img, img.propertyNames())); 770 | } 771 | exports._correctMssImg = _correctMssImg; 772 | 773 | function correctMssImgToMedianTm(col, params) { 774 | var table = ee.FeatureCollection(params.baseDir + '/mss2TmCoefCol'); 775 | var offset = ee.Image(params.baseDir + '/mss_offset'); 776 | var coefs = getMedianCoef(table); 777 | return col.map(function(img) { 778 | return img 779 | .set('mss_2_tm_coef', coefs); 780 | }) 781 | .map(_correctMssImg) 782 | .map(function(img) { 783 | return img.add(offset).round().toShort().copyProperties(img, img.propertyNames()); // NOTE: no offset - img.round().toShort().copyProperties(img, img.propertyNames()); 784 | }); 785 | } 786 | exports.correctMssImgToMedianTm = correctMssImgToMedianTm; 787 | 788 | function getFinalCorrectedMssCol(params) { 789 | var mssCol = ee.ImageCollection([]); 790 | for(var y = 1972; y <= 1982; y++) { 791 | var img = ee.Image(params.baseDir + '/WRS1_to_WRS2/' + y.toString()) 792 | .set('system:time_start', ee.Date.fromYMD(y, 1 ,1).millis()); 793 | mssCol = mssCol.merge(ee.ImageCollection(img)); 794 | } 795 | return correctMssImgToMedianTm(mssCol, params); 796 | } 797 | 798 | function exportFinalCorrectedMssCol(params) { 799 | var mssCol = getFinalCorrectedMssCol(params); 800 | var granuleGeom = msslib.getWrs1GranuleGeom(params.wrs1); 801 | // var dummy = ee.Image([0, 0, 0, 0, 0, 0, 0, 0]).selfMask().toShort() 802 | // .rename(['blue', 'green', 'red', 'nir', 'ndvi', 'tcb', 'tcg', 'tca']); 803 | 804 | for(var y = 1972; y <= 1982; y++) { 805 | var yrCol = mssCol.filter(ee.Filter.eq('year', y)); 806 | 807 | // // Deal with missing years - provide a dummy. 808 | // var outImg = ee.Algorithms.If({ 809 | // condition: yrCol.size(), 810 | // trueCase: yrCol.first().resample('bicubic'), // NOTE: not sure about the resample?, 811 | // falseCase: dummy.set('system:time_start', ee.Date.fromYMD(y, 1 ,1).millis()) 812 | // }); 813 | 814 | Export.image.toAsset({ 815 | image: yrCol.first().resample('bicubic'), //outImg, 816 | description: y.toString(), 817 | assetId: params.baseDir + '/WRS1_to_TM/' + y.toString(), 818 | region: ee.Feature(granuleGeom.get('granule')).geometry(), 819 | scale: 30, 820 | crs: params.crs, 821 | maxPixels: 1e13 822 | }); 823 | } 824 | } 825 | exports.exportFinalCorrectedMssCol = exportFinalCorrectedMssCol; 826 | 827 | 828 | // ############################################################################# 829 | // ### Final collection assembly ### 830 | // ############################################################################# 831 | 832 | function getColForLandTrendrOnTheFly(params) { // Does not rely on WRS1_to_TM assets 833 | var mssCol = getFinalCorrectedMssCol(params); 834 | 835 | var tmCol = ee.ImageCollection([]); 836 | for(var y=1983; y<=2020; y++) { 837 | params.yearRange = [y, y]; 838 | var thisYearCol = getMedoid(gatherTmCol(params), ['blue', 'green', 'red', 'nir', 'ndvi', 'tcb', 'tcg', 'tca']) 839 | .set('system:time_start', ee.Date.fromYMD(y, 1 ,1).millis()); 840 | tmCol = tmCol.merge(ee.ImageCollection(thisYearCol.toShort())); 841 | } 842 | 843 | var combinedCol = mssCol.merge(tmCol).map(function(img) { 844 | return img.select('ndvi').multiply(-1).rename('LTndvi') // TODO: move this into the run landtrandr function - get the fitting index from params 845 | .addBands(img.select(['blue', 'green', 'red', 'nir', 'ndvi', 'tcb', 'tcg', 'tca'])) // TODO: move this into the run landtrandr function - what indices should be FTV 846 | .set('system:time_start', img.get('system:time_start')); 847 | }).sort('system:time_start'); 848 | 849 | return combinedCol; 850 | } 851 | exports.getColForLandTrendrOnTheFly = getColForLandTrendrOnTheFly; 852 | 853 | function getColForLandTrendrFromAsset(params) { // Relies on WRS1_to_TM assets 854 | var mssCol = ee.ImageCollection([]); 855 | for(var y=1972; y<=1982; y++) { 856 | var img = ee.Image(params.baseDir + '/WRS1_to_TM/' + y.toString()) 857 | .set('system:time_start', ee.Date.fromYMD(y, 1 ,1).millis()); 858 | mssCol = mssCol.merge(ee.ImageCollection(img)); 859 | } 860 | 861 | var mss1983 = ee.ImageCollection(getMedoid(correctMssWrs2(params), ['blue', 'green', 'red', 'nir', 'ndvi', 'tcb', 'tcg', 'tca']) 862 | .set('system:time_start', ee.Date.fromYMD(1983, 1 ,1).millis())); 863 | 864 | var tmCol = ee.ImageCollection([]); 865 | for(var y=1984; y<=2020; y++) { 866 | params.yearRange = [y, y]; 867 | var thisYearCol = getMedoid(gatherTmCol(params), ['blue', 'green', 'red', 'nir', 'ndvi', 'tcb', 'tcg', 'tca']) 868 | .set('system:time_start', ee.Date.fromYMD(y, 1 ,1).millis()); 869 | tmCol = tmCol.merge(ee.ImageCollection(thisYearCol.toShort())); 870 | } 871 | 872 | var combinedCol = mssCol.merge(mss1983).merge(tmCol).map(function(img) { 873 | return img.select('ndvi').multiply(-1).rename('LTndvi') // TODO: move this into the run landtrandr function - get the fitting index from params 874 | .addBands(img.select(['blue', 'green', 'red', 'nir', 'ndvi', 'tcb', 'tcg', 'tca'])) // TODO: move this into the run landtrandr function - what indices should be FTV 875 | .set('system:time_start', img.get('system:time_start')); 876 | }).sort('system:time_start'); 877 | 878 | return combinedCol; 879 | } 880 | exports.getColForLandTrendrFromAsset = getColForLandTrendrFromAsset; 881 | 882 | 883 | function runLandTrendrMss2Tm(params) { 884 | var ltCol = getColForLandTrendrFromAsset(params); // alternative: getColForLandTrendrOnTheFly(params) 885 | var lt = ee.Algorithms.TemporalSegmentation.LandTrendr({ 886 | timeSeries: ltCol, 887 | maxSegments: 10, 888 | spikeThreshold: 0.7, 889 | vertexCountOvershoot: 3, 890 | preventOneYearRecovery: true, 891 | recoveryThreshold: 0.5, 892 | pvalThreshold: 0.05, 893 | bestModelProportion: 0.75, 894 | minObservationsNeeded: 6 895 | }); 896 | return lt; 897 | } 898 | exports.runLandTrendrMss2Tm = runLandTrendrMss2Tm; 899 | 900 | // ############################################################################# 901 | // ### Functions under development (Annie Taylor) ### 902 | // ############################################################################# 903 | 904 | function displayCollection(col) { 905 | var rgbviz = { 906 | bands: ['red','green','blue'], 907 | min: 100, 908 | max: 2000, 909 | gamma: [1.2] 910 | }; 911 | Map.centerObject(col.first(), 8); 912 | Map.addLayer(col, rgbviz, 'Full Landsat Collection',false); 913 | } 914 | exports.displayCollection = displayCollection; 915 | 916 | 917 | function animateCollection(col) { 918 | var rgbviz = { 919 | bands: ['red','green','blue'], 920 | min: 100, 921 | max: 2000, 922 | gamma: [1.2] 923 | }; 924 | // TODO: add year of image as label in animation 925 | // col = col.map(function(img) { 926 | // img = img.set({label: ee.String(img.get('system:id'))}) 927 | // return img 928 | // }) 929 | Map.centerObject(col.first(), 8); 930 | // run the animation 931 | animation.animate(col, { 932 | vis: rgbviz, 933 | timeStep: 1500, 934 | maxFrames: col.size() 935 | }) 936 | } 937 | exports.animateCollection = animateCollection; 938 | 939 | function displayGreatestDisturbance(lt, params) { 940 | var granuleGeom = ee.Feature(msslib.getWrs1GranuleGeom(params.wrs1) 941 | .get('granule')).geometry(); 942 | 943 | var currentYear = new Date().getFullYear(); // TODO: make sure there is not a better way to get year from image metadata eg 944 | var changeParams = { // TODO: allow a person to override these params 945 | delta: 'loss', 946 | sort: 'greatest', 947 | year: {checked:true, start:1972, end:currentYear}, // TODO: make sure there is not a better way to get years from image metadata eg 948 | mag: {checked:true, value:200, operator:'>'}, 949 | dur: {checked:true, value:4, operator:'<'}, 950 | preval: {checked:true, value:300, operator:'>'}, 951 | mmu: {checked:true, value:11}, 952 | }; 953 | // Note: add index to changeParams object this is hard coded to NDVI because currently that is the only option. 954 | changeParams.index = 'NDVI'; 955 | var changeImg = ltgee.getChangeMap(lt, changeParams); 956 | var palette = ['#9400D3', '#4B0082', '#0000FF', '#00FF00', 957 | '#FFFF00', '#FF7F00', '#FF0000']; 958 | var yodVizParms = { 959 | min: 1972, // TODO: make sure there is not a better way to get year from image metadata eg 960 | max: currentYear, // TODO: make sure there is not a better way to get year from image metadata eg 961 | palette: palette 962 | }; 963 | var magVizParms = { 964 | min: 200, 965 | max: 800, 966 | palette: palette 967 | }; 968 | Map.centerObject(granuleGeom, 12); // Zoom in pretty far otherwise the mmu filter is going to take forever (probably crash) 969 | // display two change attributes to map 970 | Map.addLayer(changeImg.select(['mag']), magVizParms, 'Magnitude of Change'); 971 | Map.addLayer(changeImg.select(['yod']), yodVizParms, 'Year of Detection'); 972 | } 973 | exports.displayGreatestDisturbance = displayGreatestDisturbance; 974 | 975 | // ############################################################################# 976 | // ### TM to MSS functions ### 977 | // ############################################################################# 978 | 979 | 980 | 981 | // // Function to make normalization function. 982 | // function getTm2mssCoefCol(img) { //function makeCorrectionFun(refImgPath) { 983 | // //var img = coCol.first(); 984 | // var yImg = msslib.addNdvi(msslib.calcToa(img)); 985 | // var xImg = ee.Image(yImg.get('coincidentTmMss')); 986 | 987 | // var xImgNdvi = xImg.normalizedDifference(['B4', 'B3']).rename(['ndvi']); 988 | // var mask = getCfmask(xImg); 989 | // xImg = xImg.addBands(xImgNdvi).updateMask(mask); 990 | 991 | // var greenCoef = calcRegression(xImg, yImg, 'B2', 'green'); 992 | // var redCoef = calcRegression(xImg, yImg, 'B3', 'red'); 993 | // var nirCoef = calcRegression(xImg, yImg, 'B4', 'nir'); 994 | // var ndviCoef = calcRegression(xImg, yImg, 'ndvi', 'ndvi'); 995 | 996 | // var granuleGeoms = msslib.getWrs1GranuleGeom('049029'); // TODO - NEED TO GET THIS FROM THE PARAMS - could really just make this a dummy geom at 0, 0 - it's not needed. 997 | // var centroid = ee.Geometry(granuleGeoms.get('centroid')); 998 | 999 | // return ee.Feature(centroid).set( 1000 | // { 1001 | // 'green_slope': greenCoef.getNumber('slope'), 1002 | // 'green_intercept': greenCoef.getNumber('intercept'), 1003 | // 'red_slope': redCoef.getNumber('slope'), 1004 | // 'red_intercept': redCoef.getNumber('intercept'), 1005 | // 'nir_slope': nirCoef.getNumber('slope'), 1006 | // 'nir_intercept': nirCoef.getNumber('intercept'), 1007 | // 'ndvi_slope': ndviCoef.getNumber('slope'), 1008 | // 'ndvi_intercept': ndviCoef.getNumber('intercept') 1009 | // }).copyProperties(yImg, ['imgID', 'year', 'path', 'row', 'pr']); // TODO: need to add RMSE. - Could just exclude the system:footprint 1010 | // } 1011 | // exports.getTm2mssCoefCol = getTm2mssCoefCol; 1012 | 1013 | 1014 | // function _correctTmImg(img) { 1015 | // var coefs = ee.Dictionary(img.get('tm_2_mss_coef')); 1016 | // return ee.Image(ee.Image.cat( 1017 | // applyCoef(img, 'B2', ee.Dictionary(coefs.get('green_coef'))).toFloat(), 1018 | // applyCoef(img, 'B3', ee.Dictionary(coefs.get('red_coef'))).toFloat(), 1019 | // applyCoef(img, 'B4', ee.Dictionary(coefs.get('nir_coef'))).toFloat(), 1020 | // applyCoef(img, 'ndvi', ee.Dictionary(coefs.get('ndvi_coef'))).toFloat()) 1021 | // .rename(['green', 'red', 'nir', 'ndvi']) 1022 | // .copyProperties(img, img.propertyNames())); 1023 | // } 1024 | 1025 | // function correctTmImg(col, params) { 1026 | // var table = ee.FeatureCollection(params.baseDir + '/tm2MssCoefCol'); 1027 | // var coefs = getMedianCoef(table); 1028 | // return col.map(function(img) { 1029 | // return img.set('tm_2_mss_coef', coefs); 1030 | // }) 1031 | // .map(_correctTmImg); 1032 | // } 1033 | // exports.correctTmImg = correctTmImg; 1034 | 1035 | 1036 | // function runLandTrendrTm2Mss() { 1037 | // var mssCol = ee.ImageCollection([]); 1038 | // for(var y=1972; y<=1982; y++) { 1039 | // var img = ee.Image("users/braaten/llr/test_proj_1/049029/" + y.toString()) 1040 | // .set('system:time_start', ee.Date.fromYMD(y, 1 ,1).millis()); 1041 | // mssCol = mssCol.merge(ee.ImageCollection(img)); 1042 | // } 1043 | 1044 | // var tmCol = ee.ImageCollection([]); 1045 | // for(var y=1983; y<=2020; y++) { 1046 | // params.yearRange = [y, y]; 1047 | // var thisYear = gatherTmCol(params); 1048 | // var thisYearCol = correctTmImg(thisYear, params).median() // TODO: this needs to be medoid. 1049 | // .set('system:time_start', ee.Date.fromYMD(y, 1 ,1).millis()); 1050 | // tmCol = tmCol.merge(ee.ImageCollection(thisYearCol)); 1051 | // } 1052 | 1053 | // var ltCol = mssCol.merge(tmCol).map(function(img) { 1054 | // return img.select('ndvi').multiply(-1).rename('LTndvi') 1055 | // .addBands(img.select(['green', 'red', 'nir'])) 1056 | // .set('system:time_start', img.get('system:time_start')); 1057 | // }); 1058 | 1059 | // var lt = ee.Algorithms.TemporalSegmentation.LandTrendr({ 1060 | // timeSeries: ltCol, 1061 | // maxSegments: 10, 1062 | // spikeThreshold: 0.7, 1063 | // vertexCountOvershoot: 3, 1064 | // preventOneYearRecovery: true, 1065 | // recoveryThreshold: 0.5, 1066 | // pvalThreshold: 0.05, 1067 | // bestModelProportion: 0.75, 1068 | // minObservationsNeeded: 6 1069 | // }); 1070 | 1071 | // return lt; 1072 | // } 1073 | -------------------------------------------------------------------------------- /landsatlinkr.py: -------------------------------------------------------------------------------- 1 | from datetime import date 2 | import copy 3 | import os 4 | import time 5 | import math 6 | import pprint 7 | from IPython.display import Image 8 | import subprocess 9 | import sys 10 | import ee 11 | 12 | from traitlets.traitlets import default 13 | def getPr(img): 14 | path = img.getNumber('WRS_PATH').format('%03d') 15 | row = img.getNumber('WRS_ROW').format('%03d') 16 | return path.cat(row) 17 | 18 | 19 | def filterById_doit(id, col): 20 | return ee.ImageCollection(col).filter( 21 | ee.Filter.neq('LANDSAT_SCENE_ID', ee.String(id))) 22 | 23 | 24 | def filterById(col, imgList): 25 | return ee.ImageCollection(ee.List(imgList).iterate(filterById_doit, col)) # TODO: use ee.Filter.inList().not() 26 | 27 | 28 | def filterCol(col, params, wrs): 29 | # Adjust band present property names depending on WRS (1 or 2). 30 | bandsPresent = { 31 | 'wrs1': [ 32 | 'PRESENT_BAND_4', 'PRESENT_BAND_5', 'PRESENT_BAND_6', 'PRESENT_BAND_7' 33 | ], 34 | 'wrs2': [ 35 | 'PRESENT_BAND_1', 'PRESENT_BAND_2', 'PRESENT_BAND_3', 'PRESENT_BAND_4' 36 | ] 37 | } 38 | 39 | if params['aoi']: 40 | col = col.filterBounds(params['aoi']) 41 | 42 | col = (col.filter(ee.Filter.neq('DATA_TYPE', 'L1G')) 43 | .filter(ee.Filter.eq(bandsPresent[wrs][0], 'Y')) 44 | .filter(ee.Filter.eq(bandsPresent[wrs][1], 'Y')) 45 | .filter(ee.Filter.eq(bandsPresent[wrs][2], 'Y')) 46 | .filter(ee.Filter.eq(bandsPresent[wrs][3], 'Y')) 47 | .filter(ee.Filter.lte('GEOMETRIC_RMSE_VERIFY', params['maxRmseVerify'])) 48 | .filter(ee.Filter.lte('CLOUD_COVER', params['maxCloudCover']))) 49 | 50 | if params['yearRange']: 51 | col = col.filter(ee.Filter.calendarRange( 52 | params['yearRange'][0], params['yearRange'][1], 'year')) 53 | 54 | if params['doyRange']: 55 | col = col.filter(ee.Filter.calendarRange( 56 | params['doyRange'][0], params['doyRange'][1], 'day_of_year')) 57 | 58 | if params['excludeIds']: 59 | col = filterById(col, params['excludeIds']) 60 | 61 | return col 62 | 63 | 64 | def getCol(params): 65 | # Define default filter parameters. 66 | _params = { 67 | 'aoi': None, 68 | 'maxRmseVerify': 0.5, 69 | 'maxCloudCover': 50, 70 | 'wrs': '1&2', 71 | 'yearRange': [1972, 2000], 72 | 'doyRange': [1, 365], 73 | 'excludeIds': None 74 | } 75 | 76 | # Replace default params with provided params. 77 | if params: 78 | for param in params: 79 | _params[param] = params[param] or _params[param] 80 | 81 | # Initialize WRS-1 and WRS-2 collections. 82 | wrs1Col = ee.ImageCollection([]) 83 | wrs2Col = ee.ImageCollection([]) 84 | 85 | # Gather MSS WRS-1 images, filter as requested, designate as 'WRS-1'. 86 | if '1' in _params['wrs']: 87 | mss1T1 = filterCol( 88 | ee.ImageCollection('LANDSAT/LM01/C02/T1'), _params, 'wrs1') 89 | mss1T2 = filterCol( 90 | ee.ImageCollection('LANDSAT/LM01/C02/T2'), _params, 'wrs1') 91 | mss2T1 = filterCol( 92 | ee.ImageCollection('LANDSAT/LM02/C02/T1'), _params, 'wrs1') 93 | mss2T2 = filterCol( 94 | ee.ImageCollection('LANDSAT/LM02/C02/T2'), _params, 'wrs1') 95 | mss3T1 = filterCol( 96 | ee.ImageCollection('LANDSAT/LM03/C02/T1'), _params, 'wrs1') 97 | mss3T2 = filterCol( 98 | ee.ImageCollection('LANDSAT/LM03/C02/T2'), _params, 'wrs1') 99 | wrs1Col = (ee.ImageCollection(ee.FeatureCollection( 100 | [mss1T1, mss1T2, mss2T1, mss2T2, mss3T1, mss3T2]).flatten()) 101 | .select(['B.|QA_PIXEL|QA_RADSAT'], 102 | ['green', 'red', 'red-edge', 'nir', 'QA_PIXEL', 'QA_RADSAT']) 103 | .map(lambda img: img.set('wrs', 'WRS-1'))) 104 | 105 | # Gather MSS WRS-2 images, filter as requested, designate as 'WRS-2'. 106 | if '2' in _params['wrs']: 107 | mss4T1 = filterCol( 108 | ee.ImageCollection('LANDSAT/LM04/C02/T1'), _params, 'wrs2'); 109 | mss4T2 = filterCol( 110 | ee.ImageCollection('LANDSAT/LM04/C02/T2'), _params, 'wrs2'); 111 | mss5T1 = filterCol( 112 | ee.ImageCollection('LANDSAT/LM05/C02/T1'), _params, 'wrs2'); 113 | mss5T2 = filterCol( 114 | ee.ImageCollection('LANDSAT/LM05/C02/T2'), _params, 'wrs2'); 115 | wrs2Col = (ee.ImageCollection(ee.FeatureCollection( 116 | [mss4T1, mss4T2, mss5T1, mss5T2]).flatten()) 117 | .select(['B.|QA_PIXEL|QA_RADSAT'], 118 | ['green', 'red', 'red-edge', 'nir', 'QA_PIXEL', 'QA_RADSAT']) 119 | .map(lambda img: img.set('wrs', 'WRS-2'))) 120 | 121 | # Return time-sorted, merged, WRS-1 and WRS-2 collection with filter params 122 | # attached. 123 | return ee.ImageCollection(ee.FeatureCollection([wrs1Col, wrs2Col]).flatten()).map(lambda img: img.set({ 124 | 'start_doy': _params['doyRange'][0], 125 | 'end_doy': _params['doyRange'][1], 126 | 'year': img.date().get('year'), 127 | 'doy': img.date().getRelative('day', 'year'), 128 | 'pr': getPr(img) 129 | # composite_year: # TODO 130 | })).sort('system:time_start') 131 | 132 | 133 | def getWrs1GranuleGeom(granuleId): 134 | granule = ee.Feature( 135 | ee.FeatureCollection('users/jstnbraaten/wrs/wrs1_descending_land') 136 | .filter(ee.Filter.eq('PR', granuleId)).first()) 137 | centroid = granule.centroid(300).geometry(300) 138 | bounds = granule.geometry(300).buffer(40000) 139 | return ee.Dictionary({ 140 | 'granule': granule, 141 | 'centroid': centroid, 142 | 'bounds': bounds 143 | }) 144 | 145 | 146 | def scaleDn(img, unit): 147 | mult = 'REFLECTANCE_MULT_BAND' 148 | add = 'REFLECTANCE_ADD_BAND' 149 | if unit == 'radiance': 150 | mult = 'RADIANCE_MULT_BAND' 151 | add = 'RADIANCE_ADD_BAND' 152 | 153 | gainBands = (ee.List(img.propertyNames()) 154 | .filter(ee.Filter.stringContains('item', mult)) 155 | .sort()) 156 | biasBands = (ee.List(img.propertyNames()) 157 | .filter(ee.Filter.stringContains('item', add)) 158 | .sort()) 159 | 160 | gainImg = ee.Image.cat( 161 | ee.Image.constant(img.get(gainBands.getString(0))), 162 | ee.Image.constant(img.get(gainBands.getString(1))), 163 | ee.Image.constant(img.get(gainBands.getString(2))), 164 | ee.Image.constant(img.get(gainBands.getString(3)))).toFloat() 165 | 166 | biasImg = ee.Image.cat( 167 | ee.Image.constant(img.get(biasBands.getString(0))), 168 | ee.Image.constant(img.get(biasBands.getString(1))), 169 | ee.Image.constant(img.get(biasBands.getString(2))), 170 | ee.Image.constant(img.get(biasBands.getString(3)))).toFloat() 171 | 172 | dnImg = img.select([0, 1, 2, 3]).multiply(gainImg).add(biasImg).toFloat() 173 | 174 | return img.addBands(dnImg, None, True) 175 | 176 | 177 | def calcToa(img): 178 | return scaleDn(img, 'reflectance') 179 | 180 | 181 | def addTc(img): 182 | bands = img.select([0, 1, 2, 3]) 183 | tcbCoeffs = ee.Image.constant([0.433, 0.632, 0.586, 0.264]) 184 | tcgCoeffs = ee.Image.constant([-0.290, -0.562, 0.600, 0.491]) 185 | tcyCoeffs = ee.Image.constant([-0.829, 0.522, -0.039, 0.194]) 186 | tcb = bands.multiply(tcbCoeffs).reduce(ee.Reducer.sum()).toFloat() 187 | tcg = bands.multiply(tcgCoeffs).reduce(ee.Reducer.sum()).toFloat() 188 | tcy = bands.multiply(tcyCoeffs).reduce(ee.Reducer.sum()).toFloat() 189 | tca = (tcg.divide(tcb)).atan().multiply(180 / math.pi).toFloat() 190 | tc = ee.Image.cat(tcb, tcg, tcy, tca).rename('tcb', 'tcg', 'tcy', 'tca') 191 | return ee.Image(img.addBands(tc).copyProperties(img, img.propertyNames())) 192 | 193 | 194 | def addNdvi(img): 195 | ndvi = img.normalizedDifference(['nir', 'red']).rename('ndvi') 196 | return ee.Image(img.addBands(ndvi).copyProperties(img, img.propertyNames())) 197 | 198 | 199 | def scaleDn(img, unit): 200 | mult = 'REFLECTANCE_MULT_BAND' 201 | add = 'REFLECTANCE_ADD_BAND' 202 | if unit == 'radiance': 203 | mult = 'RADIANCE_MULT_BAND' 204 | add = 'RADIANCE_ADD_BAND' 205 | 206 | gainBands = (ee.List(img.propertyNames()) 207 | .filter(ee.Filter.stringContains('item', mult)) 208 | .sort()) 209 | biasBands = (ee.List(img.propertyNames()) 210 | .filter(ee.Filter.stringContains('item', add)) 211 | .sort()) 212 | 213 | gainImg = ee.Image.cat( 214 | ee.Image.constant(img.get(gainBands.getString(0))), 215 | ee.Image.constant(img.get(gainBands.getString(1))), 216 | ee.Image.constant(img.get(gainBands.getString(2))), 217 | ee.Image.constant(img.get(gainBands.getString(3)))).toFloat() 218 | 219 | biasImg = ee.Image.cat( 220 | ee.Image.constant(img.get(biasBands.getString(0))), 221 | ee.Image.constant(img.get(biasBands.getString(1))), 222 | ee.Image.constant(img.get(biasBands.getString(2))), 223 | ee.Image.constant(img.get(biasBands.getString(3)))).toFloat() 224 | 225 | dnImg = img.select([0, 1, 2, 3]).multiply(gainImg).add(biasImg).toFloat() 226 | 227 | return img.addBands(dnImg, None, True) 228 | 229 | 230 | def calcRad(img): 231 | return scaleDn(img, 'radiance') 232 | 233 | 234 | def calcToa(img): 235 | return scaleDn(img, 'reflectance') 236 | 237 | 238 | visDn = { 239 | 'bands': ['nir', 'red', 'green'], 240 | 'min': [47, 20, 27], 241 | 'max': [142, 92, 71], 242 | 'gamma': [1.2, 1.2, 1.2] 243 | } 244 | 245 | 246 | visRad = { 247 | 'bands': ['nir', 'red', 'green'], 248 | 'min': [23, 15, 25], 249 | 'max': [67, 62, 64], 250 | 'gamma': [1.2, 1.2, 1.2] 251 | } 252 | 253 | 254 | visToa = { 255 | 'bands': ['nir', 'red', 'green'], 256 | 'min': [0.0896, 0.0322, 0.0464], 257 | 'max': [0.2627, 0.1335, 0.1177], 258 | 'gamma': [1.2, 1.2, 1.2] 259 | } 260 | 261 | 262 | visNdvi = { 263 | 'bands': ['ndvi'], 'min': 0.1, 'max': 0.8 264 | } 265 | 266 | def viewThumbnails(col, params=None): 267 | print('Please wait patiently, images may not load immediately\n') 268 | 269 | _params = { 270 | 'unit': 'toa', 271 | 'display': 'nir|red|green', 272 | 'visParams': None 273 | } 274 | 275 | if params: 276 | for param in params: 277 | _params[param] = params[param] or _params[param] 278 | 279 | settings = { 280 | 'unit': { 281 | 'dn': lambda img: img, 282 | 'rad': calcRad, 283 | 'toa': calcToa 284 | }, 285 | 'display': { 286 | 'nir|red|green': { 287 | 'dn': visDn, 288 | 'rad': visRad, 289 | 'toa': visToa 290 | }, 291 | 'ndvi': { 292 | 'dn': visNdvi, 293 | 'rad': visNdvi, 294 | 'toa': visNdvi 295 | } 296 | } 297 | } 298 | 299 | nImgs = col.size().getInfo() 300 | imgList = col.sort('system:time_start').toList(nImgs).getInfo() 301 | for i in range(0, nImgs): 302 | id = imgList[i]['id'] 303 | img = applyQaMask(ee.Image(id)).select(['B.'], ['green', 'red', 'red-edge', 'nir']) 304 | img = settings['unit'][_params['unit']](img) 305 | if _params['display'] == 'ndvi': 306 | img = addNdvi(img) 307 | 308 | visParams = settings['display'][_params['display']][_params['unit']] 309 | if _params['visParams']: 310 | visParams = _params['visParams'] 311 | 312 | imgVis = img.unmask(0).visualize(**visParams) 313 | date = img.date().format('YYYY-MM-dd').getInfo() 314 | sceneId = img.get('LANDSAT_SCENE_ID').getInfo() 315 | print(f'Date: {date} | Scene ID: {sceneId}') 316 | display(Image(url=imgVis.getThumbURL({ 317 | 'dimensions': 512, 318 | 'crs': 'EPSG:3857'}))) 319 | print('\n') 320 | 321 | 322 | def getQaMask(img): 323 | qaPixelMask = img.select('QA_PIXEL').bitwiseAnd(int('11111', 2)).eq(0) 324 | qaRadsatMask = img.select('QA_RADSAT').eq(0) 325 | return qaPixelMask.updateMask(qaRadsatMask).rename('QA_mask') 326 | 327 | 328 | def applyQaMask(img): # TODO: I don't think this is being applied except for thumbVis 329 | return img.updateMask(getQaMask(img)) 330 | 331 | 332 | def getDem(img): 333 | aw3d30 = ee.Image('JAXA/ALOS/AW3D30/V2_2').select('AVE_DSM').rename('elev') 334 | GMTED2010 = ee.Image('USGS/GMTED2010').rename('elev') 335 | return ee.ImageCollection([GMTED2010, aw3d30]) \ 336 | .mosaic() \ 337 | .reproject(img.projection()) 338 | 339 | def waterLayer(img): 340 | # Threshold on NDVI. 341 | mssWater = img.normalizedDifference(['nir', 'red']).lt(-0.085) 342 | 343 | # Get max extent of water 1985-2018. 344 | waterExtent = ee.Image('JRC/GSW1_1/GlobalSurfaceWater').select('max_extent') 345 | 346 | # Get intersection of MSS water and max extent. 347 | return mssWater.multiply(waterExtent) \ 348 | .reproject(img.projection()) \ 349 | .rename('water') 350 | 351 | def radians(img): 352 | return img.toFloat().multiply(math.pi).divide(180) 353 | 354 | def getIll(img, slope, aspect): 355 | # Get sun info. 356 | azimuth = img.get('SUN_AZIMUTH') 357 | zenith = ee.Number(90).subtract(img.getNumber('SUN_ELEVATION')) 358 | 359 | # Convert slope and aspect degrees to radians. 360 | slopeRad = radians(slope) 361 | aspectRad = radians(aspect) 362 | 363 | # Calculate illumination. 364 | azimuthImg = radians(ee.Image.constant(azimuth)) 365 | zenithImg = radians(ee.Image.constant(zenith)) 366 | left = zenithImg.cos().multiply(slopeRad.cos()) 367 | right = zenithImg.sin() \ 368 | .multiply(slopeRad.sin()) \ 369 | .multiply(azimuthImg.subtract(aspectRad).cos()) 370 | return left.add(right) 371 | 372 | def topoCorrB4(img, dem): 373 | # Get terrain layers. 374 | terrain = ee.Algorithms.Terrain(dem) 375 | slope = terrain.select(['slope']) 376 | aspect = terrain.select(['aspect']) 377 | 378 | # Get k image. 379 | # define polynomial coefficients to calc Minnaert value as function of slope 380 | # Ge, H., Lu, D., He, S., Xu, A., Zhou, G., & Du, H. (2008). Pixel-based 381 | # Minnaert correction method for reducing topographic effects on a Landsat 7 382 | # ETM+ image. Photogrammetric Engineering & Remote Sensing, 74(11), 383 | # 1343-1350. | 384 | # https:#orst.library.ingentaconnect.com/content/asprs/pers/2008/00000074/00000011/art00003?crawler=True&mimetype=application/pdf 385 | kImg = (slope.resample('bilinear') 386 | .where( 387 | slope.gt(50), 388 | 50) # Set max slope at 50 degrees - paper does not sample \ 389 | .polynomial([ 390 | 1.0021313684, -0.1308793751, 0.0106861276, -0.0004051135, 391 | 0.0000071825, -4.88e-8 392 | ])) 393 | 394 | # Get illumination. 395 | ill = getIll(img, slope, aspect) 396 | 397 | # Correct NIR reflectance for topography. 398 | cosTheta = radians(ee.Image.constant(ee.Number(90).subtract( 399 | ee.Number(img.get('SUN_ELEVATION'))))).cos() 400 | correction = (cosTheta.divide(ill)).pow(kImg) 401 | return img.select('nir').multiply(correction) 402 | 403 | def cloudLayer(img): 404 | # Identify cloud pixels. 405 | cloudPixels = (img.normalizedDifference(['green', 'red']) 406 | .gt(0) 407 | .multiply(img.select('green').gt(0.175)) 408 | .add(img.select('green').gt(0.39)) 409 | .gt(0)) 410 | 411 | # Nine-pixel minimum connected component sieve. 412 | cloudPixels = (cloudPixels.selfMask() 413 | .connectedPixelCount(10, True) 414 | .reproject(img.projection()) 415 | .gte(0) 416 | .unmask(0) 417 | .rename('cloudtest')) 418 | 419 | # Define kernel for buffer. 420 | kernel = ee.Kernel.circle(**{'radius': 2, 'units': 'pixels', 'normalize': True}) 421 | 422 | # Two pixel buffer, eight neighbor rule. 423 | return (cloudPixels.focalMax(**{'radius': 2, 'kernel': kernel}) 424 | .reproject(img.projection()) 425 | .rename('clouds')) 426 | 427 | def shadowLayer(img, dem, clouds): 428 | # Correct B4 reflectance for topography. 429 | b4c = topoCorrB4(img, dem) 430 | 431 | # Threshold B4 - target dark pixels. 432 | shadows = b4c.lt(0.11); # Make this True for all pixels to use full cloud projection. 433 | 434 | # Project clouds as potential shadow. 435 | shadow_azimuth = ee.Number(90).subtract(ee.Number(img.get('SUN_AZIMUTH'))) 436 | cloudProj = (clouds.directionalDistanceTransform(shadow_azimuth, 50) 437 | .reproject(**{'crs': img.projection(), 'scale': 60}) 438 | .select('distance') 439 | .gt(0) 440 | .unmask(0)) 441 | 442 | # Get water layer. 443 | water = waterLayer(img) 444 | 445 | # Exclude water pixels from intersection of cloud projection and dark pixels. 446 | return (shadows.multiply(water.Not()) 447 | .multiply(cloudProj) 448 | .focalMax(2) 449 | .reproject(img.projection())) 450 | 451 | def applyMsscvm(img): 452 | dem = getDem(img) 453 | water = waterLayer(img) 454 | b4c = topoCorrB4(img, dem) 455 | clouds = cloudLayer(img) 456 | shadows = shadowLayer(img, dem, clouds) 457 | mask = clouds.add(shadows).eq(0) 458 | return img.updateMask(mask) 459 | 460 | 461 | msslib = { 462 | 'getWrs1GranuleGeom': getWrs1GranuleGeom, 463 | 'getCol': getCol, 464 | 'calcToa': calcToa, 465 | 'addTc': addTc, 466 | 'addNdvi': addNdvi, 467 | 'viewThumbnails': viewThumbnails, 468 | 'applyQaMask': applyQaMask, 469 | 'applyMsscvm': applyMsscvm 470 | } 471 | 472 | params = None # GLOBAL dict redefined later. 473 | 474 | def getTmWrs2Col(aoi): 475 | tm4 = ee.ImageCollection("LANDSAT/LT04/C02/T1_L2") \ 476 | .filterBounds(aoi) 477 | tm5 = ee.ImageCollection("LANDSAT/LT05/C02/T1_L2") \ 478 | .filterBounds(aoi) 479 | return tm4.merge(tm5) 480 | 481 | def coincidentTmMssCol(mssWrs2Col, tmWrs2Col): 482 | filter = ee.Filter.equals(**{'leftField': 'imgID', 'rightField': 'imgID'}) 483 | join = ee.Join.saveFirst('coincidentTmMss') 484 | return ee.ImageCollection(join.apply(mssWrs2Col, tmWrs2Col, filter)) 485 | 486 | def addTmToMssJoinId(img): 487 | date = ee.Image(img).date() 488 | year = ee.Algorithms.String(date.get('year')) 489 | doy = ee.Algorithms.String(date.getRelative('day', 'year')) 490 | path = ee.Algorithms.String(img.getNumber('WRS_PATH').toInt()) 491 | row = ee.Algorithms.String(img.getNumber('WRS_ROW').toInt()) 492 | yearDoy = year.cat(doy).cat(path).cat(row) 493 | return img.set({'imgID': yearDoy, 494 | 'path': path, 495 | 'row': row 496 | }) 497 | 498 | def getFootprint(img): 499 | return ee.Geometry.Polygon(ee.Geometry(img.get('system:footprint')).coordinates()) 500 | 501 | def filterBounds(aoi): 502 | return ee.Filter.bounds(aoi) 503 | 504 | # def getCfmask(img): 505 | # cloudShadowBitMask = 1 << 3 506 | # cloudsBitMask = 1 << 5 507 | # qa = img.select('pixel_qa') 508 | # mask = qa.bitwiseAnd(cloudShadowBitMask) \ 509 | # .eq(0) \ 510 | # .And(qa.bitwiseAnd(cloudsBitMask).eq(0)) 511 | # return mask 512 | 513 | # def applyCfmask(img): 514 | # mask = getCfmask(img) 515 | # return img.updateMask(mask) 516 | 517 | 518 | def getCfmask(img): 519 | return img.select('QA_PIXEL').bitwiseAnd(int('11111', 2)).eq(0) 520 | 521 | def scaleMask(img): 522 | def getFactorImg(factorNames): 523 | factorList = img.toDictionary().select(factorNames).values() 524 | return ee.Image.constant(factorList) 525 | 526 | scaleImg = getFactorImg(['REFLECTANCE_MULT_BAND_.']) 527 | offsetImg = getFactorImg(['REFLECTANCE_ADD_BAND_.']) 528 | scaled = (img.select('SR_B.').multiply(scaleImg).add(offsetImg) 529 | .multiply(10000).round().int16()) 530 | 531 | return (img.addBands(scaled, None, True) 532 | .select('SR_B.') 533 | .updateMask(getCfmask(img))) 534 | 535 | 536 | 537 | 538 | def viewWrs1Col(params): 539 | granuleGeom = msslib['getWrs1GranuleGeom'](params['wrs1']) 540 | params['aoi'] = ee.Geometry(granuleGeom.get('centroid')) 541 | params['wrs'] = '1' 542 | mssDnCol = msslib['getCol'](params) \ 543 | .filter(ee.Filter.eq('pr', params['wrs1'])) 544 | msslib['viewThumbnails'](mssDnCol, None) 545 | 546 | def getMedoid(col, bands, parallelScale=1): 547 | col = col.select(bands) 548 | median = col.reduce(ee.Reducer.median(), parallelScale) 549 | 550 | def mapFun(img): 551 | dif = ee.Image(img).subtract(median).pow(ee.Image.constant(2)) 552 | return dif.reduce(ee.Reducer.sum()).addBands(img) 553 | 554 | difFromMedian = col.map(mapFun) 555 | bandNames = difFromMedian.first().bandNames() 556 | nBands = bandNames.length() 557 | bandsPos = ee.List.sequence(1, nBands.subtract(1)) 558 | bandNamesSub = bandNames.slice(1) 559 | return (difFromMedian.reduce(ee.Reducer.min(nBands), parallelScale) 560 | .select(bandsPos, bandNamesSub)) 561 | 562 | def getRefImg(params): 563 | granuleGeoms = msslib['getWrs1GranuleGeom'](params['wrs1']) 564 | centroid = ee.Geometry(granuleGeoms.get('centroid')) 565 | bounds = ee.Geometry(granuleGeoms.get('bounds')) 566 | 567 | refCol = msslib['getCol']({ 568 | 'aoi': bounds, 569 | 'wrs': '2', 570 | 'yearRange': [1983, 1987], # NOTE: Use five early years, want good coverage, but near to MSS WRS-1 window. 571 | 'doyRange': params['doyRange'], 572 | }).map(addTmToMssJoinId) 573 | 574 | tmCol = getTmWrs2Col(bounds) \ 575 | .filterDate('1983-01-01', '1988-01-01') \ 576 | .map(addTmToMssJoinId) 577 | 578 | def cloudMask(img): 579 | mask = getCfmask(ee.Image(img.get('coincidentTmMss'))) 580 | imgToa = msslib['calcToa'](img) 581 | return imgToa.updateMask(mask) 582 | 583 | coincident = coincidentTmMssCol(refCol, tmCol).map(cloudMask) 584 | return msslib['addTc'](msslib['addNdvi'](getMedoid(coincident, ['green', 'red', 'red-edge', 'nir']))) \ 585 | .select(['green', 'red', 'red-edge', 'nir', 'ndvi', 'tcb', 'tcg', 'tca']) \ 586 | .set('bounds', bounds) 587 | 588 | def exportMssRefImg(params): 589 | print('Exporting MSS 2nd Gen reference image, please wait.') 590 | refImg = getRefImg(params) 591 | outAsset = params['baseDir'] + '/ref' 592 | print(outAsset) 593 | task = ee.batch.Export.image.toAsset(**{ 594 | 'image': refImg, 595 | 'description': 'MSS-reference-image', 596 | 'assetId': outAsset, 597 | 'region': ee.Geometry(refImg.get('bounds')), 598 | 'scale': 60, 599 | 'crs': params['crs'], 600 | 'maxPixels': 1e13 601 | }) 602 | task.start() 603 | return task 604 | 605 | # def calcRegression(xImg, yImg, xBand, yBand, aoi, scale): 606 | # constant = ee.Image(1) 607 | # xVar = xImg.select(xBand) 608 | # yVar = yImg.select(yBand) 609 | # imgRegress = ee.Image.cat(constant, xVar, yVar) 610 | 611 | # linearRegression = imgRegress.reduceRegion(**{ 612 | # 'reducer': ee.Reducer.robustLinearRegression(**{ 613 | # 'numX': 2, 614 | # 'numY': 1 615 | # }), 616 | # 'geometry': aoi, 617 | # 'scale': scale, 618 | # 'maxPixels': 1e13 619 | # }) 620 | 621 | # coefList = ee.Array(linearRegression.get('coefficients')).toList() 622 | # intercept = ee.List(coefList.get(0)).get(0) 623 | # slope = ee.List(coefList.get(1)).get(0) 624 | # rmse = ee.Array(linearRegression.get('residuals')).toList().get(0) 625 | # return ee.Dictionary({'slope': slope, 'intercept': intercept, 'rmse': rmse}) 626 | 627 | # def applyCoef(img, band, coef): 628 | # coef = ee.Dictionary(coef) 629 | # return img.select(band) \ 630 | # .multiply(ee.Image.constant(coef.getNumber('slope'))) \ 631 | # .add(ee.Image.constant(coef.getNumber('intercept'))) 632 | 633 | # def getSampleImg(img, ref, band): 634 | # dif = img.select(band) \ 635 | # .subtract(ref.select(band)).rename('dif') 636 | 637 | # difThresh = dif.reduceRegion(**{ 638 | # 'reducer': ee.Reducer.percentile(**{ 639 | # 'percentiles': [40, 60], 640 | # 'maxRaw': 1000000, 641 | # 'maxBuckets': 1000000, 642 | # 'minBucketWidth': 0.00000000001 643 | # }), 644 | # 'geometry': img.geometry(), 645 | # 'scale': 60, 646 | # 'maxPixels': 1e13 647 | # }) 648 | 649 | # mask = dif.gt(difThresh.getNumber('dif_p40')) \ 650 | # .And(dif.lt(difThresh.getNumber('dif_p60'))) 651 | 652 | # return img.updateMask(mask) 653 | 654 | # def correctMssImg(img): 655 | # ref = ee.Image(img.get('ref_img')) 656 | # # ref = ee.Image(params['baseDir'] + '/ref') 657 | # granuleGeoms = msslib['getWrs1GranuleGeom'](img.getString('pr')) 658 | # granule = ee.Feature(granuleGeoms.get('granule')).geometry() 659 | 660 | # greenCoef = calcRegression(getSampleImg(img, ref, 'green'), ref, 'green', 'green', granule, 60) 661 | # redCoef = calcRegression(getSampleImg(img, ref, 'red'), ref, 'red', 'red', granule, 60) 662 | # nirCoef = calcRegression(getSampleImg(img, ref, 'nir'), ref, 'nir', 'nir', granule, 60) 663 | # ndviCoef = calcRegression(getSampleImg(img, ref, 'ndvi'), ref, 'ndvi', 'ndvi', granule, 60) 664 | # tcbCoef = calcRegression(getSampleImg(img, ref, 'tcb'), ref, 'tcb', 'tcb', granule, 60) 665 | # tcgCoef = calcRegression(getSampleImg(img, ref, 'tcg'), ref, 'tcg', 'tcg', granule, 60) 666 | # tcaCoef = calcRegression(getSampleImg(img, ref, 'tca'), ref, 'tca', 'tca', granule, 60) 667 | 668 | # return ee.Image(ee.Image.cat( 669 | # applyCoef(img, 'green', greenCoef).toFloat(), 670 | # applyCoef(img, 'red', redCoef).toFloat(), 671 | # applyCoef(img, 'nir', nirCoef).toFloat(), 672 | # applyCoef(img, 'ndvi', nirCoef).toFloat(), 673 | # applyCoef(img, 'tcb', tcbCoef).toFloat(), 674 | # applyCoef(img, 'tcg', tcgCoef).toFloat(), 675 | # applyCoef(img, 'tca', tcaCoef).toFloat()) \ 676 | # .rename(['green', 'red', 'nir', 'ndvi', 'tcb', 'tcg', 'tca']) \ 677 | # .copyProperties(img, img.propertyNames())) 678 | 679 | # Makes WRS1 images match the MSS WRS2 reference image using Random Forest regression 680 | # with stratified training sample based on 500 points in 6 categories of TCA defined by 681 | # percentiles 682 | def correctMssImg2(img): 683 | ref = ee.Image(img.get('ref_img')) 684 | granuleGeoms = msslib['getWrs1GranuleGeom'](img.getString('pr')) 685 | granule = ee.Feature(granuleGeoms.get('granule')).geometry() 686 | 687 | img = img.select(ref.bandNames()) 688 | dif = img.subtract(ref).pow(ee.Image.constant(2)).reduce(ee.Reducer.sum()) 689 | 690 | difThresh = dif.reduceRegion(**{ 691 | 'reducer': ee.Reducer.percentile(**{ 692 | 'percentiles': [10], 693 | 'maxRaw': 1000000, 694 | 'maxBuckets': 1000000, 695 | 'minBucketWidth': 0.00000000001 696 | }), 697 | 'geometry': granule, 698 | 'scale': 60, 699 | 'maxPixels': 1e13 700 | }) 701 | 702 | mask = dif.lt(difThresh.getNumber('sum')); 703 | 704 | tca = img.select('tca') 705 | tcaGood = tca.gt(0).And(tca.lt(45)) 706 | ndviGood = img.select('ndvi').gt(0) 707 | tcaMasked = tca.updateMask(mask)#.updateMask(tcaGood).updateMask(ndviGood) 708 | breaks = [5, 15, 30, 50, 70, 85, 95] 709 | breakNames = ee.List(breaks).map(lambda num: ee.Number(num).format('%02d')) 710 | tcaBreaks = tcaMasked.reduceRegion(**{ 711 | 'reducer': ee.Reducer.percentile(**{ 712 | 'percentiles': breaks, 713 | 'maxRaw': 1000000, 714 | 'maxBuckets': 1000000, 715 | 'minBucketWidth': 0.00000000001, 716 | 'outputNames': breakNames 717 | }), 718 | 'geometry': granule, 719 | 'scale': 60, 720 | 'maxPixels': 1e13 721 | }) 722 | 723 | breakNames = breakNames.map(lambda i: ee.String('tca_').cat(ee.String(i))) 724 | 725 | for i in range(0, len(breaks)+1): 726 | if i == 0: 727 | classImg = tcaMasked.where( 728 | tcaMasked.lt(tcaBreaks.getNumber(breakNames.get(i))), i) 729 | elif i == len(breaks): 730 | classImg = classImg.where( 731 | tcaMasked.gte(tcaBreaks.getNumber(breakNames.get(i-1))), i) 732 | else: 733 | classImg = classImg.where( 734 | tcaMasked.gte(tcaBreaks.getNumber(breakNames.get(i-1))).And( 735 | tcaMasked.lt(tcaBreaks.getNumber(breakNames.get(i)))), i) 736 | 737 | sampImg = img.addBands(classImg.rename('class').byte()).addBands(ref) 738 | sample = sampImg.stratifiedSample(**{ 739 | 'numPoints': 500, 740 | 'classBand': 'class', 741 | 'region': granule, 742 | 'scale': 60, 743 | }) 744 | 745 | def predictBand(img, samp, targetBand, inputBands): 746 | trainedClassifier = ee.Classifier.smileRandomForest(10).train(**{ 747 | 'features': samp, 748 | 'classProperty': targetBand+'_1', 749 | 'inputProperties': inputBands 750 | }).setOutputMode('REGRESSION') 751 | return img.classify(trainedClassifier).rename(targetBand) 752 | 753 | bandNames = ['green', 'red', 'red-edge', 'nir', 'ndvi', 'tcb', 'tcg', 'tca'] 754 | bands = [] 755 | for band in bandNames: 756 | bands.append(predictBand(img, sample, band, bandNames)) 757 | 758 | return ee.Image(bands).copyProperties(img, img.propertyNames()) # TODO copy only specific properties 759 | 760 | 761 | # Adds TC and NDVI bands to MSS images and applies QA and MSScvm masks to MSS images 762 | def prepMss(img): 763 | toa = msslib['calcToa'](img) 764 | toaAddBands = msslib['addTc'](msslib['addNdvi'](toa)) 765 | toaAddBandsMask = msslib['applyQaMask'](toaAddBands) 766 | return msslib['applyMsscvm'](toaAddBandsMask) 767 | #.select(['green', 'red', 'nir', 'ndvi', 'tcb', 'tcg', 'tca']) 768 | #.multiply(ee.Image([1e4, 1e4, 1e4, 1e4, 1e4, 1e4, 1e2])) 769 | #.round().toShort().copyProperties(img, img.propertyNames())) 770 | 771 | # def processMssWrs1Img(img): 772 | # toaAddBandsMsscvmMask = prepMss(img) 773 | # corrected = correctMssImg(toaAddBandsMsscvmMask) 774 | # return corrected 775 | 776 | def processMssWrs1Img2(img): 777 | toaAddBandsMsscvmMask = prepMss(img) 778 | corrected = correctMssImg2(toaAddBandsMsscvmMask) 779 | return corrected 780 | 781 | def scaleMssToInt16(img): 782 | scale = ee.Image([1e4, 1e4, 1e4, 1e4, 1e4, 1e4, 1e4, 1e2]) 783 | return (ee.Image(img.select(['green', 'red', 'red-edge', 'nir', 'ndvi', 'tcb', 'tcg', 'tca']) 784 | .multiply(scale).round().toShort() 785 | .copyProperties(img, img.propertyNames()))) # TODO copy only properties needed 786 | 787 | # Makes annual MSS image composites based on dates and regions given in params 788 | # The individual MSS WRS1 images are corrected to match the MSS WRS2 reference image 789 | # The images that come out are scaled to int16 790 | # The bands are ['green', 'red', 'red-edge', 'nir', 'ndvi', 'tcb', 'tcg', 'tca'] 791 | # Images are 60 meters clipped to the WRS1 tile specified in the params dictionary 792 | # 1983 is MSS WRS2, it is not made to match the reference image 793 | # The export is a single image with bands for each image labeled by year and band name 794 | def processMssWrs1Imgs(params): 795 | print('Exporting annual MSS composites that match the MSS 2nd Gen reference image, please wait.') 796 | granuleGeom = msslib['getWrs1GranuleGeom'](params['wrs1']) 797 | geom = ee.Feature(granuleGeom.get('granule')).geometry() 798 | params['aoi'] = ee.Geometry(granuleGeom.get('centroid')) 799 | params['wrs'] = '1' 800 | 801 | def setRefImg(img): 802 | return img.set('ref_img', ee.Image(params['baseDir'] + '/ref')) 803 | 804 | mssCol = (msslib['getCol'](params) 805 | .filter(ee.Filter.eq('pr', params['wrs1']))) 806 | 807 | mss1983 = msslib['getCol']({ 808 | 'aoi': geom, 809 | 'wrs': '2', 810 | 'yearRange': [1983, 1983], 811 | 'doyRange': params['doyRange'] 812 | }) 813 | 814 | mssCol = mssCol.merge(mss1983).map(setRefImg) 815 | 816 | dummy = (ee.Image([0, 0, 0, 0, 0, 0, 0, 0]).selfMask().toShort() 817 | .rename(['green', 'red', 'red-edge', 'nir', 'ndvi', 'tcb', 'tcg', 'tca'])) 818 | 819 | imgs = [] 820 | for y in range(1972, 1984): 821 | print('Year:', y) 822 | yrCol = mssCol.filter(ee.Filter.eq('year', y)) 823 | n_imgs = yrCol.size().getInfo() 824 | if (n_imgs == 0): 825 | print(' no images, exporting placeholder') 826 | yearImg = dummy 827 | else: 828 | if y != 1983: 829 | yrCol = yrCol.map(processMssWrs1Img2) 830 | parallelScale = 1 831 | else: 832 | yrCol = yrCol.map(prepMss) # NOTE: not 1983 normalized to ref image 833 | parallelScale = 4 834 | yearImg = getMedoid(yrCol, ['green', 'red', 'red-edge', 'nir', 'ndvi', 'tcb', 'tcg', 'tca'], parallelScale) 835 | 836 | yearImg = scaleMssToInt16(yearImg).set('year', y) 837 | yearImg = appendYearToBandnames(yearImg) 838 | imgs.append(yearImg) 839 | 840 | outImg = appendIdToBandnames(ee.ImageCollection(imgs).toBands()) 841 | outAsset = params['baseDir'] + '/MSS_WRS1_to_WRS2_stack' 842 | print(outAsset) 843 | task = ee.batch.Export.image.toAsset(**{ 844 | 'image': outImg.clip(geom), 845 | 'description': 'MSS_WRS1_to_WRS2_stack', 846 | 'assetId': outAsset, 847 | 'region': geom, 848 | 'scale': 60, 849 | 'crs': params['crs'] 850 | }) 851 | task.start() 852 | return task 853 | 854 | 855 | 856 | # def correctMss1983(params): 857 | # aoi = ee.Feature( 858 | # msslib['getWrs1GranuleGeom'](params['wrs1']).get('granule')).geometry() 859 | # mssCol = msslib['getCol']({ 860 | # 'aoi': aoi, 861 | # 'wrs': '2', 862 | # 'yearRange': [1983, 1983], 863 | # 'doyRange': params['doyRange'] 864 | # }).map(prepMss) 865 | 866 | # mssCol1983 = (mssCol.map(correctMssImg_doit) 867 | # .map(lambda img: img.resample('bicubic'))) 868 | 869 | # outImg = getMedoid(mssCol1983, ['blue', 'green', 'red', 'nir', 'ndvi', 'tcb', 'tcg', 'tcw', 'tca']) \ 870 | # .round().toShort().clip(aoi).set({ 871 | # 'dummy': False, 872 | # 'year': 1983, 873 | # 'system:time_start': ee.Date.fromYMD(1983, 1, 1) 874 | # }) 875 | 876 | # task = ee.batch.Export.image.toAsset(**{ 877 | # 'image': outImg, 878 | # 'description': 'WRS1_to_TM_1983', 879 | # 'assetId': params['baseDir'] + '/WRS1_to_TM/' + '1983', 880 | # 'region': aoi, 881 | # 'scale': 30, 882 | # 'crs': params['crs'], 883 | # 'maxPixels': 1e13 884 | # }) 885 | # task.start() 886 | # return [task] 887 | # # imgs = mssColToTm.aggregate_array('system:index').getInfo() 888 | # # tasks = [] 889 | # # for i in range(0, len(imgs)): 890 | # # fname = '1983_' + str(i).zfill(2) 891 | # # print(fname) 892 | # # thisImg = mssColToTm.filter(ee.Filter.eq('system:index', imgs[i])).first() 893 | 894 | # # task = ee.batch.Export.image.toAsset(**{ 895 | # # 'image': thisImg.clip(aoi), 896 | # # 'description': fname, 897 | # # 'assetId': params['baseDir'] + '/mss_1983_col/' + fname, 898 | # # 'region': aoi, 899 | # # 'scale': 60, # should this be 30 900 | # # 'crs': params['crs'], 901 | # # 'maxPixels': 1e13 902 | # # }) 903 | # # task.start() 904 | # # tasks.append(task) 905 | 906 | # # return tasks 907 | 908 | def renameOli(img): 909 | return img.select( 910 | ['SR_B2', 'SR_B3', 'SR_B4', 'SR_B5', 'SR_B6', 'SR_B7'], # , 'QA_PIXEL' 911 | ['blue', 'green', 'red', 'nir', 'swir1', 'swir2']) # , 'pixel_qa'] 912 | 913 | def renameTm(img): 914 | return img.select( 915 | ['SR_B1', 'SR_B2', 'SR_B3', 'SR_B4', 'SR_B5', 'SR_B7'], # , 'QA_PIXEL' 916 | ['blue', 'green', 'red', 'nir', 'swir1', 'swir2']) # , 'pixel_qa' 917 | 918 | def tmAddIndices(img): 919 | b = ee.Image(img).select(['blue', 'green', 'red', 'nir', 'swir1', 'swir2']) 920 | brt_coeffs = ee.Image.constant([0.2043, 0.4158, 0.5524, 0.5741, 0.3124, 0.2303]) 921 | grn_coeffs = ee.Image.constant([-0.1603, -0.2819, -0.4934, 0.7940, -0.0002, -0.1446]) 922 | wet_coeffs = ee.Image.constant([0.0315, 0.2021, 0.3102, 0.1594, -0.6806, -0.6109]) 923 | brightness = b.multiply(brt_coeffs).reduce(ee.Reducer.sum()).round().toShort().rename('tcb') 924 | greenness = b.multiply(grn_coeffs).reduce(ee.Reducer.sum()).round().toShort().rename('tcg') 925 | wetness = b.multiply(wet_coeffs).reduce(ee.Reducer.sum()).round().toShort().rename('tcw') 926 | angle = (greenness.divide(brightness)).atan().multiply(180 / math.pi).multiply(100).round().toShort().rename('tca') 927 | ndvi = img.normalizedDifference(['nir', 'red']).multiply(1000).round().toShort().rename('ndvi') 928 | tc = ee.Image.cat(ndvi, brightness, greenness, wetness, angle)#.rename(['ndvi', 'tcb', 'tcg', 'tcw' 'tca']) 929 | return img.addBands(tc) 930 | 931 | def gatherTmCol(params): 932 | granuleGeom = msslib['getWrs1GranuleGeom'](params['wrs1']) 933 | aoi = ee.Feature(granuleGeom.get('granule')).geometry() 934 | dateFilter = ee.Filter.calendarRange(params['doyRange'][0], params['doyRange'][1], 'day_of_year') 935 | startDate = ee.Date.fromYMD(params['yearRange'][0], 1, 1) 936 | endDate = startDate.advance(1, 'year') 937 | oli2Col = ee.ImageCollection('LANDSAT/LC09/C02/T1_L2') \ 938 | .filterBounds(aoi).filterDate(startDate, endDate).filter(dateFilter).map(prepOli) 939 | oliCol = ee.ImageCollection('LANDSAT/LC08/C02/T1_L2') \ 940 | .filterBounds(aoi).filterDate(startDate, endDate).filter(dateFilter).map(prepOli) 941 | etmCol = ee.ImageCollection('LANDSAT/LE07/C02/T1_L2') \ 942 | .filterBounds(aoi).filterDate(startDate, endDate).filter(dateFilter).map(prepTm) 943 | tm5Col = ee.ImageCollection('LANDSAT/LT05/C02/T1_L2') \ 944 | .filterBounds(aoi).filterDate(startDate, endDate).filter(dateFilter).map(prepTm) 945 | tm4Col = ee.ImageCollection('LANDSAT/LT04/C02/T1_L2') \ 946 | .filterBounds(aoi).filterDate(startDate, endDate).filter(dateFilter).map(prepTm) 947 | return ee.ImageCollection(ee.FeatureCollection([tm4Col, tm5Col, etmCol, oliCol, oli2Col]).flatten()) 948 | 949 | def prepOli(img): 950 | orig = img 951 | img = scaleMask(img) 952 | img = renameOli(img) 953 | img = tmAddIndices(img) 954 | return ee.Image(img.copyProperties(orig, orig.propertyNames())) # TODO: only copy the needed properties 955 | 956 | def prepTm(img): 957 | orig = img 958 | img = scaleMask(img) 959 | img = renameTm(img) 960 | img = tmAddIndices(img) 961 | return ee.Image(img.copyProperties(orig, orig.propertyNames())) # TODO: only copy the needed properties 962 | 963 | def getCoincidentTmMssCol(params): 964 | aoi = ee.Feature( 965 | msslib['getWrs1GranuleGeom'](params['wrs1']).get('granule')).geometry() 966 | mssCol = msslib['getCol']({ 967 | 'aoi': aoi, 968 | 'wrs': '2', 969 | 'doyRange': params['doyRange'], 970 | 'excludeIds': params['excludeIds'] 971 | }) \ 972 | .map(addTmToMssJoinId) 973 | 974 | tmCol = getTmWrs2Col(aoi).map(addTmToMssJoinId) 975 | coincident = coincidentTmMssCol(mssCol, tmCol) 976 | return coincident 977 | 978 | # Gets a sample of pixels from coincident MSS and TM images. The TM image ID 979 | # is a property of the input MSS image. The bands of TM image are added to the 980 | # MSS image and then sampled using a stratified class band based on MSS TCA 981 | # percentile bins. 982 | def getMsstoTmStratSamp(img): 983 | xImg = scaleMssToInt16(msslib['addTc'](msslib['addNdvi'](msslib['calcToa'](img)))) 984 | yImg = prepTm(ee.Image(xImg.get('coincidentTmMss'))) 985 | granule = ee.Feature(ee.FeatureCollection('users/jstnbraaten/wrs/wrs2_descending_land') \ 986 | .filter(ee.Filter.eq('PR', xImg.getString('pr'))).first()).geometry() 987 | 988 | tca = xImg.select('tca') 989 | #tcaGood = tca.gt(0).And(tca.lt(45)) 990 | ndviGood = xImg.select('ndvi').gt(-500) 991 | tcaMasked = tca#.updateMask(tcaGood).updateMask(ndviGood) 992 | breaks = [5, 15, 30, 50, 70, 85, 95] 993 | breakNames = ee.List(breaks).map(lambda num: ee.Number(num).format('%02d')) 994 | tcaBreaks = tcaMasked.reduceRegion(**{ 995 | 'reducer': ee.Reducer.percentile(**{ 996 | 'percentiles': breaks, 997 | 'maxRaw': 1000000, 998 | 'maxBuckets': 1000000, 999 | 'minBucketWidth': 0.00000000001, 1000 | 'outputNames': breakNames 1001 | }), 1002 | 'geometry': granule, 1003 | 'scale': 60, 1004 | 'maxPixels': 1e13 1005 | }) 1006 | 1007 | breakNames = breakNames.map(lambda i: ee.String('tca_').cat(ee.String(i))) 1008 | 1009 | for i in range(0, len(breaks)+1): 1010 | if i == 0: 1011 | classImg = tcaMasked.where( 1012 | tcaMasked.lt(tcaBreaks.getNumber(breakNames.get(i))), i) 1013 | elif i == len(breaks): 1014 | classImg = classImg.where( 1015 | tcaMasked.gte(tcaBreaks.getNumber(breakNames.get(i-1))), i) 1016 | else: 1017 | classImg = classImg.where( 1018 | tcaMasked.gte(tcaBreaks.getNumber(breakNames.get(i-1))).And( 1019 | tcaMasked.lt(tcaBreaks.getNumber(breakNames.get(i)))), i) 1020 | 1021 | sampImg = xImg.addBands(classImg.rename('class').byte()).addBands(yImg) 1022 | sample = sampImg.stratifiedSample(**{ 1023 | 'numPoints': 25, # Is this value enough, could it be higher, check the output table 1024 | 'classBand': 'class', 1025 | 'region': granule, 1026 | 'scale': 60, 1027 | 'geometries': True 1028 | }) 1029 | 1030 | return (sample.map(lambda f: f.set('constant', 1)) 1031 | .copyProperties(img, ['imgID', 'year', 'path', 'row', 'pr'])) # TODO: these properties are not present in output, probably not in input img 1032 | 1033 | # Creates a sample of pixels from coincident TM and MSS images to use a training 1034 | # sample in a random forest regression classifier. The sample is stratified on MSS 1035 | # TCA from 8(?) percentile bins. 1036 | def exportMss2TmCoefCol(params): 1037 | print('Exporting MSS-to-TM model training sample, please wait.') 1038 | col = getCoincidentTmMssCol(params) 1039 | sample = col.map(getMsstoTmStratSamp).flatten() 1040 | outAsset = params['baseDir'] + '/mss_to_tm_coef_fc' 1041 | print(outAsset) 1042 | task = ee.batch.Export.table.toAsset(**{ 1043 | 'collection': sample, 1044 | 'description': 'mss_to_tm_coef_fc', 1045 | 'assetId': outAsset 1046 | }) 1047 | 1048 | task.start() 1049 | return task 1050 | 1051 | def predictBand_doit(sample, img, targetBand, outName): 1052 | bands = ['green', 'red', 'red-edge', 'nir', 'ndvi', 'tcb', 'tcg', 'tca'] 1053 | trainedClassifier = ee.Classifier.smileRandomForest(10).train(**{ 1054 | 'features': sample, 1055 | 'classProperty': targetBand, 1056 | 'inputProperties': bands 1057 | }).setOutputMode('REGRESSION') 1058 | return img.classify(trainedClassifier).rename(outName).round().toShort() 1059 | 1060 | def correctMssImg_buildit(params, correctOffset): 1061 | #print('1055') 1062 | #print('params', params) 1063 | #print("params['baseDir'] + '/mss_to_tm_coef_fc'", params['baseDir'] + '/mss_to_tm_coef_fc') 1064 | def correctMssImg_doit(img): 1065 | sample = ee.FeatureCollection(params['baseDir'] + '/mss_to_tm_coef_fc') 1066 | #print('1059') 1067 | targetBands = ['blue', 'green_1', 'red_1', 'nir_1', 'swir1', 'swir2', 'ndvi_1', 'tcb_1', 'tcg_1', 'tcw', 'tca_1'] #['blue', 'green_1', 'red_1', 'nir_1', 'ndvi_1', 'tcb_1', 'tcg_1', 'tca_1'] 1068 | outBands = ['blue', 'green', 'red', 'nir', 'swir1', 'swir2', 'ndvi', 'tcb', 'tcg', 'tcw', 'tca'] # ['blue', 'green', 'red', 'nir', 'ndvi', 'tcb', 'tcg', 'tca'] 1069 | # bands = [] 1070 | bands = [None for b in range(0, len(outBands))] 1071 | for i in range(0, len(outBands)): 1072 | # print(i) 1073 | # print(targetBands[i]) 1074 | # print(outBands[i]) 1075 | band = predictBand_doit(sample, img, targetBands[i], outBands[i]) 1076 | # bands.append(band) 1077 | bands[i] = band 1078 | 1079 | outImg = ee.Image(bands) 1080 | if correctOffset: 1081 | offset = ee.Image(params['baseDir'] + '/MSS_offset') 1082 | outImg = outImg.subtract(offset) 1083 | 1084 | return outImg.copyProperties(img, img.propertyNames()) # TODO: only copy the properties needed 1085 | return correctMssImg_doit 1086 | 1087 | 1088 | def mssStackToCol(mssStackPath): 1089 | imgStack = ee.Image(mssStackPath) 1090 | imgList = [] 1091 | for y in range(1972, 1984): 1092 | img = getImgYearFromStack(imgStack, y) 1093 | imgList.append(img) 1094 | return ee.ImageCollection(imgList) 1095 | 1096 | # def getFinalCorrectedMssCol(imgStackPath): 1097 | # imgStack = ee.Image(imgStackPath) 1098 | # imgList = [] 1099 | # for y in range(1972, 1984): 1100 | # img = getImgYearFromStack(imgStack, y) 1101 | # imgList.append(img) 1102 | # 1103 | # return appendIdToBandnames(ee.ImageCollection(imgList) 1104 | # .map(correctMssImg_doit) 1105 | # .map(appendYearToBandnames) 1106 | # .toBands()) 1107 | 1108 | 1109 | def exportMssOffset(params): 1110 | print('Exporting median MSS to TM offset, please wait.') 1111 | # Create the MSS to TM correction function 1112 | correctMssImg_doit = correctMssImg_buildit(params, False) 1113 | # This calcs MSS offset from TM -to be mapped over collection 1114 | def calc_offset(img): 1115 | # Prep MSS 1116 | mssImg = scaleMssToInt16(msslib['addTc'](msslib['addNdvi'](msslib['calcToa'](img)))) 1117 | # Match MSS to TM 1118 | mssImgToTm = ee.Image(correctMssImg_doit(mssImg)) 1119 | # Prep TM 1120 | tmImg = prepTm(ee.Image(mssImgToTm.get('coincidentTmMss'))) 1121 | # Calc difference and return the image 1122 | return mssImgToTm.subtract(tmImg) 1123 | 1124 | # Get MSS / TM collection 1125 | col = getCoincidentTmMssCol(params) 1126 | 1127 | # Map the offset function over the collection and get the median 1128 | difCol = col.map(calc_offset).median().round().toShort() 1129 | 1130 | granuleGeom = msslib['getWrs1GranuleGeom'](params['wrs1']) 1131 | geom = ee.Feature(granuleGeom.get('granule')).geometry() 1132 | 1133 | outAsset = params['baseDir'] + '/MSS_offset' 1134 | print(outAsset) 1135 | task = ee.batch.Export.image.toAsset(**{ 1136 | 'image': difCol.clip(geom), 1137 | 'description': 'MSS_offset', 1138 | 'assetId': outAsset, 1139 | 'region': geom, 1140 | 'scale': 30, 1141 | 'crs': params['crs'], 1142 | 'maxPixels': 1e13 1143 | }) 1144 | task.start() 1145 | return task 1146 | 1147 | 1148 | 1149 | def exportFinalCorrectedMssCol(params): 1150 | print('Exporting annual MSS composites that match TM, please wait.') 1151 | mssCol = mssStackToCol(params['baseDir'] + '/MSS_WRS1_to_WRS2_stack') 1152 | correctMssImg_doit = correctMssImg_buildit(params, params['correctOffset']) 1153 | outImg = appendIdToBandnames(mssCol 1154 | .map(correctMssImg_doit) 1155 | .map(appendYearToBandnames) 1156 | .toBands()) 1157 | 1158 | if params['mssResample'] in ['bicubic', 'bilinear']: 1159 | outImg = outImg.resample(params['mssResample']) 1160 | 1161 | granuleGeom = msslib['getWrs1GranuleGeom'](params['wrs1']) 1162 | geom = ee.Feature(granuleGeom.get('granule')).geometry() 1163 | outAsset = params['baseDir'] + '/WRS1_to_TM_stack' 1164 | print(outAsset) 1165 | 1166 | task = ee.batch.Export.image.toAsset(**{ 1167 | 'image': outImg.clip(geom), 1168 | 'description': 'WRS1_to_TM_stack', 1169 | 'assetId': outAsset, 1170 | 'region': geom, 1171 | 'scale': params['mssScale'], 1172 | 'crs': params['crs'], 1173 | 'maxPixels': 1e13 1174 | }) 1175 | task.start() 1176 | return task 1177 | 1178 | 1179 | # def exportMss1983(params): 1180 | # aoi = ee.Feature( 1181 | # msslib['getWrs1GranuleGeom'](params['wrs1']).get('granule')).geometry() 1182 | # mssCol1983 = ee.ImageCollection(params['baseDir'] + '/mss_1983_col') \ 1183 | # .map(lambda img: img.resample('bicubic')) 1184 | 1185 | # outImg = (getMedoid(mssCol1983, ['blue', 'green', 'red', 'nir', 'swir1', 'ndvi', 'tcb', 'tcg', 'tca']) #['blue', 'green', 'red', 'nir', 'ndvi', 'tcb', 'tcg', 'tca']) \ 1186 | # .set({ 1187 | # 'dummy': False, 1188 | # 'year': 1983, 1189 | # 'system:time_start': ee.Date.fromYMD(1983, 1, 1) 1190 | # })).toShort().clip(aoi) 1191 | 1192 | # task = ee.batch.Export.image.toAsset(**{ 1193 | # 'image': outImg, 1194 | # 'description': 'WRS1_to_TM' + '1983', 1195 | # 'assetId': params['baseDir'] + '/WRS1_to_TM/' + '1983', 1196 | # 'region': aoi, 1197 | # 'scale': 30, 1198 | # 'crs': params['crs'], 1199 | # 'maxPixels': 1e13 1200 | # }) 1201 | # task.start() 1202 | # return [task] 1203 | 1204 | 1205 | def runLt(params): #exportTmComposites(params): 1206 | def add_systime(img): 1207 | millis = ee.Date.fromYMD(img.getNumber('year'), 6 ,1).millis() 1208 | date = ee.Date(millis).format('YYYY-MM-dd') 1209 | return img.set({'system:time_start': millis, 'date': date}) 1210 | 1211 | granuleGeom = msslib['getWrs1GranuleGeom'](params['wrs1']) 1212 | geom = ee.Feature(granuleGeom.get('granule')).geometry() 1213 | 1214 | mssCol = mssStackToCol(params['baseDir'] + '/WRS1_to_TM_stack').map(add_systime) # TODO: these images do not have a system:time_start - they should have one - should not need to add 1215 | # for y in range(1972, 2022): 1216 | # print(mssCol.filter(ee.Filter.eq('year', y)).first().bandNames().getInfo()) 1217 | 1218 | dummyImg = mssCol.first() 1219 | dummyImg = dummyImg.updateMask(dummyImg.mask().multiply(0)) 1220 | 1221 | todaysDate = date.today() 1222 | # tmList = [] 1223 | 1224 | def getTmYearImg(year): 1225 | params_ = copy.deepcopy(params) 1226 | params_['yearRange'] = [year, year] 1227 | bands = ['blue', 'green', 'red', 'nir', 'swir1', 'swir2', 'ndvi', 'tcb', 'tcg', 'tcw', 'tca'] # ['blue', 'green', 'red', 'nir', 'ndvi', 'tcb', 'tcg', 'tca']) 1228 | parallelScale = 8 1229 | tmCol = gatherTmCol(params_) 1230 | date = ee.Date(ee.Date.fromYMD(year, 6 ,1).millis()) 1231 | return ee.Image(ee.Algorithms.If(tmCol.size(), getMedoid(tmCol, bands, parallelScale), dummyImg)).set({ 1232 | 'system:time_start': date.millis(), 1233 | 'year': date.get('year'), 1234 | 'date': date.format('YYYY-MM-dd') 1235 | }) 1236 | tmYearRange = ee.List.sequence(1984, 2022) # NOTE: end year is set here. -todaysDate.year 1237 | tmYearImgList = tmYearRange.map(getTmYearImg) 1238 | tmYearImgCol = ee.ImageCollection.fromImages(tmYearImgList) 1239 | 1240 | 1241 | 1242 | 1243 | 1244 | # for y in range(1984, 2022): # todaysDate.year + 1 1245 | # params['yearRange'] = [y, y] 1246 | # thisYearImg = (getMedoid( 1247 | # gatherTmCol(params), ['blue', 'green', 'red', 'nir', 'swir1', 'swir2', 'ndvi', 'tcb', 'tcg', 'tcw', 'tca']) # ['blue', 'green', 'red', 'nir', 'ndvi', 'tcb', 'tcg', 'tca']) 1248 | # .set({ 1249 | # 'system:time_start': ee.Date.fromYMD(y, 1 ,1).millis(), 1250 | # 'year': y 1251 | # })) 1252 | # tmList.append(thisYearImg) 1253 | 1254 | 1255 | 1256 | def addSegBand(img): 1257 | outImg = img.select('ndvi').multiply(-1).rename('seg') 1258 | return outImg.addBands(img).copyProperties(img, ['system:time_start', 'year', 'date']) 1259 | 1260 | #tmCol = ee.ImageCollection(tmList) 1261 | #col = mssCol.merge(tmCol).map(addSegBand).sort('year') 1262 | 1263 | col = mssCol.merge(tmYearImgCol).map(addSegBand).sort('year') 1264 | years = col.aggregate_array('year')#.getInfo() 1265 | # millis = col.aggregate_array('system:time_start').getInfo() 1266 | 1267 | # print('all years', years) 1268 | # print('all millis', millis) 1269 | 1270 | 1271 | 1272 | # for y in range(1972, 2022): 1273 | # print(col.filter(ee.Filter.eq('year', y)).first().bandNames().getInfo()) 1274 | 1275 | # Deal with missing years (check is other processes are filling bads years, I think MSS is) 1276 | # year_ends = col.aggregate_array('year').reduce(ee.Reducer.minMax()).getInfo() 1277 | # print('year_ends', year_ends) 1278 | # Make a dummy image for missing years. 1279 | # bandNames = ee.List(['Blue', 'Green', 'Red', 'NIR', 'SWIR1', 'SWIR2', 'pixel_qa']) 1280 | # fillerValues = ee.List.repeat(0, bandNames.size()) 1281 | # dummyImg = ee.Image.constant(fillerValues).rename(bandNames).selfMask().int16() 1282 | 1283 | 1284 | # dummies = [] 1285 | # years = list(range(year_ends['min'], year_ends['max']+1)) 1286 | # print('years', years) 1287 | # for y in range(year_ends['min'], year_ends['max']+1): 1288 | # print('Checking year', y) 1289 | # colSize = col.filter(ee.Filter.eq('year', y)).size().getInfo() 1290 | # if colSize == 0: 1291 | # print(y, ' is a dummy') 1292 | # dummies.append(dummyImg.set({ 1293 | # 'system:time_start': ee.Date.fromYMD(y, 6 ,1).millis(), 1294 | # 'year': y 1295 | # })) 1296 | 1297 | # col = col.merge(ee.ImageCollection(dummies)).sort('year') 1298 | 1299 | 1300 | 1301 | # col = col.map(add_systime) 1302 | # print('dates', col.aggregate_array('date').getInfo()) 1303 | lt = ee.Algorithms.TemporalSegmentation.LandTrendr(**{ 1304 | 'timeSeries': col.select(['seg'] + params['ltParams']['ftvBands']), 1305 | 'maxSegments': params['ltParams']['maxSegments'], 1306 | 'spikeThreshold': params['ltParams']['spikeThreshold'], 1307 | 'vertexCountOvershoot': params['ltParams']['vertexCountOvershoot'], 1308 | 'preventOneYearRecovery': params['ltParams']['preventOneYearRecovery'], 1309 | 'recoveryThreshold': params['ltParams']['recoveryThreshold'], 1310 | 'pvalThreshold': params['ltParams']['pvalThreshold'], 1311 | 'bestModelProportion': params['ltParams']['bestModelProportion'], 1312 | 'minObservationsNeeded': params['ltParams']['minObservationsNeeded'] 1313 | }).set({'years': years}) 1314 | 1315 | return lt.int16() 1316 | 1317 | 1318 | 1319 | 1320 | def exportLt(params): 1321 | print('Exporting LandTrendr segmentation and FTV image array, please wait.') 1322 | lt = runLt(params) 1323 | years = ee.Image(lt).get('years').getInfo() 1324 | yearsStr = ['yr_' + str(year) for year in years] 1325 | granuleGeom = msslib['getWrs1GranuleGeom'](params['wrs1']) 1326 | geom = ee.Feature(granuleGeom.get('granule')).geometry() 1327 | 1328 | # tasks = [] 1329 | # for band in params['ltParams']['ftvBands']: 1330 | # print(getPaths(params['projectDir'], params['wrs1'])['fit_collection']+ '/' + band + '_fit') 1331 | # ftv_img = lt.select([band + '_fit']).arrayFlatten([yearsStr]).toShort().set('band', band) 1332 | # task = ee.batch.Export.image.toAsset(**{ 1333 | # 'image': ftv_img.clip(geom), 1334 | # 'description': band + '_fit', 1335 | # 'assetId': getPaths(params['projectDir'], params['wrs1'])['fit_collection'] + '/' + band + '_fit', 1336 | # 'region': geom, 1337 | # 'scale': 30, 1338 | # 'crs': params['crs'], 1339 | # 'maxPixels': 1e13 1340 | # }) 1341 | # task.start() 1342 | # tasks.append(task) 1343 | otherBands = ['rmse'] + [f'{band}_fit' for band in params['ltParams']['ftvBands']] 1344 | ltlt = (lt.select('LandTrendr').arrayPad([4, len(years)], 0) 1345 | .addBands(lt.select(otherBands)).int16()) 1346 | 1347 | ltltBands = ['LandTrendr'] + otherBands 1348 | pyramidingPolicy = {} 1349 | for band in ltltBands: 1350 | if band in ['LandTrendr', 'rmse']: 1351 | pyramidingPolicy[band] = 'sample' 1352 | else: 1353 | pyramidingPolicy[band] = 'mean' 1354 | 1355 | print(params['baseDir'] + '/landtrendr') 1356 | task = ee.batch.Export.image.toAsset(**{ 1357 | 'image': ltlt.clip(geom), 1358 | 'pyramidingPolicy': pyramidingPolicy, 1359 | 'description': 'landtrendr', 1360 | 'assetId': params['baseDir'] + '/landtrendr', 1361 | 'region': geom, 1362 | 'scale': params['ltParams']['scale'], 1363 | 'crs': params['crs'], 1364 | 'maxPixels': 1e13, 1365 | 'shardSize': 32 1366 | }) 1367 | task.start() 1368 | # tasks.append(task) 1369 | 1370 | return task 1371 | 1372 | 1373 | 1374 | 1375 | 1376 | 1377 | def appendYearToBandnames(img): 1378 | img = ee.Image(img) 1379 | year = img.getNumber('year').format() 1380 | names = img.bandNames() 1381 | namesYear = names.map(lambda name: ee.String(year).cat(ee.String('_')).cat(ee.String(name))) 1382 | return img.rename(namesYear) 1383 | 1384 | def appendIdToBandnames(img): 1385 | def processName(band): 1386 | parts = ee.String(band).split('_') 1387 | id = ee.String('img').cat(ee.Number.parse(parts.getString(0)).format('%02d')) 1388 | return parts.splice(0, 1, [id]).join('_') 1389 | 1390 | img = ee.Image(img) 1391 | names = img.bandNames() 1392 | namesId = names.map(processName) 1393 | return img.rename(namesId) 1394 | 1395 | def getImgYearFromStack(img, year): 1396 | search = 'img.._' + str(year) + '.*' 1397 | theseBands = img.select(search) 1398 | namesLong = theseBands.bandNames() 1399 | namesShort = namesLong.map(lambda name: ee.String(name).split('_').get(-1)) 1400 | return theseBands.rename(namesShort).set('year', year) 1401 | 1402 | 1403 | # def getColForLandTrendrOnTheFly(params): 1404 | # mssCol = getFinalCorrectedMssCol(params) 1405 | 1406 | # tmCol = ee.ImageCollection([]) 1407 | # for y in range(1983, 2022): # make this the current year 1408 | # params['yearRange'] = [y, y] 1409 | # thisYearCol = getMedoid(gatherTmCol(params), ['blue', 'green', 'red', 'nir', 'ndvi', 'tcb', 'tcg', 'tca']) \ 1410 | # .set('system:time_start', ee.Date.fromYMD(y, 1 ,1).millis()) 1411 | # tmCol = tmCol.merge(ee.ImageCollection(thisYearCol.toShort())) 1412 | 1413 | # def prepForLt(img): 1414 | # return img.select('ndvi').multiply(-1).rename('LTndvi') \ 1415 | # .addBands(img.select(['blue', 'green', 'red', 'nir', 'ndvi', 'tcb', 'tcg', 'tca'])) \ 1416 | # .set('system:time_start', img.get('system:time_start')) 1417 | 1418 | # combinedCol = mssCol.merge(tmCol).map(prepForLt).sort('system:time_start') 1419 | 1420 | # return combinedCol 1421 | 1422 | 1423 | 1424 | # def getColForLandTrendrFromAsset(params): # Relies on WRS1_to_TM assets: 1425 | # mssCol = ee.ImageCollection([]) 1426 | # for y in range(1972, 1983): 1427 | # img = ee.Image(params['baseDir'] + '/WRS1_to_TM/' + str(y)) \ 1428 | # .set('system:time_start', ee.Date.fromYMD(y, 1 ,1).millis()) 1429 | # mssCol = mssCol.merge(ee.ImageCollection(img)) 1430 | 1431 | # mss1983 = ee.ImageCollection( 1432 | # getMedoid(correctMssWrs2(params), ['blue', 'green', 'red', 'nir', 'ndvi', 'tcb', 'tcg', 'tca']) \ 1433 | # .set('system:time_start', ee.Date.fromYMD(1983, 1 ,1).millis())) 1434 | 1435 | # tmCol = ee.ImageCollection([]) 1436 | # for y in range(1984, 2020, 1): 1437 | # params['yearRange'] = [y, y] 1438 | # thisYearCol = getMedoid( 1439 | # gatherTmCol(params), ['blue', 'green', 'red', 'nir', 'ndvi', 'tcb', 'tcg', 'tca']) \ 1440 | # .set('system:time_start', ee.Date.fromYMD(y, 1 ,1).millis()) 1441 | # tmCol = tmCol.merge(ee.ImageCollection(thisYearCol.toShort())) 1442 | 1443 | # def prepForLt(img): 1444 | # return img.select('ndvi').multiply(-1).rename('LTndvi') \ 1445 | # .addBands(img.select(['blue', 'green', 'red', 'nir', 'ndvi', 'tcb', 'tcg', 'tca'])) \ 1446 | # .set('system:time_start', img.get('system:time_start')) 1447 | 1448 | # return mssCol.merge(mss1983).merge(tmCol).map(prepForLt).sort('system:time_start') 1449 | 1450 | # def runLandTrendrMss2Tm(params): 1451 | # ltCol = getColForLandTrendrFromAsset(params) 1452 | # lt = ee.Algorithms.TemporalSegmentation.LandTrendr(**{ 1453 | # 'timeSeries': ltCol, 1454 | # 'maxSegments': 10, 1455 | # 'spikeThreshold': 0.7, 1456 | # 'vertexCountOvershoot': 3, 1457 | # 'preventOneYearRecovery': True, 1458 | # 'recoveryThreshold': 0.5, 1459 | # 'pvalThreshold': 0.05, 1460 | # 'bestModelProportion': 0.75, 1461 | # 'minObservationsNeeded': 6 1462 | # }) 1463 | # return lt 1464 | 1465 | 1466 | def checkStatus(taskList): 1467 | status = [] 1468 | for i in range(0, len(taskList)): 1469 | status.append(taskList[i].status()['state'] == 'COMPLETED') 1470 | return all(status) 1471 | 1472 | 1473 | def printStatus(taskList): 1474 | for i in range(0, len(taskList)): 1475 | print(i) 1476 | print(taskList[i].status()['state']) 1477 | 1478 | def monitorTaskStatus(task): 1479 | keep_going = True 1480 | while keep_going: 1481 | time.sleep(60) 1482 | state = task.status()['state'] 1483 | if state not in ['UNSUBMITTED', 'READY', 'RUNNING']: 1484 | keep_going = False 1485 | else: 1486 | print(state) 1487 | 1488 | # Code to create and delete assets. 1489 | def getPaths(project_dir, wrs_1_granule): 1490 | base_dir = os.path.join(project_dir, wrs_1_granule) 1491 | #mss_wrs1_to_wrs2 = os.path.join(base_dir, 'WRS1_to_WRS2') 1492 | #mss_wrs1_to_tm = os.path.join(base_dir, 'WRS1_to_TM') 1493 | #post_mss = os.path.join(base_dir, 'post_mss') 1494 | #collection = os.path.join(base_dir, 'collection') 1495 | #mss_1983_col = os.path.join(base_dir, 'mss_1983_col') 1496 | #fit_collection = os.path.join(base_dir, 'fit_collection') 1497 | return { 1498 | 'wrs1_scene': base_dir, 1499 | #'mss_wrs1_to_wrs2': mss_wrs1_to_wrs2, 1500 | #'mss_wrs1_to_tm': mss_wrs1_to_tm, 1501 | #'post_mss': post_mss, 1502 | #'collection': collection, 1503 | #'mss_1983_col': mss_1983_col, 1504 | #'fit_collection': fit_collection 1505 | } 1506 | 1507 | def createProjectDir(project_dir, wrs_1_granule): 1508 | paths = getPaths(project_dir, wrs_1_granule) 1509 | os.system('earthengine create folder ' + paths['wrs1_scene']) 1510 | #os.system('earthengine create folder ' + paths['mss_wrs1_to_wrs2']) 1511 | #os.system('earthengine create folder ' + paths['mss_wrs1_to_tm']) 1512 | #os.system('earthengine create folder ' + paths['post_mss']) 1513 | #os.system('earthengine create collection ' + paths['collection']) 1514 | #os.system('earthengine create collection ' + paths['mss_1983_col']) 1515 | #os.system('earthengine create collection ' + paths['fit_collection']) 1516 | 1517 | # def rmMssWrs1ToWrs2(params): 1518 | # paths = getPaths(params['projectDir'], params['wrs1']) 1519 | # assets = ee.data.listAssets( 1520 | # params={'parent': paths['mss_wrs1_to_wrs2']})['assets'] 1521 | # for asset in assets: 1522 | # os.system(' '.join(['earthengine rm', asset['name']])) 1523 | 1524 | # def rmMssWrs1ToTm(params): 1525 | # paths = getPaths(params['projectDir'], params['wrs1']) 1526 | # assets = ee.data.listAssets( 1527 | # params={'parent': paths['mss_wrs1_to_tm']})['assets'] 1528 | # for asset in assets: 1529 | # os.system(' '.join(['earthengine rm', asset['name']])) 1530 | 1531 | # def rmPostMss(params): 1532 | # paths = getPaths(params['projectDir'], params['wrs1']) 1533 | # assets = ee.data.listAssets( 1534 | # params={'parent': paths['post_mss']})['assets'] 1535 | # for asset in assets: 1536 | # os.system(' '.join(['earthengine rm', asset['name']])) 1537 | 1538 | # def rmCollection(params): 1539 | # paths = getPaths(params['projectDir'], params['wrs1']) 1540 | # assets = ee.data.listImages( 1541 | # params={'parent': paths['collection']})['images'] 1542 | # for asset in assets: 1543 | # os.system(' '.join(['earthengine rm', asset['name']])) 1544 | 1545 | def rmFitCollection(params): 1546 | paths = getPaths(params['projectDir'], params['wrs1']) 1547 | assets = ee.data.listImages( 1548 | params={'parent': paths['fit_collection']})['images'] 1549 | for asset in assets: 1550 | os.system(' '.join(['earthengine rm', asset['name']])) 1551 | 1552 | def rmMss2TmInfo(params): 1553 | os.system(' '.join(['earthengine rm', os.path.join(params['baseDir'], 'mss_to_tm_coef_fc')])) 1554 | # os.system(' '.join(['earthengine rm', os.path.join(params['baseDir'], 'mss_offset')])) 1555 | 1556 | # def images2Col_doit(path, outCol): 1557 | # assets = ee.data.listAssets(params={'parent': path})['assets'] 1558 | # for asset in assets: 1559 | # old = asset['name'] 1560 | # year = os.path.basename(old) 1561 | # print('Moving:', year) 1562 | # new = os.path.join(outCol, year) 1563 | # os.system(' '.join(['earthengine rm', new])) 1564 | # os.system(' '.join(['earthengine cp', old, new])) 1565 | 1566 | # def images2Col(params): 1567 | # paths = getPaths(params['projectDir'], params['wrs1']) 1568 | # _images2Col(paths['mss_wrs1_to_tm'], paths['collection']) 1569 | # #_images2Col(paths['post_mss'], paths['collection']) 1570 | --------------------------------------------------------------------------------