├── geepyGLAD ├── _version.py ├── __init__.py ├── logger.py ├── alerts.py ├── utils.py └── batch.py ├── .gitignore ├── setup.py ├── README.md └── glad.py /geepyGLAD/_version.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | 3 | __version__ = '0.2.2' -------------------------------------------------------------------------------- /geepyGLAD/__init__.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | 3 | """ Python package to handle Global Forest Watch GLAD alerts using Google 4 | Earth Engine """ 5 | 6 | from ._version import __version__ -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # IDEA IDE 2 | .idea/ 3 | 4 | # any checkpoint 5 | **/.ipynb_checkpoints 6 | 7 | # Environments 8 | .env 9 | env 10 | 11 | # Distribution / packaging 12 | /build 13 | /dist/ 14 | *.egg-info/ 15 | 16 | # Unit test / coverage reports 17 | .cache 18 | 19 | /alerts/* 20 | 21 | *.ipynb 22 | notebooks/ 23 | 24 | **/__pycache__ 25 | 26 | # CONFIG 27 | config.json -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | setup( 4 | name='ipygeeGLAD', 5 | version='0.0.1', 6 | py_modules=['glad'], 7 | install_requires=[ 8 | 'Click', 9 | 'earthengine-api', 10 | 'oauth2client', 11 | 'geetools' 12 | ], 13 | entry_points=''' 14 | [console_scripts] 15 | glad=glad:main 16 | ''', 17 | ) -------------------------------------------------------------------------------- /geepyGLAD/logger.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | """ Logger module for a custom Logger """ 3 | import datetime 4 | import os 5 | from ._version import __version__ 6 | 7 | HEADER = "geepyGLAD version {}\n\n{{}}".format(__version__) 8 | 9 | 10 | class Logger(object): 11 | def __init__(self, name, folder=None, filetype='txt'): 12 | self.name = name 13 | self._logs = [] 14 | self._header = None 15 | if not folder: 16 | self.path = os.getcwd() 17 | else: 18 | self.path = os.path.join(os.getcwd(), folder) 19 | exists = os.path.exists(self.path) 20 | if not exists: 21 | os.mkdir(self.path) 22 | 23 | self.filetype = filetype 24 | if filetype == 'txt': 25 | name = '{}.txt'.format(self.name) 26 | filename = os.path.join(self.path, name) 27 | else: 28 | raise ValueError('file type {} not allowed'.format(filetype)) 29 | 30 | self.filename = filename 31 | 32 | def header(self, text): 33 | """ writer the header """ 34 | exists = os.path.exists(os.path.join(self.path, self.filename)) 35 | if not self._header and not exists: 36 | self._header = HEADER.format(text) 37 | with open(self.filename, 'ab+') as f: 38 | f.write('{}\n\n'.format(text).encode('utf-8')) 39 | 40 | def log(self, message): 41 | """ write a log into the logger """ 42 | t = datetime.datetime.today().isoformat() 43 | msg = '{time} - {msg}\n'.format(time=t, msg=message) 44 | self._logs.append(msg) 45 | with open(self.filename, 'ab+') as f: 46 | try: 47 | f.write(msg.encode('utf-8')) 48 | except Exception as e: 49 | raise e 50 | 51 | def text(self): 52 | """ get the log message as a string """ 53 | body = '\n'.join(self._logs) 54 | if self._header: 55 | body = "{}\n\n{}".format(self._header, body) 56 | return body 57 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## Global Forest Watch GLAD alerts using Google Earth Engine 2 | 3 | More information about GLAD alerts in http://glad-forest-alert.appspot.com/ 4 | 5 | *Hansen, Matthew C., Alexander Krylov, Alexandra Tyukavina, Peter V. Potapov, Svetlana Turubanova, Bryan Zutta, Suspense Ifo, Belinda Margono, Fred Stolle, and Rebecca Moore. “Humid Tropical Forest Disturbance Alerts Using Landsat Data.” Environmental Research Letters 11, no. 3 (2016): 034008. https://doi.org/10.1088/1748-9326/11/3/034008.* 6 | 7 | This code takes the GLAD alerts dataset available in Google Earth Engine and downloads a vector layer of the alerts for a desired period. 8 | 9 | ### Install in Linux 10 | 11 | 1. clone this repository: `git clone https://github.com/fitoprincipe/geepyGLAD` 12 | 2. get into the cloned repo: `cd geepyGLAD` 13 | 3. Install `virtualenv` if not installed: https://virtualenv.pypa.io/en/latest/installation/ 14 | 3. Install: 15 | 16 | ```bash 17 | $ virtualenv venv --python=python3 18 | $ . venv/bin/activate 19 | $ pip install --editable . 20 | ``` 21 | 22 | ### Install in Windows 23 | 24 | You can follow a similar approach to the Linux installation, but you'll need some previous work for which you can follow this guide: https://programwithus.com/learn-to-code/Pip-and-virtualenv-on-Windows/ 25 | 26 | Or, you can use [Anaconda](https://www.anaconda.com/). 27 | 28 | 1. Install Anaconda 29 | 30 | 2. Open Anaconda Prompt 31 | 32 | 3. Create a conda environment with python 3. Follow this guide: https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html#creating-an-environment-with-commands 33 | 34 | 4. Be sure the environment is activated, the prompt should have the name of the newly created environment between parenthesis. For example, if the name of the environment is `geepy3` the prompt should look: 35 | 36 | ``` bash 37 | (geepy3) C:/> 38 | ``` 39 | 40 | 5. If it is not activated, then activate: 41 | 42 | ``` bash 43 | (base) C:/>conda activate geepy3 44 | (geepy3) C:/> 45 | ``` 46 | 47 | 6. clone this repository 48 | 49 | ``` bash 50 | (geepy3) C:/>git clone https://github.com/fitoprincipe/geepyGLAD 51 | ``` 52 | 53 | 7. get into the cloned repository 54 | ``` bash 55 | (geepy3) C:/>cd geepyGLAD 56 | (geepy3) C:/geepyGLAD> 57 | ``` 58 | 59 | 8. Install this repository 60 | ``` bash 61 | (geepy3) C:/geepyGLAD>pip install --editable . 62 | ``` 63 | **NOTE**: The dot (.) is important 64 | 65 | 9. Install Earth Engine Python API 66 | ``` bash 67 | (geepy3) C:/geepyGLAD>conda install -c conda-forge earthengine-api 68 | ``` 69 | 10. Authenticate to Earth Engine 70 | ``` bash 71 | (geepy3) C:/geepyGLAD>earthengine authenticate 72 | ``` 73 | ### First time usage 74 | 75 | After installation 76 | 77 | 1. navigate to a folder where you want to download the alerts. Must be different from the installation folder. For example, it could be `glad_alerts`: 78 | 79 | ``` bash 80 | (geepy3) C:/geepyGLAD>cd .. 81 | (geepy3) C:/>mkdir glad_alerts 82 | (geepy3) C:/glad_alerts> 83 | ``` 84 | 85 | 2. In that empty folder check if the installation was successful: 86 | 87 | ``` bash 88 | (geepy3) C:/glad_alerts>glad --help 89 | ``` 90 | If installation was successful, this will prompt: 91 | ``` bash 92 | Usage: glad [OPTIONS] COMMAND [ARGS]... 93 | 94 | Options: 95 | --help Show this message and exit. 96 | 97 | Commands: 98 | alert Export GLAD alerts to Google Drive, Earth Engine Asset or... 99 | make-config Create a configuration file (config.json) 100 | period Export a period (from START to END) of GLAD alerts to... 101 | sites Show available site names 102 | update-config Update config.json. 103 | user Show Earth Engine user and email 104 | ``` 105 | 106 | 3. Create a configuration file 107 | 108 | ``` bash 109 | (geepy3) C:/glad_alerts>glad make-config 110 | ``` 111 | 112 | ### Configuration 113 | 114 | The configuration for this application is driven by `config.json` in the following way: 115 | 116 | ``` 117 | { 118 | "site": { 119 | "assetPath": "USDOS/LSIB_SIMPLE/2017", 120 | "propertyName": "country_na" 121 | }, 122 | 123 | "date": "today", 124 | 125 | "minArea": 1, 126 | 127 | "smooth": "max", 128 | 129 | "class": "both", 130 | 131 | "drive": { 132 | "folder": "GLAD_ALERTS", 133 | "format": "GeoJSON" 134 | }, 135 | 136 | "asset": { 137 | "folder": "GLAD_ALERTS" 138 | }, 139 | 140 | "local": { 141 | "folder": "alerts", 142 | "subfolders": "True", 143 | "format": "JSON" 144 | }, 145 | 146 | "saveTo": "drive", 147 | 148 | "rasterMask": null, 149 | 150 | "vectorMask": null 151 | } 152 | ``` 153 | - **assetPath**: the path of the asset that holds the boundaries to clip the 154 | alerts. See: https://developers.google.com/earth-engine/importing. For example, 155 | for countries you can use `USDOS/LSIB_SIMPLE/2017` 156 | - **propertyName**: the name of the property of the table that "divides" it. 157 | For example, country names in the given table are in property named `country_na` 158 | - **date**: date for the alert in format YYYY-MM-DD. If the word "today" is 159 | parsed, it'll compute the alerts for the date when the script is run. This can 160 | also be modified when running the script using parameter `-d` or `--date`. 161 | - **minArea**: the minimum area to include in results. The units are hectares. 162 | - **smooth**: the smoothing method. Can be one of: `max`, `mode` or `none`. 163 | - **class**: the class to compute. Can be one of `probable`, `confirmed` or `both` 164 | - **drive**: 165 | - **folder**: Google Drive folder to save results. It cannot be a subfolder. 166 | It can only be a folder in root. If not present, it will be created. 167 | - **format**: one of "GeoJSON", "KML", "KMZ", or "SHP" 168 | - **asset**: 169 | - **folder**: asset folder to save the results. It can be a subfolder. For 170 | example: `gfw/glad/alers` 171 | - **local**: 172 | - **folder**: folder to download the results 173 | - **subfolders**: if `True` it will create subfolders with the name of each 174 | record (given by `propertyName`) 175 | - **format**: it can only be `JSON` 176 | - **saveTo**: location to save the results. Can be one of `drive`, `asset` or 177 | `local` 178 | - **rasterMask**: the assetId for a raster mask 179 | - **vectorMask**: the assetId for a vector mask (FeatureCollection) 180 | 181 | To modify the configuration file you can (carefully) modify the file `config.json` or you can do it safely using a cmd command: 182 | 183 | ``` bash 184 | (geepy3) C:/glad_alerts>glad update-config 185 | ``` 186 | 187 | To see the help for this command type 188 | ``` bash 189 | (geepy3) C:/glad_alerts>glad update-config --help 190 | ``` 191 | 192 | ### Everyday usage 193 | 194 | Suppose you just turned on your computer, what do you do? 195 | 196 | 1. Open Anaconda Prompt 197 | 198 | 2. Activate environment. For example, 199 | 200 | ``` bash 201 | (base) C:/>conda activate geepy3 202 | (geepy3) C:/> 203 | ``` 204 | 205 | 3. navigate to alerts folder. For example, 206 | ``` bash 207 | (geepy3) C:/>cd glad_alerts 208 | (geepy3) C:/cd glad_alerts> 209 | ``` 210 | 211 | 4. If you already set up the configuration, the just run: 212 | ``` bash 213 | (geepy3) C:/cd glad_alerts>glad alert 214 | ``` 215 | To see the help for this command type 216 | ``` bash 217 | (geepy3) C:/cd glad_alerts>glad alert --help 218 | ``` -------------------------------------------------------------------------------- /geepyGLAD/alerts.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | 3 | """ Main module for GLAD alerts """ 4 | 5 | import ee 6 | import datetime 7 | from . import utils 8 | from geetools import tools 9 | 10 | ALERTS = utils.cleanup_sa19(ee.ImageCollection('projects/glad/alert/UpdResult')) 11 | TODAY = datetime.date.today() 12 | 13 | 14 | def proxy(image): 15 | """ Make a proxy (empty) image with the same bands as the parsed image """ 16 | unmasked = image.unmask() 17 | return unmasked.where(unmasked.neq(0), 0).selfMask() 18 | 19 | 20 | def get_images(collection, year): 21 | """ Get last 2 images of the given collection and date band of the last 22 | image """ 23 | # build band names 24 | year = ee.Number(year).format() 25 | suffix = year.slice(2) 26 | 27 | bandname = ee.String('conf').cat(suffix) 28 | datebandname = ee.String('alertDate').cat(suffix) 29 | 30 | last2 = collection.limit(2, 'system:time_start', False) 31 | last2list = last2.toList(2) 32 | 33 | replace = ee.Dictionary({})\ 34 | .set(bandname, 'alert')\ 35 | .set(datebandname, 'alert_doy') 36 | 37 | # Last image 38 | last = ee.Image(last2list.get(0)) 39 | last = tools.image.renameDict(last, replace) 40 | 41 | # Image before last image 42 | before = ee.Image(last2list.get(1)) 43 | before = tools.image.renameDict(before, replace) 44 | 45 | return last, before 46 | 47 | 48 | def last_confirmed_mask(collection, year): 49 | """ Get the latest confirmed mask from the given collection and year """ 50 | last, before = get_images(collection, year) 51 | 52 | last_alert = last.select('alert') 53 | before_alert = before.select('alert') 54 | 55 | # It's confirmed when last is 3 and befores is 2 56 | confirmed = last_alert.eq(3).And(before_alert.eq(2)).rename('confirmed') 57 | 58 | return last.addBands(confirmed).updateMask(confirmed).unmask() 59 | 60 | 61 | def last_probable_mask(collection, year): 62 | """ Get the latest probable mask from the given collection and year """ 63 | 64 | last, before = get_images(collection, year) 65 | 66 | last_alert = last.select('alert') 67 | before_alert = before.select('alert') 68 | 69 | probable = last_alert.eq(2).And(before_alert.eq(0)).rename('probable') 70 | return last.addBands(probable).updateMask(probable).unmask() 71 | 72 | 73 | def get_probable_OLD(site, date, limit=1, smooth='max'): 74 | """ Get the 'probable' mask for the given site and date """ 75 | try: 76 | site = site.geometry() 77 | except: 78 | pass 79 | 80 | date = ee.Date(date) 81 | year = date.get('year') 82 | month = date.get('month') 83 | day = date.get('day') 84 | 85 | # filter collection up to selected date 86 | start = ee.Date.fromYMD(year, 1, 1) 87 | col = ALERTS.filterDate(ee.Date(start), date.advance(1, 'day')) 88 | 89 | # filter bounds 90 | col = col.filterBounds(site) 91 | 92 | loss = last_probable_mask(col, year) 93 | 94 | # clip loss 95 | loss = loss.clip(site) 96 | 97 | # loss mask 98 | loss_mask = loss.select('probable') 99 | 100 | # observation date 101 | alertDate = loss.select('alert_doy') 102 | 103 | # get rid of min area 104 | final_mask = utils.get_rid_min_area(loss_mask, limit) 105 | 106 | # smooth 107 | final_mask = utils.smooth(final_mask, smooth) 108 | 109 | # add doy band 110 | smooth_alertDate = utils.smooth(alertDate, 'max').rename('alert_doy') 111 | 112 | # add date band 113 | dateBand = tools.image.doyToDate(smooth_alertDate).rename('alert_date') 114 | 115 | final = final_mask.addBands([smooth_alertDate, dateBand]) 116 | 117 | final_masked = final.updateMask(final_mask) 118 | 119 | # get days 120 | days = utils.get_days(month, year, col) 121 | 122 | return ee.Image(ee.Algorithms.If(days.contains(day), final_masked, 123 | proxy(final))) 124 | 125 | 126 | def get_confirmed_OLD(site, date, limit=1, smooth='max'): 127 | date = ee.Date(date) 128 | year = date.get('year') 129 | month = date.get('month') 130 | day = date.get('day') 131 | 132 | # filter collection up to selected date 133 | start = ee.Date.fromYMD(year, 1, 1) 134 | col = ALERTS.filterDate(ee.Date(start), date.advance(1, 'day')) 135 | 136 | col = col.filterBounds(site) 137 | 138 | # get mask 139 | loss = last_confirmed_mask(col, year) 140 | 141 | # get rid of min area 142 | final_mask = utils.get_rid_min_area(loss.select('confirmed'), limit) 143 | 144 | # smooth 145 | final_mask = utils.smooth(final_mask, smooth) 146 | 147 | # mask doy 148 | doy = loss.select('alert_doy') 149 | 150 | # focal_max doy 151 | smooth_alertDate = utils.smooth(doy, smooth) 152 | 153 | # add date band 154 | dateBand = tools.image.doyToDate(smooth_alertDate).rename('alert_date') 155 | 156 | final = final_mask.addBands([dateBand, smooth_alertDate]) 157 | 158 | final_masked = final.updateMask(final_mask) 159 | 160 | # get days 161 | days = utils.get_days(month, year, col) 162 | 163 | return ee.Image(ee.Algorithms.If(days.contains(day), final_masked, 164 | proxy(final))) 165 | 166 | 167 | def period(start, end, site, limit, year=None, eightConnected=False, 168 | useProxy=False, mask=None): 169 | """ Compute probable and confirmed alerts over a period 170 | 171 | :param start: the start date of the period 172 | :param end: the end date of the period (inclusive) 173 | :param site: the site 174 | :type site: ee.Geometry or ee.Feature or ee.FeatureCollection 175 | :param limit: the minimum area to be computed 176 | :param year: the year to compute. If None takes the year from the date of 177 | the last available image 178 | :param eightConnected: parameter to pass to ee.Kernel 179 | :param useProxy: if True, includes alerts that did not change over the 180 | given period, but were alerts before the start date. Therefore, those 181 | alerts will not have a valid confirmedDate or probableDate (both will 182 | be 0). Also, alertDate will be before start_period property 183 | :param mask: a mask to apply to results. Typically a forest mask. If a 184 | string is passed, it will try to load it as an Image asset 185 | :type mask: ee.Image or str 186 | """ 187 | if isinstance(site, (ee.Feature, ee.FeatureCollection)): 188 | region = site.geometry() 189 | else: 190 | region = site 191 | 192 | start = ee.Date(start) 193 | end = ee.Date(end).advance(1, 'day') 194 | 195 | filtered = ALERTS.filterBounds(region) 196 | 197 | if mask: 198 | if isinstance(mask, (ee.Image,)): 199 | maski = mask 200 | else: 201 | maski = ee.Image(mask) 202 | filtered = filtered.map(lambda img: img.updateMask(maski)) 203 | 204 | sort = filtered.sort('system:time_start', True) # sort ascending 205 | filteredDate = sort.filterDate(start, end) 206 | 207 | filteredDate = utils.compute_breaks(filteredDate, year) 208 | 209 | # always get a last image 210 | last = ee.Image(tools.imagecollection.getImage(filteredDate, -1)) 211 | period_first = ee.Image(filteredDate.first()) 212 | 213 | bands = utils.get_bands(last, year) 214 | confband = ee.String(bands.get('conf')) 215 | dateband = ee.String(bands.get('alertDate')) 216 | yearStr = ee.String(bands.get('suffix')) 217 | if not year: 218 | yearInt = ee.Number(end.get('year')).toInt() 219 | else: 220 | yearInt = ee.Number(year).toInt() 221 | 222 | if useProxy: 223 | proxy = tools.image.empty(0, last.bandNames()) 224 | first = proxy.copyProperties(source=last, 225 | properties=['system:footprint']) \ 226 | .set('system:time_start', start.millis()) 227 | first = ee.Image(first) 228 | else: 229 | first = period_first 230 | 231 | firstconf = first.select(confband) 232 | lastconf = last.select(confband) 233 | 234 | diff = lastconf.subtract(firstconf) 235 | 236 | probname = ee.String('probable').cat(yearStr) 237 | confname = ee.String('confirmed').cat(yearStr) 238 | 239 | probable = diff.eq(2).rename(probname) 240 | confirmed = diff.eq(1).Or(diff.eq(3)).rename(confname) 241 | 242 | probable = utils.get_rid_islands(probable, limit, eightConnected) 243 | confirmed = utils.get_rid_islands(confirmed, limit, eightConnected) 244 | 245 | area_probable = probable.select('area') 246 | area_confirmed = confirmed.select('area') 247 | 248 | probable = probable.select(probname).selfMask() 249 | confirmed = confirmed.select(confname).selfMask() 250 | 251 | area = area_probable.add(area_confirmed) 252 | mask = area.gt(0) 253 | area = area.updateMask(mask) 254 | 255 | date = tools.image.doyToDate( 256 | last.select(dateband), year=yearInt).rename(ee.String('alertDate').cat(yearStr)) 257 | date = date.updateMask(mask) 258 | 259 | # detected 260 | detected = last.select(ee.String('detectedDate').cat(yearStr)) 261 | detected = detected.updateMask(mask) 262 | 263 | # probable 264 | probD = last.select(ee.String('probableDate').cat(yearStr)) 265 | probD = probD.updateMask(mask) 266 | 267 | # confirmed 268 | confD = last.select(ee.String('confirmedDate').cat(yearStr)) 269 | confD = confD.updateMask(mask) 270 | 271 | final = probable.addBands([confirmed, area, date, detected, probD, confD]) 272 | dateformat = 'Y-MM-dd' 273 | 274 | return final.set('start_period', period_first.date().format(dateformat)) \ 275 | .set('end_period', last.date().format(dateformat)) \ 276 | .set('year', yearInt) 277 | 278 | 279 | def oneday(site, date, limit=500, year=None, eightConnected=False, mask=None): 280 | """ Compute alerts for one day. Takes the last available alerts and the 281 | alerts 1 step before """ 282 | date = ee.Date(date) 283 | 284 | if isinstance(site, (ee.Feature, ee.FeatureCollection)): 285 | region = site.geometry() 286 | else: 287 | region = site 288 | 289 | col = ALERTS.filterBounds(region) 290 | col = col.filterDate(ee.Date('1970-01-01'), date.advance(1, 'day')) 291 | 292 | last = tools.imagecollection.getImage(col, -1) 293 | before = tools.imagecollection.getImage(col, -2) 294 | 295 | return period(before.date(), last.date().advance(1,'day'), site, limit, 296 | year, eightConnected=eightConnected, mask=mask) 297 | 298 | 299 | def get_probable(site, date, limit=500, eightConnected=False, mask=None): 300 | """ Get only probable alerts """ 301 | alerts = oneday(site, date, limit, eightConnected, mask) 302 | probable_mask = alerts.select('probable') 303 | return alerts.updateMask(probable_mask) 304 | 305 | 306 | def get_confirmed(site, date, limit=500, eightConnected=False, mask=None): 307 | """ Get only confirmed alerts """ 308 | alerts = oneday(site, date, limit, eightConnected, mask) 309 | probable_mask = alerts.select('confirmed') 310 | return alerts.updateMask(probable_mask) 311 | -------------------------------------------------------------------------------- /geepyGLAD/utils.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | 3 | """ Util functions for GLAD alerts """ 4 | 5 | import ee 6 | from geetools import tools 7 | import math 8 | 9 | 10 | def cleanup_sa19(collection): 11 | """ South America alerts for 2019 have an image that should not be 12 | there """ 13 | def wrap(img, l): 14 | l = ee.List(l) 15 | theid = img.id() 16 | return ee.Algorithms.If(theid.compareTo('01_01_SBRA'), l.add(img), l) 17 | 18 | alerts = ee.List(collection.iterate(wrap, ee.List([]))) 19 | alerts = ee.ImageCollection.fromImages(alerts) 20 | return alerts 21 | 22 | 23 | def get_days(month, year, collection=None): 24 | """ Get days available for the given month and year """ 25 | if collection is None: 26 | collection = cleanup_sa19(ee.ImageCollection('projects/glad/alert/UpdResult')) 27 | 28 | def wrap(img): 29 | d = img.date() 30 | m = d.get('month') 31 | y = d.get('year') 32 | return img.set('MONTH', m, 'YEAR', y) 33 | 34 | col_date = collection.map(wrap) 35 | 36 | filtered = col_date.filterMetadata( 37 | 'MONTH', 'equals', month).filterMetadata('YEAR', 'equals', year) 38 | 39 | def iteration(img, l): 40 | l = ee.List(l) 41 | img = ee.Image(img) 42 | d = img.date().get('day') 43 | return l.add(d) 44 | 45 | days = ee.List(filtered.iterate(iteration, ee.List([]))).sort() 46 | days_str = days.map(lambda d: ee.Number(d)) 47 | 48 | return days_str.distinct() 49 | 50 | 51 | def has_image(date, collection): 52 | """ Returns True if there is at least one image for the parsed date in the 53 | parsed collection 54 | 55 | Is a server side code, so it returns a server side `True` 56 | (use `getInfo` to retrieve it) 57 | """ 58 | d = ee.Date(date) 59 | dates = collection.toList(collection.size()).map( 60 | lambda img: ee.Image(img).date()) 61 | 62 | return ee.List(dates).contains(d) 63 | 64 | 65 | def get_pixel_limit(area, scale): 66 | """ Return number of pixels in the given area (m2) for the given scale """ 67 | area = ee.Number(area).multiply(10000) 68 | scale = ee.Number(scale) 69 | return area.divide(scale.multiply(scale)).floor() 70 | 71 | 72 | def get_rid_min_area(bool_image, limit): 73 | """ Get rid of 'islands' and 'holes' less than the given limit param. 74 | 75 | :param bool_image: The boolean image that will be use to detect islands and 76 | holes. It must be boolean (ones and zeros) 77 | :param limit: all islands and holes less than this limit will be erased. 78 | This must be in m2. 79 | """ 80 | area = ee.Image.pixelArea().rename('area') 81 | 82 | # image = bool_image.addBands(area) 83 | conn = bool_image.connectedPixelCount(1000).rename('connected') 84 | finalarea = area.multiply(conn) 85 | 86 | # get holes and islands 87 | island = bool_image.eq(1).And(finalarea.lte(limit)) 88 | holes = bool_image.eq(0).And(finalarea.lte(limit)) 89 | 90 | # fill holes 91 | filled = bool_image.where(holes, 1) 92 | # get rid island 93 | no_island = filled.where(island, 0) 94 | 95 | return no_island 96 | 97 | 98 | def get_rid_islands(bool_image, limit, scale=30, eightConnected=False): 99 | """ Get rid of 'islands' and 'holes' less than the given limit param. 100 | 101 | :param bool_image: The boolean image that will be use to detect islands and 102 | holes. It must be boolean (ones and zeros) 103 | :param limit: all islands and holes less than this limit will be erased. 104 | This must be in m2. 105 | """ 106 | area = ee.Image.pixelArea().rename('area') 107 | 108 | scale = ee.Number(scale) 109 | limit = ee.Number(limit) 110 | 111 | count = limit.divide(scale.pow(2)).multiply(1.1) 112 | count = count.round() 113 | 114 | conn = bool_image.connectedPixelCount(512, eightConnected).rename('connected') 115 | finalarea = area.multiply(conn) 116 | 117 | # get holes and islands 118 | island = bool_image.eq(1).And(finalarea.lte(limit)) 119 | 120 | # get rid island 121 | no_island = bool_image.where(island, 0) 122 | 123 | finalarea = finalarea.updateMask(no_island).unmask() 124 | 125 | return no_island.addBands(finalarea) 126 | 127 | 128 | def smooth(image, algorithm='max'): 129 | """ Get the smooth algorithms given its name """ 130 | algs = { 131 | 'max': ee.Image.focal_max, 132 | 'mode': ee.Image.focal_mode, 133 | 'none': ee.Image, 134 | } 135 | result = algs[algorithm](image) 136 | 137 | return result.set('system:time_start', image.date().millis()) 138 | 139 | 140 | def histogram(alert, clas, region=None): 141 | """ Return the number of pixels equal one in the given region """ 142 | if not region: 143 | region = alert.geometry() 144 | 145 | if clas == 'both': 146 | conf = alert.select('confirmed\d{2}').unmask() 147 | prob = alert.select('probable\d{2}').unmask() 148 | image = conf.add(prob).rename('result')#.selfMask() 149 | else: 150 | pattern = '{}\d{{2}}'.format(clas) 151 | image = alert.select(pattern).rename('result') 152 | 153 | result = image.reduceRegion(**{ 154 | 'reducer': ee.Reducer.frequencyHistogram(), 155 | 'geometry': region, 156 | 'scale': alert.projection().nominalScale(), 157 | 'maxPixels': 1e13 158 | }) 159 | 160 | result = ee.Dictionary(result.get('result')) 161 | count = ee.Number( 162 | ee.Algorithms.If( 163 | result.contains('1'), 164 | result.get('1'), 165 | 0) 166 | ) 167 | return count 168 | 169 | 170 | def make_vector(image, region): 171 | """ Vectorize the given image in the given region """ 172 | reducer = ee.Reducer.max() 173 | vector = image.reduceToVectors(**{ 174 | 'geometry': region, 175 | 'reducer': reducer, 176 | 'scale': image.select([0]).projection().nominalScale(), 177 | 'maxPixels': 1e13 178 | }) 179 | vector = vector.map(lambda feat: feat.set('area_m2', 180 | feat.geometry().area(1))) 181 | return vector 182 | 183 | 184 | def make_alerts_vector(alerts, region): 185 | """ accepts the result from alerts.period function """ 186 | # band names 187 | year = ee.Number(alerts.get('year')) 188 | yearStr = year.format().slice(2,4) 189 | dateB = ee.String('alertDate').cat(yearStr) 190 | confB = ee.String('confirmed').cat(yearStr) 191 | probB = ee.String('probable').cat(yearStr) 192 | confDB = ee.String('confirmedDate').cat(yearStr) 193 | probDB = ee.String('probableDate').cat(yearStr) 194 | 195 | # period 196 | start = ee.String(alerts.get('start_period')) 197 | end = ee.String(alerts.get('end_period')) 198 | 199 | # confirmed 200 | confmask = alerts.select([confB]) 201 | confirmed = alerts.updateMask(confmask).select([dateB, confB, probDB, confDB]) 202 | 203 | # probable 204 | probmask = alerts.select([probB]) 205 | probable = alerts.updateMask(probmask).select([dateB, probB, probDB, confDB]) 206 | 207 | # make individual vectors 208 | vconf = make_vector(confirmed, region).map( 209 | lambda feat: feat.set('class', 'confirmed')) 210 | vprob = make_vector(probable, region).map( 211 | lambda feat: feat.set('class', 'probable')) 212 | 213 | def extractDate(d): 214 | condition = d.neq(0) 215 | def true(date): 216 | datestr = ee.String(date.format()) 217 | y = datestr.slice(0, 4) 218 | m = datestr.slice(4, 6) 219 | d = datestr.slice(6, 8) 220 | pattrn = '{y}-{m}-{d}' 221 | replace = {'y':y, 'm':m, 'd':d} 222 | return tools.string.format(pattrn, replace) 223 | def false(date): 224 | return date.format() 225 | 226 | return ee.String(ee.Algorithms.If(condition, true(d), false(d))) 227 | 228 | def updateDate(feat): 229 | feat = feat.set('start_period', start).set('end_period', end) 230 | date = extractDate(ee.Number(feat.get('label'))) 231 | confBand = extractDate(ee.Number(feat.get(confDB))) 232 | probBand = extractDate(ee.Number(feat.get(probDB))) 233 | feat = feat.set(dateB, date) 234 | feat = feat.set(confDB, confBand) 235 | feat = feat.set(probDB, probBand) 236 | props = ee.List(['class', dateB, confDB, probDB, 237 | 'start_period','end_period', 'area_m2']) 238 | return feat.select(props) 239 | 240 | return vconf.merge(vprob).map(updateDate) 241 | 242 | 243 | def dateFromDatetime(dt): 244 | """ ee.Date from datetime """ 245 | return ee.Date(dt.isoformat()) 246 | 247 | 248 | def get_options(featureCollection, propertyName): 249 | """ get a list of all names in property name """ 250 | def wrap(feat, l): 251 | l = ee.List(l) 252 | return l.add(feat.get(propertyName)) 253 | 254 | options = featureCollection.iterate(wrap, ee.List([])) 255 | 256 | return ee.List(options).distinct() 257 | 258 | 259 | def get_bands(image, year=None): 260 | """ Get confY and alertDateY bands from the given image """ 261 | if year: 262 | year = ee.Algorithms.String(int(year)) 263 | else: 264 | d = image.date() 265 | yearInt = ee.Number(d.get('year')) 266 | year = ee.Algorithms.String(yearInt) 267 | yearStr = year.slice(2, 4) 268 | 269 | confband = tools.string.format('conf{y}', {'y': yearStr}) 270 | dateband = tools.string.format('alertDate{y}', {'y': yearStr}) 271 | 272 | return ee.Dictionary(dict(conf=confband, alertDate=dateband, suffix=yearStr)) 273 | 274 | 275 | def compute_breaks(col, year=None): 276 | """ Compute brake dates. From nothing to 'probable' and from 'probable' to 277 | 'confirmed' """ 278 | last = tools.imagecollection.getImage(col, -1) 279 | bands = get_bands(last, year) 280 | band = ee.String(bands.get('conf')) 281 | suffix = ee.String(bands.get('suffix')) 282 | prob = ee.String('probableDate').cat(suffix) 283 | conf = ee.String('confirmedDate').cat(suffix) 284 | det = ee.String('detectedDate').cat(suffix) 285 | 286 | def makeBands(img): 287 | probDate = tools.image.emptyCopy(img.select(0)).rename(prob) 288 | confDate = tools.image.emptyCopy(img.select(0)).rename(conf) 289 | detectedDate = tools.image.emptyCopy(img.select(0)).rename(det) 290 | return img.addBands([probDate, confDate, detectedDate]) 291 | 292 | col = col.map(makeBands) 293 | 294 | collist = col.toList(col.size()) 295 | 296 | def wrap(img, accum): 297 | img = ee.Image(img) 298 | accum = ee.List(accum) 299 | before = ee.Algorithms.If(accum.size().eq(0), 300 | ee.Image(collist.get(0)), 301 | ee.Image(accum.get(-1))) 302 | before = ee.Image(before) 303 | diff = img.select(band).subtract(before.select(band)).rename('break') 304 | probable = diff.eq(2) 305 | confirmed1 = diff.eq(1) 306 | confirmed2 = diff.eq(3) 307 | confirmed = confirmed1.Or(confirmed2) 308 | detected = probable.Or(confirmed) 309 | dateband = tools.date.makeDateBand(img) 310 | probdate = dateband.where(probable.Not(), before.select(prob)).rename(prob) 311 | confdate = dateband.where(confirmed.Not(), before.select(conf)).rename(conf) 312 | detecteddate = dateband.where(detected.Not(), before.select(det)).rename(det) 313 | newi = img.addBands([probdate, confdate, detecteddate], overwrite=True) 314 | return accum.add(newi) 315 | 316 | collist = ee.List(collist.iterate(wrap, ee.List([]))) 317 | return ee.ImageCollection.fromImages(collist) -------------------------------------------------------------------------------- /geepyGLAD/batch.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | 3 | """ Batch module """ 4 | 5 | import ee 6 | from . import alerts, utils 7 | import requests 8 | import os 9 | from geetools import batch as gbatch 10 | 11 | 12 | FUNCTIONS = { 13 | 'probable': alerts.get_probable, 14 | 'confirmed': alerts.get_confirmed, 15 | 'both': alerts.oneday, 16 | 'period': alerts.period 17 | } 18 | 19 | 20 | def mask(image, vector, raster): 21 | """ Mask out a vector mask or a raster mask """ 22 | if vector: 23 | return image.clip(vector) 24 | elif raster: 25 | return image.updateMask(raster) 26 | else: 27 | return image 28 | 29 | 30 | def downloadFile(url, name, ext, path=None): 31 | """ Download a file from a given url 32 | 33 | :param url: full url 34 | :type url: str 35 | :param name: name for the file (can contain a path) 36 | :type name: str 37 | :param ext: extension for the file 38 | :type ext: str 39 | :return: the created file (closed) 40 | :rtype: file 41 | """ 42 | response = requests.get(url, stream=True) 43 | code = response.status_code 44 | 45 | if path is None: 46 | path = os.getcwd() 47 | 48 | path = os.path.join(path, name) 49 | 50 | while code != 200: 51 | if code == 400: 52 | return None 53 | response = requests.get(url, stream=True) 54 | code = response.status_code 55 | size = response.headers.get('content-length', 0) 56 | if size: print('size:', size) 57 | 58 | filename = '{}.{}'.format(path, ext) 59 | 60 | with open(filename, "wb") as handle: 61 | for data in response.iter_content(): 62 | handle.write(data) 63 | 64 | return handle 65 | 66 | 67 | def _download(vector, name, extension='JSON', path=None, verbose=True, 68 | logger=None): 69 | if extension in ['JSON', 'json', 'geojson', 'geoJSON']: 70 | try: 71 | gbatch.Download.table.toGeoJSON(vector, name, path) 72 | except Exception as e: 73 | msg = 'Download method failed: {} \n\ntrying another method...'.format(e) 74 | if verbose: 75 | print(msg) 76 | if logger: 77 | logger.log(msg) 78 | try: 79 | gbatch.Download.table.toLocal(vector, name, 'geojson', 80 | path=path) 81 | except Exception as e: 82 | msg = "Download failed: {}".format(e) 83 | if verbose: 84 | print(msg) 85 | if logger: 86 | logger.log(msg) 87 | else: 88 | print('Format {} not supported'.format(extension)) 89 | 90 | 91 | def _toDrive(vector, filename, folder, extension, **kwargs): 92 | verbose = kwargs.get('verbose', False) 93 | logger = kwargs.get('logger', None) 94 | 95 | try: 96 | task = ee.batch.Export.table.toDrive(vector, filename, 97 | folder, filename, 98 | extension) 99 | task.start() 100 | msg = 'uploading {} to {} in GDrive'.format(filename, folder) 101 | if verbose: 102 | print(msg) 103 | if logger: 104 | logger.log(msg) 105 | 106 | except Exception as e: 107 | msg = 'ERROR writing {} - {}'.format(filename, e) 108 | if verbose: 109 | print(msg) 110 | if logger: 111 | logger.log(msg) 112 | 113 | 114 | def _toAsset(vector, filename, folder, **kwargs): 115 | verbose = kwargs.get('verbose', False) 116 | logger = kwargs.get('logger', None) 117 | 118 | user = ee.data.getAssetRoots()[0]['id'] 119 | path = '{}/{}'.format(user, folder) 120 | 121 | assetId = '{}/{}'.format(path, filename) 122 | 123 | try: 124 | # task = ee.batch.Export.table.toAsset(vector, filename, assetId) 125 | # task.start() 126 | gbatch.Export.table.toAsset(vector, path, filename) 127 | msg = 'uploading {} to {} in Assets'.format(filename, path) 128 | if verbose: 129 | print(msg) 130 | if logger: 131 | logger.log(msg) 132 | 133 | except Exception as e: 134 | msg = 'ERROR in {} to {} in Assets - {}'.format(filename, path, e) 135 | if verbose: 136 | print(msg) 137 | if logger: 138 | logger.log(msg) 139 | 140 | 141 | def _toLocal(vector, filename, folder=None, extension='geojson', 142 | subfolders=True, subname=None, **kwargs): 143 | 144 | verbose = kwargs.get('verbose', True) 145 | logger = kwargs.get('logger', None) 146 | 147 | # MANAGE ALERTS PATH 148 | if folder is None: 149 | folder = os.path.join(os.getcwd(), 'alerts') 150 | 151 | # make path if not present 152 | if not os.path.isdir(folder): 153 | if verbose: 154 | print('creating {} folder'.format(folder)) 155 | os.mkdir(folder) 156 | 157 | if subfolders: 158 | subpath = os.path.join(folder, subname) 159 | if not os.path.isdir(subpath): 160 | if verbose: 161 | print('creating {}'.format(subpath)) 162 | os.mkdir(subpath) 163 | else: 164 | subpath = folder 165 | 166 | msg = '{}: Downloading "{}" to "{}"'.format(subname, filename, subpath) 167 | if verbose: 168 | print(msg) 169 | if logger: 170 | logger.log(msg) 171 | 172 | try: 173 | _download(vector, filename, extension, subpath) 174 | except Exception as e: 175 | msg = '{}: ERROR writing {}'.format(subname, filename) 176 | if logger: 177 | logger.log(msg) 178 | else: 179 | msg = '{}: "{}" downloaded to "{}"'.format(subname, filename, subpath) 180 | if logger: 181 | logger.log(msg) 182 | 183 | 184 | def _are_alerts(alert, name, date, clas, region, verbose, logger): 185 | try: 186 | count = utils.histogram(alert, clas, region).getInfo() 187 | except Exception as e: 188 | msg = '{}: ERROR getting histogram - {}'.format(name, e) 189 | if logger: 190 | logger.log(msg) 191 | return False 192 | 193 | if count == 0: 194 | msg = '{}: no alerts for {}'.format(name, date) 195 | if verbose: 196 | print(msg) 197 | if logger: 198 | logger.log(msg) 199 | return False 200 | else: 201 | return True 202 | 203 | 204 | def _process_period(start, end, geometry, limit, year=None, 205 | eightConnected=False, useProxy=False, mask=None, 206 | destination='local', name=None, folder=None, **kwargs): 207 | verbose = kwargs.get('verbose', True) 208 | logger = kwargs.get('logger', None) 209 | 210 | try: 211 | alert = FUNCTIONS['period'](start, end, geometry, limit, year, 212 | eightConnected, useProxy, mask) 213 | except Exception as e: 214 | msg = 'ERROR while getting period alert {} to {}'.format(start, end) 215 | if verbose: 216 | print(msg) 217 | if logger: 218 | logger.log(msg) 219 | raise e 220 | 221 | date_str = '{} to {}'.format(start, end) 222 | 223 | are_alerts = _are_alerts(alert, name, date_str, 'both', geometry, **kwargs) 224 | if not are_alerts: 225 | return None 226 | 227 | filename = '{}_{}_to_{}'.format(name, start, end) 228 | 229 | vector = utils.make_alerts_vector(alert, geometry) 230 | # LOCAL 231 | if destination == 'local': 232 | subfolders = kwargs.get('subfolders', True) 233 | ext = kwargs.get('extension', 'geojson') 234 | _toLocal(vector, filename, folder, ext, subfolders, 235 | name, **kwargs) 236 | 237 | elif destination == 'drive': 238 | filename = filename.encode().decode('ascii', errors='ignore') 239 | ext = kwargs.get('extension', 'geojson') 240 | _toDrive(vector, filename, folder, ext, **kwargs) 241 | 242 | elif destination == 'asset': 243 | _toAsset(vector, filename, folder, **kwargs) 244 | 245 | 246 | def _process(geometry, date, clas, limit, folder, raster_mask, destination, 247 | filename, name, **kwargs): 248 | verbose = kwargs.get('verbose', True) 249 | logger = kwargs.get('logger', None) 250 | 251 | try: 252 | alert = FUNCTIONS[clas](geometry, date, limit, mask=raster_mask) 253 | except Exception as e: 254 | msg = 'ERROR while getting alert for {}'.format(date) 255 | if verbose: 256 | print(msg) 257 | if logger: 258 | logger.log(msg) 259 | raise e 260 | 261 | # SKIP IF EMPTY ALERT 262 | are_alerts = _are_alerts(alert, name, date, clas, geometry, **kwargs) 263 | if not are_alerts: 264 | return None 265 | 266 | vector = utils.make_alerts_vector(alert, geometry) 267 | # LOCAL 268 | if destination == 'local': 269 | subfolders = kwargs.get('subfolders', True) 270 | ext = kwargs.get('extension', 'geojson') 271 | _toLocal(vector, filename, folder, ext, subfolders, 272 | name, **kwargs) 273 | 274 | elif destination == 'drive': 275 | filename = filename.encode().decode('ascii', errors='ignore') 276 | ext = kwargs.get('extension', 'geojson') 277 | _toDrive(vector, filename, folder, ext, **kwargs) 278 | 279 | elif destination == 'asset': 280 | _toAsset(vector, filename, folder, **kwargs) 281 | 282 | 283 | def period(site, start, end, limit, year=None, proxy=False, eightConnected=False, 284 | folder=None, property_name=None, raster_mask=None, 285 | destination='local', verbose=True, logger=None): 286 | """ General download function for a period """ 287 | 288 | args = dict(verbose=verbose, logger=logger) 289 | 290 | # START PROCESS 291 | # If it is a FeatureCollection and there is a property name 292 | if isinstance(site, ee.FeatureCollection) and property_name: 293 | names = utils.get_options(site, property_name) 294 | names_cli = names.getInfo() 295 | for name in names_cli: 296 | 297 | geom = site.filterMetadata( 298 | property_name, 'equals', name).first().geometry() 299 | 300 | _process_period(start, end, geom, limit, year, eightConnected, 301 | proxy, raster_mask, destination, name, folder, 302 | **args) 303 | else: 304 | if isinstance(site, ee.Feature) and property_name: 305 | name = ee.String(site.get(property_name)).getInfo() 306 | else: 307 | name = 'N/A' 308 | 309 | # GET GEOMETRY 310 | if isinstance(site, (ee.FeatureCollection, ee.Feature)): 311 | geom = site.geometry() 312 | else: 313 | geom = site 314 | 315 | _process_period(start, end, geom, limit, year, eightConnected, 316 | proxy, raster_mask, destination, name, folder, 317 | **args) 318 | 319 | 320 | def download(site, date, clas, limit, folder=None, property_name=None, 321 | raster_mask=None, destination='local', verbose=True, logger=None): 322 | """ General download function """ 323 | if not utils.has_image(date, alerts.ALERTS).getInfo(): 324 | msg = 'GLAD alerts not available for date {}'.format(date) 325 | if logger: 326 | logger.log(msg) 327 | return None 328 | 329 | if clas not in ['probable', 'confirmed', 'both']: 330 | clas = 'both' 331 | 332 | # BASE NAME FOR OUTPUT FILE 333 | if clas == 'both': 334 | basename = 'alerts_for' 335 | else: 336 | basename = '{}_alerts_for'.format(clas) 337 | 338 | args = dict(verbose=verbose, logger=logger) 339 | 340 | # START PROCESS 341 | # If it is a FeatureCollection and there is a property name 342 | if isinstance(site, ee.FeatureCollection) and property_name: 343 | names = utils.get_options(site, property_name) 344 | names_cli = names.getInfo() 345 | for name in names_cli: 346 | filename = '{}_{}_{}'.format(basename, date, name) 347 | 348 | geom = site.filterMetadata( 349 | property_name, 'equals', name).first().geometry() 350 | 351 | _process(geom, date, clas, limit, folder, raster_mask, destination, 352 | filename, name, **args) 353 | else: 354 | if isinstance(site, ee.Feature) and property_name: 355 | name = ee.String(site.get(property_name)).getInfo() 356 | filename = '{}_{}_{}'.format(basename, date, name) 357 | else: 358 | name = 'N/A' 359 | filename = '{}_{}'.format(basename, date) 360 | 361 | # GET GEOMETRY 362 | if isinstance(site, (ee.FeatureCollection, ee.Feature)): 363 | geom = site.geometry() 364 | else: 365 | geom = site 366 | 367 | _process(geom, date, clas, limit, folder, raster_mask, destination, 368 | filename, name, **args) 369 | -------------------------------------------------------------------------------- /glad.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | import click 3 | from datetime import date as dt 4 | import json 5 | import os 6 | 7 | CONFIG = { 8 | 'class': 'both', 9 | 'site': { 10 | 'assetPath': '', 11 | 'propertyName': '' 12 | }, 13 | 'date': 'today', 14 | 'minArea': 1000, # m2 15 | 'vectorMask': '', 16 | 'rasterMask': '', 17 | 'drive': { 18 | 'folder': 'gladAlerts', 19 | 'format': 'GeoJSON' 20 | }, 21 | 'asset': { 22 | 'folder': 'gladAlerts' 23 | }, 24 | 'local': { 25 | 'folder': 'alerts', 26 | 'subfolders': True, 27 | 'format': 'JSON' 28 | }, 29 | 'saveTo': 'local' 30 | } 31 | 32 | HEADER = """Config file: 33 | 34 | {} 35 | 36 | Run command: 37 | 38 | {} 39 | """ 40 | 41 | 42 | # HELPERS 43 | def load_config(name): 44 | exists = os.path.isfile(name) 45 | if exists: 46 | # load config file 47 | with open(name, 'r') as conf: 48 | config = json.load(conf) 49 | else: 50 | msg = "Configuration file {} doesn't exist, to make it "\ 51 | "run:\n\nglad make-config\n" 52 | print(msg.format(name)) 53 | config = None 54 | return config 55 | 56 | 57 | def initEE(logger=None): 58 | import ee 59 | try: 60 | ee.Initialize() 61 | except Exception as e: 62 | msg = "Couldn't connect to Earth Engine. Check your internet connection - {}".format(e) 63 | print(msg) 64 | if logger: 65 | logger.log(msg) 66 | raise e 67 | else: 68 | if logger: 69 | logger.log('Earth Engine initialized successfully') 70 | 71 | 72 | @click.group() 73 | def main(): 74 | pass 75 | 76 | 77 | @main.command() 78 | def make_config(): 79 | """ Create a configuration file (config.json) """ 80 | def create(name): 81 | with open(name, 'w') as f: 82 | json.dump(CONFIG, f, indent=2) 83 | print('{} file has been created into {}'.format(name, os.getcwd())) 84 | 85 | fname = 'config.json' 86 | 87 | exists = os.path.isfile(fname) 88 | if exists: 89 | msg = 'Configuration file already exists, do you really want ' \ 90 | 'to overwrite it? (y/n)' 91 | really = click.prompt(msg, type=bool, default=False) 92 | if really: create(fname) 93 | else: 94 | create(fname) 95 | 96 | 97 | @main.command() 98 | @click.argument('parameter') 99 | @click.argument('value') 100 | def update_config(parameter, value): 101 | """ Update config.json. Options are: 102 | 103 | - sitePath: the Asset path of the site to process\n 104 | - siteProperty: the name of the property that holds the name of the sites\n 105 | - date: date to process\n 106 | - minArea: minimum area in square meters\n 107 | - vectorMask: the Asset path of the mask (ee.FeatureCollection) to apply\n 108 | - rasterMask: the Asset path of the mask (ee.Image) to apply\n 109 | - driveFolder: the folder name to upload the results to Google Drive\n 110 | - driveFormat: the format for the file to upload to Google Drive\n 111 | - assetFolder: the Asset path to upload the results\n 112 | - localFolder: the local folder to download the results\n 113 | - localFormat: the file format to download the results\n 114 | - localSub: if True creates subfolders for each site (given by siteProperty)\n 115 | - saveTo: where to save results (drive, asset or local)\n 116 | """ 117 | endpoints = { 118 | 'class': ['class'], 119 | 'sitePath': ['site', 'assetPath'], 120 | 'siteProperty': ['site', 'propertyName'], 121 | 'date': ['date'], 122 | 'minArea': ['minArea'], 123 | 'vectorMask': ['vectorMask'], 124 | 'rasterMask': ['rasterMask'], 125 | 'driveFolder': ['drive', 'folder'], 126 | 'driveFormat': ['drive', 'format'], 127 | 'assetFolder': ['asset', 'folder'], 128 | 'localFolder': ['local', 'folder'], 129 | 'localFormat': ['local', 'format'], 130 | 'localSub': ['local', 'subfolders'], 131 | 'saveTo': ['saveTo'] 132 | } 133 | 134 | if parameter in ['minArea']: 135 | value = int(value) 136 | 137 | fname = 'config.json' 138 | exists = os.path.isfile(fname) 139 | if not exists: 140 | msg = 'Do you want to create a configuration file in {}' 141 | make = click.prompt(msg.format(os.getcwd()), default=False) 142 | if make: 143 | make_config() 144 | else: 145 | return None 146 | 147 | # load config file 148 | with open('config.json', 'r') as conf: 149 | config = json.load(conf) 150 | 151 | endpoint = endpoints.get(parameter) 152 | if endpoint: 153 | upd = config 154 | for end in endpoint: 155 | v = upd[end] 156 | if (not isinstance(v, dict)): 157 | upd[end] = value 158 | else: 159 | upd = v 160 | 161 | with open(fname, 'w') as f: 162 | json.dump(config, f, indent=2) 163 | 164 | else: 165 | print('parameter {} not available'.format(parameter)) 166 | 167 | 168 | @main.command() 169 | def user(): 170 | """ Show Earth Engine user and email """ 171 | import ee 172 | try: 173 | ee.Initialize() 174 | except: 175 | print("Couldn't connect to Earth Engine. Check your internet connection") 176 | 177 | uid = ee.data.getAssetRoots()[0]['id'] 178 | email = ee.data.getAssetAcl(uid)['owners'][0] 179 | us = uid.split('/')[1] 180 | msg = 'User: {}\nEmail: {}'.format(us, email) 181 | print(msg) 182 | 183 | 184 | @main.command() 185 | def sites(): 186 | """ Show available site names """ 187 | import ee 188 | initEE() 189 | config = load_config('config.json') 190 | if not config: return None 191 | 192 | siteAP = config['site']['assetPath'] 193 | site = ee.FeatureCollection(siteAP) 194 | prop = config['site']['propertyName'] 195 | 196 | options = site.aggregate_array(prop) 197 | print(options.getInfo()) 198 | 199 | 200 | @main.command() 201 | @click.argument('start')#, help='Start date for the period') 202 | @click.argument('end')#, help='Start date for the period') 203 | @click.option('-y', '--year', default=None, help='Year of the alerts. If None will use the last image year') 204 | @click.option('-p', '--proxy', default=False, help='use proxy? If True start date will be dismissed') 205 | @click.option('-s', '--savein', default=None, help='where to save the files. Takes default from config.json') 206 | @click.option('--site', default=None, help='The name of the site to process, must be present in the parsed property') 207 | @click.option('-m', '--mask', default=True, type=bool, help='Whether to use the mask in config file or not') 208 | @click.option('-v', '--verbose', default=True, type=bool) 209 | @click.option('--config', default=None, help='The name of the configuration file. Defaults to "config.json"') 210 | def period(start, end, year, proxy, savein, site, mask, verbose, config): 211 | """ Export a period (from START to END) of GLAD alerts to Google Drive, 212 | Earth Engine Asset or Local files. Takes configuration parameters from 213 | `config.json`. 214 | """ 215 | # LOAD CONFIG FILE 216 | configname = config # change variable name 217 | if not configname: 218 | configname = 'config.json' 219 | 220 | config = load_config(configname) 221 | if not config: return None 222 | 223 | # SITE PARAMS 224 | site_params = config['site'] 225 | asset_path = site_params['assetPath'] 226 | property_name = site_params['propertyName'] 227 | usersite = site # change variable name 228 | 229 | # SAVE PARAMS 230 | destination = savein or config['saveTo'] 231 | save_params = config[destination] 232 | soptions = ['drive', 'asset', 'local'] 233 | 234 | # MIN AREA 235 | limit = config['minArea'] 236 | 237 | # RUN COMMAND AND HASH 238 | command = 'glad period {} {} --proxy {} -s {} -m {} -v {}'.format( 239 | start, end, proxy, savein, mask, verbose) 240 | if usersite: 241 | command += ' --site {}'.format(usersite) 242 | 243 | config_str = json.dumps(config, indent=2) 244 | tohash = '{} {}'.format(config_str, command) 245 | tohash = tohash.encode('utf-8') 246 | import hashlib 247 | h = hashlib.sha256() 248 | h.update(tohash) 249 | hexcode = h.hexdigest() 250 | logname = 'period {} to {} {}'.format(start, end, hexcode) 251 | 252 | header = HEADER.format(config_str, command) 253 | 254 | # LOGGER 255 | from geepyGLAD.logger import Logger 256 | logdir = 'logs' 257 | logger = Logger(logname, logdir) 258 | 259 | logger.header(header) 260 | 261 | if destination not in soptions: 262 | msg = 'savein parameter must be one of {}'.format(soptions) 263 | logger.log(msg) 264 | print(msg) 265 | return None 266 | 267 | # INITIALIZE EE 268 | import ee 269 | initEE(logger) 270 | try: 271 | from geepyGLAD import utils, alerts, batch 272 | except Exception as e: 273 | msg = 'ERROR while importing geepyGLAD - {}'.format(e) 274 | logger.log(msg) 275 | raise e 276 | 277 | site = ee.FeatureCollection(asset_path) 278 | 279 | if usersite: 280 | site = site.filterMetadata(property_name, 'equals', usersite) 281 | site = ee.Feature(site.first()) 282 | 283 | args = dict( 284 | start=start, 285 | end=end, 286 | year=int(year), 287 | proxy=bool(proxy), 288 | site=site, 289 | limit=limit, 290 | property_name=property_name, 291 | verbose=verbose, 292 | folder=save_params['folder'], 293 | logger=logger 294 | ) 295 | 296 | raster_mask_id = config['rasterMask'] 297 | if raster_mask_id and mask: 298 | raster_mask = ee.Image(raster_mask_id) 299 | args['raster_mask'] = raster_mask 300 | 301 | # COMPUTE ALERTS 302 | try: 303 | batch.period(**args, destination=destination) 304 | except Exception as e: 305 | msg = 'ERROR: {}'.format(str(e)) 306 | logger.log(msg) 307 | raise e 308 | 309 | 310 | @main.command() 311 | @click.option('-s', '--savein', default=None, help='where to save the files. Takes default from config.json') 312 | @click.option('-c', '--clas', default=None, help='The class to export. Can be "probable", "confirmed" or "both"') 313 | @click.option('-d', '--date', default=None, help='If this param is not set, it will use the date for today') 314 | @click.option('--site', default=None, help='The name of the site to process, must be present in the parsed property') 315 | @click.option('-m', '--mask', default=True, type=bool, help='Whether to use the mask in config file or not') 316 | @click.option('-v', '--verbose', default=True, type=bool) 317 | @click.option('--config', default=None, help='The name of the configuration file. Defaults to "config.json"') 318 | def alert(savein, clas, date, site, mask, verbose, config): 319 | """ Export GLAD alerts to Google Drive, Earth Engine Asset or Local files. 320 | Takes configuration parameters from `config.json`. 321 | """ 322 | # LOAD CONFIG FILE 323 | configname = config # change variable name 324 | if not configname: 325 | configname = 'config.json' 326 | 327 | config = load_config(configname) 328 | if not config: return None 329 | 330 | # SITE PARAMS 331 | site_params = config['site'] 332 | asset_path = site_params['assetPath'] 333 | property_name = site_params['propertyName'] 334 | usersite = site # change variable name 335 | 336 | # SAVE PARAMS 337 | destination = savein or config['saveTo'] 338 | save_params = config[destination] 339 | soptions = ['drive', 'asset', 'local'] 340 | 341 | # DATE PARAMS 342 | if not date: 343 | date = config['date'] 344 | alert_date = dt.today().isoformat() if date == 'today' else date 345 | 346 | # MIN AREA 347 | limit = config['minArea'] 348 | 349 | # CLASS 350 | if not clas: 351 | clas = config['class'] 352 | 353 | # RUN COMMAND AND HASH 354 | command = 'glad alert -s {} -c {} -d {} -m {} -v {}'.format( 355 | savein, clas, date, mask, verbose) 356 | if usersite: 357 | command += ' --site {}'.format(usersite) 358 | 359 | config_str = json.dumps(config, indent=2) 360 | tohash = '{} {}'.format(config_str, command) 361 | tohash = tohash.encode('utf-8') 362 | import hashlib 363 | h = hashlib.sha256() 364 | h.update(tohash) 365 | hexcode = h.hexdigest() 366 | logname = '{} {}'.format(date, hexcode) 367 | 368 | header = HEADER.format(config_str, command) 369 | 370 | # LOGGER 371 | from geepyGLAD.logger import Logger 372 | logdir = 'logs' 373 | logger = Logger(logname, logdir) 374 | 375 | logger.header(header) 376 | 377 | if destination not in soptions: 378 | msg = 'savein parameter must be one of {}'.format(soptions) 379 | logger.log(msg) 380 | print(msg) 381 | return None 382 | 383 | # INITIALIZE EE 384 | import ee 385 | initEE(logger) 386 | try: 387 | from geepyGLAD import utils, alerts, batch 388 | except Exception as e: 389 | msg = 'ERROR while importing geepyGLAD - {}'.format(e) 390 | logger.log(msg) 391 | raise e 392 | 393 | site = ee.FeatureCollection(asset_path) 394 | 395 | if usersite: 396 | site = site.filterMetadata(property_name, 'equals', usersite) 397 | site = ee.Feature(site.first()) 398 | 399 | # Check for available alert image in the given date 400 | has_images = utils.has_image(alert_date, alerts.ALERTS).getInfo() 401 | if not has_images: 402 | msg = 'GLAD alerts not available for date {}'.format(date) 403 | logger.log(msg) 404 | print(msg) 405 | return None 406 | 407 | args = dict( 408 | site=site, 409 | date=alert_date, 410 | clas=clas, 411 | limit=limit, 412 | property_name=property_name, 413 | verbose=verbose, 414 | folder=save_params['folder'], 415 | logger=logger 416 | ) 417 | 418 | raster_mask_id = config['rasterMask'] 419 | if raster_mask_id and mask: 420 | raster_mask = ee.Image(raster_mask_id) 421 | args['raster_mask'] = raster_mask 422 | 423 | # COMPUTE ALERTS 424 | try: 425 | batch.download(**args, destination=destination) 426 | except Exception as e: 427 | msg = 'ERROR: {}'.format(str(e)) 428 | logger.log(msg) 429 | raise e 430 | 431 | 432 | if __name__ == '__main__': 433 | main() --------------------------------------------------------------------------------