├── .bumpversion.cfg ├── .github └── workflows │ ├── cd.yaml │ └── ci.yaml ├── .gitignore ├── LICENSE ├── README.md ├── eepackages ├── __init__.py ├── applications │ ├── __init__.py │ ├── bathymetry.py │ └── waterbody_area.py ├── assets.py ├── geometry.py ├── gl.py ├── multiprocessing │ ├── __init__.py │ └── download_image.py ├── tiler.py └── utils.py ├── notebooks └── assets_cloud_band.ipynb ├── requirements.txt ├── setup.py ├── tests └── test_sample.py └── tox.ini /.bumpversion.cfg: -------------------------------------------------------------------------------- 1 | [bumpversion] 2 | current_version = 0.28.0 3 | 4 | [bumpversion:file:setup.py] 5 | search = __version__ = "{current_version}" 6 | replace = __version__ = "{new_version}" 7 | -------------------------------------------------------------------------------- /.github/workflows/cd.yaml: -------------------------------------------------------------------------------- 1 | name: CD 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | workflow_dispatch: 8 | 9 | jobs: 10 | release: 11 | runs-on: ubuntu-latest 12 | steps: 13 | - uses: actions/checkout@v3 14 | 15 | - name: Evaluate version bump 16 | run: | 17 | case "${{ github.event.head_commit.message }}" in 18 | PATCH*) 19 | part=patch;; 20 | MAJOR*) 21 | part=major;; 22 | *) 23 | part=minor;; 24 | esac 25 | 26 | echo "part=$(echo $part)" >> $GITHUB_ENV 27 | 28 | - name: Set up Python 29 | uses: actions/setup-python@v3 30 | with: 31 | python-version: '3.x' 32 | 33 | - name: Install dependencies 34 | run: | 35 | python -m pip install --upgrade pip 36 | pip install build bump2version==1.0.1 37 | 38 | - name: bump version 39 | run: | 40 | git config --global user.name 'version bump' 41 | git config --global user.email 'version-bumper@users.noreply.github.com' 42 | bump2version \ 43 | --commit \ 44 | --tag \ 45 | ${{ env.part }} 46 | git push --atomic origin main v$(cat .bumpversion.cfg | grep -Po "current_version = \K\d+\.\d+\.\d+") 47 | 48 | - name: Build package 49 | run: python -m build 50 | - name: Publish package 51 | uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29 52 | with: 53 | user: __token__ 54 | password: ${{ secrets.PYPI_API_TOKEN }} 55 | -------------------------------------------------------------------------------- /.github/workflows/ci.yaml: -------------------------------------------------------------------------------- 1 | # This workflow will install Python dependencies, run tests and lint with a variety of Python versions 2 | # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions 3 | 4 | name: CI 5 | 6 | on: 7 | push: 8 | branches: [ "main" ] 9 | pull_request: 10 | branches: [ "main" ] 11 | 12 | jobs: 13 | build: 14 | 15 | runs-on: ubuntu-latest 16 | strategy: 17 | fail-fast: false 18 | matrix: 19 | python-version: [3.7, 3.8, 3.9] 20 | 21 | steps: 22 | - uses: actions/checkout@v3 23 | - name: Set up Python ${{ matrix.python-version }} 24 | uses: actions/setup-python@v3 25 | with: 26 | python-version: ${{ matrix.python-version }} 27 | - name: Install dependencies 28 | run: | 29 | python -m pip install --upgrade pip 30 | python -m pip install flake8 pytest 31 | if [ -f requirements.txt ]; then pip install -r requirements.txt; fi 32 | - name: Lint with flake8 33 | run: | 34 | # stop the build if there are Python syntax errors or undefined names 35 | flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics 36 | # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide 37 | flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics 38 | - name: Test with pytest 39 | run: | 40 | pytest -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.py[cod] 2 | 3 | # C extensions 4 | *.so 5 | 6 | # Packages 7 | *.egg 8 | *.egg-info 9 | dist 10 | build 11 | eggs 12 | parts 13 | bin 14 | var 15 | sdist 16 | develop-eggs 17 | .installed.cfg 18 | lib 19 | lib64 20 | __pycache__ 21 | 22 | # Installer logs 23 | pip-log.txt 24 | 25 | # Unit test / coverage reports 26 | .coverage 27 | .tox 28 | nosetests.xml 29 | 30 | # Translations 31 | *.mo 32 | 33 | # Mr Developer 34 | .mr.developer.cfg 35 | .project 36 | .pydevproject 37 | 38 | # pyenv virtualenv 39 | .python-version 40 | 41 | # act 42 | .secrets 43 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2021 Gennadii Donchyts 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # eepackages 2 | 3 | [![PyPi version](https://badgen.net/pypi/v/eepackages/)](https://pypi.com/project/eepackages) 4 | [![CI](https://github.com/gee-community/ee-packages-py/actions/workflows/ci.yaml/badge.svg?branch=main&event=push)](https://github.com/gee-community/ee-packages-py/actions/workflows/ci.yaml) 5 | [![CD](https://github.com/gee-community/ee-packages-py/actions/workflows/cd.yaml/badge.svg?branch=main&event=release)](https://github.com/gee-community/ee-packages-py/actions/workflows/cd.yaml) 6 | 7 | A set of utilities built on top of Google Earth Engine (migrated from JavaScript) 8 | 9 | This repository is basically a Python version of a set of utilities developed by me (@gena) while working with Google Earth Engine and migrated from JavaScript repository: https://code.earthengine.google.com/?accept_repo=users/gena/packages 10 | 11 | For a GitHub clone of this package see: https://github.com/gee-community/ee-packages-js 12 | 13 | There are not docs (yet), some of the utilities are demonstrated in the following presentation: https://docs.google.com/presentation/d/103gfl3gS8rokkrcMJV9qMBnkV7oLLbDf8ygDu6QaIZQ/edit?usp=sharing 14 | 15 | 16 | 17 | ## Usage 18 | 19 | ## Installation 20 | 21 | ## Requirements 22 | 23 | ## Compatibility 24 | 25 | ## Licence 26 | 27 | ## Authors 28 | 29 | `eepackages` was written by `Gennadii Donchyts `_. 30 | 31 | ## TODO 32 | 33 | - unit testing 34 | - coverage 35 | -------------------------------------------------------------------------------- /eepackages/__init__.py: -------------------------------------------------------------------------------- 1 | """eepackages - A set of utilities built on top of Google Earth Engine (migrated from JavaScript)""" 2 | 3 | __version__ = "0.1.0" 4 | __author__ = "Gennadii Donchyts " 5 | __all__ = [] 6 | -------------------------------------------------------------------------------- /eepackages/applications/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gee-community/ee-packages-py/6a822d50fe817a3aa821c68c1a7feb8ca8290329/eepackages/applications/__init__.py -------------------------------------------------------------------------------- /eepackages/applications/bathymetry.py: -------------------------------------------------------------------------------- 1 | from typing import Any, Dict, List, Optional 2 | from datetime import datetime 3 | 4 | import ee 5 | 6 | from eepackages import assets 7 | from eepackages import gl 8 | from eepackages import utils 9 | 10 | 11 | class Bathymetry(object): 12 | def __init__(self, waterIndexMin: float = -0.15, waterIndexMax: float = 0.35): 13 | self.waterIndexMin = waterIndexMin 14 | self.waterIndexMax = waterIndexMax 15 | 16 | @staticmethod 17 | def _remove_all_zero_images(image: ee.Image): 18 | mask: ee.Image = ( 19 | image.select(["blue", "green", "red", "nir", "swir"]) 20 | .mask() 21 | .reduce(ee.Reducer.allNonZero()) 22 | ) 23 | return image.updateMask(mask) 24 | 25 | def compute_inverse_depth( 26 | self, 27 | bounds, 28 | start: datetime, 29 | stop: datetime, 30 | filter_masked: bool, 31 | scale: float, 32 | missions: List[str] = ["S2", "L8"], 33 | cloud_frequency_threshold_data: float = 0.15, 34 | pansharpen: bool = False, 35 | skip_neighborhood_search: bool = False, 36 | skip_scene_boundary_fix: bool = False, 37 | bounds_buffer: int = 10000, 38 | ) -> ee.Image: 39 | images: ee.ImageCollection = self.get_images( 40 | bounds=bounds, 41 | start=start, 42 | stop=stop, 43 | filter_masked=filter_masked, 44 | scale=scale, 45 | missions=missions, 46 | cloud_frequency_threshold_delta=cloud_frequency_threshold_data, 47 | ) 48 | # save loaded images in class as raw_images 49 | self._raw_images = images 50 | images = images.map(self._remove_all_zero_images) 51 | 52 | bounds = bounds.buffer(bounds_buffer, ee.Number(bounds_buffer).divide(10)) 53 | 54 | return self._compute_inverse_depth( 55 | images=images, 56 | bounds=bounds, 57 | scale=scale, 58 | pansharpen=pansharpen, 59 | skip_neighborhood_search=skip_neighborhood_search, 60 | skip_scene_boundary_fix=skip_scene_boundary_fix, 61 | ) 62 | 63 | def _compute_inverse_depth( 64 | self, 65 | images: ee.ImageCollection, 66 | bounds, 67 | scale: int, 68 | pansharpen: bool, 69 | skip_neighborhood_search: bool, 70 | skip_scene_boundary_fix: bool, 71 | ) -> ee.Image: 72 | bands: List[str] = ["red", "green", "blue"] 73 | green_max: float = 0.4 74 | 75 | def _set_image_area_properties(image: ee.Image) -> ee.Image: 76 | water: ee.Image = ( 77 | image.normalizedDifference(["green", "nir"]) 78 | .rename("water") 79 | .unitScale(0, 0.1) 80 | ) 81 | 82 | if pansharpen: 83 | # TODO: Not implemented 84 | raise NotImplementedError() 85 | image = assets.pansharpen(image) 86 | 87 | water_area: ee.Number = ( 88 | water.gt(0.01) 89 | .multiply(ee.Image.pixelArea()) 90 | .reduceRegion( 91 | reducer=ee.Reducer.sum(), 92 | geometry=bounds, 93 | scale=ee.Number(scale).multiply(5), 94 | tileScale=4, 95 | ) 96 | .values() 97 | .get(0) 98 | ) 99 | 100 | land_area: ee.Number = ( 101 | water.lt(0) 102 | .multiply(ee.Image.pixelArea()) 103 | .reduceRegion( 104 | reducer=ee.Reducer.sum(), 105 | geometry=bounds, 106 | scale=ee.Number(scale).multiply(5), 107 | tileScale=4, 108 | ) 109 | .values() 110 | .get(0) 111 | ) 112 | 113 | dark: ee.Image = image 114 | 115 | dark = dark.updateMask(water.gt(0)).reduceRegion( 116 | reducer=ee.Reducer.percentile([0]), 117 | geometry=bounds, 118 | scale=scale, 119 | maxPixels=1e10, 120 | tileScale=4, 121 | ) 122 | 123 | image = image.set(dark).set( 124 | {"water": water, "waterArea": water_area, "landArea": land_area} 125 | ) 126 | 127 | return image 128 | 129 | images: ee.ImageCollection = images.map(_set_image_area_properties) 130 | self._images_area_properties = images 131 | 132 | # Filter images with negative RGB values 133 | images = images.filter( 134 | ee.Filter.And( 135 | ee.Filter.gt(bands[0], 0), 136 | ee.Filter.gt(bands[1], 0), 137 | ee.Filter.gt(bands[2], 0), 138 | ) 139 | ) 140 | 141 | if skip_neighborhood_search: 142 | images = assets.addCdfQualityScore( 143 | images=images, 144 | opt_thresholdMin=70, 145 | opt_thresholdMax=80, 146 | opt_includeNeighborhood=False, 147 | ) 148 | else: 149 | images = assets.addCdfQualityScore( 150 | images=images, 151 | opt_thresholdMin=70, 152 | opt_thresholdMax=80, 153 | opt_includeNeighborhood=True, 154 | opt_neighborhoodOptions={"erosion": 0, "dilation": 0, "weight": 200}, 155 | ) 156 | 157 | def fix_scene_boundaries(image: ee.Image) -> ee.Image: 158 | weight: ee.Image = image.select("weight") 159 | mask = image.select(0).mask() 160 | 161 | mask: ee.Image = ( 162 | utils.focalMin(mask, 10) 163 | .reproject(ee.Projection("EPSG:3857").atScale(scale)) 164 | .resample("bicubic") 165 | ) 166 | mask = ( 167 | utils.focalMaxWeight(mask.Not(), 10) 168 | .reproject(ee.Projection("EPSG:3857").atScale(scale)) 169 | .resample("bicubic") 170 | ) 171 | mask = ee.Image.constant(1).subtract(mask) 172 | 173 | weight = weight.multiply(mask) 174 | 175 | return image.addBands(weight, None, True) 176 | 177 | if not skip_scene_boundary_fix: 178 | images = images.map(fix_scene_boundaries) 179 | 180 | def image_map_func(i: ee.Image): 181 | t: str = i.get("system:time_start") 182 | weight: ee.Image = i.select("weight") 183 | 184 | dark_image: ee.Image = ee.Image.constant( 185 | list(map(lambda n: i.get(n), bands)) 186 | ).rename(bands) 187 | mission: str = i.get("MISSION") 188 | scale_water_to: str = "percentiles" 189 | scale_land_to: str = "percentiles" 190 | 191 | range_percentiles_water: List[int] = [2, 98] 192 | range_percentiles_land: List[int] = [2, 98] 193 | 194 | range_sigma_water: List[int] = [1, 1] 195 | range_sigma_land: List[int] = [2, 2] 196 | 197 | water: ee.Image = ee.Image(i.get("water")) 198 | non_water: ee.Image = water.subtract(1).multiply(-1) # Not being used 199 | 200 | i = i.select(bands).subtract(dark_image).max(0.0001) 201 | 202 | i_all: ee.Image = i 203 | 204 | water2: ee.Image = gl.smoothStep(-0.05, 0.2, water) 205 | non_water2: ee.Image = water2.subtract(1).multiply(-1) 206 | 207 | i = i.log() 208 | 209 | stat1: ee.Image = i 210 | 211 | stat1 = stat1.updateMask( 212 | water2.multiply(i_all.select("green").lt(green_max)) 213 | ) 214 | 215 | if scale_water_to == "percentiles": 216 | stat1 = stat1.reduceRegion( 217 | reducer=ee.Reducer.percentile(range_percentiles_water), 218 | geometry=bounds, 219 | scale=ee.Number(scale).multiply(3), 220 | maxPixels=1e10, 221 | ) 222 | 223 | min1: List[str] = [ 224 | ee.String(stat1.get(f"{band}_p{range_percentiles_water[0]}")) 225 | for band in bands 226 | ] 227 | max1: List[str] = [ 228 | ee.String(stat1.get(f"{band}_p{range_percentiles_water[1]}")) 229 | for band in bands 230 | ] 231 | 232 | if scale_water_to == "sigma": 233 | stat1mean = stat1.reduceRegion( 234 | reducer=ee.Reducer.mean(), 235 | geometry=bounds, 236 | scale=ee.Number(scale).multiply(3), 237 | maxPixels=1e10, 238 | ) 239 | 240 | stat1sigma = stat1.reduceregion( # not being used. 241 | reducer=ee.Reducer.stDev(), 242 | geometry=bounds, 243 | scale=ee.Number(scale).multiply(3), 244 | maxPixels=1e10, 245 | ) 246 | 247 | # Not sure whether this si tested, min1 should always be zero 248 | min1: List[ee.Number] = [ 249 | ee.Number(stat1mean.get(band)).subtract( 250 | ee.Number(stat1mean.get(band)).multiply(range_sigma_water[0]) 251 | ) 252 | for band in bands 253 | ] 254 | max1: List[ee.Number] = [ 255 | ee.Number(stat1mean.get(band)).add( 256 | ee.Number(stat1mean.get(band)).multiply(range_sigma_water[1]) 257 | ) 258 | for band in bands 259 | ] 260 | 261 | min1 = self._fix_null(min1, 0) 262 | max1 = self._fix_null(max1, 0.001) 263 | 264 | stat2: ee.Image = i_all.updateMask( 265 | non_water2.multiply(i_all.select("green").lt(green_max)) 266 | ) 267 | 268 | if scale_land_to == "percentiles": 269 | stat2 = stat2.reduceRegion( 270 | reducer=ee.Reducer.percentile(range_percentiles_land), 271 | geometry=bounds, 272 | scale=ee.Number(scale).multiply(3), 273 | maxPixels=1e10, 274 | ) 275 | 276 | min2 = [ 277 | ee.String(stat2.get(f"{band}_p{range_percentiles_land[0]}")) 278 | for band in bands 279 | ] 280 | max2 = [ 281 | ee.String(stat2.get(f"{band}_p{range_percentiles_land[1]}")) 282 | for band in bands 283 | ] 284 | 285 | if scale_land_to == "sigma": 286 | stat2mean = stat2.reduceRegion( 287 | reducer=ee.Reducer.mean(), 288 | geometry=bounds, 289 | scale=ee.Number(scale).multiply(3), 290 | maxPixels=1e10, 291 | ) 292 | stat2sigma = stat2.reduceRegion( # not used? 293 | reducer=ee.Reducer.stDev(), 294 | geometry=bounds, 295 | scale=ee.Number(scale).multiply(3), 296 | maxPixels=1e10, 297 | ) 298 | 299 | min2: List[ee.Number] = [ 300 | ee.Number(stat2mean) 301 | .get(band) 302 | .subtract( 303 | ee.Number(stat2mean.get(band)).multiply(range_sigma_land[0]) 304 | ) 305 | for band in bands 306 | ] 307 | max2: List[ee.Number] = [ 308 | ee.Number(stat2mean.get(band)).add( 309 | ee.Number(stat2mean.get(band)).multiply(range_sigma_land[1]) 310 | ) 311 | for band in bands 312 | ] 313 | 314 | min2 = self._fix_null(min2, 0) 315 | max2 = self._fix_null(max2, 0.001) 316 | 317 | i_water = self._unit_scale(i.select(bands), min1, max1).updateMask(water2) 318 | 319 | i_land = self._unit_scale(i_all.select(bands), min2, max2).updateMask( 320 | non_water2 321 | ) 322 | 323 | i = i_water.blend(i_land).addBands(water) 324 | 325 | i = i.addBands(weight) 326 | 327 | return i.set( 328 | { 329 | "label": ee.Date(t).format().cat(", ").cat(mission), 330 | "system:time_start": t, 331 | } 332 | ) 333 | 334 | images = images.map(image_map_func) 335 | 336 | self._images_with_statistics = images 337 | 338 | # mean = sum(w * x) / sum(w) 339 | image: ee.Image = ( 340 | images.map( 341 | lambda i: i.select(bands + ["water"]).multiply(i.select("weight")) 342 | ) 343 | .sum() 344 | .divide(images.select("weight").sum()) 345 | ) 346 | 347 | return image 348 | 349 | def compute_intertidal_depth( 350 | self, 351 | bounds, 352 | start: datetime, 353 | stop: datetime, 354 | scale: float, 355 | filter_masked: bool, 356 | filter_masked_fraction: Optional[float] = None, 357 | filter: Optional[ee.Filter] = None, 358 | bounds_buffer: Optional[float] = None, 359 | water_index_min: Optional[float] = None, 360 | water_index_max: Optional[float] = None, 361 | missions: List[str] = ["S2", "L8"], 362 | skip_scene_boundary_fix=False, 363 | skip_neighborhood_search: bool = False, 364 | neighborhood_search_parameters: Dict[str, float] = { 365 | "erosion": 0, 366 | "dilation": 0, 367 | "weight": 100, 368 | }, 369 | lower_cdf_boundary: float = 70, 370 | upper_cdf_boundary: float = 80, 371 | cloud_frequency_threshold_data: float = 0.15, 372 | clip: bool = False, 373 | ) -> ee.Image: 374 | if water_index_min: 375 | self.waterIndexMin = water_index_min 376 | water_index_min = self.waterIndexMin 377 | if water_index_max: 378 | self.waterIndexMax = water_index_max 379 | water_index_max = self.waterIndexMax 380 | if bounds_buffer: 381 | bounds = bounds.buffer(bounds_buffer, bounds_buffer / 10) 382 | 383 | images: ee.ImageCollection = self.get_images( 384 | bounds=bounds, 385 | start=start, 386 | stop=stop, 387 | filter_masked=filter_masked, 388 | filter_masked_fraction=filter_masked_fraction, 389 | scale=scale, 390 | missions=missions, 391 | cloud_frequency_threshold_delta=cloud_frequency_threshold_data, 392 | filter=filter, 393 | ) 394 | 395 | self._raw_images = images 396 | 397 | bands: List[str] = ["blue", "green", "red", "nir", "swir"] 398 | 399 | # Mask all zero images 400 | def mask_zero_images(i: ee.Image) -> ee.Image: 401 | mask: ee.Image = ( 402 | i.select(bands).mask().reduce(ee.Reducer.allNonZero()).eq(1) 403 | ) 404 | return i.updateMask(mask) 405 | 406 | images = images.map(mask_zero_images) 407 | 408 | if not bounds_buffer: 409 | bounds_buffer = 10000 # Not being used. 410 | 411 | if skip_neighborhood_search: 412 | images = assets.addCdfQualityScore(images, 70, 80, False) 413 | else: 414 | images = assets.addCdfQualityScore( 415 | images=images, 416 | opt_thresholdMin=lower_cdf_boundary, 417 | opt_thresholdMax=upper_cdf_boundary, 418 | opt_includeNeighborhood=True, 419 | opt_neighborhoodOptions=neighborhood_search_parameters, 420 | ) 421 | 422 | if not skip_scene_boundary_fix: 423 | 424 | def fix_scene_boundaries(i: ee.Image) -> ee.Image: 425 | weight: ee.Image = i.select("weight") 426 | mask: ee.Image = i.select(0).mask() 427 | 428 | mask = ( 429 | utils.focalMin(mask, 10) 430 | .reproject(ee.Projection("EPSG:3857").atScale(scale)) 431 | .resample("bicubic") 432 | ) 433 | mask = ( 434 | utils.focalMaxWeight(mask.Not(), 10) 435 | .reproject(ee.Projection("EPSG:3857").atScale(scale)) 436 | .resample("bicubic") 437 | ) 438 | mask = ee.Image.constant(1).subtract(mask) 439 | 440 | weight = weight.multiply(mask) 441 | 442 | return i.addBands(srcImg=weight, overwrite=True) 443 | 444 | images = images.map(fix_scene_boundaries) 445 | 446 | self._refined_images = images 447 | 448 | # mean = sum(w * x) / sum (w) 449 | self.composite = ( 450 | images.map(lambda i: i.select(bands).multiply(i.select("weight"))) 451 | .sum() 452 | .divide(images.select("weight").sum()) 453 | .select(["red", "green", "blue", "swir", "nir"]) 454 | ) 455 | 456 | bands = ["ndwi", "indvi", "mndwi"] 457 | 458 | def calculate_water_statistics(i: ee.Image) -> ee.Image: 459 | t = i.get("system:time_start") # Not used 460 | weight: ee.Image = i.select("weight") 461 | 462 | ndwi: ee.Image = i.normalizedDifference(["green", "nir"]).rename("ndwi") 463 | indvi: ee.Image = i.normalizedDifference(["red", "nir"]).rename("indvi") 464 | mndwi: ee.Image = i.normalizedDifference(["green", "swir"]).rename("mndwi") 465 | 466 | ndwi = ndwi.clamp(water_index_min, water_index_max) 467 | indvi = indvi.clamp(water_index_min, water_index_max) 468 | mndwi = mndwi.clamp(water_index_min, water_index_max) 469 | 470 | return ee.Image([ndwi, indvi, mndwi]).addBands(weight) 471 | 472 | images = images.map(calculate_water_statistics) 473 | 474 | self._images_with_statistics = images 475 | 476 | image = ( 477 | images.map(lambda i: i.select(bands).multiply(i.select("weight"))) 478 | .sum() 479 | .divide(images.select("weight").sum()) 480 | ) 481 | 482 | if clip == True: 483 | return image.clip(bounds) 484 | else: 485 | return image 486 | 487 | @staticmethod 488 | def get_images( 489 | bounds, 490 | start: datetime, 491 | stop: datetime, 492 | filter_masked: bool, 493 | scale: float, 494 | missions: List[str], 495 | filter_masked_fraction: Optional[float] = None, 496 | cloud_frequency_threshold_delta: float = 0.15, 497 | filter: Optional[ee.Filter] = None, 498 | ) -> ee.ImageCollection: 499 | date_filter: ee.Filter = ee.Filter.date(start, stop) 500 | if filter: 501 | filter: ee.Filter = ee.Filter.And(filter, date_filter) 502 | else: 503 | filter: ee.Filter = date_filter 504 | 505 | options_get_images: Dict[str, Any] = { 506 | "missions": missions, 507 | "filter": filter, 508 | "filterMasked": filter_masked, 509 | "filterMaskedFraction": filter_masked_fraction, 510 | "scale": ee.Number(scale).multiply(10), # why *10? 511 | "resample": True, 512 | } 513 | 514 | images: ee.ImageCollection = assets.getImages(bounds, options_get_images) 515 | 516 | options_get_mostly_clean_images: Dict[str, Any] = { 517 | "cloudFrequencyThresholdDelta": cloud_frequency_threshold_delta 518 | } 519 | 520 | return assets.getMostlyCleanImages( 521 | images, bounds, options_get_mostly_clean_images 522 | ) 523 | 524 | @staticmethod 525 | def _fix_null(values, v) -> ee.List: 526 | return ee.List(values).map( 527 | lambda o: ee.Algorithms.If(ee.Algorithms.IsEqual(o, None), v, o) 528 | ) 529 | 530 | @staticmethod 531 | def _unit_scale(image: ee.Image, min: float, max: float) -> ee.Image: 532 | min_image: ee.Image = ee.Image.constant(min) 533 | max_image: ee.Image = ee.Image.constant(max) 534 | return image.subtract(min_image).divide(max_image.subtract(min_image)) 535 | -------------------------------------------------------------------------------- /eepackages/applications/waterbody_area.py: -------------------------------------------------------------------------------- 1 | import ee 2 | from eepackages import assets 3 | from eepackages import utils 4 | 5 | 6 | def computeSurfaceWaterArea( 7 | waterbody, 8 | start_filter, 9 | start, 10 | stop, 11 | scale, 12 | waterOccurrence, 13 | opt_missions=None, 14 | quality_score_attributes=None, 15 | mosaic_by_day=True, 16 | min_overlap_fraction=None, 17 | ): 18 | """ 19 | compute the surface area of given waterbody between start and stop, using images between 20 | start_filter and stop to obtain a representative set of images to filter cloudy images. 21 | 22 | args: 23 | waterbody: waterbody feature 24 | start_filter: start date of images using to filter cloudy images 25 | start: start date of analysis 26 | stop: stop date of analysis 27 | scale: scale of analysis 28 | waterOccurrence: feature collection of reference water occurrence data. 29 | opt_missions: mission codes to use for the input imagery. Leave empty for a recommended 30 | set of missions. 31 | quality_score_attribute: if the quality_score is precomputed, a ranked list of attributes of 32 | the waterbody to use as quality_score. Otherwise None. 33 | min_overlap_fraction (Optional[float]): minimum overlap fraction between image footprint 34 | and waterbody. Leave as None to skip filtering. 35 | 36 | returns: 37 | image collection containing water layers. 38 | """ 39 | 40 | geom = ee.Feature(waterbody).geometry() 41 | 42 | missions = ["L4", "L5", "L7", "L8", "S2"] 43 | props = waterbody.getInfo()["properties"] 44 | quality_score_threshold = None 45 | if quality_score_attributes: 46 | for prop in quality_score_attributes: 47 | att = props.get(prop) 48 | if att: 49 | quality_score_threshold = att 50 | break 51 | 52 | if opt_missions: 53 | missions = opt_missions 54 | 55 | images = assets.getImages( 56 | geom, 57 | { 58 | "resample": True, 59 | "filter": ee.Filter.date(start_filter, stop), 60 | "missions": missions, 61 | "scale": scale * 10, 62 | }, 63 | ) 64 | 65 | # TODO: mosaic by day 66 | # images = assets.mosaic_by_day(images) 67 | 68 | options = { 69 | # 'cloudFrequencyThresholdDelta': -0.15 70 | "scale": scale * 5, 71 | "cloud_frequency": props.get("cloud_frequency"), 72 | "quality_score_cloud_threshold": quality_score_threshold, 73 | } 74 | 75 | g = geom.buffer(300, scale) 76 | 77 | images = assets.getMostlyCleanImages(images, g, options).sort("system:time_start") 78 | images = images.filterDate(start, stop) 79 | 80 | if mosaic_by_day: 81 | images = assets.mosaic_by_day(images) 82 | 83 | if min_overlap_fraction: 84 | images = filter_waterbody_overlap(images, waterbody, min_overlap_fraction) 85 | 86 | water = images.map( 87 | lambda i: computeSurfaceWaterArea_SingleImage( 88 | i, waterbody, scale, waterOccurrence 89 | ) 90 | ) 91 | 92 | water = water.filter(ee.Filter.neq("area", 0)) 93 | 94 | return water 95 | 96 | 97 | def computeSurfaceWaterArea_SingleImage(i, waterbody, scale, waterOccurrence): 98 | geom = ee.Feature(waterbody).geometry() 99 | 100 | fillPercentile = 50 # // we don't trust our prior 101 | 102 | ndwiBands = ["green", "swir"] 103 | # var ndwiBands = ['green', 'nir'] 104 | 105 | waterMaxImage = ee.Image().float().paint(waterbody.buffer(150), 1) 106 | 107 | maxArea = waterbody.area(scale) 108 | 109 | t = i.get("system:time_start") 110 | 111 | i = i.updateMask(waterMaxImage).updateMask( 112 | i.select("swir").min(i.select("nir")).gt(0.001) 113 | ) 114 | 115 | ndwi = i.normalizedDifference(ndwiBands) 116 | 117 | # var water = ndwi.gt(0) 118 | 119 | th = utils.computeThresholdUsingOtsu(ndwi, scale, geom, 0.5, 0.7, -0.2) 120 | water = ndwi.gt(th) 121 | 122 | area = ( 123 | ee.Image.pixelArea() 124 | .mask(water) 125 | .reduceRegion(**{"reducer": ee.Reducer.sum(), "geometry": geom, "scale": scale}) 126 | .get("area") 127 | ) 128 | 129 | # # fill missing, estimate water probability as the lowest percentile. 130 | # var waterEdge = ee.Algorithms.CannyEdgeDetector(water, 0.1, 0) 131 | waterEdge = ee.Algorithms.CannyEdgeDetector(ndwi, 0.5, 0.7) 132 | 133 | # image mask 134 | imageMask = ndwi.mask() 135 | 136 | # var imageMask = i.select(ndwiBands).reduce(ee.Reducer.allNonZero()) 137 | 138 | # imageMask = utils.focalMin(imageMask, ee.Number(scale) * 1.5) 139 | imageMask = imageMask.focal_min(ee.Number(scale).multiply(1.5), "square", "meters") 140 | # .multiply(waterMaxImage) 141 | 142 | # TODO: exclude non-water/missing boundsry 143 | waterEdge = waterEdge.updateMask(imageMask) 144 | 145 | # # clip by clouds 146 | # var bad = ee.Image(1).float().subtract(i.select('weight')) 147 | # bad = utils.focalMax(bad, 90).not() 148 | # waterEdge = waterEdge.updateMask(bad) 149 | 150 | # get water probability around edges 151 | # P(D|W) = P(D|W) * P(W) / P(D) ~= P(D|W) * P(W) 152 | p = ( 153 | waterOccurrence.mask(waterEdge) 154 | .reduceRegion( 155 | **{ 156 | "reducer": ee.Reducer.percentile([fillPercentile]), 157 | "geometry": geom, 158 | "scale": scale, 159 | } 160 | ) 161 | .values() 162 | .get(0) 163 | ) 164 | 165 | # TODO: exclude edges belonging to cloud/water or cloud/land 166 | 167 | # TODO: use multiple percentiles (confidence margin) 168 | 169 | p = ee.Algorithms.If(ee.Algorithms.IsEqual(p, None), 101, p) 170 | 171 | waterFill = waterOccurrence.gt(ee.Image.constant(p)).updateMask( 172 | water.unmask(0, False).Not() 173 | ) 174 | 175 | # exclude false-positive, where we're sure in a non-water 176 | nonWater = ndwi.lt(-0.15).unmask(0, False) 177 | waterFill = waterFill.updateMask(nonWater.Not()) 178 | 179 | fill = ( 180 | ee.Image.pixelArea() 181 | .mask(waterFill) 182 | .reduceRegion(**{"reducer": ee.Reducer.sum(), "geometry": geom, "scale": scale}) 183 | .get("area") 184 | ) 185 | 186 | area_filled = ee.Number(area).add(fill) 187 | 188 | filled_fraction = ee.Number(fill).divide(area_filled) 189 | 190 | return ( 191 | i.addBands(waterFill.rename("water_fill")) 192 | .addBands(waterEdge.rename("water_edge")) 193 | .addBands(ndwi.rename("ndwi")) 194 | .addBands(water.rename("water")) 195 | .set( 196 | { 197 | "p": p, 198 | "area": area, 199 | "area_filled": area_filled, 200 | "filled_fraction": filled_fraction, 201 | "system:time_start": t, 202 | "ndwi_threshold": th, 203 | "waterOccurrenceExpected": waterOccurrence.mask(waterEdge), 204 | } 205 | ) 206 | ) 207 | 208 | 209 | def computeSurfaceWaterAreaJRC(waterbody, start, stop, scale): 210 | geom = ee.Feature(waterbody).geometry() 211 | 212 | jrcMonthly = ee.ImageCollection("JRC/GSW1_0/MonthlyHistory") 213 | 214 | def compute_area(i): 215 | area = ( 216 | ee.Image.pixelArea() 217 | .mask(i.eq(2)) 218 | .reduceRegion( 219 | {"reducer": ee.Reducer.sum(), "geometry": geom, "scale": scale} 220 | ) 221 | ) 222 | 223 | return i.set({"area": area.get("area")}) 224 | 225 | water = jrcMonthly.filterDate(start, stop).map(compute_area) 226 | 227 | return water 228 | 229 | 230 | def filter_waterbody_overlap(images, waterbody, min_overlap_fraction=0.4): 231 | """ 232 | filter images based on the overlap of the image footprint with the waterbody. 233 | If the overlap consists of less than `min_overlap_fraction` of the waterbody area, the image 234 | if filtered out. 235 | 236 | Args: 237 | images: ImageCollection of input images 238 | waterbody: Feature with waterbody 239 | min_overlap_fraction: minumum fraction of overlap to accept. 240 | returns: 241 | filtered ImageCollection 242 | """ 243 | 244 | def set_intersection(i): 245 | intersection_area = ( 246 | ee.Geometry(i.get("system:footprint")) 247 | .intersection(waterbody.geometry()) 248 | .area() 249 | ) 250 | waterbody_area = waterbody.geometry().area() 251 | i = i.set("overlap_fraction", intersection_area.divide(waterbody_area)) 252 | return i 253 | 254 | images = images.map(set_intersection) 255 | images = images.filter(ee.Filter.gte("overlap_fraction", min_overlap_fraction)) 256 | 257 | return images 258 | 259 | 260 | def extrapolate_JRC_wo(waterbody, scale, max_trusted_occurrence=0.97): 261 | """ 262 | extrapolate the JRC dataset based on a maximum value that is trusted. 263 | This is needed when a waterbody experiences an unprecedented drought, in which gap filling 264 | fails when using the JRC dataset as-is. 265 | """ 266 | water_occurrence = ( 267 | ee.Image("JRC/GSW1_4/GlobalSurfaceWater") 268 | .select("occurrence") 269 | .mask(1) # fixes JRC masking lacking 270 | .resample("bicubic") 271 | .divide(100) 272 | ) 273 | 274 | # we don't trust water occurrence dataset when water level is below this (values > th) 275 | min_water = water_occurrence.gt(max_trusted_occurrence) 276 | 277 | # Calculate distance from water Occurence max 278 | dist = ( 279 | min_water.Not() 280 | .fastDistanceTransform(150) 281 | .reproject(ee.Projection("EPSG:4326").atScale(100)) 282 | .resample("bicubic") 283 | .sqrt() 284 | .multiply(100) 285 | ) 286 | 287 | # calculate max distance for scaling to 1 288 | max_distance = dist.reduceRegion( 289 | reducer=ee.Reducer.max(), 290 | geometry=waterbody.geometry(), 291 | scale=scale, 292 | bestEffort=True, 293 | ).get("distance") 294 | 295 | # scale distance values from min_trusted_occurrence to 1 296 | extrap_scale = ee.Number(1).subtract(max_trusted_occurrence).divide(max_distance) 297 | water_occurrence_extrapolated = dist.multiply(extrap_scale).add( 298 | max_trusted_occurrence 299 | ) 300 | 301 | return water_occurrence.where( 302 | water_occurrence.gt(max_trusted_occurrence), water_occurrence_extrapolated 303 | ) 304 | -------------------------------------------------------------------------------- /eepackages/assets.py: -------------------------------------------------------------------------------- 1 | import ee 2 | 3 | from eepackages import utils 4 | 5 | 6 | def cloudMaskAlgorithms_Landsat(image): 7 | imageWithCloud = ee.Algorithms.Landsat.simpleCloudScore(ee.Image(image)) 8 | 9 | return imageWithCloud.addBands( 10 | ee.Image(imageWithCloud.select("cloud").divide(100)), None, True 11 | ) 12 | 13 | 14 | def cloudMaskAlgorithms_S2(image): 15 | qa = image.select("QA60") 16 | # Bits 10 and 11 are clouds and cirrus, respectively. 17 | cloudBitMask = 1 << 10 18 | cirrusBitMask = 1 << 11 19 | # Both flags should be set to zero, indicating clear conditions. 20 | mask = qa.bitwiseAnd(cloudBitMask).eq(0).And(qa.bitwiseAnd(cirrusBitMask).eq(0)) 21 | mask = mask.subtract(1).multiply(-1).rename("cloud") 22 | return image.addBands(mask) 23 | 24 | 25 | cloudMaskAlgorithms = { 26 | "L8": cloudMaskAlgorithms_Landsat, 27 | "S2": cloudMaskAlgorithms_S2, 28 | "L7": cloudMaskAlgorithms_Landsat, 29 | "L5": cloudMaskAlgorithms_Landsat, 30 | "L4": cloudMaskAlgorithms_Landsat, 31 | } 32 | 33 | 34 | def getImages(g, options): 35 | g = ee.Geometry(g) 36 | 37 | resample = False 38 | 39 | s2MergeByTime = True 40 | 41 | if options and "s2MergeByTime" in options: 42 | s2MergeByTime = options["s2MergeByTime"] 43 | 44 | cloudMask = False 45 | 46 | if options and "cloudMask" in options: 47 | cloudMask = options["cloudMask"] 48 | 49 | cloudMaskAlgorithms_ = cloudMaskAlgorithms 50 | if options and "cloudMaskAlgorithms" in options: 51 | cloudMaskAlgorithms_ = options["cloudMaskAlgorithms"] 52 | 53 | missions = ["S2", "L8", "L9"] 54 | 55 | if options and "missions" in options: 56 | missions = options["missions"] 57 | 58 | if options and "resample" in options: 59 | resample = options["resample"] 60 | 61 | bands = { 62 | "S2": { 63 | "from": ["B11", "B8", "B4", "B3", "B2"], 64 | "to": ["swir", "nir", "red", "green", "blue"], 65 | }, 66 | "L8": { 67 | "from": ["B6", "B5", "B4", "B3", "B2"], 68 | "to": ["swir", "nir", "red", "green", "blue"], 69 | }, 70 | "L9": { 71 | "from": ["B6", "B5", "B4", "B3", "B2"], 72 | "to": ["swir", "nir", "red", "green", "blue"], 73 | }, 74 | "L5": { 75 | "from": ["B5", "B4", "B3", "B2", "B1"], 76 | "to": ["swir", "nir", "red", "green", "blue"], 77 | }, 78 | "L4": { 79 | "from": ["B5", "B4", "B3", "B2", "B1"], 80 | "to": ["swir", "nir", "red", "green", "blue"], 81 | }, 82 | "L7": { 83 | "from": ["B5", "B4", "B3", "B2", "B1"], 84 | "to": ["swir", "nir", "red", "green", "blue"], 85 | }, 86 | } 87 | 88 | if options and "includeTemperature" in options: 89 | bands["L8"]["from"].append("B10") 90 | bands["L8"]["to"].append("temp") 91 | bands["L9"]["from"].append("B10") 92 | bands["L9"]["to"].append("temp") 93 | bands["L5"]["from"].append("B6") 94 | bands["L5"]["to"].append("temp") 95 | bands["L4"]["from"].append("B6") 96 | bands["L4"]["to"].append("temp") 97 | bands["L7"]["from"].append("B6_VCID_1") 98 | bands["L7"]["to"].append("temp") 99 | 100 | # used only for a single sensor 101 | if options and "bandsAll" in options: 102 | bands["S2"] = { 103 | "from": [ 104 | "B11", 105 | "B8", 106 | "B3", 107 | "B2", 108 | "B4", 109 | "B5", 110 | "B6", 111 | "B7", 112 | "B8A", 113 | "B9", 114 | "B10", 115 | ], 116 | "to": [ 117 | "swir", 118 | "nir", 119 | "green", 120 | "blue", 121 | "red", 122 | "red2", 123 | "red3", 124 | "red4", 125 | "nir2", 126 | "water_vapour", 127 | "cirrus", 128 | ], 129 | } 130 | 131 | s2 = ee.ImageCollection("COPERNICUS/S2_HARMONIZED").filterBounds(g) 132 | 133 | if options and "filter" in options: 134 | s2 = s2.filter(options["filter"]) 135 | 136 | # apply custom cloud masking 137 | if cloudMask: 138 | s2 = s2.map(cloudMaskAlgorithms_["S2"]) 139 | bands["S2"]["from"].append("cloud") 140 | bands["S2"]["to"].append("cloud") 141 | 142 | s2 = s2.select(bands["S2"]["from"], bands["S2"]["to"]) 143 | 144 | l9 = ee.ImageCollection("LANDSAT/LC09/C02/T1_TOA") 145 | 146 | l9 = l9.filterBounds(g) 147 | 148 | if options and "filter" in options: 149 | l9 = l9.filter(options["filter"]) 150 | 151 | if cloudMask: 152 | l9 = l9.map(cloudMaskAlgorithms_["L9"]) 153 | bands["L9"]["from"].append("cloud") 154 | bands["L9"]["to"].append("cloud") 155 | 156 | l9 = l9.select(bands["L9"]["from"], bands["L9"]["to"]) 157 | 158 | if options and "real_time" in options and options.get("real_time"): # default to RT 159 | l8 = ee.ImageCollection("LANDSAT/LC08/C02/T1_RT_TOA") 160 | else: 161 | l8 = ee.ImageCollection("LANDSAT/LC08/C02/T1_TOA") 162 | 163 | if options and "includeTier2" in options: 164 | l8 = l8.merge(ee.ImageCollection("LANDSAT/LC08/C02/T2_TOA")) 165 | 166 | l8 = l8.filterBounds(g) 167 | 168 | if options and "filter" in options: 169 | l8 = l8.filter(options["filter"]) 170 | 171 | if cloudMask: 172 | l8 = l8.map(cloudMaskAlgorithms_["L8"]) 173 | bands["L8"]["from"].append("cloud") 174 | bands["L8"]["to"].append("cloud") 175 | 176 | l8 = l8.select(bands["L8"]["from"], bands["L8"]["to"]) 177 | 178 | l5 = ee.ImageCollection("LANDSAT/LT05/C02/T1_TOA") 179 | 180 | if options and "includeTier2" in options: 181 | l5 = l5.merge(ee.ImageCollection("LANDSAT/LT05/C02/T2_TOA")) 182 | 183 | l5 = l5.filterBounds(g) 184 | 185 | if options and "filter" in options: 186 | l5 = l5.filter(options["filter"]) 187 | 188 | if cloudMask: 189 | l5 = l5.map(cloudMaskAlgorithms_["L5"]) 190 | bands["L5"]["from"].append("cloud") 191 | bands["L5"]["to"].append("cloud") 192 | 193 | l5 = l5.select(bands["L5"]["from"], bands["L5"]["to"]) 194 | 195 | l4 = ee.ImageCollection("LANDSAT/LT04/C02/T1_TOA") 196 | 197 | if options and "includeTier2" in options: 198 | l4 = l4.merge(ee.ImageCollection("LANDSAT/LT04/C02/T2_TOA")) 199 | 200 | l4 = l4.filterBounds(g) 201 | 202 | if options and "filter" in options: 203 | l4 = l4.filter(options["filter"]) 204 | 205 | if cloudMask: 206 | l4 = l4.map(cloudMaskAlgorithms_["L4"]) 207 | bands["L4"]["from"].append("cloud") 208 | bands["L4"]["to"].append("cloud") 209 | 210 | l4 = l4.select(bands["L4"]["from"], bands["L4"]["to"]) 211 | 212 | if options and "real_time" in options and options.get("real_time"): # default to RT 213 | l7 = ee.ImageCollection("LANDSAT/LE07/C02/T1_RT_TOA") 214 | else: 215 | l7 = ee.ImageCollection("LANDSAT/LE07/C02/T1_TOA") 216 | 217 | if options and "includeTier2" in options: 218 | l7 = l7.merge(ee.ImageCollection("LANDSAT/LE07/C02/T2_TOA")) 219 | 220 | l7 = l7.filterBounds(g) 221 | 222 | if options and "filter" in options: 223 | l7 = l7.filter(options["filter"]) 224 | 225 | if cloudMask: 226 | l7 = l7.map(cloudMaskAlgorithms_["L7"]) 227 | bands["L7"]["from"].append("cloud") 228 | bands["L7"]["to"].append("cloud") 229 | 230 | l7 = l7.select(bands["L7"]["from"], bands["L7"]["to"]) 231 | 232 | if options and "clipBufferSizeL7" in options: 233 | 234 | def f(i): 235 | mask = ( 236 | i.select(["green", "red", "nir", "swir"]) 237 | .mask() 238 | .reduce(ee.Reducer.allNonZero()) 239 | ) 240 | mask = mask.focal_min( 241 | options["clipBufferSizeL7"], "square", "meters" 242 | ).reproject(i.select("nir").projection()) 243 | return i.updateMask(mask) 244 | 245 | l7 = l7.map(f) 246 | 247 | clipBufferSize = 6000 248 | 249 | if options and "clipBufferSize" in options: 250 | clipBufferSize = options["clipBufferSize"] 251 | 252 | scale = 100 253 | if options and "scale" in options: 254 | scale = options["scale"] 255 | 256 | def clipNegativeFootprint(i): 257 | return i.clip( 258 | i.select(0).geometry().buffer(ee.Number(clipBufferSize).multiply(-1), 1000) 259 | ) 260 | 261 | l4 = l4.map(clipNegativeFootprint) 262 | l5 = l5.map(clipNegativeFootprint) 263 | l7 = l7.map(clipNegativeFootprint) 264 | 265 | resample = "bicubic" 266 | 267 | s2 = s2.map(lambda i: i.resample(resample)) 268 | 269 | # merge by time (remove duplicates) 270 | if s2MergeByTime: 271 | s2 = mosaicByTime(s2) 272 | 273 | def f2(i): 274 | return ( 275 | i.addBands(i.multiply(0.0001).float(), i.bandNames().remove("cloud"), True) 276 | .copyProperties(i) 277 | .set( 278 | { 279 | "SUN_ELEVATION": ee.Number(90).subtract( 280 | i.get("MEAN_SOLAR_ZENITH_ANGLE") 281 | ) 282 | } 283 | ) 284 | .set({"MISSION": "S2"}) 285 | .set({"SUN_AZIMUTH": i.get("MEAN_SOLAR_AZIMUTH_ANGLE")}) 286 | .set({"MULTIPLIER": 0.0001}) 287 | .set({"SPACECRAFT_ID": i.get("SPACECRAFT_NAME")}) 288 | ) 289 | 290 | s2 = s2.map(f2) 291 | 292 | images = ee.ImageCollection([]) 293 | 294 | if "L5" in missions: 295 | 296 | def f(i): 297 | return i.set( 298 | { 299 | "MISSION": "L5", 300 | "BANDS_FROM": bands["L5"]["from"], 301 | "BANDS_TO": bands["L5"]["to"], 302 | "MULTIPLIER": 1, 303 | } 304 | ) 305 | 306 | l5 = l5.map(f) 307 | images = images.merge(l5) 308 | 309 | if "L4" in missions: 310 | 311 | def f(i): 312 | return i.set( 313 | { 314 | "MISSION": "L4", 315 | "BANDS_FROM": bands["L4"]["from"], 316 | "BANDS_TO": bands["L4"]["to"], 317 | "MULTIPLIER": 1, 318 | } 319 | ) 320 | 321 | l4 = l4.map(f) 322 | images = images.merge(l4) 323 | 324 | if "L7" in missions: 325 | 326 | def f(i): 327 | return i.set( 328 | { 329 | "MISSION": "L7", 330 | "BANDS_FROM": bands["L7"]["from"], 331 | "BANDS_TO": bands["L7"]["to"], 332 | "MULTIPLIER": 1, 333 | } 334 | ) 335 | 336 | l7 = l7.map(f) 337 | images = images.merge(l7) 338 | 339 | if "L8" in missions: 340 | 341 | def f(i): 342 | return i.set( 343 | { 344 | "MISSION": "L8", 345 | "BANDS_FROM": bands["L8"]["from"], 346 | "BANDS_TO": bands["L8"]["to"], 347 | "MULTIPLIER": 1, 348 | } 349 | ) 350 | 351 | l8 = l8.map(f) 352 | images = images.merge(l8) 353 | 354 | if "L9" in missions: 355 | 356 | def f(i): 357 | return i.set( 358 | { 359 | "MISSION": "L9", 360 | "BANDS_FROM": bands["L9"]["from"], 361 | "BANDS_TO": bands["L9"]["to"], 362 | "MULTIPLIER": 1, 363 | } 364 | ) 365 | 366 | l9 = l9.map(f) 367 | images = images.merge(l9) 368 | 369 | images = ee.ImageCollection(images) 370 | 371 | if resample: 372 | images = images.map(lambda i: i.resample(resample)) 373 | 374 | if "S2" in missions: 375 | 376 | def f(i): 377 | return i.set( 378 | { 379 | "MISSION": "S2", 380 | "BANDS_FROM": bands["S2"]["from"], 381 | "BANDS_TO": bands["S2"]["to"], 382 | "MULTIPLIER": 0.0001, 383 | } 384 | ) 385 | 386 | s2 = s2.map(f) 387 | images = images.merge(s2) 388 | 389 | images = ee.ImageCollection(images) 390 | 391 | if options and "filterMasked" in options: 392 | if "filterMaskedFraction" in options and options["filterMaskedFraction"]: 393 | # get images coverting bounds at least filterMaskedFraction% 394 | area = g.area(scale) 395 | 396 | def f(i): 397 | maskedArea = ( 398 | ee.Image.pixelArea() 399 | .updateMask(i.select(0).mask()) 400 | .reduceRegion(ee.Reducer.sum(), g, ee.Number(scale).multiply(10)) 401 | .values() 402 | .get(0) 403 | ) 404 | return i.set({"maskedFraction": ee.Number(maskedArea).divide(area)}) 405 | 406 | images = images.map(f).filter( 407 | ee.Filter.gt("maskedFraction", options["filterMaskedFraction"]) 408 | ) 409 | else: 410 | # get images covering bounds 100% 411 | images = images.map( 412 | lambda i: i.set( 413 | { 414 | "complete": i.select(0) 415 | .mask() 416 | .reduceRegion( 417 | ee.Reducer.allNonZero(), g, ee.Number(scale).multiply(10) 418 | ) 419 | .values() 420 | .get(0) 421 | } 422 | ) 423 | ).filter(ee.Filter.eq("complete", 1)) 424 | 425 | # exclude night images 426 | images = images.filter(ee.Filter.gt("SUN_ELEVATION", 0)) 427 | 428 | return images 429 | 430 | 431 | # /*** 432 | # * Sentinel-2 produces multiple images, resulting sometimes 4x more images than the actual size. 433 | # * This is bad for any statistical analysis. 434 | # * 435 | # * This function mosaics images by time. 436 | # */ 437 | def mosaicByTime(images): 438 | TIME_FIELD = "system:time_start" 439 | 440 | distinct = images.distinct([TIME_FIELD]) 441 | 442 | filter = ee.Filter.equals(**{"leftField": TIME_FIELD, "rightField": TIME_FIELD}) 443 | join = ee.Join.saveAll("matches") 444 | results = join.apply(distinct, images, filter) 445 | 446 | # mosaic 447 | def merge_matches(i): 448 | matchedImages = ee.ImageCollection.fromImages(i.get("matches")) 449 | mosaic = ( 450 | matchedImages.sort("system:index") 451 | .mosaic() 452 | .set({"system:footprint": matchedImages.geometry()}) 453 | ) 454 | 455 | return mosaic.copyProperties(i).set(TIME_FIELD, i.get(TIME_FIELD)) 456 | 457 | results = results.map(merge_matches) 458 | 459 | return ee.ImageCollection(results) 460 | 461 | 462 | def mosaic_by_day(images): 463 | """ 464 | Mosaic the image collection based on the day of the image. 465 | Args: 466 | images: input ImageCollection 467 | Returns: 468 | ImageCollection with merged images by date. 469 | """ 470 | 471 | images = ee.ImageCollection(images) 472 | 473 | def merge_daily_images(i): 474 | matches = ee.ImageCollection.fromImages(i.get("images")) 475 | return ( 476 | matches.sort("system:index") 477 | .mosaic() 478 | .copyProperties(i, exclude=["system:time_start"]) 479 | .set( 480 | { 481 | "system:time_start": ee.Date(i.get("date")).millis(), 482 | "system:footprint": matches.geometry().dissolve(10), 483 | } 484 | ) 485 | ) 486 | 487 | def set_image_properties(i): 488 | d = i.toDictionary() 489 | return i.set( 490 | { 491 | "date": i.date().format("yyyy-MM-dd"), 492 | "SENSING_ORBIT_NUMBER": ee.Number(d.get("SENSING_ORBIT_NUMBER", -1)), 493 | } 494 | ) 495 | 496 | images = images.map(set_image_properties) 497 | distinct = images.distinct(["date"]) 498 | return ee.ImageCollection( 499 | ee.ImageCollection( 500 | ee.Join.saveAll("images").apply( 501 | primary=distinct, 502 | secondary=images, 503 | condition=ee.Filter.And( 504 | ee.Filter.equals(leftField="date", rightField="date"), 505 | ee.Filter.equals( 506 | leftField="SPACECRAFT_ID", rightField="SPACECRAFT_ID" 507 | ), 508 | ee.Filter.equals( 509 | leftField="SENSING_ORBIT_NUMBER", 510 | rightField="SENSING_ORBIT_NUMBER", 511 | ), # unique for S2 512 | ), 513 | ) 514 | ).map(merge_daily_images) 515 | ) 516 | 517 | 518 | def addQualityScore(images, g, options): 519 | scorePercentile = 75 520 | scale = 500 521 | mask = None 522 | qualityBand = "green" 523 | 524 | if options: 525 | if "percentile" in options: 526 | scorePercentile = options["percentile"] 527 | if "scale" in options: 528 | scale = options["scale"] 529 | if "mask" in options: 530 | mask = options["mask"] 531 | if "qualityBand" in options: 532 | qualityBand = options["qualityBand"] 533 | 534 | def add_quality_score(i): 535 | score = ee.Image(i).select( 536 | qualityBand 537 | ) # //.where(i.select('green').gt(0.5), 0.5) 538 | 539 | if mask: 540 | score = score.updateMask(mask) 541 | 542 | score = ( 543 | score.reduceRegion(ee.Reducer.percentile([scorePercentile]), g, scale) 544 | .values() 545 | .get(0) 546 | ) 547 | 548 | # // var score = i.select('green').add(i.select('blue')) 549 | # // .reduceRegion(ee.Reducer.percentile([scorePercentile]), g, scale).values().get(0) 550 | 551 | # // var cloudScore = computeCloudScore(i) 552 | # // var score = cloudScore.gt(cloudThreshold) 553 | # // .reduceRegion(ee.Reducer.sum(), g, scale).values().get(0) 554 | 555 | return i.set({"quality_score": score}) 556 | 557 | return images.map(add_quality_score) 558 | 559 | 560 | def getMostlyCleanImages(images, g, options=None): 561 | g = ee.Geometry(g) 562 | 563 | scale = 500 564 | 565 | if options and "scale" in options: 566 | scale = options["scale"] 567 | 568 | p = 85 569 | 570 | if options and "percentile" in options: 571 | p = options["percentile"] 572 | 573 | # // http://www.earthenv.org/cloud 574 | modis_clouds = ee.Image("users/gena/MODCF_meanannual") 575 | 576 | # If cloud frequency already known for waterbody 577 | if options and options.get("cloud_frequency"): 578 | cloud_frequency = ee.Number(options["cloud_frequency"]) 579 | else: 580 | cloud_frequency = ( 581 | modis_clouds.divide(10000) 582 | .reduceRegion( 583 | ee.Reducer.percentile([p]), 584 | g.buffer(10000, ee.Number(scale).multiply(10)), 585 | ee.Number(scale).multiply(10), 586 | ) 587 | .values() 588 | .get(0) 589 | ) 590 | 591 | # print('Cloud frequency (over AOI):', cloud_frequency.getInfo()) 592 | 593 | # decrease cloudFrequency, include some more partially-cloudy images then clip based on a quality metric 594 | # also assume inter-annual variability of the cloud cover 595 | cloud_frequency = ee.Number(cloud_frequency).subtract(0.15).max(0.0) 596 | 597 | if options and options.get("cloudFrequencyThresholdDelta"): 598 | cloud_frequency = cloud_frequency.add(options["cloudFrequencyThresholdDelta"]) 599 | 600 | images = images.filterBounds(g) 601 | 602 | # size = images.size() # not being used? 603 | 604 | images = addQualityScore(images, g, options).filter( 605 | ee.Filter.gt("quality_score", 0) 606 | ) # sometimes null?! 607 | 608 | # clip collection based on quality score and cloud frequency 609 | # If quality score to filter clouds is already known for waterbody 610 | if options and options.get("quality_score_cloud_threshold"): 611 | images = images.filter( 612 | ee.Filter.lte("quality_score", options["quality_score_cloud_threshold"]) 613 | ) 614 | # Else calculate based on available images 615 | else: 616 | images = images.sort("quality_score").limit( 617 | images.size().multiply(ee.Number(1).subtract(cloud_frequency)).toInt() 618 | ) 619 | 620 | # # remove too dark images 621 | # images = images.sort('quality_score', false) 622 | # .limit(images.size().multiply(0.99).toInt()) 623 | 624 | # print('size, filtered: ', images.size()) 625 | 626 | return images 627 | # .set({scoreMax: scoreMax}) 628 | 629 | 630 | def addCdfQualityScore( 631 | images, 632 | opt_thresholdMin=75, 633 | opt_thresholdMax=95, 634 | opt_includeNeighborhood=False, 635 | opt_neighborhoodOptions={"erosion": 5, "dilation": 50, "weight": 50}, 636 | opt_qualityBand="green", 637 | ): 638 | """ 639 | Compute one or more high percentile for every pixel and add quality score if pixel value is hither/lower than the threshold % 640 | """ 641 | 642 | images = images.map(lambda i: i.addBands(i.select(opt_qualityBand).rename("q"))) 643 | 644 | # P(bad | I < min) = 0, P(bad | I >= max) = 1 645 | pBad = ( 646 | images.select("q") 647 | .reduce(ee.Reducer.percentile([opt_thresholdMin, opt_thresholdMax])) 648 | .rename(["qmin", "qmax"]) 649 | ) 650 | 651 | pBadRange = pBad.select("qmax").subtract(pBad.select("qmin")) 652 | 653 | def remove_bad_pixels(i): 654 | # probability of bad due to high reflactance values 655 | badLinear = ( 656 | i.select("q") 657 | .max(pBad.select("qmin")) 658 | .min(pBad.select("qmax")) 659 | .subtract(pBad.select("qmin")) 660 | .divide(pBadRange) 661 | .clamp(0, 1) 662 | ) 663 | 664 | # too low values 665 | badLow = i.select("q").lt(0.0001) 666 | 667 | badWeight = badLinear.multiply(badLow.Not()) 668 | 669 | if opt_includeNeighborhood: 670 | radius = opt_neighborhoodOptions 671 | 672 | # opening 673 | bad = badWeight.gt(0.5) 674 | 675 | if radius.get("erosion"): 676 | bad = utils.focalMin(bad, radius["erosion"]) 677 | 678 | bad = utils.focalMax(bad, radius["dilation"]) 679 | 680 | bad = utils.focalMaxWeight(bad, radius["weight"]) 681 | 682 | badWeight = badWeight.max(bad) 683 | 684 | # smoothen scene boundaries 685 | # badWeight = badWeight 686 | # .multiply(utils.focalMin(i.select('blue').mask(), 10).convolve(ee.Kernel.gaussian(5, 3))) 687 | 688 | # compute bad pixel probability 689 | weight = ee.Image(1).float().subtract(badWeight).rename("weight") 690 | 691 | return i.addBands(weight).addBands(pBad) 692 | 693 | # bad pixel removal (costly) 694 | images = images.map(remove_bad_pixels) 695 | 696 | return images 697 | 698 | 699 | def otsu(histogram): 700 | """ 701 | The script computes surface water mask using Canny Edge detector and Otsu thresholding 702 | See the following paper for details: http://www.mdpi.com/2072-4292/8/5/386 703 | 704 | Author: Gennadii Donchyts (gennadiy.donchyts@gmail.com) 705 | Contributors: Nicholas Clinton (nclinton@google.com) - re-implemented otsu() using ee.Array 706 | 707 | Example: 708 | thresholding = require('users/gena/packages:thresholding') 709 | 710 | th = thresholding.computeThresholdUsingOtsu(image, scale, bounds, cannyThreshold, cannySigma, minValue, ...) 711 | 712 | Returns: 713 | the DN that maximizes interclass variance in B5 (in the region). 714 | """ 715 | histogram = ee.Dictionary(histogram) 716 | 717 | counts = ee.Array(histogram.get("histogram")) 718 | means = ee.Array(histogram.get("bucketMeans")) 719 | size = means.length().get([0]) 720 | total = counts.reduce(ee.Reducer.sum(), [0]).get([0]) 721 | sum = means.multiply(counts).reduce(ee.Reducer.sum(), [0]).get([0]) 722 | mean = sum.divide(total) 723 | 724 | indices = ee.List.sequence(1, size) 725 | 726 | # Compute between sum of squares, where each mean partitions the data. 727 | def f(i): 728 | aCounts = counts.slice(0, 0, i) 729 | aCount = aCounts.reduce(ee.Reducer.sum(), [0]).get([0]) 730 | aMeans = means.slice(0, 0, i) 731 | aMean = ( 732 | aMeans.multiply(aCounts) 733 | .reduce(ee.Reducer.sum(), [0]) 734 | .get([0]) 735 | .divide(aCount) 736 | ) 737 | bCount = total.subtract(aCount) 738 | bMean = sum.subtract(aCount.multiply(aMean)).divide(bCount) 739 | 740 | return aCount.multiply(aMean.subtract(mean).pow(2)).add( 741 | bCount.multiply(bMean.subtract(mean).pow(2)) 742 | ) 743 | 744 | bss = indices.map(f) 745 | 746 | # Return the mean value corresponding to the maximum BSS. 747 | return means.sort(bss).get([-1]) 748 | -------------------------------------------------------------------------------- /eepackages/geometry.py: -------------------------------------------------------------------------------- 1 | import ee 2 | import math 3 | 4 | 5 | def createTransectAtCentroid(line, length, crs, maxError): 6 | """ 7 | Creates transect in the middle of line geometry 8 | """ 9 | if not crs: 10 | crs = "EPSG:3857" 11 | 12 | if not maxError: 13 | maxError = 1 14 | 15 | line = ee.Geometry(line).transform(crs, maxError) 16 | origin = line.centroid(maxError, crs) 17 | 18 | length = ee.Number(length) 19 | 20 | # compute angle from two points 21 | coords = line.coordinates() 22 | pt0 = coords.slice(0, 1) 23 | pt1 = coords.slice(-1) 24 | delta = ee.Array(pt1).subtract(pt0) 25 | dx = delta.get([0, 0]) 26 | dy = delta.get([0, 1]) 27 | angle = dx.atan2(dy).add(math.pi / 2) 28 | 29 | ptOrigin = ee.Array([origin.coordinates()]).transpose() 30 | 31 | # get coordinates as a list 32 | proj1 = origin.projection().translate(length.multiply(-0.5), 0) 33 | pt1 = ee.Array([origin.transform(proj1).coordinates()]).transpose() 34 | 35 | # translate 36 | proj2 = origin.projection().translate(length.multiply(0.5), 0) 37 | pt2 = ee.Array([origin.transform(proj2).coordinates()]).transpose() 38 | 39 | # define rotation matrix 40 | cosa = angle.cos() 41 | sina = angle.sin() 42 | M = ee.Array([[cosa, sina.multiply(-1)], [sina, cosa]]) 43 | 44 | # rotate 45 | pt1 = M.matrixMultiply(pt1.subtract(ptOrigin)).add(ptOrigin) 46 | pt2 = M.matrixMultiply(pt2.subtract(ptOrigin)).add(ptOrigin) 47 | 48 | # get points 49 | pt1 = pt1.transpose().project([1]).toList() 50 | pt2 = pt2.transpose().project([1]).toList() 51 | 52 | # construct line 53 | line = ee.Algorithms.GeometryConstructors.LineString([pt1, pt2], ee.Projection(crs)) 54 | 55 | return line 56 | 57 | 58 | def createVector(origin, angle, length): 59 | # get coordinates as a list 60 | pt1 = ee.Array([ee.Feature(origin).geometry().coordinates()]).transpose() 61 | 62 | # translate 63 | proj = origin.projection().translate(length, 0) 64 | pt2 = ee.Array([origin.transform(proj).coordinates()]).transpose() 65 | 66 | # define rotation matrix 67 | angle = ee.Number(angle).multiply(math.pi).divide(180) 68 | cosa = angle.cos() 69 | sina = angle.sin() 70 | M = ee.Array([[cosa, sina.multiply(-1)], [sina, cosa]]) 71 | 72 | # rotate 73 | pt2 = M.matrixMultiply(pt2.subtract(pt1)).add(pt1) 74 | 75 | # get end point 76 | pt2 = pt2.transpose().project([1]).toList() 77 | 78 | # construct line 79 | line = ee.Algorithms.GeometryConstructors.LineString( 80 | [origin, pt2], origin.projection() 81 | ) 82 | 83 | return line 84 | 85 | 86 | def angle(pt0, pt1): 87 | pt0 = ee.List(pt0) 88 | pt1 = ee.List(pt1) 89 | 90 | x0 = ee.Number(pt0.get(0)) 91 | x1 = ee.Number(pt1.get(0)) 92 | y0 = ee.Number(pt0.get(1)) 93 | y1 = ee.Number(pt1.get(1)) 94 | 95 | dy = y1.subtract(y0) 96 | dx = x1.subtract(x0) 97 | 98 | return dy.atan2(dx) 99 | -------------------------------------------------------------------------------- /eepackages/gl.py: -------------------------------------------------------------------------------- 1 | import ee 2 | 3 | 4 | def smoothStep(edge0: float, edge1: float, x: ee.Image): 5 | t: ee.Image = x.subtract(edge0).divide(ee.Image(edge1).subtract(edge0)).clamp(0, 1) 6 | return t.multiply(t).multiply( 7 | ee.Image.constant(3).subtract(ee.Image.constant(2).multiply(t)) 8 | ) 9 | -------------------------------------------------------------------------------- /eepackages/multiprocessing/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gee-community/ee-packages-py/6a822d50fe817a3aa821c68c1a7feb8ca8290329/eepackages/multiprocessing/__init__.py -------------------------------------------------------------------------------- /eepackages/multiprocessing/download_image.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from pathlib import Path 3 | import requests 4 | import shutil 5 | from typing import Any, Callable, Dict, Optional 6 | 7 | from pathos import logger 8 | from pathos.core import getpid 9 | from retry import retry 10 | 11 | 12 | @retry(tries=10, delay=5, backoff=10) 13 | def download_image( 14 | i_ee, 15 | index: int, 16 | image_download_method: Callable, 17 | serialized_image_list: Dict[str, Any], 18 | name_prefix: str, 19 | out_dir: Optional[Path], 20 | download_kwargs: Optional[Dict[str, Any]], 21 | ) -> None: 22 | """ 23 | Hidden function to be used with download_image_collection. As we want compatibility with 24 | windows, we need to use dill to pickle the initialized earthengine module. Pathos uses 25 | dill pickle data uses to start new processes with its multiprocessing pool, while the python 26 | multiprocessing module uses pickle, which cannot pickle module objects. 27 | """ 28 | 29 | if not out_dir: 30 | out_dir: Path = Path.cwd() / "output" 31 | if not download_kwargs: 32 | download_kwargs: Dict[str, str] = {} 33 | 34 | pid = getpid() 35 | log_file: Path = out_dir / "logging" / f"{pid}.log" 36 | fh = logging.FileHandler(log_file) 37 | fh.setLevel(logging.DEBUG) 38 | formatter = logging.Formatter("%(asctime)s %(name)-12s %(levelname)-8s %(message)s") 39 | fh.setFormatter(formatter) 40 | 41 | plogger = logger(level=logging.DEBUG, handler=fh) 42 | 43 | ee_api_url: str = i_ee.data._cloud_api_base_url 44 | ee_hv_url: str = "https://earthengine-highvolume.googleapis.com" 45 | if ee_api_url is not ee_hv_url: 46 | i_ee.Initialize(opt_url=ee_hv_url) 47 | 48 | image_list: i_ee.List = i_ee.deserializer.fromJSON(serialized_image_list) 49 | 50 | img: i_ee.Image = i_ee.Image(image_list.get(index)) 51 | url: str = image_download_method(img, download_kwargs) 52 | r: requests.Response = requests.get(url, stream=True) 53 | 54 | # File format chain 55 | format: Optional[str] = download_kwargs.get("format") 56 | if format: 57 | if format == "GEO_TIFF": 58 | extention: str = ".tif" 59 | elif format == "NPY": 60 | extention: str = ".npy" 61 | elif format == "PNG": 62 | extention: str = ".png" 63 | elif format == "JPG": 64 | extention: str = ".jpg" 65 | else: 66 | extention: str = ".tif.zip" 67 | elif image_download_method == i_ee.Image.getDownloadURL: 68 | extention: str = ".tif.zip" 69 | elif image_download_method == i_ee.Image.getThumbURL: 70 | extention: str = ".png" 71 | else: 72 | raise RuntimeError(f"image download method {image_download_method} unknown.") 73 | 74 | # Image naming chain 75 | img_props: Dict[str, Any] = img.getInfo()["properties"] 76 | t0: Optional[int] = img_props.get("system:time_start") 77 | img_index: Optional[int] = img_props.get("system:index") 78 | if t0: 79 | file_id: str = t0 80 | elif img_index: 81 | file_id: str = img_index 82 | else: 83 | file_id: str = index 84 | 85 | # File name 86 | filename: Path = out_dir / f"{name_prefix}{file_id}{extention}" 87 | 88 | with open(filename, "wb") as out_file: 89 | shutil.copyfileobj(r.raw, out_file) 90 | 91 | plogger.info(f"Done: {index}") 92 | plogger.removeHandler(fh) 93 | -------------------------------------------------------------------------------- /eepackages/tiler.py: -------------------------------------------------------------------------------- 1 | import math 2 | from typing import Optional 3 | 4 | import ee 5 | 6 | """ 7 | creating a tiler using earthengine classes, so it can be used on-cluster in a parallel way. 8 | """ 9 | 10 | TILE_SIZE: float = 256 11 | ORIGIN: float = math.pi * 6378137 # earth depth 12 | C: float = 40075016.686 # earth circumpherence 13 | PROJECTION: str = "EPSG:3857" 14 | 15 | 16 | def zoom_to_scale(zoom: ee.Number) -> ee.Number: 17 | zoom: ee.Number = ee.Number(zoom) 18 | return ee.Number(C).divide(ee.Number(2).pow(zoom)).divide(TILE_SIZE) 19 | 20 | 21 | def scale_to_zoom(scale: ee.Number) -> ee.Number: 22 | scale: ee.Number = ee.Number(scale) 23 | zoom: ee.Number = ( 24 | ee.Number(C).divide(scale.multiply(TILE_SIZE)).log().divide(ee.Number(2).log()) 25 | ) 26 | return zoom.ceil() 27 | 28 | 29 | def to_radians(degrees: ee.Number) -> ee.Number: 30 | degrees: ee.Number = ee.Number(degrees) 31 | 32 | return degrees.multiply(math.pi).divide(180) 33 | 34 | 35 | def pixels_to_meters(px: ee.Number, py: ee.Number, zoom: ee.Number) -> ee.List: 36 | px: ee.Number = ee.Number(px) 37 | py: ee.Number = ee.Number(py) 38 | zoom = ee.Number(zoom) 39 | 40 | resolution: ee.Number = zoom_to_scale(zoom) 41 | x: ee.Number = px.multiply(resolution).subtract(ORIGIN) 42 | y: ee.Number = py.multiply(resolution).subtract(ORIGIN) 43 | return ee.List([x, y]) 44 | 45 | 46 | def meters_to_pixels(x: ee.Number, y: ee.Number, zoom: ee.Number) -> ee.List: 47 | x: ee.Number = ee.Number(x) 48 | y: ee.Number = ee.Number(y) 49 | zoom = ee.Number(zoom) 50 | 51 | resolution: ee.Number = zoom_to_scale(zoom) 52 | px: ee.Number = x.add(ORIGIN).divide(resolution) 53 | py: ee.Number = y.add(ORIGIN).divide(resolution) 54 | return ee.List([px, py]) 55 | 56 | 57 | def degrees_to_tiles(lon: ee.Number, lat: ee.Number, zoom: ee.Number) -> ee.List: 58 | lon: ee.Number = ee.Number(lon) 59 | lat: ee.Number = ee.Number(lat) 60 | zoom: ee.Number = ee.Number(zoom) 61 | 62 | tx: ee.Number = lon.add(180).divide(360).multiply(ee.Number(2).pow(zoom)).floor() 63 | ty: ee.Number = ( 64 | ee.Number(1) 65 | .subtract( 66 | to_radians(lat) 67 | .tan() 68 | .add(ee.Number(1).divide(to_radians(lat).cos())) 69 | .log() 70 | .divide(ee.Number(math.pi)) 71 | ) 72 | .divide(2) 73 | .multiply(ee.Number(2).pow(zoom)) 74 | .floor() 75 | ) 76 | return ee.List([tx, ty]) 77 | 78 | 79 | def get_tile_bounds(tx: ee.Number, ty: ee.Number, zoom: ee.Number) -> ee.List: 80 | tx: ee.Number = ee.Number(tx) 81 | ty: ee.Number = ee.Number(ty) 82 | zoom: ee.Number = ee.Number(zoom) 83 | 84 | ty_flip: ee.Number = ( 85 | ee.Number(2).pow(zoom).subtract(ty).subtract(1) 86 | ) # TMS -> XYZ, flip y index 87 | min: ee.Number = pixels_to_meters( 88 | ee.Number(tx).multiply(TILE_SIZE), ty_flip.multiply(TILE_SIZE), zoom 89 | ) 90 | max: ee.Number = pixels_to_meters( 91 | ee.Number(tx).add(1).multiply(TILE_SIZE), 92 | ty_flip.add(1).multiply(TILE_SIZE), 93 | zoom, 94 | ) 95 | return ee.List([min, max]) 96 | 97 | 98 | def get_tiles_for_geometry( 99 | geometry: ee.Geometry, zoom: ee.Number, opt_bounds: Optional[ee.Geometry] = None 100 | ) -> ee.FeatureCollection: 101 | zoom: ee.Number = ee.Number(zoom) 102 | 103 | bounds: ee.Geometry = ee.Geometry( 104 | ee.Algorithms.If(opt_bounds, opt_bounds, geometry) 105 | ) 106 | bounds_list: ee.List = ee.List(bounds.bounds().coordinates().get(0)) 107 | 108 | ll: ee.List = ee.List(bounds_list.get(0)) 109 | ur: ee.List = ee.List(bounds_list.get(2)) 110 | 111 | tmin: ee.List = ee.List( 112 | degrees_to_tiles(ee.Number(ll.get(0)), ee.Number(ll.get(1)), zoom) 113 | ) 114 | tmax: ee.List = ee.List( 115 | degrees_to_tiles(ee.Number(ur.get(0)), ee.Number(ur.get(1)), zoom) 116 | ) 117 | 118 | # create ranges for server-side mapping 119 | tx_range: ee.List = ee.List.sequence(tmin.get(0), ee.Number(tmax.get(0)).add(1)) 120 | ty_range: ee.List = ee.List.sequence(tmax.get(1), ee.Number(tmin.get(1)).add(1)) 121 | 122 | def create_tile(tx: ee.Number, ty: ee.Number, zoom: ee.Number) -> ee.Feature: 123 | """ 124 | Add a tile to input tiles 125 | """ 126 | tx: ee.Number = ee.Number(tx) 127 | ty: ee.Number = ee.Number(ty) 128 | zoom: ee.Number = ee.Number(zoom) 129 | 130 | tile_bounds: ee.List = get_tile_bounds(tx, ty, zoom) 131 | rect: ee.Geometry = ee.Geometry.Rectangle(tile_bounds, PROJECTION, False) 132 | return ee.Feature(rect).set( 133 | {"tx": tx.format(), "ty": ty.format(), "zoom": zoom.format()} 134 | ) 135 | 136 | tiles: ee.List = ee.List( 137 | tx_range.map( 138 | lambda tx: ty_range.map(lambda ty: create_tile(tx=tx, ty=ty, zoom=zoom)) 139 | ) 140 | ).flatten() 141 | 142 | return ee.FeatureCollection(tiles).filterBounds(geometry) 143 | -------------------------------------------------------------------------------- /eepackages/utils.py: -------------------------------------------------------------------------------- 1 | from datetime import timedelta 2 | from itertools import repeat 3 | import math 4 | from pathlib import Path 5 | from time import sleep, time 6 | from typing import Any, Callable, Dict, Optional 7 | 8 | import ee 9 | from pathos.multiprocessing import ProcessPool 10 | 11 | from eepackages.multiprocessing.download_image import download_image 12 | 13 | # /*** 14 | # * The script computes surface water mask using Canny Edge detector and Otsu thresholding 15 | # * See the following paper for details: http://www.mdpi.com/2072-4292/8/5/386 16 | # * 17 | # * Author: Gennadii Donchyts (gennadiy.donchyts@gmail.com) 18 | # * Contributors: Nicholas Clinton (nclinton@google.com) - re-implemented otsu() using ee.Array 19 | # * 20 | # * Usage: 21 | # * 22 | # * var thresholding = require('users/gena/packages:thresholding') 23 | # * 24 | # * var th = thresholding.computeThresholdUsingOtsu(image, scale, bounds, cannyThreshold, cannySigma, minValue, ...) 25 | # * 26 | # */ 27 | 28 | # /*** 29 | # * Return the DN that maximizes interclass variance in B5 (in the region). 30 | # */ 31 | 32 | 33 | def otsu(histogram): 34 | histogram = ee.Dictionary(histogram) 35 | 36 | counts = ee.Array(histogram.get("histogram")) 37 | means = ee.Array(histogram.get("bucketMeans")) 38 | size = means.length().get([0]) 39 | total = counts.reduce(ee.Reducer.sum(), [0]).get([0]) 40 | sum = means.multiply(counts).reduce(ee.Reducer.sum(), [0]).get([0]) 41 | mean = sum.divide(total) 42 | 43 | indices = ee.List.sequence(1, size) 44 | 45 | # Compute between sum of squares, where each mean partitions the data. 46 | def f(i): 47 | aCounts = counts.slice(0, 0, i) 48 | aCount = aCounts.reduce(ee.Reducer.sum(), [0]).get([0]) 49 | aMeans = means.slice(0, 0, i) 50 | aMean = ( 51 | aMeans.multiply(aCounts) 52 | .reduce(ee.Reducer.sum(), [0]) 53 | .get([0]) 54 | .divide(aCount) 55 | ) 56 | bCount = total.subtract(aCount) 57 | bMean = sum.subtract(aCount.multiply(aMean)).divide(bCount) 58 | 59 | return aCount.multiply(aMean.subtract(mean).pow(2)).add( 60 | bCount.multiply(bMean.subtract(mean).pow(2)) 61 | ) 62 | 63 | bss = indices.map(f) 64 | 65 | # Return the mean value corresponding to the maximum BSS. 66 | return means.sort(bss).get([-1]) 67 | 68 | 69 | # /*** 70 | # * Compute a threshold using Otsu method (bimodal) 71 | # */ 72 | 73 | 74 | def computeThresholdUsingOtsu( 75 | image, 76 | scale, 77 | bounds, 78 | cannyThreshold, 79 | cannySigma, 80 | minValue, 81 | debug=False, 82 | minEdgeLength=None, 83 | minEdgeGradient=None, 84 | minEdgeValue=None, 85 | ): 86 | # clip image edges 87 | mask = ( 88 | image.mask() 89 | .gt(0) 90 | .clip(bounds) 91 | .focal_min(ee.Number(scale).multiply(3), "circle", "meters") 92 | ) 93 | 94 | # detect sharp changes 95 | edge = ee.Algorithms.CannyEdgeDetector(image, cannyThreshold, cannySigma) 96 | edge = edge.multiply(mask) 97 | 98 | if minEdgeLength: 99 | connected = edge.mask(edge).lt(cannyThreshold).connectedPixelCount(200, True) 100 | 101 | edgeLong = connected.gte(minEdgeLength) 102 | 103 | # if debug: 104 | # print('Edge length: ', ui.Chart.image.histogram(connected, bounds, scale, buckets)) 105 | # Map.addLayer(edge.mask(edge), {palette:['ff0000']}, 'edges (short)', false); 106 | 107 | edge = edgeLong 108 | 109 | # buffer around NDWI edges 110 | edgeBuffer = edge.focal_max(ee.Number(scale), "square", "meters") 111 | 112 | if minEdgeValue: 113 | edgeMin = image.reduceNeighborhood( 114 | ee.Reducer.min(), ee.Kernel.circle(ee.Number(scale), "meters") 115 | ) 116 | 117 | edgeBuffer = edgeBuffer.updateMask(edgeMin.gt(minEdgeValue)) 118 | 119 | # if debug: 120 | # Map.addLayer(edge.updateMask(edgeBuffer), {palette:['ff0000']}, 'edge min', false); 121 | 122 | if minEdgeGradient: 123 | edgeGradient = ( 124 | image.gradient() 125 | .abs() 126 | .reduce(ee.Reducer.max()) 127 | .updateMask(edgeBuffer.mask()) 128 | ) 129 | 130 | edgeGradientTh = ee.Number( 131 | edgeGradient.reduceRegion( 132 | ee.Reducer.percentile([minEdgeGradient]), bounds, scale 133 | ) 134 | .values() 135 | .get(0) 136 | ) 137 | 138 | # if debug: 139 | # print('Edge gradient threshold: ', edgeGradientTh) 140 | 141 | # Map.addLayer(edgeGradient.mask(edgeGradient), {palette:['ff0000']}, 'edge gradient', false); 142 | 143 | # print('Edge gradient: ', ui.Chart.image.histogram(edgeGradient, bounds, scale, buckets)) 144 | 145 | edgeBuffer = edgeBuffer.updateMask(edgeGradient.gt(edgeGradientTh)) 146 | 147 | edge = edge.updateMask(edgeBuffer) 148 | edgeBuffer = edge.focal_max(ee.Number(scale).multiply(1), "square", "meters") 149 | imageEdge = image.mask(edgeBuffer) 150 | 151 | # if debug: 152 | # Map.addLayer(imageEdge, {palette:['222200', 'ffff00']}, 'image edge buffer', false) 153 | 154 | # compute threshold using Otsu thresholding 155 | buckets = 100 156 | hist = ee.Dictionary( 157 | ee.Dictionary( 158 | imageEdge.reduceRegion(ee.Reducer.histogram(buckets), bounds, scale) 159 | ) 160 | .values() 161 | .get(0) 162 | ) 163 | 164 | threshold = ee.Algorithms.If(hist.contains("bucketMeans"), otsu(hist), minValue) 165 | threshold = ee.Number(threshold) 166 | 167 | if debug: 168 | # // experimental 169 | # // var jrc = ee.Image('JRC/GSW1_0/GlobalSurfaceWater').select('occurrence') 170 | # // var jrcTh = ee.Number(ee.Dictionary(jrc.updateMask(edge).reduceRegion(ee.Reducer.mode(), bounds, scale)).values().get(0)) 171 | # // var water = jrc.gt(jrcTh) 172 | # // Map.addLayer(jrc, {palette: ['000000', 'ffff00']}, 'JRC') 173 | # // print('JRC occurrence (edge)', ui.Chart.image.histogram(jrc.updateMask(edge), bounds, scale, buckets)) 174 | 175 | # Map.addLayer(edge.mask(edge), {palette:['ff0000']}, 'edges', true); 176 | 177 | print("Threshold: ", threshold) 178 | 179 | # print('Image values:', ui.Chart.image.histogram(image, bounds, scale, buckets)); 180 | # print('Image values (edge): ', ui.Chart.image.histogram(imageEdge, bounds, scale, buckets)); 181 | # Map.addLayer(mask.mask(mask), {palette:['000000']}, 'image mask', false); 182 | 183 | if minValue is not None: 184 | return threshold.max(minValue) 185 | else: 186 | return threshold 187 | 188 | 189 | def focalMin(image: ee.Image, radius: float): 190 | erosion: ee.Image = ( 191 | image.Not().fastDistanceTransform(radius).sqrt().lte(radius).Not() 192 | ) 193 | return erosion 194 | 195 | 196 | def focalMax(image: ee.Image, radius: float): 197 | dilation: ee.Image = image.fastDistanceTransform(radius).sqrt().lte(radius) 198 | return dilation 199 | 200 | 201 | def focalMaxWeight(image: ee.Image, radius: float): 202 | distance: ee.Image = image.fastDistanceTransform(radius).sqrt() 203 | dilation: ee.Image = distance.where(distance.gte(radius), radius) 204 | dilation = ee.Image(radius).subtract(dilation).divide(radius) 205 | return dilation 206 | 207 | 208 | def _batch_download_ic( 209 | ic: ee.ImageCollection, 210 | img_download_method: Callable, 211 | name_prefix: str, 212 | out_dir: Optional[Path], 213 | pool_size: int, 214 | download_kwargs: Optional[Dict[str, Any]], 215 | ): 216 | """ 217 | does the actual work batch downloading images in an ee.ImageCollection using the python 218 | multiprocessing module. Takes the img download method as a callable to implement different 219 | ee.Image methods. 220 | """ 221 | 222 | out_dir.mkdir(exist_ok=True, parents=True) 223 | log_dir: Path = out_dir / "logging" 224 | log_dir.mkdir(exist_ok=True, parents=True) 225 | 226 | num_images: int = ic.size().getInfo() 227 | image_list: ee.List = ic.toList(num_images) 228 | 229 | serialized_il: str = image_list.serialize() 230 | 231 | pool: ProcessPool = ProcessPool(nodes=pool_size) 232 | result = pool.amap( 233 | download_image, 234 | repeat(ee), 235 | range(num_images), 236 | repeat(img_download_method), 237 | repeat(serialized_il), 238 | repeat(name_prefix), 239 | repeat(out_dir), 240 | repeat(download_kwargs), 241 | ) 242 | 243 | start_time: time = time() 244 | elapsed: timedelta = timedelta(0) 245 | result.get() # needed to trigger execution? 246 | 247 | while not result.ready() and elapsed.total_seconds() < 3600: # 1h timeout 248 | sleep(5) 249 | elapsed = timedelta(seconds=time() - start_time) 250 | 251 | pool.close() 252 | pool.join() 253 | pool.clear() 254 | 255 | 256 | def download_image_collection( 257 | ic: ee.ImageCollection, 258 | name_prefix: str = "ic_", 259 | out_dir: Optional[Path] = None, 260 | pool_size: int = 25, 261 | download_kwargs: Optional[Dict[str, Any]] = None, 262 | ) -> None: 263 | """ 264 | Download images in image collection. Only works for images in the collection that are < 32M 265 | and grid dimension < 10000, documented at 266 | https://developers.google.com/earth-engine/apidocs/ee-image-getdownloadurl. 267 | 268 | args: 269 | ic (ee.ImageCollection): ImageCollection to download. 270 | name_prefix (str): prefix for the filename of the downloaded objects. 271 | out_dir (Optional(Path)): pathlib object referring to output dir. 272 | pool_size (int): multiprocessing pool size. 273 | download_kwargs (Optional(Dict(str, Any))): keyword arguments used in 274 | [getDownloadUrl](https://developers.google.com/earth-engine/apidocs/ee-image-getdownloadurl). 275 | """ 276 | _batch_download_ic( 277 | ic, ee.Image.getDownloadURL, name_prefix, out_dir, pool_size, download_kwargs 278 | ) 279 | 280 | 281 | def download_image_collection_thumb( 282 | ic: ee.ImageCollection, 283 | name_prefix: str = "ic_", 284 | out_dir: Optional[Path] = None, 285 | pool_size: int = 25, 286 | download_kwargs: Optional[Dict[str, Any]] = None, 287 | ) -> None: 288 | """ 289 | Download thumb images in and image collection. Only works for images in the collection that are < 32M 290 | and grid dimension < 10000, documented at 291 | https://developers.google.com/earth-engine/apidocs/ee-image-getthumburl. 292 | 293 | args: 294 | ic (ee.ImageCollection): ImageCollection to download. 295 | name_prefix (str): prefix for the filename of the downloaded objects. 296 | out_dir (Optional(Path)): pathlib object referring to output dir. 297 | pool_size (int): multiprocessing pool size. 298 | download_kwargs (Optional(Dict(str, Any))): keyword arguments used in 299 | [getDownloadUrl](https://developers.google.com/earth-engine/apidocs/ee-image-getthumburl). 300 | """ 301 | _batch_download_ic( 302 | ic, ee.Image.getThumbURL, name_prefix, out_dir, pool_size, download_kwargs 303 | ) 304 | 305 | 306 | def radians(img): 307 | """Converts image from degrees to radians""" 308 | return img.toFloat().multiply(math.pi).divide(180) 309 | 310 | 311 | def hillshade(az, ze, slope, aspect): 312 | """Computes hillshade""" 313 | azimuth = radians(ee.Image.constant(az)) 314 | zenith = radians(ee.Image.constant(90).subtract(ee.Image.constant(ze))) 315 | 316 | return ( 317 | azimuth.subtract(aspect) 318 | .cos() 319 | .multiply(slope.sin()) 320 | .multiply(zenith.sin()) 321 | .add(zenith.cos().multiply(slope.cos())) 322 | ) 323 | 324 | 325 | def hillshadeRGB( 326 | image, 327 | elevation, 328 | weight=1, 329 | height_multiplier=5, 330 | azimuth=0, 331 | zenith=45, 332 | contrast=0, 333 | brightness=0, 334 | saturation=1, 335 | castShadows=False, 336 | customTerrain=False, 337 | ): 338 | """Styles RGB image using hillshading, mixes RGB and hillshade using HSV<->RGB transform""" 339 | 340 | hsv = image.visualize().unitScale(0, 255).rgbToHsv() 341 | 342 | z = elevation.multiply(ee.Image.constant(height_multiplier)) 343 | 344 | terrain = ee.Algorithms.Terrain(z) 345 | slope = radians(terrain.select(["slope"])).resample("bicubic") 346 | aspect = radians(terrain.select(["aspect"])).resample("bicubic") 347 | 348 | if customTerrain: 349 | raise NotImplementedError("customTerrain argument is not implemented yet") 350 | 351 | hs = hillshade(azimuth, zenith, slope, aspect).resample("bicubic") 352 | 353 | if castShadows: 354 | hysteresis = True 355 | neighborhoodSize = 256 356 | 357 | hillShadow = ee.Algorithms.HillShadow( 358 | elevation, 359 | azimuth, 360 | ee.Number(90).subtract(zenith), 361 | neighborhoodSize, 362 | hysteresis, 363 | ).float() 364 | 365 | hillShadow = ee.Image(1).float().subtract(hillShadow) 366 | 367 | # opening 368 | # hillShadow = hillShadow.multiply(hillShadow.focal_min(3).focal_max(6)) 369 | 370 | # cleaning 371 | hillShadow = hillShadow.focal_mode(3) 372 | 373 | # smoothing 374 | hillShadow = hillShadow.convolve(ee.Kernel.gaussian(5, 3)) 375 | 376 | # transparent 377 | hillShadow = hillShadow.multiply(0.7) 378 | 379 | hs = hs.subtract(hillShadow).rename("shadow") 380 | 381 | intensity = hs.multiply(ee.Image.constant(weight)).add( 382 | hsv.select("value").multiply(ee.Image.constant(1).subtract(weight)) 383 | ) 384 | 385 | sat = hsv.select("saturation").multiply(saturation) 386 | 387 | hue = hsv.select("hue") 388 | 389 | result = ( 390 | ee.Image.cat(hue, sat, intensity) 391 | .hsvToRgb() 392 | .multiply(ee.Image.constant(1).float().add(contrast)) 393 | .add(ee.Image.constant(brightness).float()) 394 | ) 395 | 396 | if customTerrain: 397 | mask = elevation.mask().focal_min(2) 398 | 399 | result = result.updateMask(mask) 400 | 401 | return result 402 | 403 | 404 | def getIsolines(image, levels=None): 405 | """Adds isolines to an ee.Image""" 406 | 407 | def addIso(image, level): 408 | crossing = image.subtract(level).focal_median(3).zeroCrossing() 409 | exact = image.eq(level) 410 | return ee.Image(level).float().mask(crossing.Or(exact)).set({"level": level}) 411 | 412 | if not levels: 413 | levels = ee.List.sequence(0, 1, 0.1) 414 | 415 | levels = ee.List(levels) 416 | 417 | isoImages = ee.ImageCollection(levels.map(lambda l: addIso(image, ee.Number(l)))) 418 | 419 | return isoImages 420 | -------------------------------------------------------------------------------- /notebooks/assets_cloud_band.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import ee\n", 10 | "from eepackages import assets\n", 11 | "from IPython.display import Image" 12 | ] 13 | }, 14 | { 15 | "cell_type": "code", 16 | "execution_count": 2, 17 | "metadata": {}, 18 | "outputs": [], 19 | "source": [ 20 | "ee.Initialize()" 21 | ] 22 | }, 23 | { 24 | "cell_type": "code", 25 | "execution_count": 3, 26 | "metadata": {}, 27 | "outputs": [], 28 | "source": [ 29 | "bounds = ee.Geometry.Polygon(\n", 30 | " [[[-39.20961463729369, -5.266625790689956],\n", 31 | " [-39.20961463729369, -5.450526132170581],\n", 32 | " [-38.90474403182494, -5.450526132170581],\n", 33 | " [-38.90474403182494, -5.266625790689956]]], None, False)" 34 | ] 35 | }, 36 | { 37 | "cell_type": "code", 38 | "execution_count": 4, 39 | "metadata": {}, 40 | "outputs": [], 41 | "source": [ 42 | "images = (assets.getImages(bounds, {\n", 43 | " 'missions': ['L8'],\n", 44 | " 'cloudMask': True,\n", 45 | " 'filter': ee.Filter.date('2020-01-01', '2022-01-01')\n", 46 | "}))\n", 47 | "\n", 48 | "image = images.first()" 49 | ] 50 | }, 51 | { 52 | "cell_type": "code", 53 | "execution_count": 6, 54 | "metadata": {}, 55 | "outputs": [], 56 | "source": [ 57 | "rgb = image.visualize().blend(image.select('cloud').selfMask().visualize(**{'palette': ['ffff00']}))\n", 58 | "\n", 59 | "url = rgb.getThumbUrl({'dimensions': [800, 800]})\n", 60 | "Image(url=url, embed=True, format='png')" 61 | ] 62 | } 63 | ], 64 | "metadata": { 65 | "interpreter": { 66 | "hash": "01d30bbae3913ec8e7030c6a5e0e9ea4f7d540fbe620af3bde4b389e5cb4c8cf" 67 | }, 68 | "kernelspec": { 69 | "display_name": "Python 3.7.6 ('work')", 70 | "language": "python", 71 | "name": "python3" 72 | }, 73 | "language_info": { 74 | "codemirror_mode": { 75 | "name": "ipython", 76 | "version": 3 77 | }, 78 | "file_extension": ".py", 79 | "mimetype": "text/x-python", 80 | "name": "python", 81 | "nbconvert_exporter": "python", 82 | "pygments_lexer": "ipython3", 83 | "version": "3.7.6" 84 | }, 85 | "orig_nbformat": 4 86 | }, 87 | "nbformat": 4, 88 | "nbformat_minor": 2 89 | } 90 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | earthengine-api>=0.1.282 2 | pathos>=0.2.8 3 | retry>=0.9.2 4 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import io 2 | import os 3 | import re 4 | 5 | from setuptools import find_packages 6 | from setuptools import setup 7 | 8 | __version__ = "0.28.0" 9 | 10 | 11 | def read(filename): 12 | filename = os.path.join(os.path.dirname(__file__), filename) 13 | text_type = type("") 14 | with io.open(filename, mode="r", encoding="utf-8") as fd: 15 | return re.sub(text_type(r":[a-z]+:`~?(.*?)`"), text_type(r"``\1``"), fd.read()) 16 | 17 | 18 | setup( 19 | name="eepackages", 20 | version=__version__, 21 | url="https://github.com/gee-community/ee-packages-py", 22 | license="MIT", 23 | author="Gennadii Donchyts", 24 | author_email="gennadiy.donchyts@gmail.com", 25 | description="A set of utilities built on top of Google Earth Engine (migrated from JavaScript)", 26 | long_description=read("README.md"), 27 | packages=find_packages(exclude=("tests",)), 28 | install_requires=["earthengine-api>=0.1.284", "pathos>=0.2.8", "retry>=0.9.2"], 29 | classifiers=[ 30 | "Development Status :: 2 - Pre-Alpha", 31 | "License :: OSI Approved :: MIT License", 32 | "Programming Language :: Python", 33 | "Programming Language :: Python :: 2", 34 | "Programming Language :: Python :: 2.7", 35 | "Programming Language :: Python :: 3", 36 | "Programming Language :: Python :: 3.4", 37 | "Programming Language :: Python :: 3.5", 38 | "Programming Language :: Python :: 3.6", 39 | "Programming Language :: Python :: 3.7", 40 | ], 41 | ) 42 | -------------------------------------------------------------------------------- /tests/test_sample.py: -------------------------------------------------------------------------------- 1 | # Sample Test passing with nose and pytest 2 | 3 | 4 | def test_pass(): 5 | assert True, "dummy sample test" 6 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py27,py34,py35,py36,py37 3 | 4 | [testenv] 5 | commands = py.test eepackages 6 | deps = pytest 7 | --------------------------------------------------------------------------------