├── .github └── workflows │ ├── docs.yml │ └── pypi-publish.yml ├── .gitignore ├── LICENSE ├── README.md ├── docs ├── core.md ├── images.md ├── img │ ├── domain_example.png │ ├── imgcollection_example.png │ ├── modis_ndvi_example.png │ └── table_example.png ├── index.md ├── installation.md ├── tables.md ├── usage.md └── values.md ├── mkdocs.yml ├── restee ├── __init__.py ├── core.py ├── images.py ├── tables.py └── values.py └── setup.py /.github/workflows/docs.yml: -------------------------------------------------------------------------------- 1 | name: docs 2 | on: 3 | push: 4 | branches: 5 | - main 6 | jobs: 7 | deploy: 8 | runs-on: ubuntu-latest 9 | steps: 10 | - uses: actions/checkout@v2 11 | - uses: actions/setup-python@v2 12 | with: 13 | python-version: 3.6 14 | - run: pip install mkdocs-material mkdocstrings restee 15 | - run: mkdocs gh-deploy --force -------------------------------------------------------------------------------- /.github/workflows/pypi-publish.yml: -------------------------------------------------------------------------------- 1 | # This workflows will upload a Python Package using Twine when a release is created 2 | # For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries 3 | 4 | name: Upload Python Package 5 | 6 | on: 7 | release: 8 | types: [created] 9 | 10 | jobs: 11 | deploy: 12 | 13 | runs-on: ubuntu-latest 14 | 15 | steps: 16 | - uses: actions/checkout@v2 17 | - name: Set up Python 18 | uses: actions/setup-python@v2 19 | with: 20 | python-version: '3.x' 21 | - name: Install dependencies 22 | run: | 23 | python -m pip install --upgrade pip 24 | pip install setuptools wheel twine 25 | - name: Build and publish 26 | env: 27 | TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} 28 | TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} 29 | run: | 30 | python setup.py sdist bdist_wheel 31 | twine upload dist/* -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Kel Markert 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # restee 2 | 3 | 🚨 This repository is no longer under development or maintined, therefor it is archived. The [google/Xee](https://github.com/google/Xee) repository serves a similar purpose and is actively maintained, please use that package instead. 🚨 4 | 5 | Python package to call process EE objects via the REST API to local data 6 | 7 | [![PyPI version](https://badge.fury.io/py/restee.svg)](https://badge.fury.io/py/restee) 8 | [![docs](https://github.com/KMarkert/restee/actions/workflows/docs.yml/badge.svg)](https://github.com/KMarkert/restee/actions/workflows/docs.yml) 9 | [![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](https://opensource.org/licenses/MIT) 10 | 11 | 12 | `restee` is a package that aims to make plugging Earth Engine (EE) computations into downstream Python processing easier. The EE REST API allows user to interface with EE using REST API calls that allow for . There are many more features to the EE REST API, however, `restee` aims to simply provide a user-friendly means to access computed server-side objects (like image data) from the [Python `earthengine-api`](https://developers.google.com/earth-engine/guides/python_install) API to a local Python enviroment (client-side). 13 | 14 | It should be noted that `restee` relies on fairly new and advanced EE features that may not be suitable for all users (see [warning from the EE team](https://developers.google.com/earth-engine/reference#audience)). If you are new to Earth Engine, please get started with the [JavaScript guide](https://developers.google.com/earth-engine/getstarted). 15 | 16 | ## Installation 17 | 18 | `restee` relies heavily on the geospatial Python ecosystem to manage different geospatial data formats and execute geospatial processes. It is recommended to use [`conda`](https://docs.anaconda.com/anaconda/install/) to handle the package dependencies and create a virtual environment. To do this run the following command: 19 | 20 | ```sh 21 | conda create -n restee -c conda-forge -y \ 22 | python>=3.6 \ 23 | numpy \ 24 | scipy \ 25 | pandas \ 26 | xarray \ 27 | rasterio \ 28 | geopandas \ 29 | pyproj \ 30 | requests \ 31 | backoff \ 32 | earthengine-api \ 33 | tqdm 34 | ``` 35 | 36 | Once all of the dependencies are installed, the `restee` package can be installed using `pip`: 37 | 38 | ```sh 39 | pip install restee 40 | ``` 41 | 42 | It is strongly recommended to read the [Installation documentation]() 43 | 44 | ## Getting Started 45 | 46 | This section is meant purely as a demonstration of what is possible, please see the [Installation page](/installation) for how to install package and setup the authentication then the [Usage page](/usage) for in depth information. 47 | 48 | ```python 49 | import ee 50 | ee.Initialize() 51 | 52 | import restee as ree 53 | 54 | # get an authenticated session with GCP for REST API calls 55 | session = ree.EESession("","") 56 | 57 | # use ee to get a featurecollection for USA 58 | countries = ee.FeatureCollection("USDOS/LSIB_SIMPLE/2017") 59 | camerica= countries.filter(ee.Filter.eq("wld_rgn", "Central America")) 60 | 61 | # define the domain imagery will be requested for 62 | # in this case it is the computed USA featurecollection 63 | domain = ree.Domain.from_ee_geometry(session,camerica,resolution=0.01) 64 | 65 | # define some computations 66 | # here we calculate median NDVI for the summer months over the USA 67 | modis = ( 68 | ee.ImageCollection("MODIS/006/MOD09GA") 69 | .filterDate("2020-06-01","2020-09-01") 70 | .map(lambda x: x.normalizedDifference(["sur_refl_b02","sur_refl_b01"])) 71 | .median() 72 | .rename("NDVI") 73 | ) 74 | 75 | # request the ee.Image pixels as a xarray dataset 76 | ndvi_ds = ree.img_to_xarray(session,domain,modis,no_data_value=0) 77 | 78 | # inspect the local xarray Dataset object 79 | ndvi_ds 80 | 81 | # output 82 | # 83 | # Dimensions: (lat: 1130, lon: 1509) 84 | # Coordinates: 85 | # * lon (lon) float64 -92.23 -92.22 -92.21 -92.2 ... -77.17 -77.16 -77.15 86 | # * lat (lat) float64 18.48 18.47 18.46 18.45 ... 7.225 7.215 7.205 7.195 87 | # Data variables: 88 | # NDVI (lat, lon) float32 nan nan nan nan nan nan ... nan nan nan nan nan 89 | ``` 90 | 91 | From this point on the computed data is local to your system so you can do with it what you want. This allows the data to be plotted, persisted, or fed into another downstream process. For the sake of example, here we will plot the result. 92 | 93 | ```python 94 | ndvi_ds.NDVI.plot(robust=True,cmap="viridis") 95 | ``` 96 | 97 | ![MODIS Summer NDVI](docs/img/modis_ndvi_example.png) 98 | 99 | Again, this quick example was to highlight how a user may define an EE computation using the `earthengine-api` and request the data into a local data structure. One may use `restee` to get zonal statitics calculated for feature collections or even explore collection metadata, any format on EE can be requested locally. For more details, please see the [Usage page](/usage). 100 | 101 | ## Get in touch 102 | 103 | Please report any bugs, ask questions, or suggest new features on [GitHub](https://github.com/KMarkert/restee/issues). 104 | 105 | ## Contribute 106 | 107 | Contributions are welcome, and they are greatly appreciated! Every little bit helps, and credit will always be given. 108 | 109 | ## License 110 | 111 | `restee` is available under the open source [MIT License](https://github.com/KMarkert/restee/blob/main/LICENSE). 112 | -------------------------------------------------------------------------------- /docs/core.md: -------------------------------------------------------------------------------- 1 | ::: restee.core 2 | rendering: 3 | show_root_heading: true 4 | show_source: true -------------------------------------------------------------------------------- /docs/images.md: -------------------------------------------------------------------------------- 1 | ::: restee.images 2 | rendering: 3 | show_root_heading: true 4 | show_source: true -------------------------------------------------------------------------------- /docs/img/domain_example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KMarkert/restee/60d1291bf8be6d3df7eea99eee886d1855956227/docs/img/domain_example.png -------------------------------------------------------------------------------- /docs/img/imgcollection_example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KMarkert/restee/60d1291bf8be6d3df7eea99eee886d1855956227/docs/img/imgcollection_example.png -------------------------------------------------------------------------------- /docs/img/modis_ndvi_example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KMarkert/restee/60d1291bf8be6d3df7eea99eee886d1855956227/docs/img/modis_ndvi_example.png -------------------------------------------------------------------------------- /docs/img/table_example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KMarkert/restee/60d1291bf8be6d3df7eea99eee886d1855956227/docs/img/table_example.png -------------------------------------------------------------------------------- /docs/index.md: -------------------------------------------------------------------------------- 1 | # Welcome to the restee documentation 2 | 3 | `restee` is a package that aims to make plugging Earth Engine (EE) computations into downstream Python processing easier. The EE REST API allows user to interface with EE using REST API calls which allows for more flexibility in working with EE, however, without detailed knowleged of the API use it can be somewhat cryptic. `restee` aims to provide a user-friendly means to access computed server-side objects (like image data) from the [`earthengine-api`](https://developers.google.com/earth-engine/guides/python_install) API to a local Python enviroment (client-side). 4 | 5 | It should be noted that `restee` relies on fairly new and advanced EE features that may not be suitable for all users (see [warning from the EE team](https://developers.google.com/earth-engine/reference#audience)). If you are new to Earth Engine, please get started with the [JavaScript guide](https://developers.google.com/earth-engine/getstarted). 6 | 7 | ## Getting Started 8 | 9 | This section is meant purely as a demonstration of what is possible, please see the [Installation page](/restee/installation) for how to install package and setup the authentication then the [Usage page](/restee/usage) for in depth information. 10 | 11 | In this case, we will compute a median NDVI composite for the Summer months over Central America using MODIS. Then we will request the data locally as a xarray.Dataset object so we can then use for local processing. 12 | 13 | ```python 14 | import ee 15 | ee.Initialize() 16 | 17 | import restee as ree 18 | 19 | # get an authenticated session with GCP for REST API calls 20 | session = ree.EESession("","") 21 | 22 | # use ee to get a featurecollection for USA 23 | countries = ee.FeatureCollection("USDOS/LSIB_SIMPLE/2017") 24 | camerica= countries.filter(ee.Filter.eq("wld_rgn", "Central America")) 25 | 26 | # define the domain imagery will be requested for at ~1km resolution 27 | # in this case it is the computed Central America FeatureCollection 28 | domain = ree.Domain.from_ee_geometry(session,camerica,resolution=0.01) 29 | 30 | # define some image computations 31 | # here we calculate median NDVI for the summer months 32 | modis = ( 33 | ee.ImageCollection("MODIS/006/MOD09GA") 34 | .filterDate("2020-06-01","2020-09-01") 35 | .map(lambda x: x.normalizedDifference(["sur_refl_b02","sur_refl_b01"])) 36 | .median() 37 | .rename("NDVI") 38 | ) 39 | 40 | # request the ee.Image pixels as a xarray dataset for the domain 41 | ndvi_ds = ree.img_to_xarray(session,domain,modis,no_data_value=0) 42 | 43 | # inspect the local xarray Dataset object 44 | ndvi_ds 45 | 46 | # output 47 | # 48 | # Dimensions: (lat: 1130, lon: 1509) 49 | # Coordinates: 50 | # * lon (lon) float64 -92.23 -92.22 -92.21 -92.2 ... -77.17 -77.16 -77.15 51 | # * lat (lat) float64 18.48 18.47 18.46 18.45 ... 7.225 7.215 7.205 7.195 52 | # Data variables: 53 | # NDVI (lat, lon) float32 nan nan nan nan nan nan ... nan nan nan nan nan 54 | ``` 55 | 56 | From this point on the computed data is local to your system so you can do with it what you want. This allows the data to be plotted, persisted, or fed into another downstream process. For the sake of example, here we will plot the result. 57 | 58 | ```python 59 | ndvi_ds.NDVI.plot(robust=True,cmap="viridis") 60 | ``` 61 | 62 | ![MODIS Summer NDVI](img/modis_ndvi_example.png) 63 | 64 | Again, this quick example was to highlight how a user may define an EE computation using the `earthengine-api` and request the data into a local data structure. One may use `restee` to get zonal statitics calculated for feature collections or even explore collection metadata, any format on EE can be requested locally. For more details, please see the [Usage page](/restee/usage). 65 | 66 | ## Get in touch 67 | 68 | Please report any bugs, ask questions, or suggest new features on [GitHub](https://github.com/KMarkert/restee/issues). 69 | 70 | ## Contribute 71 | 72 | Contributions are welcome, and they are greatly appreciated! Every little bit helps, and credit will always be given. 73 | 74 | ## License 75 | 76 | `restee` is available under the open source [MIT License](https://github.com/KMarkert/restee/blob/main/LICENSE). 77 | -------------------------------------------------------------------------------- /docs/installation.md: -------------------------------------------------------------------------------- 1 | # restee Installation 2 | 3 | This pages walks you through setting up the `restee` package as well as points users to the documentation for setting up a GCP project for use with the EE REST API. 4 | 5 | ## Before beginning 6 | 7 | Before proceeding with the package installation, please see the instructions to setup a service account for access to the REST API. Instructions for setting up your service account can be found [here](https://developers.google.com/earth-engine/reference/Quickstart#before-you-begin) 8 | 9 | Once you have a Google Cloud Project and whitelisted service accout for the cloud project, you will need to create a private key so that your system can securely communicate with the Google Cloud. Instructions for creating a private key can be found [here](https://developers.google.com/earth-engine/reference/Quickstart#obtain-a-private-key-file-for-your-service-account) 10 | 11 | Lastly, test the setup by following the instructions [here](https://developers.google.com/earth-engine/reference/Quickstart#accessing-and-testing-your-credentials) 12 | 13 | ## Installing the package 14 | 15 | `restee` relies heavily on the geospatial Python ecosystem to manage different geospatial data formats and execute geospatial processes. It is recommended to use [`conda`](https://docs.anaconda.com/anaconda/install/) to handle the package dependencies and create a virtual environment to work with `restee`. To do this run the following command: 16 | 17 | ```sh 18 | conda create -n restee -c conda-forge -y \ 19 | python>=3.6 \ 20 | numpy \ 21 | scipy \ 22 | pandas \ 23 | xarray \ 24 | rasterio \ 25 | geopandas \ 26 | pyproj \ 27 | requests \ 28 | backoff \ 29 | earthengine-api \ 30 | tqdm 31 | ``` 32 | 33 | Once all of the dependencies are installed, the `restee` package can be installed using `pip`: 34 | 35 | ```sh 36 | pip install restee 37 | ``` 38 | -------------------------------------------------------------------------------- /docs/tables.md: -------------------------------------------------------------------------------- 1 | ::: restee.tables 2 | rendering: 3 | show_root_heading: true 4 | show_source: true -------------------------------------------------------------------------------- /docs/usage.md: -------------------------------------------------------------------------------- 1 | # Using restee 2 | 3 | The page serves as more in depth examples of using `restee` to get data from Earth Engine results. Before walking through these examples, please make sure you have gone throught the [Installation page](/restee/installation) and installed everything correctly. 4 | 5 | To begin, we will need to import the `ee` and `restee` packages. Next, we need to authenticate Earth Engine and create a authenticated cloud session so we can request the data from the server to our local system. 6 | 7 | ```python 8 | import ee 9 | ee.Initialize() 10 | 11 | import restee as ree 12 | 13 | # get and authenticated cloud session for requesting data 14 | session = ree.EESession("","") 15 | ``` 16 | 17 | Now we are ready to crunch some data! 18 | 19 | ## Requesting Images 20 | 21 | Earth Engines core capabilities come from massive geospatial raster processing. It makes working with and processing imagery much easier but often times, one would like to use the results with some other process. `restee` allows for users to request processed imagery to local data structures such as 22 | 23 | ```python 24 | states = ee.FeatureCollection('TIGER/2018/States') 25 | maine = states.filter(ee.Filter.eq('NAME', 'Maine')) 26 | 27 | # get a domain for the state of Maine at ~500m resolution 28 | domain = ree.Domain.from_ee_geometry(session, maine, resolution=0.005) 29 | 30 | imshow(domain.mask) 31 | ``` 32 | 33 | ![Maine domain](img/domain_example.png) 34 | 35 | Now that we have a domain we can use that to control where on the globe we request imagery. Here we simply grab the first image in the MOD13Q1 NDVI collection and request the data for Maine. 36 | 37 | ```python 38 | # get an ee.Image object 39 | img = ( 40 | ee.ImageCollection('MODIS/006/MOD13Q1') 41 | .select("NDVI") 42 | .first() 43 | ) 44 | 45 | # request the image as an np.ndarray using the domain 46 | ndvi_arr = ree.img_to_ndarray(session,domain,img) 47 | 48 | # inspect the data structure of ndvi_arr 49 | ndvi_arr 50 | 51 | # array([[( 3905,), ( 3880,), ( 2823,), ..., ( 2522,), ( 1714,), ( 1714,)], 52 | # [( 3605,), ( 3447,), ( 3447,), ..., ( 3845,), ( 2324,), ( 2324,)], 53 | # [( 3399,), ( 3315,), ( 3315,), ..., ( 4584,), ( 2238,), ( 2238,)], 54 | # ..., 55 | # [( 5711,), ( 5611,), ( 4905,), ..., (-3000,), (-3000,), (-3000,)], 56 | # [( 4845,), ( 4791,), ( 4606,), ..., (-3000,), (-3000,), (-3000,)], 57 | # [( 5101,), ( 4443,), ( 4610,), ..., (-3000,), (-3000,), (-3000,)]], 58 | # dtype=[('NDVI', '=0).NDVI.plot(col='time',col_wrap=5,robust=True,cmap='Greens') 77 | ``` 78 | 79 | ![MODIS NDVI Timeseries](img/imgcollection_example.png) 80 | 81 | This function is handy as it allows for geographic as well as time information to be retained. 82 | 83 | It is planned to allow users to define an ImageCollection by other properties than `system:time_start`, however, since ImageCollections are most often time based this functionality should serve most purposes. 84 | 85 | ## Requesting Tables 86 | 87 | Imagery is not the only thing that Earth Engine can process. There are many vector based workflows that are useful on Earth Engine such as sampling values from an image or calculating zonal statistics. To access computed a computed feature or feature collection, `restee` has methods to request the data in a local table structure. 88 | 89 | In this example, we will calculate the average NDVI from a MODIS image for the countries in Southeast Asia: 90 | 91 | ```python 92 | countries = ee.FeatureCollection("USDOS/LSIB_SIMPLE/2017") 93 | seasia = countries.filter(ee.Filter.eq("wld_rgn", "SE Asia")) 94 | 95 | computation = img.reduceRegions( 96 | collection=seasia, 97 | reducer=ee.Reducer.mean().setOutputs(["NDVI"]), 98 | scale=1000 99 | ) 100 | 101 | gdf = ree.features_to_geodf(session,computation) 102 | 103 | gdf.plot(column="NDVI",cmap="Greens") 104 | ``` 105 | 106 | ![SE Asia NDVI](img/table_example.png) 107 | 108 | Not all computed ee.Feature or ee.FeatureCollections have geometry information, to support the lack of geometry data, `restee` also has a function [`restee.features_to_df`](/restee/tables/#restee.features_to_df) to request the data as a `pandas.DataFrame`. 109 | 110 | ## Requesting Values 111 | 112 | Not all information from Earth Engine is a spatial computation. Sometimes calculating statistics or reading metadata from the image/collections is needed. We can request essentially any ee.ComputedObject to a local Python data type and continue using on our local system. 113 | 114 | Here is a quick example of getting a list of dates from an image collection: 115 | 116 | ``` 117 | ic = ( 118 | ee.ImageCollection('MODIS/006/MOD13Q1') 119 | .limit(10,"system:time_start") 120 | .select("NDVI") 121 | ) 122 | 123 | # compute the string format of the image dates 124 | ee_dates = ( 125 | ic.aggregate_array("system:time_start") 126 | .map(lambda x: ee.Date(x).format("YYYY-MM-dd HH:mm:ss")) 127 | ) 128 | 129 | # get this list of date strings 130 | dates = ree.get_value(session, ee_dates) 131 | 132 | dates 133 | # output 134 | # ['2000-02-18 00:00:00', 135 | # '2000-03-05 00:00:00', 136 | # '2000-03-21 00:00:00', 137 | # '2000-04-06 00:00:00', 138 | # '2000-04-22 00:00:00', 139 | # '2000-05-08 00:00:00', 140 | # '2000-05-24 00:00:00', 141 | # '2000-06-09 00:00:00', 142 | # '2000-06-25 00:00:00', 143 | # '2000-07-11 00:00:00'] 144 | ``` 145 | 146 | The `restee.get_value` function takes essentially any ee.ComputedObject (such as ee.List, ee.Array, ee.Dictionary, ee.String)and will convert it to the Python equivalent for use locally. 147 | 148 | ## Caveauts 149 | 150 | There is a limit to how much data/computations you can request. Typically this manifests itself in 500 error. Please note that this is an Earth Image limit and has little to do with `restee`. 151 | 152 | Please refer to the [Quotas and Limits section](https://developers.google.com/earth-engine/reference#quota-and-limits) of the REST API documentation to learn more. 153 | -------------------------------------------------------------------------------- /docs/values.md: -------------------------------------------------------------------------------- 1 | ::: restee.values 2 | rendering: 3 | show_root_heading: true 4 | show_source: true -------------------------------------------------------------------------------- /mkdocs.yml: -------------------------------------------------------------------------------- 1 | site_name: restee Documentation 2 | 3 | nav: 4 | - Overview: index.md 5 | - Installation: installation.md 6 | - Usage: usage.md 7 | - API Reference: 8 | - core module: core.md 9 | - images module: images.md 10 | - tables module: tables.md 11 | - values module: values.md 12 | 13 | theme: 14 | name: material 15 | 16 | # markdown_extensions: 17 | # - markdown.extensions.codehilite: 18 | # guess_lang: false 19 | plugins: 20 | - search 21 | - mkdocstrings -------------------------------------------------------------------------------- /restee/__init__.py: -------------------------------------------------------------------------------- 1 | from restee.core import * 2 | from restee.images import * 3 | from restee.values import * 4 | from restee.tables import * 5 | 6 | __version__ = "0.0.3" 7 | -------------------------------------------------------------------------------- /restee/core.py: -------------------------------------------------------------------------------- 1 | import ee 2 | import copy 3 | import json 4 | import backoff 5 | import requests 6 | import numpy as np 7 | from io import StringIO 8 | import geopandas as gpd 9 | from affine import Affine 10 | from rasterio import features 11 | from pyproj import Transformer 12 | from collections.abc import Iterable 13 | from scipy import interpolate, ndimage 14 | 15 | from google.auth.transport.requests import AuthorizedSession 16 | from google.oauth2 import service_account 17 | 18 | 19 | class Domain: 20 | """Domain class to define spatial region for image requests 21 | 22 | example: 23 | Defines a psuedo-global domain at 1 degree resolution 24 | >>> coords = [-180,-60,180,85] 25 | >>> domain = restee.Domain(coords,resolution=1) 26 | >>> domain.shape 27 | (145, 360) 28 | """ 29 | 30 | def __init__( 31 | self, bbox: Iterable, resolution: float = 0.25, crs: str = "EPSG:4326" 32 | ): 33 | """Initialize Domain class 34 | 35 | args: 36 | bbox (Iterable): bounding box to create domain as [W,S,E,N] 37 | resolution (float): resolution to make domain. default = 0.25 38 | crs (str): string name of coordinate reference system of domain. 39 | resolution units must match the crs units. default = "EPSG:4326" 40 | """ 41 | # set crs 42 | self._crs = crs 43 | 44 | # set resolution info 45 | self.resolution = resolution 46 | 47 | # set the bounding box 48 | if np.any((np.array(bbox) % self.resolution)) != 0: 49 | self.bbox = Domain._round_out(bbox, self.resolution) 50 | else: 51 | self.bbox = bbox 52 | 53 | minx, miny, maxx, maxy = self.bbox 54 | y_coords = np.arange(miny+self.resolution, maxy+self.resolution, self.resolution)[::-1] 55 | x_coords = np.arange(minx, maxx, self.resolution) 56 | 57 | self.x_size = x_coords.size 58 | self.y_size = y_coords.size 59 | 60 | self.x_coords = x_coords 61 | self.y_coords = y_coords 62 | 63 | self._construct_grid() 64 | 65 | self._mask = np.ones(self.shape) 66 | 67 | return 68 | 69 | def _construct_grid(self): 70 | """Helper function to create a grid of coordinates and geotransform""" 71 | 72 | self.xx, self.yy = np.meshgrid(self.x_coords, self.y_coords) 73 | 74 | self.shape = self.xx.shape 75 | 76 | self.transform = Affine.from_gdal( 77 | *(self.bbox[0], self.resolution, 0.0, self.bbox[-1], 0.0, -self.resolution) 78 | ) 79 | 80 | return 81 | 82 | @property 83 | def pixelgrid(self): 84 | """Property for json/dict represenstion of pixel grid info for requesting EE rasters. 85 | Used to define the spatial domain and shape of imagery for requests 86 | 87 | returns: 88 | dict: dictionary representation of pixel grid (https://developers.google.com/earth-engine/reference/rest/v1beta/PixelGrid) 89 | """ 90 | at_keys = ("translateX", "scaleX", "shearX", "translateY", "shearY", "scaleY") 91 | gt = self.transform.to_gdal() 92 | affinetranform = {k: gt[i] for i, k in enumerate(at_keys)} 93 | 94 | dims = dict(width=self.x_size, height=self.y_size) 95 | 96 | pgrid = dict(affineTransform=affinetranform, dimensions=dims, crsCode=self.crs) 97 | 98 | return pgrid 99 | 100 | @property 101 | def crs(self): 102 | return self._crs 103 | 104 | @property 105 | def mask(self): 106 | return self._mask 107 | 108 | @mask.setter 109 | def mask(self, value): 110 | if value.shape == self.shape: 111 | self._mask = value 112 | else: 113 | raise AttributeError( 114 | f"provided mask has a shape of {value.shape} which does not match the domain shape of {self.shape}" 115 | ) 116 | return 117 | 118 | def to_ee_bbox(self): 119 | """Converts the domain bounding box to and ee.Geometry 120 | 121 | returns: 122 | ee.Geometry: bounding box of domain 123 | """ 124 | return ee.Geometry.Rectangle(self.bbox) 125 | 126 | def resample(self, factor: float): 127 | """Function to resample domain shape and coordinates to a new resolution. 128 | Useful for requesting imagery over the same domain but at different spatial 129 | resolutions 130 | 131 | args: 132 | factor (float): factor to scale the shape and coordinates. For example, 133 | if factor = 2, then the resolution will half and shape doubles. 134 | 135 | returns: 136 | restee.Domain: domain object with new resolution/coordinates 137 | """ 138 | new = copy.deepcopy(self) 139 | new.resolution = float(new.resolution / factor) 140 | # interpolate the x coordinates 141 | old_x = np.arange(self.x_size) * factor 142 | new_x = np.arange(self.x_size * factor) 143 | f_x = interpolate.interp1d( 144 | old_x, self.x_coords, bounds_error=False, fill_value="extrapolate" 145 | ) 146 | new.x_coords = f_x(new_x) 147 | 148 | old_x = np.arange(self.y_size) * factor 149 | new_x = np.arange(self.y_size * factor) 150 | f_y = interpolate.interp1d( 151 | old_x, self.y_coords, bounds_error=False, fill_value="extrapolate" 152 | ) 153 | new.y_coords = f_y(new_x) 154 | 155 | new.x_size = new.x_coords.size 156 | new.y_size = new.y_coords.size 157 | 158 | new._construct_grid() 159 | 160 | interp_mask = ndimage.zoom(self.mask, factor, order=0, mode="nearest") 161 | new.mask = interp_mask.astype(np.bool) 162 | 163 | return new 164 | 165 | @staticmethod 166 | def from_geopandas(gdf, resolution: float = 0.25): 167 | """Domain constructor function that takes a GeoDataFrame and returns a domain 168 | object with the vector as a mask. 169 | 170 | args: 171 | gdf (geopandas.GeoDataFrame): GeoDataFrame to create the domain from 172 | resolution (float): resolution to make domain, must match units of vector crs. 173 | default = 0.25 174 | 175 | returns: 176 | restee.Domain: domain object with mask from vector 177 | """ 178 | bbox = Domain._round_out(gdf.total_bounds, res=resolution) 179 | crs = gdf.crs.srs 180 | d = Domain(bbox, resolution, crs) 181 | d.mask = features.geometry_mask( 182 | gdf.geometry, d.shape, transform=d.transform, all_touched=True, invert=True 183 | ) 184 | return d 185 | 186 | @staticmethod 187 | def from_rasterio(ds, mask_val=None): 188 | """Domain contructor function that takes a rasterio object and returns a domain 189 | with the same geotransform and crs 190 | 191 | args: 192 | ds (rasterio): rasterio object to model domain from 193 | 194 | returns: 195 | restee.Domain: domain object with same geotransform and crs as the input 196 | """ 197 | resolution = np.mean(ds.res) 198 | crs = ds.crs.data["init"] 199 | bbox = tuple(ds.bounds) 200 | d = Domain(bbox, resolution, crs) 201 | # TODO: add code to automatically mask no data values 202 | return d 203 | 204 | @staticmethod 205 | def from_ee_geometry(session, geom, resolution: float = 0.25): 206 | """Domain contructor function that takes a ee.Geometry, ee.Feature, or ee.FeatureCollection 207 | and returns a domain object with the geometry as a mask. Useful for using ee to process 208 | a region and use as domain for requsting imagery. 209 | 210 | args: 211 | session (EESession): restee session autheticated to make requests 212 | geom (ee.Geometry|ee.Feature|ee.FeatureCollection): ee object to create the domain from 213 | resolution (float): resolution to make domain, must match units of vector crs. 214 | default = 0.25 215 | 216 | returns: 217 | restee.Domain: domain object with mask from vector 218 | """ 219 | if isinstance(geom, ee.Geometry): 220 | fc = ee.FeatureCollection([ee.Feature(geom)]) 221 | elif isinstance(geom, ee.Feature): 222 | fc = ee.FeatureCollection([geom]) 223 | else: 224 | fc = geom 225 | 226 | project = session.cloud_project 227 | url = f"https://earthengine.googleapis.com/v1beta/projects/{project}/table:computeFeatures" 228 | serialized = ee.serializer.encode(fc, for_cloud_api=True) 229 | payload = dict(expression=serialized) 230 | 231 | response = session.send_request(url, payload) 232 | 233 | gdf = gpd.read_file(StringIO(response.content.decode())) 234 | 235 | return Domain.from_geopandas(gdf, resolution=resolution) 236 | 237 | # TODO: write a Domain constructor from xarray dataarray/dataset 238 | 239 | @staticmethod 240 | def _round_out(bb: Iterable, res: float): 241 | """Function to round bounding box to nearest resolution 242 | args: 243 | bb (iterable): list of bounding box coordinates in the order of [W,S,E,N] 244 | res: (float): resolution of pixels to round bounding box to 245 | """ 246 | minx = bb[0] - (bb[0] % res) 247 | miny = bb[1] - (bb[1] % res) 248 | maxx = bb[2] + (res - (bb[2] % res)) 249 | maxy = bb[3] + (res - (bb[3] % res)) 250 | return minx, miny, maxx, maxy 251 | 252 | 253 | # @staticmethod 254 | def _fatal_code(e): 255 | """Helper function defining a fatal error for backoff decorator to stop requests 256 | """ 257 | # return 400 <= e.response.status_code < 500 258 | return e.response.status_code == 404 259 | 260 | 261 | class EESession: 262 | """EESession class that handles GCP/EE REST API info to make authenticated requests to Google Cloud. 263 | Users provides credentials that are used to create an authorized session to make HTTP requests 264 | 265 | """ 266 | def __init__(self, project: str, key: str): 267 | """Initialization function for the EESession class 268 | 269 | args: 270 | project (str): Google Cloud project name with service account whitelisted to use Earth Engine 271 | key (str): path to private key file for your whitelisted service account 272 | """ 273 | self._PROJECT = project 274 | self._SESSION = self._get_session(key) 275 | 276 | @property 277 | def cloud_project(self): 278 | return self._PROJECT 279 | 280 | @property 281 | def session(self): 282 | return self._SESSION 283 | 284 | @backoff.on_exception( 285 | backoff.expo, 286 | requests.exceptions.RequestException, 287 | max_tries=5, 288 | max_time=300, 289 | giveup=_fatal_code, 290 | ) 291 | def send_request(self, url, data): 292 | """Method to send authenticated requests to google cloud. 293 | This is wrapped with a backoff decorator that will try multiple requests 294 | if the initial ones fail. 295 | 296 | args: 297 | url (str): EE REST API endpoint to send request. 298 | See https://developers.google.com/earth-engine/reference/rest for more info 299 | data (dict): Dictionary object to send in the body of the Request. 300 | 301 | returns: 302 | response: Reponse object with information on status and content 303 | """ 304 | return self.session.post(url=url, data=json.dumps(data)) 305 | 306 | @staticmethod 307 | def _get_session(key): 308 | """Helper function to authenticate""" 309 | credentials = service_account.Credentials.from_service_account_file(key) 310 | scoped_credentials = credentials.with_scopes( 311 | ["https://www.googleapis.com/auth/cloud-platform"] 312 | ) 313 | 314 | return AuthorizedSession(scoped_credentials) 315 | -------------------------------------------------------------------------------- /restee/images.py: -------------------------------------------------------------------------------- 1 | import ee 2 | import time 3 | import json 4 | import requests 5 | import numpy as np 6 | import pandas as pd 7 | import xarray as xr 8 | from tqdm import tqdm 9 | from io import BytesIO 10 | from pyproj import CRS 11 | from pathlib import Path 12 | from collections.abc import Iterable 13 | from concurrent.futures import ThreadPoolExecutor 14 | 15 | from restee.core import Domain, EESession 16 | from restee.values import get_value 17 | 18 | 19 | def img_to_xarray( 20 | session: EESession, 21 | domain: Domain, 22 | image: ee.Image, 23 | bands: Iterable = None, 24 | apply_mask: bool = True, 25 | no_data_value: float = None, 26 | ): 27 | """Function to request ee.Image as a xarray Dataset. This function 28 | wraps img_to_ndarray. 29 | 30 | args: 31 | session (EESession): Earth Engine cloud session used to manage REST API requests 32 | domain (Domain): Domain object defining the spatial region to request image 33 | image (ee.Image): computed ee.Image object to request 34 | bands (Iterable[str]): list or tuple or band names to request from image, if None then 35 | all bands will be requested. default = None 36 | apply_mask (bool): mask pixels based on domain mask. default = True 37 | no_data_value (float): no data value to mask in returned dataset, typically 0. if None, 38 | then no data will be masked by value. default = None 39 | 40 | returns: 41 | xarray.Dataset: dataset with geocoordinates and each band as a variable 42 | 43 | example: 44 | >>> img = ( 45 | ee.ImageCollection('MODIS/006/MOD13Q1') 46 | .first() 47 | ) 48 | >>> states = ee.FeatureCollection('TIGER/2018/States') 49 | >>> maine = states.filter(ee.Filter.eq('NAME', 'Maine')) 50 | >>> domain = restee.Domain.from_ee_geometry(session,maine,0.01) 51 | >>> ds_ndvi = ree.img_to_xarray(session,domain,img,no_data_value=0) 52 | """ 53 | 54 | if bands is None: 55 | bands = get_value(session, image.bandNames()) 56 | 57 | pixels = img_to_ndarray(session, domain, image, bands=bands) 58 | 59 | bandnames = pixels.dtype.names 60 | 61 | if CRS.from_string(domain.crs).is_geographic: 62 | x_name, y_name = "lon", "lat" 63 | x_long, y_long = "Longitude", "Latitude" 64 | x_units, y_units = "degrees_east", "degrees_north" 65 | 66 | else: 67 | x_name, y_name = ( 68 | "x", 69 | "y", 70 | ) 71 | x_long, y_long = "Eastings", "Northings" 72 | # assumes all non-geographic projections have m units... 73 | x_units, y_units = "meters", "meters" 74 | 75 | # CF conventions are coordinates for center pixels 76 | # assign domain coordinates and shift to center 77 | coords = { 78 | x_name: ( 79 | [x_name], 80 | domain.x_coords + (domain.resolution / 2), 81 | {"units": x_units, "long_name": x_long}, 82 | ), 83 | y_name: ( 84 | [y_name], 85 | domain.y_coords - (domain.resolution / 2), 86 | {"units": y_units, "long_name": y_long}, 87 | ), 88 | } 89 | 90 | data_dict = {band: ([y_name, x_name], pixels[band]) for band in bandnames} 91 | 92 | ds = xr.Dataset(data_dict, coords=coords) 93 | 94 | if no_data_value is not None: 95 | ds = ds.where(ds != no_data_value) 96 | 97 | if apply_mask: 98 | ds = ds.where(domain.mask == 1) 99 | 100 | return ds 101 | 102 | 103 | def imgcollection_to_xarray( 104 | session, 105 | domain: Domain, 106 | imagecollection: ee.ImageCollection, 107 | bands: Iterable = None, 108 | max_workers: int = 5, 109 | verbose: bool = False, 110 | apply_mask: bool = True, 111 | no_data_value: float = None, 112 | ): 113 | """Function to request ee.ImageCollection as a xarray Dataset. This function assumes 114 | the image collection is distinguished by time (i.e. 'system:time_start' property). 115 | Sends multiple concurrent requests to speed up data transfer. This function wraps 116 | img_to_ndarray. 117 | 118 | args: 119 | session (EESession): Earth Engine cloud session used to manage REST API requests 120 | domain (Domain): Domain object defining the spatial region to request image 121 | imagecollection (ee.ImageCollection): computed ee.ImageCollection object to request, 122 | images must have `system:time_start` property 123 | bands (Iterable[str]): list or tuple or band names to request from image, if None then 124 | all bands will be requested. default = None 125 | max_workers (int): number of concurrent requests to send. default = 5, 126 | verbose (bool): flag to determine if a request progress bar should be shown. default = False 127 | apply_mask (bool): mask pixels based on domain mask. default = True 128 | no_data_value (float): no data value to mask in returned dataset, typically 0. if None, 129 | then no data will be masked by value. default = None 130 | 131 | returns: 132 | xarray.Dataset: dataset with multiple images along time dimesions, each band is a variable 133 | 134 | example: 135 | >>> ic = ( 136 | ee.ImageCollection('MODIS/006/MOD13Q1') 137 | .limit(10,"system:time_start") 138 | ) 139 | >>> states = ee.FeatureCollection('TIGER/2018/States') 140 | >>> maine = states.filter(ee.Filter.eq('NAME', 'Maine')) 141 | >>> domain = restee.Domain.from_ee_geometry(session,maine,0.01) 142 | >>> ds_ndvi = ree.imgcollection_to_xarray(session,domain,img,no_data_value=0,verbose=True) 143 | """ 144 | 145 | #TODO: write functionality to allow the definition of ImageCollections by other properties than time 146 | 147 | dates = get_value(session, imagecollection.aggregate_array("system:time_start")) 148 | dates = pd.to_datetime(list(map(lambda x: x / 1e-6, dates))) 149 | 150 | coll_id = get_value(session, imagecollection.get("system:id")) 151 | 152 | n_imgs = get_value(session, imagecollection.size()) 153 | 154 | if bands is None: 155 | bands = get_value(session, ee.Image(imagecollection.first()).bandNames()) 156 | 157 | imgseq = range(n_imgs) 158 | imglist = imagecollection.toList(n_imgs) 159 | 160 | def request_func(x): 161 | return img_to_ndarray(session, domain, ee.Image(imglist.get(x)), bands=bands) 162 | 163 | if n_imgs < max_workers: 164 | gen = map(request_func, imgseq) 165 | 166 | if verbose: 167 | series = tuple(tqdm(gen, total=n_imgs, desc=f"{coll_id} progress")) 168 | else: 169 | series = tuple(gen) 170 | 171 | else: 172 | with ThreadPoolExecutor(max_workers) as executor: 173 | gen = executor.map(request_func, imgseq) 174 | 175 | if verbose: 176 | series = tuple(tqdm(gen, total=n_imgs, desc=f"{coll_id} progress")) 177 | else: 178 | series = tuple(gen) 179 | 180 | if CRS.from_string(domain.crs).is_geographic: 181 | x_name, y_name = "lon", "lat" 182 | x_long, y_long = "Longitude", "Latitude" 183 | x_units, y_units = "degrees_east", "degrees_north" 184 | 185 | else: 186 | x_name, y_name = ( 187 | "x", 188 | "y", 189 | ) 190 | x_long, y_long = "Eastings", "Northings" 191 | # assumes all non-geographic projections have m units... 192 | x_units, y_units = "meters", "meters" 193 | 194 | # CF conventions are coordinates for center pixels 195 | # assign domain coordinates and shift to center 196 | data_dict = { 197 | "time": {"dims": ("time"), "data": dates}, 198 | x_name: { 199 | "dims": (x_name), 200 | "data": domain.x_coords + (domain.resolution / 2), 201 | "attrs": {"long_name": x_long, "units": x_units}, 202 | }, 203 | y_name: { 204 | "dims": (y_name), 205 | "data": domain.y_coords - (domain.resolution / 2), 206 | "attrs": {"long_name": y_long, "units": y_units}, 207 | }, 208 | } 209 | 210 | bandnames = series[0].dtype.names 211 | series_shp = (n_imgs, domain.y_size, domain.x_size) 212 | 213 | for i in range(n_imgs): 214 | for band in bandnames: 215 | if i == 0: 216 | data_dict[band] = { 217 | "dims": ("time", y_name, x_name), 218 | "data": np.zeros(series_shp), 219 | } 220 | data_dict[band]["data"][i, :, :] = series[i][band][:, :] 221 | 222 | ds = xr.Dataset.from_dict(data_dict) 223 | 224 | if no_data_value is not None: 225 | ds = ds.where(ds != no_data_value) 226 | 227 | if apply_mask: 228 | ds = ds.where(domain.mask == 1) 229 | 230 | return ds 231 | 232 | 233 | def img_to_ndarray( 234 | session: EESession, 235 | domain: Domain, 236 | image: ee.Image, 237 | bands: Iterable = None, 238 | ): 239 | """Function to request ee.Image as a numpy.ndarray 240 | 241 | args: 242 | session (EESession): Earth Engine cloud session used to manage REST API requests 243 | domain (Domain): Domain object defining the spatial region to request image 244 | image (ee.Image): computed ee.Image object to request 245 | bands (Iterable[str]): list or tuple or band names to request from image, if None then 246 | all bands will be requested. default = None 247 | 248 | returns: 249 | numpy.ndarray: structured numpy array where each band from image is a named field 250 | 251 | example: 252 | >>> img = ( 253 | ee.ImageCollection('MODIS/006/MOD13Q1') 254 | .select("NDVI") 255 | .first() 256 | ) 257 | >>> states = ee.FeatureCollection('TIGER/2018/States') 258 | >>> maine = states.filter(ee.Filter.eq('NAME', 'Maine')) 259 | >>> domain = restee.Domain.from_ee_geometry(session,maine,0.01) 260 | >>> ndvi_arr = ree.img_to_ndarray(session,domain,img) 261 | """ 262 | if bands is None: 263 | bands = get_value(session, image.bandNames()) 264 | 265 | pixels = _get_image(session, domain, image, bands, dataformat="NPY") 266 | 267 | return np.load(BytesIO(pixels)) 268 | 269 | 270 | def img_to_geotiff( 271 | session: EESession, 272 | domain: Domain, 273 | image: ee.Image, 274 | outfile: str, 275 | bands: Iterable = None, 276 | ): 277 | """Function to save requested ee.Image to a file in a GeoTIFF format 278 | 279 | args: 280 | session (EESession): Earth Engine cloud session used to manage REST API requests 281 | domain (Domain): Domain object defining the spatial region to request image 282 | image (ee.Image): computed ee.Image object to request 283 | outfile (str): path to write requested data to 284 | bands (Iterable[str]): list or tuple or band names to request from image, if None then 285 | all bands will be requested. default = None 286 | 287 | example: 288 | >>> img = ( 289 | ee.ImageCollection('MODIS/006/MOD13Q1') 290 | .select("NDVI") 291 | .first() 292 | ) 293 | >>> states = ee.FeatureCollection('TIGER/2018/States') 294 | >>> maine = states.filter(ee.Filter.eq('NAME', 'Maine')) 295 | >>> domain = restee.Domain.from_ee_geometry(session,maine,0.01) 296 | >>> ree.img_to_geotiff(session,domain,img,"maine_ndvi.tiff") 297 | """ 298 | 299 | outfile = Path(outfile) 300 | 301 | if bands is None: 302 | bands = get_value(session, image.bandNames()) 303 | 304 | pixels = _get_image(session, domain, image, bands, dataformat="GEO_TIFF") 305 | 306 | outfile.write_bytes(pixels) 307 | 308 | return 309 | 310 | 311 | def _get_image( 312 | session: EESession, 313 | domain: Domain, 314 | image: ee.Image, 315 | bands: Iterable = None, 316 | dataformat: str = "NPY", 317 | ): 318 | """Base function to request ee.Image object for a specified domain 319 | 320 | args: 321 | session (EESession): Earth Engine cloud session used to manage REST API requests 322 | domain (Domain): Domain object defining the spatial region to request image 323 | image (ee.Image): computed ee.Image object to request 324 | bands (Iterable[str]): list or tuple or band names to request from image, if None then 325 | all bands will be requested. default = None 326 | dataformat (str): data format to return compute image data for domain. current options are 327 | 'NPY' or 'GEO_TIFF'. default = 'NPY' 328 | 329 | returns: 330 | bytes: raw bytes in the format specified by dataformat 331 | 332 | raises: 333 | RequestException: when request status code is not 200 334 | NotImplementedError: when defined dataformat is not "NPY" or "GEO_TIFF" 335 | """ 336 | project = session.cloud_project 337 | if bands is None: 338 | bands = get_value(session, project, image.bandNames()) 339 | 340 | url = f"https://earthengine.googleapis.com/v1beta/projects/{project}/image:computePixels" 341 | 342 | serialized = ee.serializer.encode(image, for_cloud_api=True) 343 | 344 | payload = dict( 345 | expression=serialized, 346 | fileFormat=dataformat, 347 | bandIds=bands, 348 | grid=domain.pixelgrid, 349 | ) 350 | 351 | response = session.send_request(url, payload) 352 | 353 | if response.status_code != 200: 354 | raise requests.exceptions.RequestException( 355 | f"received the following bad status code: {response.status_code}\nServer message: {response.json()['error']['message']}" 356 | ) 357 | 358 | if dataformat in ["NPY", "GEO_TIFF"]: 359 | result = response.content 360 | elif dataformat == "TF_RECORD_IMAGE": 361 | raise NotImplementedError() 362 | else: 363 | raise NotImplementedError( 364 | f"select dataformat {dataformat} is not implemented.Options are 'NPY','GEO_TIFF', or 'TF_RECORD_IMAGE'" 365 | ) 366 | 367 | return result 368 | -------------------------------------------------------------------------------- /restee/tables.py: -------------------------------------------------------------------------------- 1 | import ee 2 | import json 3 | import requests 4 | from io import StringIO 5 | from pathlib import Path 6 | import pandas as pd 7 | import geopandas as gpd 8 | 9 | from restee.core import EESession 10 | 11 | 12 | def features_to_file( 13 | session: EESession, features: ee.FeatureCollection, outfile: str, driver: str = "GeoJSON" 14 | ): 15 | """Wrapper fuction to save requested ee.Feature or ee.FeatureCollection in a vector format 16 | 17 | args: 18 | session (EESession): restee session autheticated to make requests 19 | features (ee.Feature | ee.FeatureCollection): ee.Feature or ee.FeatureCollections save as file 20 | outfile (str): path to save features 21 | driver (str): valid vector driver name to save file, see `import fiona; fiona.supported_drivers` 22 | for full list of supported drivers . default = "GeoJSON" 23 | 24 | example: 25 | >>> img = ( 26 | ee.ImageCollection('MODIS/006/MOD13Q1') 27 | .select("NDVI") 28 | .first() 29 | ) 30 | >>> states = ee.FeatureCollection('TIGER/2018/States') 31 | >>> features = image.reduceRegions( 32 | collection=maine, 33 | reducer=ee.Reducer.mean().setOutputs(["NDVI"]), 34 | scale=image.projection().nominalScale() 35 | ) 36 | >>> restee.features_to_file(session,features,"state_ndvi.geojson") 37 | """ 38 | gdf = features_to_geopandas(session, features) 39 | 40 | gdf.to_file(outfile,driver=driver) 41 | 42 | return 43 | 44 | 45 | def features_to_geodf(session: EESession, features: ee.FeatureCollection): 46 | """Fuction to request ee.Feature or ee.FeatureCollection as a geopandas GeoDataFrame 47 | 48 | args: 49 | session (EESession): restee session autheticated to make requests 50 | features (ee.Feature | ee.FeatureCollection): ee.Feature or ee.FeatureCollections to 51 | request as a GeoDataFrame 52 | 53 | returns: 54 | geopandas.GeoDataFrame: ee.FeatureCollection as GeoDataFrame 55 | 56 | example: 57 | >>> img = ( 58 | ee.ImageCollection('MODIS/006/MOD13Q1') 59 | .select("NDVI") 60 | .first() 61 | ) 62 | >>> states = ee.FeatureCollection('TIGER/2018/States') 63 | >>> features = image.reduceRegions( 64 | collection=maine, 65 | reducer=ee.Reducer.mean().setOutputs(["NDVI"]), 66 | scale=image.projection().nominalScale() 67 | ) 68 | >>> gdf = restee.features_to_geopandas(session,features) 69 | """ 70 | if isinstance(features, ee.Feature): 71 | features = ee.FeatureCollection([features]) 72 | 73 | table = _get_table(session, features) 74 | 75 | return gpd.read_file(StringIO(table.decode())) 76 | 77 | 78 | def features_to_df(session: EESession, features: ee.FeatureCollection): 79 | """Fuction to request ee.Feature or ee.FeatureCollection without coordinates as a pandas DataFrame 80 | 81 | args: 82 | session (EESession): restee session autheticated to make requests 83 | features (ee.Feature | ee.FeatureCollection): ee.Feature or ee.FeatureCollections to 84 | request as a DataFrame 85 | 86 | returns: 87 | pandas.DataFrame: ee.FeatureCollection as DataFrame 88 | 89 | example: 90 | >>> ndvi = ( 91 | ee.ImageCollection('MODIS/006/MOD13Q1') 92 | .select("NDVI") 93 | .first() 94 | ) 95 | >>> temp = ee.ImageCollection('OREGONSTATE/PRISM/AN81m') 96 | .filter(ee.Filter.date('2018-07-01', '2018-07-31')); 97 | 98 | >>> states = ee.FeatureCollection('TIGER/2018/States') 99 | >>> features = image.reduceRegions( 100 | collection=maine, 101 | reducer=ee.Reducer.mean().setOutputs(["NDVI"]), 102 | scale=image.projection().nominalScale() 103 | ) 104 | >>> gdf = restee.features_to_geopandas(session,features) 105 | """ 106 | if isinstance(features, ee.Feature): 107 | features = ee.FeatureCollection([features]) 108 | 109 | table = _get_table(session, features) 110 | 111 | return pd.read_file(StringIO(table.decode())) 112 | 113 | 114 | 115 | def _get_table(session: EESession, featurecollection: ee.FeatureCollection): 116 | """Base fuction to request ee.Feature or ee.FeatureCollection 117 | 118 | args: 119 | session (EESession): restee session autheticated to make requests 120 | featurecollection (ee.FeatureCollection): ee.FeatureCollections to request data from 121 | 122 | returns: 123 | bytes: raw byte data of table in geojson format requested 124 | """ 125 | project = session.cloud_project 126 | url = f"https://earthengine.googleapis.com/v1beta/projects/{project}/table:computeFeatures" 127 | 128 | serialized = ee.serializer.encode(featurecollection, for_cloud_api=True) 129 | 130 | payload = dict(expression=serialized) 131 | 132 | response = session.send_request(url, payload) 133 | 134 | if response.status_code != 200: 135 | raise requests.exceptions.RequestException( 136 | f"received the following bad status code: {response.status_code}\nServer message: {response.json()['error']['message']}" 137 | ) 138 | 139 | return response.content 140 | -------------------------------------------------------------------------------- /restee/values.py: -------------------------------------------------------------------------------- 1 | import ee 2 | import time 3 | import json 4 | import requests 5 | 6 | from restee.core import EESession 7 | 8 | def get_value(session: EESession, value): 9 | """General EE REST API wrapper to request any ee.ComputedObject value 10 | 11 | args: 12 | session (EESession): restee session autheticated to make requests 13 | value (Any): any ee.ComputedObject to request the value for 14 | 15 | returns: 16 | Any: Python evaluated equivalent of ee object 17 | 18 | example: 19 | >>> img = ee.Image("NASA/NASADEM_HGT/001") 20 | >>> ee_bnames = img.bandNames() 21 | >>> band_names = restee.get_value(session, ee_bnames) 22 | >>> print(band_names) 23 | ['elevation', 'num', 'swb'] 24 | """ 25 | project = session.cloud_project 26 | url = f'https://earthengine.googleapis.com/v1beta/projects/{project}/value:compute' 27 | 28 | serialized = ee.serializer.encode(value, for_cloud_api=True) 29 | 30 | payload = dict(expression=serialized) 31 | 32 | response = session.send_request(url, payload) 33 | 34 | if response.status_code != 200: 35 | raise requests.exceptions.RequestException( 36 | f"received the following bad status code: {response.status_code}\nServer message: {response.json()['error']['message']}") 37 | 38 | return response.json()['result'] 39 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import setuptools 2 | from setuptools import setup 3 | 4 | with open("README.md", "r") as fh: 5 | long_description = fh.read() 6 | 7 | setup(name='restee', 8 | version='0.0.3', 9 | description='Python package to call processed EE objects via the REST API to local data', 10 | long_description=long_description, 11 | long_description_content_type="text/markdown", 12 | url='http://github.com/kmarkert/restee', 13 | packages=setuptools.find_packages(), 14 | author='Kel Markert', 15 | author_email='kel.markert@gmail.com', 16 | license='MIT', 17 | zip_safe=False, 18 | include_package_data=True, 19 | install_requires=[ 20 | 'numpy', 21 | 'scipy', 22 | 'pandas', 23 | 'xarray', 24 | 'rasterio', 25 | 'geopandas', 26 | 'pyproj', 27 | 'requests', 28 | 'backoff', 29 | 'earthengine-api', 30 | 'tqdm', 31 | ], 32 | ) 33 | --------------------------------------------------------------------------------