├── Procfile ├── runtime.txt ├── .gitignore ├── uwsgi.ini ├── requirements.txt ├── main.py ├── code_editor.js ├── README.md └── checker.py /Procfile: -------------------------------------------------------------------------------- 1 | web: uwsgi uwsgi.ini -------------------------------------------------------------------------------- /runtime.txt: -------------------------------------------------------------------------------- 1 | python-3.8.12 -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .env 2 | .idea 3 | /credential 4 | /__pycache__ -------------------------------------------------------------------------------- /uwsgi.ini: -------------------------------------------------------------------------------- 1 | [uwsgi] 2 | http-socket = :$(PORT) 3 | master = true 4 | die-on-term = true 5 | module = app:app 6 | memory-report = true -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | certifi==2021.10.8 2 | charset-normalizer==2.0.7 3 | click==8.0.3 4 | colorama==0.4.4 5 | Flask==2.0.2 6 | geojson==2.5.0 7 | geomet==0.3.0 8 | html2text==2020.1.16 9 | idna==3.3 10 | itsdangerous==2.0.1 11 | Jinja2==3.0.2 12 | MarkupSafe==2.0.1 13 | python-dotenv==0.19.1 14 | requests==2.26.0 15 | sentinelsat==1.1.0 16 | six==1.16.0 17 | tqdm==4.62.3 18 | urllib3==1.26.7 19 | Werkzeug==2.0.2 20 | wincertstore==0.2 21 | uwsgi 22 | -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | from flask import Flask, request 2 | from checker import Checker, get_api 3 | import os 4 | from dotenv import load_dotenv 5 | 6 | load_dotenv() # Load environment variable from .env 7 | 8 | app = Flask( 9 | __name__, 10 | static_folder='.', 11 | root_path='/home/runner' 12 | ) 13 | 14 | 15 | @app.route('/') 16 | def root(): 17 | return "App made by Rodrigo E. Principe (fitoprincipe82@gmail.com)" 18 | 19 | 20 | @app.route('/error') 21 | def error(): 22 | return "An error occurred, sorry =)" 23 | 24 | 25 | @app.route('/error2') 26 | def error2(): 27 | error = request.args.get('error') 28 | return "Ocurrió el siguiente error \n\n {}".format(error) 29 | 30 | 31 | @app.route('/check') 32 | def check(): 33 | coords = request.args.get('coords') 34 | level = request.args.get('level') 35 | ingee = request.args.get('ingee') 36 | start = request.args.get('start') 37 | end = request.args.get('end') 38 | 39 | user = os.environ['HUB_USER'] 40 | password = os.environ['HUB_PASS'] 41 | api = get_api(user, password) 42 | 43 | ch = Checker(coords, start, end, level, ingee, api) 44 | 45 | html = ch.create_html() 46 | 47 | return html 48 | 49 | if __name__ == '__main__': 50 | app.run(host='0.0.0.0', port='3000', debug=True) -------------------------------------------------------------------------------- /code_editor.js: -------------------------------------------------------------------------------- 1 | // Draw a geometry 2 | var endpoint = 'http://127.0.0.1:5000/check' 3 | var start = '2021-09-20' 4 | var end = '2021-09-21' 5 | 6 | var format_footprint = function(geom) { 7 | var newcoords = [] 8 | var coords = geom.bounds().coordinates().getInfo()[0] 9 | for (var i in coords) { 10 | var coord = coords[i] 11 | var lon = coord[0] 12 | var lat = coord[1] 13 | newcoords.push(lon) 14 | newcoords.push(lat) 15 | } 16 | return newcoords.join(' ') 17 | } 18 | 19 | var check_S2_available = function(col, roi, start, end, level) { 20 | var start = ee.Date(start) 21 | var end = ee.Date(end) 22 | var coords = format_footprint(roi) 23 | var start = start.format('yMMdd').getInfo() 24 | var end = end.format('yMMdd').getInfo() 25 | var ingee = col.aggregate_array('PRODUCT_ID').join(' ').getInfo() 26 | 27 | var params = "?coords="+coords+"&start="+start+"&end="+end+"&ingee="+ingee+"&level="+level 28 | var url = endpoint+params 29 | return url 30 | } 31 | 32 | var check_S2TOA_available = function(roi, start, end) { 33 | start = ee.Date(start) 34 | end = ee.Date(end).advance(1, 'day') 35 | roi = roi.bounds() 36 | Map.addLayer(roi) 37 | var col = ee.ImageCollection('COPERNICUS/S2') 38 | .filterBounds(roi) 39 | .filterDate(start, end) 40 | return check_S2_available(col, roi, start, end, 'toa') 41 | } 42 | 43 | var url = check_S2TOA_available(geometry, start, end) 44 | print(ui.Label('CHECK', null, url)) -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # GEE Sentinel Checker 2 | 3 | Check if Google Earth Engine (GEE) is missing a Sentinel image by 4 | calling the Copernicus Open Hub API. 5 | 6 | This is a Flask (python) application that you can deploy or run locally. 7 | 8 | The query parameters are: 9 | - **coords**: the coordinates of the polygon to check. Must be the coordinates of a simple rectangle. Must be a string as follows: "lat lon lat lon..." 10 | - **start**: the start date (inclusive). Format: yyyyMMdd 11 | - **end**: the end date (exclusive). Format: yyyyMMdd 12 | - **ingee**: the ids list of the images available on GEE taken from "PRODUCT_ID" property of the images. Must be a string as follows: "id1 id2 id3..." 13 | - **level**: the processing level of the images (options: 'toa', 'sr') 14 | 15 | The file `code_editor.js` contains an example on how to use this web app. 16 | 17 | ## Usage 18 | ### Alternative 1 (without python) 19 | Avoid the python part and use the already running endpoint. It is running 20 | on Heroku so it has its limitations. 21 | 22 | #### In the code editor 23 | ``` javascript 24 | var checker = require('users/fitoprincipe/s2checker:main') 25 | 26 | // Local endpoint. Uncomment this line if the server is running locally 27 | //var endpoint = 'http://127.0.0.1:5000/check'; // local 28 | //var endpoint2 = 'https://checkSentinelGee.rodrigoprincipe.repl.co/check'; 29 | var endpoint = null; 30 | 31 | // DRAW A GEOMETRY to get `geometry` variable 32 | var level = 'toa' // or 'sr' 33 | var start = '2021-09-15' 34 | var end = '2021-09-21' 35 | var check = new checker.Checker(geometry, level, start, end, endpoint); 36 | 37 | print(ui.Label('CHECK', null, check.url())) 38 | ``` 39 | https://code.earthengine.google.com/4a5472d6d80990ce58afa385cf5f9adc 40 | 41 | If you hit an error that says: 42 | 43 | > **Application error**: An error occurred in the application 44 | and your page could not be served. If you are the application 45 | owner, check your logs for details. You can do this from the 46 | Heroku CLI with the command 47 | 48 | is because you are asking too much, to solve it decrease the area 49 | of the geometry or the time window. 50 | 51 | ### Alternative 2 (with python) 52 | If you run the server locally you can fetch more data, so you won't 53 | hit the _Application error_. Also, you can modify the code to your needs 54 | and even contribute to this project to make it better for everyone. 55 | 56 | Steps: 57 | 1. Clone this repository 58 | 2. Create a virtual environment (for example: `s2checker`) 59 | 3. Activate the environment 60 | 4. Navigate to the repository folder 61 | 5. Install dependencies: 62 | >pip install -r requirements.txt 63 | 6. Create a new file in the repo folder called `.env` (text file) 64 | 7. Add the following to that file: 65 | > FLASK_ENV=development 66 | > HUB_USER=your_copernicus_hub_user 67 | > HUB_PASS=your_copernicus_hub_password 68 | 8. Run the server: 69 | >flask run 70 | 9. The server is running, now you can uncomment the `endpoint` line 71 | in the [code editor code](#in-the-code-editor) -------------------------------------------------------------------------------- /checker.py: -------------------------------------------------------------------------------- 1 | from sentinelsat import SentinelAPI, geojson_to_wkt 2 | from dotenv import load_dotenv 3 | import os 4 | from requests.auth import HTTPBasicAuth 5 | import requests 6 | import base64 7 | 8 | load_dotenv() # Load environment variable from .env 9 | 10 | coords = '-68.45317820662159 -26.35042608088762 -62.36675242537159 -26.35042608088762 -62.36675242537159 -22.002031348439377 -68.45317820662159 -22.002031348439377 -68.45317820662159 -26.35042608088762' 11 | start = '20210920' 12 | end = '20210922' 13 | ingee = 'S2B_MSIL1C_20210920T143729_N0301_R096_T19JEL_20210920T180332 S2B_MSIL1C_20210920T143729_N0301_R096_T19JEM_20210920T180332 S2B_MSIL1C_20210920T143729_N0301_R096_T19JEN_20210920T180332 S2B_MSIL1C_20210920T143729_N0301_R096_T19JFL_20210920T180332 S2B_MSIL1C_20210920T143729_N0301_R096_T19JFM_20210920T180332 S2B_MSIL1C_20210920T143729_N0301_R096_T19JFN_20210920T180332 S2B_MSIL1C_20210920T143729_N0301_R096_T19KEP_20210920T180332 S2B_MSIL1C_20210920T143729_N0301_R096_T19KEQ_20210920T180332 S2B_MSIL1C_20210920T143729_N0301_R096_T19KER_20210920T180332 S2B_MSIL1C_20210920T143729_N0301_R096_T19KFP_20210920T180332 S2B_MSIL1C_20210920T143729_N0301_R096_T19KFQ_20210920T180332 S2B_MSIL1C_20210920T143729_N0301_R096_T19KFR_20210920T180332 S2B_MSIL1C_20210920T143729_N0301_R096_T19KGQ_20210920T180332 S2B_MSIL1C_20210920T143729_N0301_R096_T19KGR_20210920T180332 S2B_MSIL1C_20210921T141049_N0301_R110_T20JMS_20210921T172924 S2B_MSIL1C_20210921T141049_N0301_R110_T20KNU_20210921T172924' 14 | level = 'toa' 15 | 16 | 17 | MATCH = { 18 | 'identifier': 'PRODUCT_ID', 19 | } 20 | 21 | 22 | def get_api(user, password): 23 | return SentinelAPI(user, password, 'https://apihub.copernicus.eu/apihub') 24 | 25 | 26 | def _date_gee(date): 27 | y = date[0:4] 28 | m = date[4:6] 29 | d = date[6:8] 30 | return '{}-{}-{}'.format(y, m, d) 31 | 32 | 33 | class Checker: 34 | TEMPLATE = """ 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | {content} 44 |
ESA IDIs available in GEE?Is online in S2 Hub?Code EditorPreview
45 | """ 46 | ROW = """ 47 | 48 | {esaid} 49 | {ingee} 50 | {online} 51 | {codeeditor} 52 | {preview} 53 | """ 54 | 55 | def __init__(self, coords, start, end, level, ingee, api, 56 | match='identifier'): 57 | self.coords = coords 58 | self.level = level 59 | self.ingee = ingee 60 | self.start = start 61 | self.end = end 62 | self.api = api 63 | self.match = match 64 | self._products = None 65 | 66 | @property 67 | def start_gee(self): 68 | return _date_gee(self.start) 69 | 70 | @property 71 | def end_gee(self): 72 | return _date_gee(self.end) 73 | 74 | @property 75 | def ingeelist(self): 76 | return self.ingee.split(' ') 77 | 78 | @property 79 | def coordlist(self): 80 | coordlist = self.coords.split(' ') 81 | points = len(coordlist) 82 | indices = range(0, int(points), 2) 83 | final = [] 84 | for i in indices: 85 | lon = float(coordlist[i]) 86 | lat = float(coordlist[i+1]) 87 | final.append((lon, lat)) 88 | return final 89 | 90 | @property 91 | def products(self): 92 | if not self._products: 93 | products = self.api.query( 94 | self.footprint, 95 | date = (self.start, self.end), 96 | platformname = 'Sentinel-2', 97 | processinglevel = self.plevel 98 | ) 99 | self._products = products 100 | return self._products 101 | 102 | def preview_tag(self, pid): 103 | """ Return a tag for the requested product id """ 104 | data = self.api.get_product_odata(pid) 105 | url = data['quicklook_url'] 106 | user = os.environ['HUB_USER'] 107 | password = os.environ['HUB_PASS'] 108 | r = requests.get(url, auth=HTTPBasicAuth(user, password)) 109 | c = r.content 110 | c64 = base64.b64encode(c) 111 | tag_template = '' 112 | return tag_template.format(c64.decode('utf-8')) 113 | 114 | def _isTOA(self): 115 | if self.level.lower() in ['l1c', 'level-1c', 'toa', 'level1c']: 116 | return True 117 | else: 118 | return False 119 | 120 | @property 121 | def plevel(self): 122 | return 'Level-1C' if self._isTOA() else 'Level-2A' 123 | 124 | def gee_ic_id(self): 125 | """ return GEE image collection id """ 126 | toa = 'COPERNICUS/S2' 127 | sr = 'COPERNICUS/S2_SR' 128 | return toa if self._isTOA() else sr 129 | 130 | def gee_ic(self): 131 | return "ee.ImageCollection({})".format(self.gee_ic_id()) 132 | 133 | def _product_id_to_gee_id(self, pid): 134 | pinfo = self.products[pid] 135 | identifier = pinfo['identifier'] 136 | datastrip = pinfo['datastripidentifier'] 137 | one = identifier.split('_')[2] 138 | two = datastrip.split('_')[7][1:] 139 | three = identifier.split('_')[5] 140 | return '{}_{}_{}'.format(one, two, three) 141 | 142 | def gee_image(self, pid): 143 | icid = self.gee_ic_id() 144 | iid = self._product_id_to_gee_id(pid) 145 | return "ee.Image('{}/{}')".format(icid, iid) 146 | 147 | def code_editor(self, pid): 148 | template = 'var i = {}; print(i)' 149 | return template.format(self.gee_image(pid)) 150 | 151 | def _createGeoJSON(self): 152 | polyDict = { 153 | 'type': 'Polygon', 154 | 'coordinates': [self.coordlist] 155 | } 156 | return polyDict 157 | 158 | @property 159 | def footprint(self): 160 | """ Get footprint from coordinates """ 161 | # coords format: 'lon lat lon lat....' 162 | geojson = self._createGeoJSON() 163 | return geojson_to_wkt(geojson, 14) 164 | 165 | def _get_hub_ids(self): 166 | ids = [] 167 | for pid, v in dict(self.products).items(): 168 | for key, value in v.items(): 169 | if key == self.match: 170 | if value not in ids: 171 | ids.append([value, pid]) 172 | return ids 173 | 174 | def check(self): 175 | hubIDs = self._get_hub_ids() 176 | data = [] 177 | for hid, pid in hubIDs: 178 | online = self.api.is_online(pid) 179 | code = self.code_editor(pid) 180 | img = self.preview_tag(pid) 181 | final = [hid, hid in self.ingeelist, online, code, img] 182 | data.append(final) 183 | return data 184 | 185 | def _format_row(self, row): 186 | esaid, ingee, online, code, img = row 187 | av = 'YES' if ingee else 'NO' 188 | onl = 'YES' if online else 'NO' 189 | color = 'green' if ingee else 'red' 190 | return self.ROW.format(color=color, esaid=esaid, 191 | ingee=av, online=onl, 192 | codeeditor=code, preview=img) 193 | 194 | def create_html(self): 195 | """ Products is a list of ((esaID, available on gee, online in ESA, 196 | code in code editor), ...) """ 197 | rows = self.check() 198 | htmlrows = "" 199 | for row in rows: 200 | data = self._format_row(row) 201 | htmlrows += data 202 | 203 | return self.TEMPLATE.format(content=htmlrows) 204 | --------------------------------------------------------------------------------