├── .dockerignore ├── .gitignore ├── .secret └── .gitkeep ├── Dockerfile ├── LICENSE ├── README.md ├── docker-compose.yml ├── examples ├── conversion.py └── download.py ├── requirements.txt ├── scripts └── sen2cor_install.sh ├── sentinel2download ├── __init__.py ├── downloader.py ├── grid │ ├── readme.txt │ ├── sentinel2grid.dbf │ ├── sentinel2grid.prj │ ├── sentinel2grid.shp │ └── sentinel2grid.shx └── overlap.py ├── sentinel2preprocessing ├── __init__.py └── conversion.py ├── setup.py └── test_geojson ├── Kharkiv.geojson ├── Ohio.geojson └── osnova_lake.geojson /.dockerignore: -------------------------------------------------------------------------------- 1 | # Python cache 2 | __pycache__ 3 | 4 | # Environments 5 | venv/ 6 | .idea 7 | 8 | sentinel2imagery 9 | docker-compose.yml 10 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Python cache 2 | __pycache__ 3 | 4 | # Environments 5 | venv/ 6 | .idea 7 | 8 | *.secret/sentinel2_google_api_key.json 9 | sentinel2imagery 10 | -------------------------------------------------------------------------------- /.secret/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/QuantuMobileSoftware/sentinel2tools/ca232cb66106db6cac729defdab91aad9aecb15b/.secret/.gitkeep -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.7 2 | 3 | ENV PATH=/opt/sen2cor/bin:$PATH 4 | ENV PYTHONPATH=/code 5 | 6 | RUN apt-get update \ 7 | && apt-get upgrade -y \ 8 | && apt-get install -y gcc libspatialindex-dev python3-dev 9 | 10 | # Sen2cor installation 11 | RUN wget -nv http://step.esa.int/thirdparties/sen2cor/2.8.0/Sen2Cor-02.08.00-Linux64.run && \ 12 | bash Sen2Cor-02.08.00-Linux64.run --target /opt/sen2cor && \ 13 | rm Sen2Cor-02.08.00-Linux64.run 14 | 15 | RUN mkdir /code 16 | 17 | WORKDIR /code 18 | 19 | COPY requirements.txt /code/ 20 | RUN pip install -r requirements.txt 21 | 22 | COPY . /code/ 23 | 24 | CMD ["python3", "examples/download.py"] 25 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # sentinel2tools 2 | 3 | ## Repository for downloading, processing Sentinel-2 satellite images 4 | 5 | * Downloading images from [Sentinel-2 google data storage](https://cloud.google.com/storage/docs/public-datasets/sentinel-2) 6 | * Using [Google Api Key](https://support.google.com/googleapi/answer/6251787?hl=en) 7 | 8 | ### Installation 9 | 1. Install dependencies, see Dockerfile 10 | 2. Add to requirements.txt in the project or use `pip install`:
11 | `git+https://github.com/QuantuMobileSoftware/sentinel2tools.git@` 12 |
or 13 | `git+ssh://git@github.com/QuantumobileSoftware/sentinel2tools.git@` 14 | 3. If the only dependency that you didn't have installed is `Sen2Cor`, then you can use `sen2cor_install.sh` script. 15 | It is added as a part of python package, so you can simply run sen2cor_install.sh - Sen2Cor will be installed in your home directory 16 | 17 | ### Usage 18 | 19 | See `examples/download.py` on how to download Sentinel-2 images. 20 | For more details see docs in `Sentinel2Downloader.download()` method 21 | 22 | See `examples/conversion.py` on how to convert raw Sentinel-2 L1C products into L2A products. 23 | In order to convert Sentinel-2 products need to be fully downloaded (.SAFE folder), 24 | to achieve it set _**full_download=True**_ option of `Sentinel2Downloader`. 25 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3" 2 | services: 3 | example: 4 | build: . 5 | volumes: 6 | - ./:/code 7 | -------------------------------------------------------------------------------- /examples/conversion.py: -------------------------------------------------------------------------------- 1 | from sentinel2download.downloader import Sentinel2Downloader 2 | from sentinel2download.overlap import Sentinel2Overlap 3 | from sentinel2preprocessing.conversion import Sentinel2Converter 4 | 5 | if __name__ == '__main__': 6 | verbose = True 7 | aoi_path = "./test_geojson/osnova_lake.geojson" 8 | 9 | overlap = Sentinel2Overlap(aoi_path, verbose=verbose) 10 | tiles = overlap.overlap() 11 | 12 | print(f"Overlapped tiles: {tiles}") 13 | 14 | api_key = f"./.secret/sentinel2_google_api_key.json" 15 | 16 | loader = Sentinel2Downloader(api_key, verbose=verbose) 17 | 18 | product_type = 'L1C' 19 | start_date = "2018-05-01" 20 | end_date = "2018-05-05" 21 | download_dir = './sentinel2imagery/l1c_products' 22 | conversion_dir = './sentinel2imagery/l2a_products' 23 | cores = 3 24 | BANDS = {'TCI', 'B04', } 25 | CONSTRAINTS = {'NODATA_PIXEL_PERCENTAGE': 15.0, 'CLOUDY_PIXEL_PERCENTAGE': 10.0, } 26 | 27 | loaded = loader.download(product_type, 28 | tiles, 29 | start_date=start_date, 30 | end_date=end_date, 31 | output_dir=download_dir, 32 | cores=cores, 33 | bands=BANDS, 34 | constraints=CONSTRAINTS, 35 | full_download=True) 36 | 37 | converter = Sentinel2Converter(verbose=verbose) 38 | 39 | converted_products = converter.convert(download_dir, conversion_dir) 40 | print(f"Total number of converted products: {len(converted_products)}") 41 | 42 | print("Execution ended") 43 | -------------------------------------------------------------------------------- /examples/download.py: -------------------------------------------------------------------------------- 1 | from sentinel2download.downloader import Sentinel2Downloader 2 | from sentinel2download.overlap import Sentinel2Overlap 3 | 4 | if __name__ == '__main__': 5 | verbose = True 6 | aoi_path = "./test_geojson/osnova_lake.geojson" 7 | 8 | overlap = Sentinel2Overlap(aoi_path, verbose=verbose) 9 | tiles = overlap.overlap() 10 | 11 | print(f"Overlapped tiles: {tiles}") 12 | 13 | api_key = f"./.secret/sentinel2_google_api_key.json" 14 | 15 | loader = Sentinel2Downloader(api_key, verbose=verbose) 16 | 17 | product_type = 'L2A' # or L1C 18 | start_date = "2020-10-01" 19 | end_date = "2020-10-20" 20 | output_dir = './sentinel2imagery' 21 | cores = 3 22 | BANDS = {'TCI', 'B04', } 23 | CONSTRAINTS = {'NODATA_PIXEL_PERCENTAGE': 15.0, 'CLOUDY_PIXEL_PERCENTAGE': 10.0, } 24 | 25 | loaded = loader.download(product_type, 26 | tiles, 27 | start_date=start_date, 28 | end_date=end_date, 29 | output_dir=output_dir, 30 | cores=cores, 31 | bands=BANDS, 32 | constraints=CONSTRAINTS) 33 | 34 | print(f"Load information") 35 | for item in loaded: 36 | print(item) 37 | 38 | print("Execution ended") 39 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | google-cloud-storage==1.32.0 2 | geopandas==0.8.1 3 | Rtree==0.9.4 4 | 5 | -------------------------------------------------------------------------------- /scripts/sen2cor_install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | if ! command -v L2A_Process &> /dev/null 4 | then 5 | echo "Installing sen2cor for top-atmosphere correction of Sentinel-2" 6 | wget -nv http://step.esa.int/thirdparties/sen2cor/2.8.0/Sen2Cor-02.08.00-Linux64.run 7 | install_dir=~/sen2cor 8 | bash Sen2Cor-02.08.00-Linux64.run --target $install_dir 9 | full_path=$(cd "$(dirname $install_dir)"; pwd -P)/$(basename $install_dir) 10 | echo "export PATH=\"$full_path/bin:\$PATH\"" >> ~/.bashrc 11 | . ~/.bashrc 12 | rm Sen2Cor-02.08.00-Linux64.run 13 | else 14 | echo "L2A_Process for Sentinel-2 correction is already installed" 15 | fi -------------------------------------------------------------------------------- /sentinel2download/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/QuantuMobileSoftware/sentinel2tools/ca232cb66106db6cac729defdab91aad9aecb15b/sentinel2download/__init__.py -------------------------------------------------------------------------------- /sentinel2download/downloader.py: -------------------------------------------------------------------------------- 1 | import os 2 | import re 3 | import logging 4 | import time 5 | 6 | from datetime import datetime, timedelta 7 | from collections import namedtuple 8 | from types import MappingProxyType 9 | from typing import Optional, List, Tuple 10 | from xml.dom import minidom 11 | from google.cloud import storage 12 | from pathlib import Path 13 | from concurrent.futures import ThreadPoolExecutor, as_completed 14 | 15 | logger = logging.getLogger(__name__) 16 | logging.basicConfig() 17 | 18 | PRODUCT_TYPE = namedtuple('type', 'L2A L1C')('L2A', 'L1C') 19 | 20 | BANDS = frozenset(('TCI', 'B01', 'B02', 'B03', 'B04', 'B05', 'B06', 21 | 'B07', 'B08', 'B8A', 'B09', 'B10', 'B11', 'B12', 'CLD')) 22 | 23 | CONSTRAINTS = MappingProxyType({'CLOUDY_PIXEL_PERCENTAGE': 100.0, 'NODATA_PIXEL_PERCENTAGE': 100.0, }) 24 | 25 | FOLDER_SUFFIX = "_$folder$" 26 | 27 | 28 | class Sentinel2Downloader: 29 | """ 30 | Class for loading Sentinel2 L1C or L2A images 31 | """ 32 | 33 | def __init__(self, api_key: str, verbose: bool = False): 34 | """ 35 | :param api_key: str, path to google key, https://cloud.google.com/storage/docs/public-datasets/sentinel-2 36 | :param verbose: bool, flag, print logging information, default: False 37 | """ 38 | if verbose: 39 | logger.setLevel(logging.INFO) 40 | else: 41 | logger.setLevel(logging.CRITICAL) 42 | 43 | os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = api_key 44 | self.client = storage.Client() 45 | self.bucket = self.client.get_bucket('gcp-public-data-sentinel-2') 46 | self.metadata_suffix = 'MTD_TL.xml' 47 | 48 | def _filter_by_dates(self, safe_prefixes) -> List[str]: 49 | # acquired date: 20200812T113607 50 | date_pattern = r"_(\d+)T\d+_" 51 | filtered = list() 52 | for safe_prefix in safe_prefixes: 53 | search = re.search(date_pattern, safe_prefix) 54 | date = search.group(1) 55 | date = datetime.strptime(date, '%Y%m%d') 56 | if date in self.date_range: 57 | filtered.append(safe_prefix) 58 | return filtered 59 | 60 | def _tile_prefix(self, tile): 61 | prefix = f"tiles/{tile[:2]}/{tile[2]}/{tile[3:]}/" 62 | if self.product_type == PRODUCT_TYPE.L2A: 63 | prefix = "L2/" + prefix 64 | return prefix 65 | 66 | @staticmethod 67 | def is_dir(blob): 68 | return blob.name.endswith(FOLDER_SUFFIX) 69 | 70 | @staticmethod 71 | def _date_range(start_date, end_date): 72 | days = (end_date - start_date).days 73 | date_range = [start_date + timedelta(days=delta) for delta in range(0, days + 1)] 74 | return date_range 75 | 76 | def _get_safe_prefixes(self, prefix, delimiter='/'): 77 | iterator = self.client.list_blobs(self.bucket, prefix=prefix, delimiter=delimiter) 78 | prefixes = set() 79 | for page in iterator.pages: 80 | prefixes.update(page.prefixes) 81 | return prefixes 82 | 83 | def _file_suffixes(self): 84 | if self.product_type == 'L2A': 85 | file_suffixes = list() 86 | for band in self.bands: 87 | if band in ('TCI', 'B02', 'B03', 'B04', 'B08'): 88 | suffix = f"{band}_10m.jp2" 89 | elif band in ('B05', 'B06', 'B07', 'B8A', 'B11', 'B12'): 90 | suffix = f"{band}_20m.jp2" 91 | elif band == 'CLD': 92 | suffix = "MSK_CLDPRB_20m.jp2" 93 | else: 94 | suffix = f"{band}_60m.jp2" 95 | file_suffixes.append(suffix) 96 | else: 97 | file_suffixes = [f"{band}.jp2" for band in self.bands] 98 | return file_suffixes 99 | 100 | def _match_constraints(self, metadata_blob): 101 | try: 102 | metadata = metadata_blob.download_as_string() 103 | xml_dom = minidom.parseString(metadata) 104 | 105 | for constraint, value in self.constraints.items(): 106 | xml_node = xml_dom.getElementsByTagName(constraint) 107 | if xml_node: 108 | parsed_value = float(xml_node[0].firstChild.data) 109 | logger.info(f"Blob: {metadata_blob.name}, " 110 | f"constraint {constraint}:{value}, parsed value: {parsed_value}") 111 | if parsed_value > value: 112 | return False 113 | else: 114 | logger.info(f"Constraint: {constraint} not present in metadata: {metadata_blob.name}") 115 | except Exception as ex: 116 | logger.info(f"Error parsing blob metadata: {metadata_blob.name}: {str(ex)}") 117 | return False 118 | else: 119 | return True 120 | 121 | def get_save_path(self, blob): 122 | if self.full_download: 123 | name = blob.name 124 | # extract full path, for ex: S2A_MSIL1C_20201001T084801_N0209_R107_T36UYA_20201001T094101.SAFE/rep_info_$folder$ 125 | search = re.search(r"([^/]+\.SAFE.*)", name) 126 | file_path = search.group(1) 127 | if self.is_dir(blob): 128 | file_path = file_path.replace(FOLDER_SUFFIX, "") 129 | save_path = Path(self.output_dir) / Path(file_path) 130 | else: 131 | name = blob.name 132 | # extract dirname, for ex: S2A_MSIL2A_20200703T084601_N0214_R107_T36UYA_20200703T113817 133 | search = re.search(r"/([^/]+)\.SAFE", name) 134 | save_dir = search.group(1) 135 | save_path = Path(self.output_dir) / Path(save_dir) / Path(name).name 136 | return save_path 137 | 138 | def _filter_by_suffix(self, blobs, file_suffixes): 139 | blobs_to_load = set() 140 | for blob in blobs: 141 | if blob.name.endswith(self.metadata_suffix): 142 | if not self._match_constraints(blob): 143 | return 144 | else: 145 | blobs_to_load.add(blob) 146 | for suffix in file_suffixes: 147 | if blob.name.endswith(suffix): 148 | blobs_to_load.add(blob) 149 | return blobs_to_load 150 | 151 | def _get_blobs_to_load(self, prefixes): 152 | blobs_to_load = set() 153 | file_suffixes = self._file_suffixes() 154 | for prefix in prefixes: 155 | granule_prefix = prefix + "GRANULE/" 156 | blobs = list(self.client.list_blobs(self.bucket, prefix=granule_prefix)) 157 | 158 | granule_blobs = self._filter_by_suffix(blobs, file_suffixes) 159 | if granule_blobs: 160 | blobs_to_load.update(granule_blobs) 161 | 162 | return blobs_to_load 163 | 164 | def _get_filtered_prefixes(self, tile_prefix) -> List[str]: 165 | # filter store items by base prefix, ex: tiles/36/U/YA/ 166 | safe_prefixes = self._get_safe_prefixes(tile_prefix) 167 | # filter .SAFE paths by date range 168 | filtered_prefixes = self._filter_by_dates(safe_prefixes) 169 | return filtered_prefixes 170 | 171 | def _download_blob(self, blob, save_path) -> Tuple[str, str]: 172 | # check if file exists 173 | if save_path.is_file(): 174 | logger.info(f"Blob {save_path} exists, skipping download") 175 | # update mtime thus tile is not evicted from cache 176 | save_path.touch() 177 | return str(save_path), blob.name 178 | if self.is_dir(blob): 179 | Path.mkdir(save_path, parents=True, exist_ok=True) 180 | return str(save_path), blob.name 181 | 182 | Path.mkdir(save_path.parent, parents=True, exist_ok=True) 183 | 184 | with open(save_path, 'wb') as file: 185 | blob.download_to_file(file) 186 | logger.info(f"Loaded {blob.name}") 187 | return str(save_path), blob.name 188 | 189 | def _download_blobs_mult(self, blobs) -> List[Tuple[str, str]]: 190 | results = list() 191 | with ThreadPoolExecutor(max_workers=self.cores) as executor: 192 | future_to_blob = {executor.submit(self._download_blob, blob, self.get_save_path(blob)): blob.name 193 | for blob in blobs} 194 | for future in as_completed(future_to_blob): 195 | blob_name = future_to_blob[future] 196 | try: 197 | result = future.result() 198 | results.append(result) 199 | except Exception as ex: 200 | logger.info(f"Error while loading {blob_name}: {str(ex)}") 201 | results.append((None, blob_name)) 202 | return results 203 | 204 | def _setup(self, product_type, tiles, start_date, end_date, bands, 205 | constraints, output_dir, cores, full_download): 206 | if product_type not in PRODUCT_TYPE: 207 | raise ValueError(f"Provide proper Sentinel2 type: {PRODUCT_TYPE}") 208 | self.product_type = product_type 209 | 210 | self.tiles = tiles 211 | 212 | format = '%Y-%m-%d' 213 | if end_date: 214 | end_date = datetime.strptime(end_date, format) 215 | else: 216 | now = datetime.now() 217 | end_date = datetime(now.year, now.month, now.day) 218 | if start_date: 219 | start_date = datetime.strptime(start_date, format) 220 | else: 221 | delta = 10 222 | start_date = end_date - timedelta(days=delta) 223 | 224 | self.date_range = self._date_range(start_date, end_date) 225 | logger.info(f"Search date range from {start_date} to {end_date}") 226 | 227 | bands = set(bands).intersection(BANDS) 228 | if not bands: 229 | raise ValueError(f"Provide bands from available set: {BANDS}") 230 | else: 231 | self.bands = bands 232 | 233 | self.constraints = constraints 234 | self.output_dir = output_dir 235 | self.cores = cores 236 | self.full_download = full_download 237 | 238 | def download(self, 239 | product_type: str, 240 | tiles: list, 241 | *, 242 | start_date: Optional[str] = None, 243 | end_date: Optional[str] = None, 244 | bands: set = BANDS, 245 | constraints: dict = CONSTRAINTS, 246 | output_dir: str = './sentinel2imagery', 247 | cores: int = 5, 248 | full_download: bool = False) -> Optional[List]: 249 | """ 250 | :param product_type: str, "L2A" or "L1C" Sentinel2 products 251 | :param tiles: list, tiles to load (ex: {36UYA, 36UYB}) 252 | :param start_date: str, format: 2020-01-01, start date to search and load blobs, default: (today - 10 days) 253 | :param end_date: str, format: 2020-01-02, end date to search and load blobs, default: today 254 | :param bands: set, selected bands for loading, default: {'TCI', 'B01', 'B02', 'B03', 'B04', 'B05', 'B06', 255 | 'B07', 'B08', 'B8A', 'B09', 'B10', 'B11', 'B12', } 256 | :param constraints: dict, constraints that blobs must match, default: {'CLOUDY_PIXEL_PERCENTAGE': 100.0, }, 257 | for L2A product_type, 'NODATA_PIXEL_PERCENTAGE' can be added 258 | :param output_dir: str, path to loading dir, default: './sentinel2imagery' 259 | :param cores: int, number of cores, default: 5 260 | :param full_download: bool, option for full download of Sentinel-2 .SAFE folder, default: False 261 | :return: [tuple, None], tuples (save_path, blob_name), if save_path is None, the blob not loaded 262 | or None if nothing to load 263 | """ 264 | 265 | self._setup(product_type, tiles, start_date, end_date, bands, constraints, output_dir, cores, full_download) 266 | 267 | logger.info("Start downloading...") 268 | start_time = time.time() 269 | results = list() 270 | for tile in tiles: 271 | logger.info(f"Loading blobs for tile {tile}...") 272 | 273 | tile_prefix = self._tile_prefix(tile) 274 | filtered_prefixes = self._get_filtered_prefixes(tile_prefix) 275 | if self.full_download: 276 | for prefix in filtered_prefixes: 277 | blobs = list(self.client.list_blobs(self.bucket, prefix=prefix)) 278 | if blobs: 279 | result = self._download_blobs_mult(blobs) 280 | results.extend(result) 281 | else: 282 | blobs_to_load = self._get_blobs_to_load(filtered_prefixes) 283 | result = self._download_blobs_mult(blobs_to_load) 284 | results.extend(result) 285 | logger.info(f"Finished loading blobs for tile {tile}") 286 | 287 | logger.info(f"Loaded: {len([r[0] for r in results if r[0]])} blobs") 288 | logger.info(f"Finished loading at {time.strftime('%H:%M:%S', time.gmtime(time.time() - start_time))}") 289 | 290 | return results 291 | -------------------------------------------------------------------------------- /sentinel2download/grid/readme.txt: -------------------------------------------------------------------------------- 1 | © Copernicus Sentinel data 2016 downloaded from https://scihub.copernicus.eu/ 2 | 3 | Sentinel-2 tiling grid adapted from the kml file downloaded from ESA (https://sentinel.esa.int/web/sentinel/missions/sentinel-2/data-products) at the link https://sentinel.esa.int/documents/247904/1955685/S2A_OPER_GIP_TILPAR_MPC__20151209T095117_V20150622T000000_21000101T000000_B00.kml -------------------------------------------------------------------------------- /sentinel2download/grid/sentinel2grid.dbf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/QuantuMobileSoftware/sentinel2tools/ca232cb66106db6cac729defdab91aad9aecb15b/sentinel2download/grid/sentinel2grid.dbf -------------------------------------------------------------------------------- /sentinel2download/grid/sentinel2grid.prj: -------------------------------------------------------------------------------- 1 | GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137,298.257223563]],PRIMEM["Greenwich",0],UNIT["Degree",0.017453292519943295]] -------------------------------------------------------------------------------- /sentinel2download/grid/sentinel2grid.shp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/QuantuMobileSoftware/sentinel2tools/ca232cb66106db6cac729defdab91aad9aecb15b/sentinel2download/grid/sentinel2grid.shp -------------------------------------------------------------------------------- /sentinel2download/grid/sentinel2grid.shx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/QuantuMobileSoftware/sentinel2tools/ca232cb66106db6cac729defdab91aad9aecb15b/sentinel2download/grid/sentinel2grid.shx -------------------------------------------------------------------------------- /sentinel2download/overlap.py: -------------------------------------------------------------------------------- 1 | import os 2 | import geopandas as gp 3 | import logging 4 | from shapely.geometry import box 5 | from typing import Optional, List 6 | 7 | logger = logging.getLogger(__name__) 8 | logging.basicConfig() 9 | 10 | GRID_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "./grid")) 11 | 12 | 13 | class Sentinel2Overlap: 14 | def __init__(self, aoi_path: str, *, grid_path: str = os.path.join(GRID_DIR, "sentinel2grid.shp"), verbose: bool = False): 15 | self.crs = "epsg:4326" 16 | 17 | if verbose: 18 | logger.setLevel(logging.INFO) 19 | else: 20 | logger.setLevel(logging.CRITICAL) 21 | aoi = gp.read_file(aoi_path) 22 | 23 | if len(aoi) > 1 or aoi.geometry[0].geom_type != 'Polygon': 24 | logger.info(f"Input file contains more than 1 features or feature is not Polygon." 25 | f"Bound box will be created.") 26 | bbox = box(*aoi.total_bounds) 27 | logger.info(f"Bound box: {bbox}") 28 | aoi = gp.GeoDataFrame(geometry=[bbox], crs=self.crs) 29 | 30 | self.aoi = aoi 31 | self.grid = gp.read_file(grid_path) 32 | 33 | def _intersect(self, limit): 34 | """ 35 | Find all tiles that intersects given region with area >= limit km2 36 | :param limit: float, min intersection area in km2 37 | :return: (GeoDataFrame, epsg), precised intersected tiles and UTM zone code 38 | """ 39 | 40 | # Get the indices of the tiles that are likely to be inside the bounding box of the given Polygon 41 | aoi = self.aoi 42 | grid = self.grid 43 | geometry = aoi.geometry[0] 44 | 45 | tiles_indexes = list(grid.sindex.intersection(geometry.bounds)) 46 | grid = grid.loc[tiles_indexes] 47 | # Make the precise tiles in Polygon query 48 | 49 | grid = grid.loc[grid.intersects(geometry)] 50 | 51 | # intersection area 52 | epsg = self.epsg_code(geometry.centroid.x, geometry.centroid.y) 53 | 54 | # to UTM projection in meters 55 | aoi['geometry'] = aoi.geometry.to_crs(epsg=epsg) 56 | grid['geometry'] = grid.geometry.to_crs(epsg=epsg) 57 | 58 | grid['area'] = grid.geometry.apply(lambda g: g.intersection(aoi.geometry[0]).area / 1e6) 59 | grid = grid.loc[grid['area'] >= limit] 60 | grid = grid.sort_values(by=['area', 'Name'], ascending=[False, True]) 61 | 62 | return grid, epsg 63 | 64 | def overlap(self, *, limit: float = 0.001) -> Optional[List]: 65 | """ 66 | Find unique tiles that intersects given aoi, area 67 | :param limit: float, min intersection area in km2 68 | :return: list, list of tiles 69 | """ 70 | 71 | logger.info(f"Start finding overlapping tiles") 72 | 73 | grid, epsg = self._intersect(limit) 74 | 75 | aoi = self.aoi 76 | overlap_tiles = list() 77 | for row in grid.itertuples(): 78 | start_area = aoi.geometry[0].area 79 | aoi.geometry[0] = aoi.geometry[0].difference(row.geometry) 80 | if start_area != aoi.geometry[0].area: 81 | overlap_tiles.append(dict(Name=row.Name, geometry=row.geometry)) 82 | 83 | if not overlap_tiles: 84 | return 85 | 86 | tiles = gp.GeoDataFrame(overlap_tiles, crs=epsg) 87 | tiles = tiles.to_crs(self.crs) 88 | 89 | tile_names = sorted(tiles.Name) 90 | logger.info(f"Found {len(tile_names)} tiles: {', '.join(tile_names)}") 91 | return tile_names 92 | 93 | @staticmethod 94 | def epsg_code(longitude, latitude): 95 | """ 96 | Generates EPSG code from lon, lat 97 | :param longitude: float 98 | :param latitude: float 99 | :return: int, EPSG code 100 | """ 101 | 102 | def _zone_number(lat, lon): 103 | if 56 <= lat < 64 and 3 <= lon < 12: 104 | return 32 105 | if 72 <= lat <= 84 and lon >= 0: 106 | if lon < 9: 107 | return 31 108 | elif lon < 21: 109 | return 33 110 | elif lon < 33: 111 | return 35 112 | elif lon < 42: 113 | return 37 114 | 115 | return int((lon + 180) / 6) + 1 116 | 117 | zone = _zone_number(latitude, longitude) 118 | 119 | if latitude > 0: 120 | return 32600 + zone 121 | else: 122 | return 32700 + zone 123 | -------------------------------------------------------------------------------- /sentinel2preprocessing/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/QuantuMobileSoftware/sentinel2tools/ca232cb66106db6cac729defdab91aad9aecb15b/sentinel2preprocessing/__init__.py -------------------------------------------------------------------------------- /sentinel2preprocessing/conversion.py: -------------------------------------------------------------------------------- 1 | import subprocess 2 | import os 3 | import logging 4 | import time 5 | from typing import List 6 | 7 | logger = logging.getLogger(__name__) 8 | logging.basicConfig() 9 | 10 | 11 | class Sentinel2Converter: 12 | """ 13 | Class for converting Sentinel2 L1C to L2A images 14 | """ 15 | 16 | def __init__(self, verbose): 17 | """ 18 | :param verbose: bool, flag, print logging information, default: False 19 | """ 20 | if verbose: 21 | logger.setLevel(logging.INFO) 22 | else: 23 | logger.setLevel(logging.CRITICAL) 24 | 25 | @staticmethod 26 | def __convert_l1c_to_l2a(input_tile_path, output_dir_path, sen2cor_path) -> bool: 27 | if not os.path.exists(input_tile_path): 28 | logger.error(f"Check that your input tile directory exists: {input_tile_path}") 29 | return False 30 | # Creating these folders is required for correct sen2cor processing 31 | os.makedirs(os.path.join(input_tile_path, "AUX_DATA"), exist_ok=True) 32 | os.makedirs(os.path.join(input_tile_path, "HTML"), exist_ok=True) 33 | logger.info(f"Started converting {input_tile_path}") 34 | process = subprocess.run([sen2cor_path, f'--output_dir={output_dir_path}', f'{input_tile_path}'], 35 | stdout=subprocess.PIPE, 36 | stderr=subprocess.PIPE, 37 | universal_newlines=True) 38 | success = process.returncode == 0 39 | if success: 40 | logger.info(f"Successfully processed {input_tile_path}, results are stored at {output_dir_path}") 41 | else: 42 | logger.error(f"Something went wrong when trying to convert L1C product {input_tile_path} " 43 | f"to L2A product: {process.stderr}") 44 | return success 45 | 46 | def convert(self, input_dir_path, output_dir_path, sen2cor_path='L2A_Process') -> List[str]: 47 | """ 48 | :param input_dir_path: str, path to a directory with downloaded Sentinel-2 L1C products 49 | :param output_dir_path: list, tiles to load (ex: {36UYA, 36UYB}) 50 | :param sen2cor_path: str, path to L2A_Process executable 51 | :return: List[str], 52 | """ 53 | start_time = time.time() 54 | logger.info(f"Started converting L1C products into L2A products") 55 | if not os.path.exists(input_dir_path): 56 | logger.info(f"Check that your input directory exists: {input_dir_path}") 57 | os.makedirs(output_dir_path, exist_ok=True) 58 | results = [] 59 | for tile_dir in os.listdir(input_dir_path): 60 | tile_dir_path = os.path.join(input_dir_path, tile_dir) 61 | status = self.__convert_l1c_to_l2a(tile_dir_path, output_dir_path, sen2cor_path) 62 | if status: 63 | results.append(tile_dir_path) 64 | logger.info(f"Finished converting at {time.strftime('%H:%M:%S', time.gmtime(time.time() - start_time))}") 65 | return results 66 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | 3 | __version__ = '1.1' 4 | __author__ = 'Quantumobile' 5 | 6 | install_requires = ['google-cloud-storage==1.32.0', 7 | 'geopandas==0.8.1', 8 | 'Rtree==0.9.4', ] 9 | 10 | setup( 11 | name='sentinel2tools', 12 | version=__version__, 13 | author=__author__, 14 | description="Repository for downloading, processing Sentinel-2 satellite images", 15 | url="https://github.com/QuantuMobileSoftware/sentinel2tools/", 16 | packages=find_packages(), 17 | install_requires=install_requires, 18 | package_data={'sentinel2download': ['grid/*', ]}, 19 | python_requires='>=3.7', 20 | scripts=['scripts/sen2cor_install.sh'] 21 | ) 22 | -------------------------------------------------------------------------------- /test_geojson/Kharkiv.geojson: -------------------------------------------------------------------------------- 1 | { 2 | "type": "FeatureCollection", 3 | "features": [ 4 | { 5 | "type": "Feature", 6 | "properties": {}, 7 | "geometry": { 8 | "type": "Polygon", 9 | "coordinates": [ 10 | [ 11 | [ 12 | 35.4638671875, 13 | 50.43651601698633 14 | ], 15 | [ 16 | 34.661865234375, 17 | 50.0289165635219 18 | ], 19 | [ 20 | 34.78271484375, 21 | 48.95497369808868 22 | ], 23 | [ 24 | 36.37573242187499, 25 | 48.378145469762444 26 | ], 27 | [ 28 | 37.41943359375, 29 | 48.785151998043155 30 | ], 31 | [ 32 | 38.3038330078125, 33 | 49.44312875803005 34 | ], 35 | [ 36 | 37.91931152343749, 37 | 50.41201824668217 38 | ], 39 | [ 40 | 36.6009521484375, 41 | 50.55532498251967 42 | ], 43 | [ 44 | 35.4638671875, 45 | 50.43651601698633 46 | ] 47 | ] 48 | ] 49 | } 50 | } 51 | ] 52 | } -------------------------------------------------------------------------------- /test_geojson/Ohio.geojson: -------------------------------------------------------------------------------- 1 | {"type":"FeatureCollection","features":[ 2 | {"type":"Feature","id":"39","properties":{"name":"Ohio","density":281.9},"geometry":{"type":"Polygon","coordinates":[[[-80.518598,41.978802],[-80.518598,40.636951],[-80.666475,40.582182],[-80.595275,40.472643],[-80.600752,40.319289],[-80.737675,40.078303],[-80.830783,39.711348],[-81.219646,39.388209],[-81.345616,39.344393],[-81.455155,39.410117],[-81.57017,39.267716],[-81.685186,39.273193],[-81.811156,39.0815],[-81.783771,38.966484],[-81.887833,38.873376],[-82.03571,39.026731],[-82.221926,38.785745],[-82.172634,38.632391],[-82.293127,38.577622],[-82.331465,38.446175],[-82.594358,38.424267],[-82.731282,38.561191],[-82.846298,38.588575],[-82.890113,38.758361],[-83.032514,38.725499],[-83.142052,38.626914],[-83.519961,38.703591],[-83.678792,38.632391],[-83.903347,38.769315],[-84.215533,38.807653],[-84.231963,38.895284],[-84.43461,39.103408],[-84.817996,39.103408],[-84.801565,40.500028],[-84.807042,41.694001],[-83.454238,41.732339],[-83.065375,41.595416],[-82.933929,41.513262],[-82.835344,41.589939],[-82.616266,41.431108],[-82.479343,41.381815],[-82.013803,41.513262],[-81.739956,41.485877],[-81.444201,41.672093],[-81.011523,41.852832],[-80.518598,41.978802],[-80.518598,41.978802]]]}} 3 | ]} -------------------------------------------------------------------------------- /test_geojson/osnova_lake.geojson: -------------------------------------------------------------------------------- 1 | {"type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[36.223726, 49.938323], [36.218963, 49.937273], [36.218233, 49.935588], [36.215873, 49.933986], [36.215186, 49.93197], [36.216774, 49.930174], [36.222439, 49.926582], [36.224585, 49.925919], [36.229434, 49.927688], [36.230035, 49.930533], [36.226988, 49.935809], [36.225572, 49.937577], [36.223726, 49.938323]]]}, "properties": {}, "id": "c1047236-dd7d-4b74-baf5-e1ad663d03da", "name": "Blue", "start_date": "2020-05-23T02:19:42", "end_date": "2020-10-06T05:18:14"} --------------------------------------------------------------------------------