├── .DS_Store ├── .dccache ├── .gitattributes ├── .github └── workflows │ └── python-package.yml ├── .gitignore ├── .vscode └── settings.json ├── LICENSE.md ├── README.md ├── examples ├── examples_clone_build │ ├── calibrate_grd.ipynb │ ├── download_calibrate_speckle.ipynb │ ├── download_sentinel1_grd.ipynb │ ├── download_sentinel1_raw.ipynb │ ├── download_sentinel1_slc.ipynb │ ├── load_image.ipynb │ └── speckle_filter.ipynb └── examples_pypi_build │ └── download_calibrate_speckle.ipynb ├── figs ├── calibrate.png ├── extent.png ├── s1.gif └── slc_thumn.png ├── landmask ├── README.txt ├── simplified_land_polygons.cpg ├── simplified_land_polygons.dbf ├── simplified_land_polygons.prj ├── simplified_land_polygons.shp └── simplified_land_polygons.shx ├── pyproject.toml ├── requirements.txt ├── setup.py ├── src └── sentinel_1_python │ ├── __init__.py │ ├── download │ ├── __init__.py │ ├── _utilities_dwl.py │ ├── download_s1grd.py │ └── satellite_download.py │ ├── metadata │ ├── __init__.py │ ├── _masking.py │ ├── _utilities.py │ ├── query_data.py │ └── sentinel_metadata.py │ ├── pre_process_grd │ ├── Process.py │ ├── README.md │ ├── __init__.py │ ├── _get_functions.py │ ├── _load_image.py │ ├── _proces_tools.py │ ├── filters.py │ └── load_data.py │ └── visualize │ ├── __init__.py │ └── show.py └── test_environment.py /.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aalling93/Sentinel_1_python/d9f5079111627644720f985d7fcc1121c3c548f1/.DS_Store -------------------------------------------------------------------------------- /.dccache: -------------------------------------------------------------------------------- 1 | {"/Users/kaaso/Documents/phd/coding/Sentinel_1_python/test_environment.py":[641,1665965848839.3801,"6c774cc2da6e9d6054b89d4caf1431f5950dba7d25231e39d9ac3f4837605844"],"/Users/kaaso/Documents/phd/coding/Sentinel_1_python/src/sentinel_1_python/__init__.py":[137,1665940595968.2097,"87d9c09cd5e2f39504c7bdb64a0f81aba30077b6a9a01511123f3cae6349da5d"],"/Users/kaaso/Documents/phd/coding/Sentinel_1_python/src/sentinel_1_python/download/__init__.py":[94,1665766478549.3625,"83a31ef58e58a372f414927dd0e1731183dc5181b074bc16bed979d5f9347c15"],"/Users/kaaso/Documents/phd/coding/Sentinel_1_python/src/sentinel_1_python/download/_utilities_dwl.py":[2002,1665940628732.902,"85cacbe0be92d738ee5f7f770762abb95d6d3f5ec24d824e16aac63197037f92"],"/Users/kaaso/Documents/phd/coding/Sentinel_1_python/src/sentinel_1_python/download/download_s1grd.py":[54341,1665950664574.585,"82f84e12c33f4986c1773bf1aebdedbc165ca4f2a92e19a0cc35a9551e7567b2"],"/Users/kaaso/Documents/phd/coding/Sentinel_1_python/src/sentinel_1_python/download/satellite_download.py":[1067,1665950501965.4956,"4350ea236ea462b0348e0b07e91d420c63d81a25a648b51f0ee8515441d8a38b"],"/Users/kaaso/Documents/phd/coding/Sentinel_1_python/src/sentinel_1_python/pre_process_grd/Process.py":[16026,1665929878906.5054,"8940eba7a6afee68b0b012564ee279a9ceeceebc6da7652aa3d57a7b67568693"],"/Users/kaaso/Documents/phd/coding/Sentinel_1_python/src/sentinel_1_python/pre_process_grd/__init__.py":[320,1665930100261.08,"ce2f5dfc79443f9886e41fc77507d43e4d0a5038deaeee0041aa88812cb12098"],"/Users/kaaso/Documents/phd/coding/Sentinel_1_python/src/sentinel_1_python/pre_process_grd/_get_functions.py":[2952,1665929851418.5913,"d0874e54aa9c1df6c56bed3ba81a586c77a7c9d76b316b9bd6d67727b0d8b503"],"/Users/kaaso/Documents/phd/coding/Sentinel_1_python/src/sentinel_1_python/pre_process_grd/_load_image.py":[9427,1665929874951.7104,"52e0ab1a686c3ccd3637d626c3602a9bcea5318452532591b555b7f2a5a4c58f"],"/Users/kaaso/Documents/phd/coding/Sentinel_1_python/src/sentinel_1_python/pre_process_grd/_proces_tools.py":[1605,1665929871282.0786,"422e19afa111689212f6e3e62fb56d4e1cf8f473cb354440364b1ea594bef21d"],"/Users/kaaso/Documents/phd/coding/Sentinel_1_python/src/sentinel_1_python/pre_process_grd/filters.py":[915,1665929866842.5588,"20fef664433bbff4accb4df92244e0a067bec57b8595fb2abfe154c6f6f642ad"],"/Users/kaaso/Documents/phd/coding/Sentinel_1_python/src/sentinel_1_python/pre_process_grd/load_data.py":[16504,1665767203974.8152,"34af39b784a60db1075e43267c5228b2a40b81bcdc037b99f312517050828e1e"],"/Users/kaaso/Documents/phd/coding/Sentinel_1_python/src/sentinel_1_python/metadata/__init__.py":[108,1665766228448.3696,"98fd2716236b138e03696cff737c7ed2df5739510f2520c47c2f36fd7eeb2fad"],"/Users/kaaso/Documents/phd/coding/Sentinel_1_python/src/sentinel_1_python/metadata/_masking.py":[1199,1665765966401.5696,"37eda0d3653b806e54cc78b2f497658d46b8a24bcde015fec780658a8242030b"],"/Users/kaaso/Documents/phd/coding/Sentinel_1_python/src/sentinel_1_python/metadata/_utilities.py":[1508,1665929784273.8462,"5c630a012fe96056a76344144ec2878cac5b2c8d962b7e25a1406a5f9f0608bb"],"/Users/kaaso/Documents/phd/coding/Sentinel_1_python/src/sentinel_1_python/metadata/query_data.py":[2125,1665929800531.341,"80687d2b80e355e63057f6a716853c847752f4167759eed46f0035320b05193f"],"/Users/kaaso/Documents/phd/coding/Sentinel_1_python/src/sentinel_1_python/metadata/sentinel_metadata.py":[4568,1665950525286.867,"41b4838aaf963861f10c0eb2df771bd4222a916ed5d3158a76f6c337ebe42416"],"/Users/kaaso/Documents/phd/coding/Sentinel_1_python/src/sentinel_1_python/visualize/__init__.py":[19,1665766242758.0427,"4d7d5f913f097bc1e683c5e103abc0b5294e3e5b7c09508c559070202f935f2b"],"/Users/kaaso/Documents/phd/coding/Sentinel_1_python/src/sentinel_1_python/visualize/show.py":[4032,1665929895801.5596,"fac25c5539ffe39f7c13fce746c60c86eca2f70f4fffde52b60a34c15a3c5cfc"]} -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | *.py linguist-language=Python -------------------------------------------------------------------------------- /.github/workflows/python-package.yml: -------------------------------------------------------------------------------- 1 | # This workflow will install Python dependencies, run tests and lint with a variety of Python versions 2 | # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python 3 | 4 | name: Python package 5 | 6 | on: 7 | push: 8 | branches: [ "main" ] 9 | pull_request: 10 | branches: [ "main" ] 11 | 12 | jobs: 13 | build: 14 | 15 | runs-on: ubuntu-latest 16 | strategy: 17 | fail-fast: false 18 | matrix: 19 | python-version: ["3.8", "3.9", "3.10"] 20 | 21 | steps: 22 | - uses: actions/checkout@v3 23 | - name: Set up Python ${{ matrix.python-version }} 24 | uses: actions/setup-python@v3 25 | with: 26 | python-version: ${{ matrix.python-version }} 27 | - name: Install dependencies 28 | run: | 29 | python -m pip install --upgrade pip 30 | python -m pip install flake8 pytest 31 | if [ -f requirements.txt ]; then pip install -r requirements.txt; fi 32 | - name: Lint with flake8 33 | run: | 34 | # stop the build if there are Python syntax errors or undefined names 35 | flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics 36 | # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide 37 | flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics 38 | - name: Test with pytest 39 | run: | 40 | pytest 41 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | .dist 12 | dist/ 13 | build/ 14 | .coveralls.yml 15 | .coveralls 16 | .env 17 | data/ 18 | .env/ 19 | pyproject.toml 20 | pyproject 21 | setup.py 22 | .setup.py 23 | .coveralls.yml 24 | coveralls.yml 25 | .gitattributes 26 | .dccache 27 | dccache 28 | data 29 | develop-eggs/ 30 | dist/ 31 | downloads/ 32 | eggs/ 33 | .eggs/ 34 | lib/ 35 | lib64/ 36 | parts/ 37 | sdist/ 38 | var/ 39 | wheels/ 40 | pip-wheel-metadata/ 41 | share/python-wheels/ 42 | *.egg-info/ 43 | .installed.cfg 44 | *.egg 45 | MANIFEST 46 | 47 | # PyInstaller 48 | # Usually these files are written by a python script from a template 49 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 50 | *.manifest 51 | *.spec 52 | 53 | # Installer logs 54 | pip-log.txt 55 | pip-delete-this-directory.txt 56 | 57 | # Unit test / coverage reports 58 | htmlcov/ 59 | .tox/ 60 | .nox/ 61 | .coverage 62 | .coverage.* 63 | .cache 64 | nosetests.xml 65 | coverage.xml 66 | *.cover 67 | .hypothesis/ 68 | .pytest_cache/ 69 | 70 | # Translations 71 | *.mo 72 | *.pot 73 | 74 | # Django stuff: 75 | *.log 76 | local_settings.py 77 | db.sqlite3 78 | db.sqlite3-journal 79 | 80 | # Flask stuff: 81 | instance/ 82 | .webassets-cache 83 | 84 | # Scrapy stuff: 85 | .scrapy 86 | 87 | # Sphinx documentation 88 | docs/_build/ 89 | 90 | # PyBuilder 91 | target/ 92 | 93 | # Jupyter Notebook 94 | .ipynb_checkpoints 95 | 96 | # IPython 97 | profile_default/ 98 | ipython_config.py 99 | 100 | # pyenv 101 | .python-version 102 | 103 | # pipenv 104 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 105 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 106 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 107 | # install all needed dependencies. 108 | #Pipfile.lock 109 | 110 | # celery beat schedule file 111 | celerybeat-schedule 112 | 113 | # SageMath parsed files 114 | *.sage.py 115 | 116 | # Environments 117 | .env 118 | .venv 119 | env/ 120 | venv/ 121 | ENV/ 122 | env.bak/ 123 | venv.bak/ 124 | 125 | # Spyder project settings 126 | .spyderproject 127 | .spyproject 128 | 129 | # Rope project settings 130 | .ropeproject 131 | 132 | # mkdocs documentation 133 | /site 134 | 135 | # mypy 136 | .mypy_cache/ 137 | .dmypy.json 138 | dmypy.json 139 | 140 | # Pyre type checker 141 | .pyre/ 142 | How_to_make_pypi_package.md 143 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | {} -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | 2 | The MIT License (MIT) 3 | 4 | Copyright (c) 2022 Mac 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | 1) Cite me in your work! something like: Kristian Aalling Sørensen, 2022, kaaso@space.dtu.dk 14 | 2) Get as many as possible to follow me on Github. You and your colleagues who use this at the very least. I am a like-hunter. 15 | 3) Star this repository, conditioned to the same as above. 16 | 4) Maybe write me an email or two, telling me how amazing job I did? 17 | 5) Help me with improving the work. I am always looking for collaborators. 18 | 19 | The above copyright notice and this permission notice shall be included in all 20 | copies or substantial portions of the Software. 21 | 22 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 23 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 24 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 25 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 26 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 27 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 28 | SOFTWARE. 29 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | [![Coverage Status](https://coveralls.io/repos/github/aalling93/Sentinel_1_python/badge.svg)](https://coveralls.io/github/aalling93/Sentinel_1_python) 5 | ![Repo Size](https://img.shields.io/github/repo-size/aalling93/Sentinel_1_python) 6 | [![Known Vulnerabilities](https://snyk.io/test/github/aalling93/Sentinel_1_python//badge.svg)](https://snyk.io/test/github/aalling93/Sentinel_1_python/) 7 | ![Python](https://img.shields.io/badge/python-3.9-blue.svg) 8 | 9 | 10 | Kristian Aalling Sørensen 11 | 12 | kaaso@space.dtu.dk 13 | 14 | # Brief Description 15 | 16 | 17 | 18 | This is a Python module for working with Sentinel-1 satellite images, purly in Python. It allows you to find the images you want, download them and work with them (calibrate, speckle fitler etc.).. I use the SentinelSAT package for the metadata. The data is then downloaded from NASA ASF. 19 | 20 | Why? Because I don't to work with ESA SNAP. Also, in this was it is easier to have my entire workflow in Python.. 21 | 22 | I make no guarantees for the quality, security or anything. Use it as you wish. 23 | 24 | 25 | 26 | 27 | 28 | 29 | # Table Of Contents 30 | 31 | 32 | - [Introduction](#Introduction) 33 | - [Requirements](#Requirements) 34 | - [Install and Run](#Install-and-Run) 35 | * [Use Sentinel-1 images in Python](#use) 36 | * [SAR, briefly](#sar) 37 | - [Acknowledgments](#Acknowledgments) 38 | 39 | 40 | 41 | # Requirements 42 | 43 | 44 | - [numpy](https://github.com/numpy) 45 | - [geopandas](https://github.com/geopandas) 46 | - [mgrs](https://github.com/mgrs) (should be removed in later version.. sry..) 47 | - [scikit-learn](https://github.com/scikit-learn) (should be removed in later version.. sry..) 48 | - [scipy](https://github.com/scipy) (should be removed in later version.. sry..) 49 | - [cartopy](https://github.com/cartopy) 50 | - [Pillow](https://github.com/Pillow) 51 | - [pandas](https://github.com/pandas) 52 | - [sentinelsat](https://github.com/sentinelsat) 53 | - [matplotlib](https://github.com/matplotlib) 54 | 55 | 56 | # Install and Run 57 | 58 | 59 | This repo can be installed using either git clone OR pypi.. Currently, I have only placed it in pypi-test, so lets hope it stays there.. 60 | 61 | 62 | **Using Pypi** 63 | 64 | 1. GDAL. Make sure your gdal bindings are working... 65 | 66 | 2. Install sentinel_1_python using pypy test 67 | ``` 68 | python3 -m pip install sentinel-1-python --extra-index-url=https://test.pypi.org/simple/ 69 | ``` 70 | 71 | 72 | **Using clone** 73 | 74 | 1. Install all requirements 75 | 76 | 2. Clone 77 | ``` 78 | git clone https://github.com/aalling93/sentinel_1_python.git 79 | 80 | ``` 81 | 82 | 83 | 84 | 85 | 86 | ## Use Sentinel-1 images in Python 87 | Go back to [Table of Content](#content) 88 | 89 | 1. Get metadata of images 90 | ------------ 91 | 92 | ```python 93 | with Sentinel_metadata() as met: 94 | met.area([29.9,21,56.7,58]) 95 | met.get_metadata(sensor='s1_slc',start_data='20220109',end_date='20221010') 96 | ``` 97 | 98 | 2. Filter the images if you want 99 | ---------------- 100 | ```python 101 | met.iw() #filer so we only have IW 102 | ``` 103 | 104 | 105 | 3. Displaying the images before download: 106 | ```python 107 | met.plot_image_areas() # Showing extent of images 108 | met.show_cross_pol(4) 109 | ``` 110 | We can then see then extent of the images. 111 | 112 | 113 | 114 | And display the images before downloading them... 115 | 116 | 117 | 118 | 4. Download the images 119 | -------------- 120 | ```python 121 | folder = f'{os.getenv("raw_data_dir")}/slc_sweden' 122 | with Satellite_download(met.products_df) as dwl: 123 | os.makedirs(folder, exist_ok=True) 124 | #save metadata 125 | dwl.products_df.to_pickle(f'{folder}/slc_dataframe.pkl') 126 | #download the thumbnails 127 | dwl.download_thumbnails(folder=f'{folder}/slc_thumbnails') 128 | #download the slc images in .zip format and extract to .SAFE format.. 129 | dwl.download_sentinel_1(f'{folder}/slc') 130 | ``` 131 | 132 | 133 | 5. Load, calibrate, speckle filter image in python 134 | 135 | ```python 136 | image_paths = glob.glob(f'{os.getenv("raw_data_dir")}/*/*/*.SAFE') 137 | img = s1_load(image_paths[0]) 138 | img =img.calibrate(mode='gamma') # could also use, e.g., 'sigma_0' 139 | img = img.boxcar(5) #could easily make, e.g., a Lee filter.. 140 | img.simple_plot(band_index=0) 141 | ``` 142 | 143 | 144 | 145 | 146 | we can now exctract a region of the image, defined by either index or coordinate set. 147 | ```python 148 | indx = img.get_index(lat=57.0047,long=19.399) 149 | img[indx[0]-125:indx[0]+125,indx[1]-125:indx[1]+125].simple_plot(band_index=1) 150 | ``` 151 | 152 | 153 | ------------ 154 | 155 | 156 | 157 | ## SAR satellites 158 | Go back to [Table of Content](#content) 159 | 160 | A Synthetic Aperture Radar (SAR) is an active instrument that can be used for e.g. non-cooperative surveillance tasks. Its biggest advantages over e.g. MSI, is that it works day and night, and that it can see though clouds and rain. By placing the SAR instrument on a satellite, it is possible to acquire global coverage with design-specific temporal and spatial resolution. Consequently, by combining, e.g., AIS and SAR instruments, cooperative and non-cooperative surveillance can be acquired. 161 | 162 | 163 | A radar is an instrument that is emitting electromagnetic pulses with a specific signature in the microwave spectrum. For a mono-static radar, the radar instrument is both transmitting and receiving the backscatter signal from the pulse. The backscatter signal depends on the structure of the target it illuminated and thus, by comparing the well-known transmitted and received signal, it is possible to describe both the geometrical and underlying characteristics of the target using the mono-static radar equation: 164 | 165 | $P_r = \frac{P_t \lambda^2 G(\phi, \theta)^2}{(4 \pi )^3 R^4}\sigma (\phi,\theta),$ 166 | 167 | 168 | 169 | where 𝑃𝑟 is the received signal derived from the transmitted signal, 𝑃𝑡. The variable 𝜆 is the design specific wavelength of the radar, and 𝐺(𝜙,𝜃) the radar Gain pattern. The signal is dispersed according to the distance travelled, 𝑅. The radar cross-section, 𝜎(𝜙, 𝜃), can therefore be derived and is describing the target’s dielectric and geometrical characteristics and is dependant on the angles 𝜙 and 𝜃. However, in the presence of noise, another contrubution must be added to the mono-static radar equation. In my other Repo, https://github.com/aalling93/Finding-on-groud-Radars-in-SAR-images, I work with Radio Frequency Interfence ( RFI) . A phenomenan where other signals from other radars interfer with the SAR signal. 170 | Generally speaking, 𝜎(𝜙,𝜃) is describing the available energy within the target area and must therefore be normalised with the area. The radar backscattering coefficient is found by: 171 | 172 | $\sigma^0(\phi, \theta) = \frac{\sigma (\phi, \theta)}{Area}, $ 173 | 174 | where different areas can be used depending on the problem at hand. When using a SAR as an imaging radar, each pixel in the image has a phase and an amplitude value. By calibrating the image, it is possible to get the radar backscattering coefficient as seen in the equation. . In this module, it is possible to download load and calibrate Sentinel-1 images without the need of external software or, e.g., the (infamous) Snappy package. 175 | 176 | 177 | Since a SAR is getting a backscatter contribution from all objects within the area illuminated, a noise-like phenomena called speckle arises. This results in a granular image where each pixel is a combination of the backscatter from the individual object in the area. In my repo, https://github.com/aalling93/Custom-made-SAR-speckle-reduction, I have implemented several differente Speckle filters and show the difference under varying conditions. . 178 | 179 | A SAR imaging radar differs from a normal radar, by utilising the movement of its platform to synthesise a better resolution, hence the name Synthetic Aperture Radar. When taking pictures of a stationary target, a doppler frequency is found from the velocity of the platform. The SAR is emitting and receiving several pulses to and from the same target. When the SAR is flying towards its target, it will measure a positive doppler frequency which is decreasing until it is perpendicular to the target whereafter it will experience an increasing negative doppler frequency 180 | 181 | 182 | The electromagnetic signal is transmitted with either a horizontal or a vertical polar- isation, with full parametric SARs being capable of transmitting both horizontal and vertical polarisation. Due to the interaction of the transmitted pulse with the target, both a vertical and horizontal signal is reflected back to the SAR. This causes several different scattering mechanism to occur. Several types of scattering mechanisms ex- ists. For ship detection, the most prominent are Surface scattering and Double bounce scattering. 183 | 184 | 185 | 186 | #### Surfance scattering 187 | 188 | A transmitted signal will be partly absorbed, and partly reflected by the object it illuminates. Surface scattering is the scattering describing the reflected signal. If a surface is completely smooth(specular), no backscatter is reflected back to the SAR. If the surface is rough, a scattering occurs and part of the incident pulse is scattered back to the SAR. Rough surfaces have a higher backscatter as compared to smoother surfaces. Moreover, VV and HH has a higher backscatter compared to VH and HV(HV and VHthey are almost always the same) for both rough and smooth surfaces. A moist surface results in a higher Radar Cross Section. The backscatter of a surface depends on the roughness and dielectric constant of the target it illuminates. The ocean surface will therefore often result in a small backscatter due to its wet and relatively smooth surface (at low wind speeds), even considering its high dielectric constant at SAR frequencies. 189 | 190 | 191 | #### Double bounce scattering 192 | 193 | 194 | Double bounce scattering and occurs when the transmitted pulse is reflected specularly twice from a corner back to the SAR. This results in a very high backscatter. Ships often have many corners and are very smooth, resulting in an especially high backscatter. It is therefore often easy to differentiate e.g. ships with the ocean surface. For more information on the scattering mechanisms on the oceans. As aforementioned, several other scattering mechanisms exist and when detecting e.g. ships in SAR images in the Arctic, volume scattering has to be considers as well. 195 | 196 | #### SAR and moving targets 197 | 198 | 199 | Due to the geometry of the SAR and its moving platform, typical SAR imaging sensors are designed to take focused images with good resolution under the assumption that their target is stationary during image acquisition. This focusing can not be made on moving targets, and normal SAR instruments are therefore ill suited to detect fast moving objects, such as ships. The results is a well resolved static background and poorly resolved moving target. In non-cooperative surveillance tasks, this is a significant problem. Under the assumption that a target is moving perpendicular to the line of sight of the SAR with a constant acceleration, it is possible to reduce the problem by taking the doppler shift of the SAR images into consideration. Maritime vessels do not normally follow such patterns. Hence, more complex trajectory patterns must be accounted for when looking at ships with SAR instruments. 200 | 201 | In summary, using the capabilities of a SAR instrument, it should be possible to detect ships on the ocean surface. 202 | 203 | 204 | 205 | 206 | # Acknowledgments 207 | 208 | Myself, 209 | Simon Lupemba, 210 | Eigil Lippert 211 | 212 | # Licence 213 | See License file. In short: 214 | 215 | 1. Cite me in your work! something like: 216 | Kristian Aalling Sørensen (2020) sentinel_1_python [Source code]. https://github.com/aalling93/sentinel_1_python. email: kaaso@space.dtu.dk 217 | 2. Get as many as possible to follow me on Github. You and your colleagues who use this at the very least. I am a like-hunter. 218 | 3. Star this repository, conditioned to the same as above. 219 | 4. Maybe write me an email or two, telling me how amazing job I did? 220 | 5. Help me with improving the work. I am always looking for collaborators. -------------------------------------------------------------------------------- /figs/calibrate.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aalling93/Sentinel_1_python/d9f5079111627644720f985d7fcc1121c3c548f1/figs/calibrate.png -------------------------------------------------------------------------------- /figs/extent.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aalling93/Sentinel_1_python/d9f5079111627644720f985d7fcc1121c3c548f1/figs/extent.png -------------------------------------------------------------------------------- /figs/s1.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aalling93/Sentinel_1_python/d9f5079111627644720f985d7fcc1121c3c548f1/figs/s1.gif -------------------------------------------------------------------------------- /figs/slc_thumn.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aalling93/Sentinel_1_python/d9f5079111627644720f985d7fcc1121c3c548f1/figs/slc_thumn.png -------------------------------------------------------------------------------- /landmask/README.txt: -------------------------------------------------------------------------------- 1 | 2 | This data was downloaded from osmdata.openstreetmap.de which offers 3 | extracts and processings of OpenStreetMap data. 4 | 5 | See https://osmdata.openstreetmap.de/ for details. 6 | 7 | 8 | PACKAGE CONTENT 9 | =============== 10 | 11 | This package contains OpenStreetMap data of the 12 | coastline land polygons, simplified for rendering at low zooms. 13 | 14 | Layers contained are: 15 | 16 | simplified_land_polygons.shp: 17 | 18 | 63530 Polygon features 19 | Mercator projection (EPSG: 3857) 20 | Extent: (-20037507, -20037507) - (20037508, 18461504) 21 | In geographic coordinates: (-180.000, -85.051) - (180.000, 83.666) 22 | 23 | Date of the data used is 2022-06-06T00:00:00Z 24 | 25 | You can find more information on this data set at 26 | 27 | https://osmdata.openstreetmap.de/data/land-polygons.html 28 | 29 | 30 | LICENSE 31 | ======= 32 | 33 | This data is Copyright 2022 OpenStreetMap contributors. It is 34 | available under the Open Database License (ODbL). 35 | 36 | For more information see https://www.openstreetmap.org/copyright 37 | 38 | -------------------------------------------------------------------------------- /landmask/simplified_land_polygons.cpg: -------------------------------------------------------------------------------- 1 | UTF-8 2 | -------------------------------------------------------------------------------- /landmask/simplified_land_polygons.dbf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aalling93/Sentinel_1_python/d9f5079111627644720f985d7fcc1121c3c548f1/landmask/simplified_land_polygons.dbf -------------------------------------------------------------------------------- /landmask/simplified_land_polygons.prj: -------------------------------------------------------------------------------- 1 | PROJCS["WGS_1984_Web_Mercator_Auxiliary_Sphere",GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137.0,298.257223563]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.0174532925199433]],PROJECTION["Mercator_Auxiliary_Sphere"],PARAMETER["False_Easting",0.0],PARAMETER["False_Northing",0.0],PARAMETER["Central_Meridian",0.0],PARAMETER["Standard_Parallel_1",0.0],PARAMETER["Auxiliary_Sphere_Type",0.0],UNIT["Meter",1.0]] -------------------------------------------------------------------------------- /landmask/simplified_land_polygons.shp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aalling93/Sentinel_1_python/d9f5079111627644720f985d7fcc1121c3c548f1/landmask/simplified_land_polygons.shp -------------------------------------------------------------------------------- /landmask/simplified_land_polygons.shx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aalling93/Sentinel_1_python/d9f5079111627644720f985d7fcc1121c3c548f1/landmask/simplified_land_polygons.shx -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools>=61.0"] 3 | build-backend = "setuptools.build_meta" 4 | 5 | 6 | 7 | [project] 8 | name = "sentinel_1_python" 9 | version = "0.0.8" 10 | authors = [ 11 | { name="Aalling93", email="kaaso@space.dtu.dk" }, 12 | ] 13 | description = "Work with Sentinel-1 SAR images in Python: Get Metadata, Filter info, Download images, Load and Calibrate images and a bunch of other things." 14 | readme = "README.md" 15 | requires-python = ">=3.9" 16 | classifiers = [ 17 | "Programming Language :: Python :: 3", 18 | "License :: OSI Approved :: MIT License", 19 | "Operating System :: OS Independent", 20 | ] 21 | 22 | 23 | 24 | [project.urls] 25 | "Homepage" = "https://github.com/aalling93/Sentinel_1_python" -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | # You need Python3.. 2 | 3 | # external requirements 4 | python-dotenv>=0.5.1 5 | numpy 6 | geopandas 7 | mgrs #should remove later... 8 | scikit-learn 9 | scipy 10 | cartopy 11 | Pillow 12 | pandas 13 | sentinelsat 14 | matplotlib 15 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from distutils.core import setup 2 | 3 | setup( 4 | install_requires=[ 5 | "numpy", 6 | "geopandas", 7 | "mgrs", 8 | "scikit-learn", 9 | "scipy", 10 | "cartopy", 11 | "Pillow", 12 | "pandas", 13 | "sentinelsat", 14 | "matplotlib", 15 | ], 16 | ) 17 | -------------------------------------------------------------------------------- /src/sentinel_1_python/__init__.py: -------------------------------------------------------------------------------- 1 | #from .download import * 2 | #from .metadata import * 3 | #from .pre_process_grd import * 4 | #from .visualize import * 5 | #from . import _general_util 6 | -------------------------------------------------------------------------------- /src/sentinel_1_python/download/__init__.py: -------------------------------------------------------------------------------- 1 | from ._utilities_dwl import * 2 | from .download_s1grd import * 3 | from .satellite_download import * 4 | -------------------------------------------------------------------------------- /src/sentinel_1_python/download/_utilities_dwl.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | import geopandas as gpd 5 | import requests 6 | 7 | from .download_s1grd import bulk_downloader 8 | 9 | import os, zipfile 10 | 11 | 12 | def unzip_path(dir_name): 13 | cuurent_dir = os.getcwd() 14 | os.chdir(dir_name) 15 | for file in os.listdir(os.getcwd()): 16 | if zipfile.is_zipfile(file): 17 | with zipfile.ZipFile(file) as item: 18 | item.extractall() 19 | 20 | os.chdir(cuurent_dir) 21 | 22 | 23 | def signal_handler(sig, frame): 24 | global abort 25 | sys.stderr.output("\n > Caught Signal. Exiting!\n") 26 | abort = True # necessary to cause the program to stop 27 | raise SystemExit # this will only abort the thread that the ctrl+c was caught in 28 | 29 | 30 | def download_sentinel_1_function( 31 | gdf: gpd.geodataframe.GeoDataFrame = None, data_folder: str = "sentinel_images" 32 | ): 33 | 34 | original_path = os.getcwd() 35 | 36 | if not os.path.exists(data_folder): 37 | os.makedirs(data_folder) 38 | 39 | os.chdir(data_folder) 40 | if "sentinel-1" in gdf.platformname.iloc[0].lower(): 41 | downloader = bulk_downloader(gdf) 42 | downloader.download_files() 43 | downloader.print_summary() 44 | 45 | os.chdir(original_path) 46 | unzip_path(data_folder) 47 | 48 | 49 | def download_thumbnails_function( 50 | grd, 51 | index: list = None, 52 | folder: str = "s1_thumbnails", 53 | username: str = "", 54 | password="", 55 | ): 56 | """ """ 57 | # print(folder) 58 | if not os.path.exists(folder): 59 | os.makedirs(folder) 60 | 61 | if index: 62 | for link_in in index: 63 | link = grd.link_icon.iloc[link_in] 64 | name = grd.uuid.iloc[link_in] 65 | im = requests.get(link, stream=True, auth=(username, password)).content 66 | 67 | with open(f"{folder}/{name}.jpg", "wb") as handler: 68 | handler.write(im) 69 | else: 70 | for i in range(len(grd)): 71 | link = grd.link_icon.iloc[i] 72 | name = grd.uuid.iloc[i] 73 | im = requests.get(link, stream=True, auth=(username, password)).content 74 | 75 | with open(f"{folder}/{name}.jpg", "wb") as handler: 76 | handler.write(im) 77 | -------------------------------------------------------------------------------- /src/sentinel_1_python/download/download_s1grd.py: -------------------------------------------------------------------------------- 1 | # This next block is a bunch of Python 2/3 compatability 2 | # For more information on bulk downloads, navigate to: 3 | # https://asf.alaska.edu/how-to/data-tools/data-tools/#bulk_download 4 | # 5 | # 6 | # 7 | # This script was generated by the Alaska Satellite Facility's bulk download service. 8 | # For more information on the service, navigate to: 9 | # http://bulk-download.asf.alaska.edu/help 10 | 11 | try: 12 | # Python 2.x Libs 13 | from cookielib import MozillaCookieJar 14 | from StringIO import StringIO 15 | from urllib2 import ( 16 | HTTPCookieProcessor, 17 | HTTPError, 18 | HTTPHandler, 19 | HTTPSHandler, 20 | Request, 21 | URLError, 22 | build_opener, 23 | install_opener, 24 | urlopen, 25 | ) 26 | 27 | except ImportError as e: 28 | 29 | # Python 3.x Libs 30 | from http.cookiejar import MozillaCookieJar 31 | from io import StringIO 32 | from urllib.error import HTTPError, URLError 33 | from urllib.request import ( 34 | HTTPCookieProcessor, 35 | HTTPHandler, 36 | HTTPSHandler, 37 | Request, 38 | build_opener, 39 | install_opener, 40 | urlopen, 41 | ) 42 | 43 | ### 44 | # Global variables intended for cross-thread modification 45 | abort = False 46 | 47 | ### 48 | # A routine that handles trapped signals 49 | 50 | 51 | def get_download_name(gpd): 52 | try: 53 | name = gpd.identifier.split("_") 54 | if gpd.producttype == "GRD": 55 | dwl_link = f"https://datapool.asf.alaska.edu/GRD_HD/S{name[0][-1]}/{gpd.identifier}.zip" 56 | else: 57 | dwl_link = f"https://datapool.asf.alaska.edu/{name[2]}/S{name[0][-1]}/{gpd.identifier}.zip" 58 | 59 | return dwl_link 60 | except Exception as e: 61 | print(e) 62 | pass 63 | 64 | 65 | class bulk_downloader: 66 | """ 67 | Class to download Sentinel-1 files... Currectly, only GRD_HD files can be downloaded.. It takes like 10 min to add the other features, but I dont wanna now.. 68 | 69 | 70 | """ 71 | 72 | def __init__(self, gdf): 73 | # List of files to download 74 | signal.signal(signal.SIGINT, signal_handler) 75 | self.gdf = gdf 76 | 77 | download_files = [get_download_name(row) for ix, row in self.gdf.iterrows()] 78 | self.files = download_files 79 | 80 | # Local stash of cookies so we don't always have to ask 81 | self.cookie_jar_path = os.path.join( 82 | os.path.expanduser("~"), ".bulk_download_cookiejar.txt" 83 | ) 84 | self.cookie_jar = None 85 | 86 | self.asf_urs4 = { 87 | "url": "https://urs.earthdata.nasa.gov/oauth/authorize", 88 | "client": "BO_n7nTIlMljdvU6kRRB3g", 89 | "redir": "https://auth.asf.alaska.edu/login", 90 | } 91 | 92 | # Make sure we can write it our current directory 93 | if os.access(os.getcwd(), os.W_OK) is False: 94 | print( 95 | "WARNING: Cannot write to current path! Check permissions for {0}".format( 96 | os.getcwd() 97 | ) 98 | ) 99 | exit(-1) 100 | 101 | # For SSL 102 | self.context = {} 103 | 104 | # Check if user handed in a Metalink or CSV: 105 | if len(sys.argv) > 0: 106 | download_files = [] 107 | input_files = [] 108 | for arg in sys.argv[1:]: 109 | if arg == "--insecure": 110 | try: 111 | ctx = ssl.create_default_context() 112 | ctx.check_hostname = False 113 | ctx.verify_mode = ssl.CERT_NONE 114 | self.context["context"] = ctx 115 | except AttributeError: 116 | # Python 2.6 won't complain about SSL Validation 117 | pass 118 | 119 | elif arg.endswith(".metalink") or arg.endswith(".csv"): 120 | if os.path.isfile(arg): 121 | input_files.append(arg) 122 | if arg.endswith(".metalink"): 123 | new_files = self.process_metalink(arg) 124 | else: 125 | new_files = self.process_csv(arg) 126 | if new_files is not None: 127 | for file_url in new_files: 128 | download_files.append(file_url) 129 | else: 130 | print( 131 | " > I cannot find the input file you specified: {0}".format( 132 | arg 133 | ) 134 | ) 135 | else: 136 | print( 137 | " > Command line argument '{0}' makes no sense, ignoring.".format( 138 | arg 139 | ) 140 | ) 141 | 142 | if len(input_files) > 0: 143 | if len(download_files) > 0: 144 | print( 145 | " > Processing {0} downloads from {1} input files. ".format( 146 | len(download_files), len(input_files) 147 | ) 148 | ) 149 | self.files = download_files 150 | else: 151 | print( 152 | " > I see you asked me to download files from {0} input files, but they had no downloads!".format( 153 | len(input_files) 154 | ) 155 | ) 156 | print(" > I'm super confused and exiting.") 157 | exit(-1) 158 | 159 | # Make sure cookie_jar is good to go! 160 | self.get_cookie() 161 | 162 | # summary 163 | self.total_bytes = 0 164 | self.total_time = 0 165 | self.cnt = 0 166 | self.success = [] 167 | self.failed = [] 168 | self.skipped = [] 169 | 170 | # Get and validate a cookie 171 | def get_cookie(self): 172 | if os.path.isfile(self.cookie_jar_path): 173 | self.cookie_jar = MozillaCookieJar() 174 | self.cookie_jar.load(self.cookie_jar_path) 175 | 176 | # make sure cookie is still valid 177 | if self.check_cookie(): 178 | print(" > Reusing previous cookie jar.") 179 | return True 180 | else: 181 | print(" > Could not validate old cookie Jar") 182 | 183 | # We don't have a valid cookie, prompt user or creds 184 | print( 185 | "No existing URS cookie found, please enter Earthdata username & password:" 186 | ) 187 | print("(Credentials will not be stored, saved or logged anywhere)") 188 | 189 | # Keep trying 'till user gets the right U:P 190 | while self.check_cookie() is False: 191 | self.get_new_cookie() 192 | 193 | return True 194 | 195 | # Validate cookie before we begin 196 | def check_cookie(self): 197 | 198 | if self.cookie_jar is None: 199 | print(" > Cookiejar is bunk: {0}".format(self.cookie_jar)) 200 | return False 201 | 202 | # File we know is valid, used to validate cookie 203 | file_check = "https://urs.earthdata.nasa.gov/profile" 204 | 205 | # Apply custom Redirect Hanlder 206 | opener = build_opener( 207 | HTTPCookieProcessor(self.cookie_jar), 208 | HTTPHandler(), 209 | HTTPSHandler(**self.context), 210 | ) 211 | install_opener(opener) 212 | 213 | # Attempt a HEAD request 214 | request = Request(file_check) 215 | request.get_method = lambda: "HEAD" 216 | try: 217 | print(" > attempting to download {0}".format(file_check)) 218 | response = urlopen(request, timeout=30) 219 | resp_code = response.getcode() 220 | # Make sure we're logged in 221 | if not self.check_cookie_is_logged_in(self.cookie_jar): 222 | return False 223 | 224 | # Save cookiejar 225 | self.cookie_jar.save(self.cookie_jar_path) 226 | 227 | except HTTPError: 228 | # If we ge this error, again, it likely means the user has not agreed to current EULA 229 | print("\nIMPORTANT: ") 230 | print( 231 | "Your user appears to lack permissions to download data from the ASF Datapool." 232 | ) 233 | print( 234 | "\n\nNew users: you must first log into Vertex and accept the EULA. In addition, your Study Area must be set at Earthdata https://urs.earthdata.nasa.gov" 235 | ) 236 | exit(-1) 237 | 238 | # This return codes indicate the USER has not been approved to download the data 239 | if resp_code in (300, 301, 302, 303): 240 | try: 241 | redir_url = response.info().getheader("Location") 242 | except AttributeError: 243 | redir_url = response.getheader("Location") 244 | 245 | # Funky Test env: 246 | if ( 247 | "vertex-retired.daac.asf.alaska.edu" in redir_url 248 | and "test" in self.asf_urs4["redir"] 249 | ): 250 | print("Cough, cough. It's dusty in this test env!") 251 | return True 252 | 253 | print("Redirect ({0}) occured, invalid cookie value!".format(resp_code)) 254 | return False 255 | 256 | # These are successes! 257 | if resp_code in (200, 307): 258 | return True 259 | 260 | return False 261 | 262 | def get_new_cookie(self): 263 | # Start by prompting user to input their credentials 264 | 265 | # Another Python2/3 workaround 266 | try: 267 | new_username = raw_input("Username: ") 268 | except NameError: 269 | new_username = input("Username: ") 270 | new_password = getpass.getpass(prompt="Password (will not be displayed): ") 271 | 272 | # Build URS4 Cookie request 273 | auth_cookie_url = ( 274 | self.asf_urs4["url"] 275 | + "?client_id=" 276 | + self.asf_urs4["client"] 277 | + "&redirect_uri=" 278 | + self.asf_urs4["redir"] 279 | + "&response_type=code&state=" 280 | ) 281 | 282 | try: 283 | # python2 284 | user_pass = base64.b64encode(bytes(new_username + ":" + new_password)) 285 | except TypeError: 286 | # python3 287 | user_pass = base64.b64encode( 288 | bytes(new_username + ":" + new_password, "utf-8") 289 | ) 290 | user_pass = user_pass.decode("utf-8") 291 | 292 | # Authenticate against URS, grab all the cookies 293 | self.cookie_jar = MozillaCookieJar() 294 | opener = build_opener( 295 | HTTPCookieProcessor(self.cookie_jar), 296 | HTTPHandler(), 297 | HTTPSHandler(**self.context), 298 | ) 299 | request = Request( 300 | auth_cookie_url, headers={"Authorization": "Basic {0}".format(user_pass)} 301 | ) 302 | 303 | # Watch out cookie rejection! 304 | try: 305 | response = opener.open(request) 306 | except HTTPError as e: 307 | if ( 308 | "WWW-Authenticate" in e.headers 309 | and "Please enter your Earthdata Login credentials" 310 | in e.headers["WWW-Authenticate"] 311 | ): 312 | print( 313 | " > Username and Password combo was not successful. Please try again." 314 | ) 315 | return False 316 | else: 317 | # If an error happens here, the user most likely has not confirmed EULA. 318 | print("\nIMPORTANT: There was an error obtaining a download cookie!") 319 | print( 320 | "Your user appears to lack permission to download data from the ASF Datapool." 321 | ) 322 | print( 323 | "\n\nNew users: you must first log into Vertex and accept the EULA. In addition, your Study Area must be set at Earthdata https://urs.earthdata.nasa.gov" 324 | ) 325 | exit(-1) 326 | except URLError as e: 327 | print( 328 | "\nIMPORTANT: There was a problem communicating with URS, unable to obtain cookie. " 329 | ) 330 | print("Try cookie generation later.") 331 | exit(-1) 332 | 333 | # Did we get a cookie? 334 | if self.check_cookie_is_logged_in(self.cookie_jar): 335 | # COOKIE SUCCESS! 336 | self.cookie_jar.save(self.cookie_jar_path) 337 | return True 338 | 339 | # if we aren't successful generating the cookie, nothing will work. Stop here! 340 | print( 341 | "WARNING: Could not generate new cookie! Cannot proceed. Please try Username and Password again." 342 | ) 343 | print("Response was {0}.".format(response.getcode())) 344 | print( 345 | "\n\nNew users: you must first log into Vertex and accept the EULA. In addition, your Study Area must be set at Earthdata https://urs.earthdata.nasa.gov" 346 | ) 347 | exit(-1) 348 | 349 | # make sure we're logged into URS 350 | def check_cookie_is_logged_in(self, cj): 351 | for cookie in cj: 352 | if cookie.name == "urs_user_already_logged": 353 | # Only get this cookie if we logged in successfully! 354 | return True 355 | 356 | return False 357 | 358 | # Download the file 359 | def download_file_with_cookiejar(self, url, file_count, total, recursion=False): 360 | # see if we've already download this file and if it is that it is the correct size 361 | download_file = os.path.basename(url).split("?")[0] 362 | if os.path.isfile(download_file): 363 | try: 364 | request = Request(url) 365 | request.get_method = lambda: "HEAD" 366 | response = urlopen(request, timeout=30) 367 | remote_size = self.get_total_size(response) 368 | # Check that we were able to derive a size. 369 | if remote_size: 370 | local_size = os.path.getsize(download_file) 371 | if remote_size < ( 372 | local_size + (local_size * 0.01) 373 | ) and remote_size > (local_size - (local_size * 0.01)): 374 | print( 375 | " > Download file {0} exists! \n > Skipping download of {1}. ".format( 376 | download_file, url 377 | ) 378 | ) 379 | return None, None 380 | # partial file size wasn't full file size, lets blow away the chunk and start again 381 | print( 382 | " > Found {0} but it wasn't fully downloaded. Removing file and downloading again.".format( 383 | download_file 384 | ) 385 | ) 386 | os.remove(download_file) 387 | 388 | except ssl.CertificateError as e: 389 | print(" > ERROR: {0}".format(e)) 390 | print( 391 | " > Could not validate SSL Cert. You may be able to overcome this using the --insecure flag" 392 | ) 393 | return False, None 394 | 395 | except HTTPError as e: 396 | if e.code == 401: 397 | print( 398 | " > IMPORTANT: Your user may not have permission to download this type of data!" 399 | ) 400 | else: 401 | print(" > Unknown Error, Could not get file HEAD: {0}".format(e)) 402 | 403 | except URLError as e: 404 | print("URL Error (from HEAD): {0}, {1}".format(e.reason, url)) 405 | if "ssl.c" in "{0}".format(e.reason): 406 | print( 407 | "IMPORTANT: Remote location may not be accepting your SSL configuration. This is a terminal error." 408 | ) 409 | return False, None 410 | 411 | # attempt https connection 412 | try: 413 | request = Request(url) 414 | response = urlopen(request, timeout=30) 415 | 416 | # Watch for redirect 417 | if response.geturl() != url: 418 | 419 | # See if we were redirect BACK to URS for re-auth. 420 | if ( 421 | "https://urs.earthdata.nasa.gov/oauth/authorize" 422 | in response.geturl() 423 | ): 424 | 425 | if recursion: 426 | print(" > Entering seemingly endless auth loop. Aborting. ") 427 | return False, None 428 | 429 | # make this easier. If there is no app_type=401, add it 430 | new_auth_url = response.geturl() 431 | if "app_type" not in new_auth_url: 432 | new_auth_url += "&app_type=401" 433 | 434 | print(" > While attempting to download {0}....".format(url)) 435 | print(" > Need to obtain new cookie from {0}".format(new_auth_url)) 436 | old_cookies = [cookie.name for cookie in self.cookie_jar] 437 | opener = build_opener( 438 | HTTPCookieProcessor(self.cookie_jar), 439 | HTTPHandler(), 440 | HTTPSHandler(**self.context), 441 | ) 442 | request = Request(new_auth_url) 443 | try: 444 | response = opener.open(request) 445 | for cookie in self.cookie_jar: 446 | if cookie.name not in old_cookies: 447 | print(" > Saved new cookie: {0}".format(cookie.name)) 448 | 449 | # A little hack to save session cookies 450 | if cookie.discard: 451 | cookie.expires = ( 452 | int(time.time()) + 60 * 60 * 24 * 30 453 | ) 454 | print( 455 | " > Saving session Cookie that should have been discarded! " 456 | ) 457 | 458 | self.cookie_jar.save( 459 | self.cookie_jar_path, 460 | ignore_discard=True, 461 | ignore_expires=True, 462 | ) 463 | except HTTPError as e: 464 | print("HTTP Error: {0}, {1}".format(e.code, url)) 465 | return False, None 466 | 467 | # Okay, now we have more cookies! Lets try again, recursively! 468 | print(" > Attempting download again with new cookies!") 469 | return self.download_file_with_cookiejar( 470 | url, file_count, total, recursion=True 471 | ) 472 | 473 | print( 474 | " > 'Temporary' Redirect download @ Remote archive:\n > {0}".format( 475 | response.geturl() 476 | ) 477 | ) 478 | 479 | # seems to be working 480 | print("({0}/{1}) Downloading {2}".format(file_count, total, url)) 481 | 482 | # Open our local file for writing and build status bar 483 | tf = tempfile.NamedTemporaryFile(mode="w+b", delete=False, dir=".") 484 | self.chunk_read(response, tf, report_hook=self.chunk_report) 485 | 486 | # Reset download status 487 | sys.stdout.write("\n") 488 | 489 | tempfile_name = tf.name 490 | tf.close() 491 | 492 | # handle errors 493 | except HTTPError as e: 494 | print("HTTP Error: {0}, {1}".format(e.code, url)) 495 | 496 | if e.code == 401: 497 | print( 498 | " > IMPORTANT: Your user does not have permission to download this type of data!" 499 | ) 500 | 501 | if e.code == 403: 502 | print(" > Got a 403 Error trying to download this file. ") 503 | print(" > You MAY need to log in this app and agree to a EULA. ") 504 | 505 | return False, None 506 | 507 | except URLError as e: 508 | print("URL Error (from GET): {0}, {1}, {2}".format(e, e.reason, url)) 509 | if "ssl.c" in "{0}".format(e.reason): 510 | print( 511 | "IMPORTANT: Remote location may not be accepting your SSL configuration. This is a terminal error." 512 | ) 513 | return False, None 514 | 515 | except socket.timeout as e: 516 | print(" > timeout requesting: {0}; {1}".format(url, e)) 517 | return False, None 518 | 519 | except ssl.CertificateError as e: 520 | print(" > ERROR: {0}".format(e)) 521 | print( 522 | " > Could not validate SSL Cert. You may be able to overcome this using the --insecure flag" 523 | ) 524 | return False, None 525 | 526 | # Return the file size 527 | shutil.copy(tempfile_name, download_file) 528 | os.remove(tempfile_name) 529 | file_size = self.get_total_size(response) 530 | actual_size = os.path.getsize(download_file) 531 | if file_size is None: 532 | # We were unable to calculate file size. 533 | file_size = actual_size 534 | return actual_size, file_size 535 | 536 | def get_redirect_url_from_error(self, error): 537 | find_redirect = re.compile(r"id=\"redir_link\"\s+href=\"(\S+)\"") 538 | print("error file was: {}".format(error)) 539 | redirect_url = find_redirect.search(error) 540 | if redirect_url: 541 | print("Found: {0}".format(redirect_url.group(0))) 542 | return redirect_url.group(0) 543 | 544 | return None 545 | 546 | # chunk_report taken from http://stackoverflow.com/questions/2028517/python-urllib2-progress-hook 547 | def chunk_report(self, bytes_so_far, file_size): 548 | if file_size is not None: 549 | percent = float(bytes_so_far) / file_size 550 | percent = round(percent * 100, 2) 551 | sys.stdout.write( 552 | " > Downloaded %d of %d bytes (%0.2f%%)\r" 553 | % (bytes_so_far, file_size, percent) 554 | ) 555 | else: 556 | # We couldn't figure out the size. 557 | sys.stdout.write(" > Downloaded %d of unknown Size\r" % (bytes_so_far)) 558 | 559 | # chunk_read modified from http://stackoverflow.com/questions/2028517/python-urllib2-progress-hook 560 | def chunk_read(self, response, local_file, chunk_size=8192, report_hook=None): 561 | file_size = self.get_total_size(response) 562 | bytes_so_far = 0 563 | 564 | while 1: 565 | try: 566 | chunk = response.read(chunk_size) 567 | except: 568 | sys.stdout.write("\n > There was an error reading data. \n") 569 | break 570 | 571 | try: 572 | local_file.write(chunk) 573 | except TypeError: 574 | local_file.write(chunk.decode(local_file.encoding)) 575 | bytes_so_far += len(chunk) 576 | 577 | if not chunk: 578 | break 579 | 580 | if report_hook: 581 | report_hook(bytes_so_far, file_size) 582 | 583 | return bytes_so_far 584 | 585 | def get_total_size(self, response): 586 | try: 587 | file_size = response.info().getheader("Content-Length").strip() 588 | except AttributeError: 589 | try: 590 | file_size = response.getheader("Content-Length").strip() 591 | except AttributeError: 592 | print("> Problem getting size") 593 | return None 594 | 595 | return int(file_size) 596 | 597 | # Get download urls from a metalink file 598 | def process_metalink(self, ml_file): 599 | print("Processing metalink file: {0}".format(ml_file)) 600 | with open(ml_file, "r") as ml: 601 | xml = ml.read() 602 | 603 | # Hack to remove annoying namespace 604 | it = ET.iterparse(StringIO(xml)) 605 | for _, el in it: 606 | if "}" in el.tag: 607 | el.tag = el.tag.split("}", 1)[1] # strip all namespaces 608 | root = it.root 609 | 610 | dl_urls = [] 611 | ml_files = root.find("files") 612 | for dl in ml_files: 613 | dl_urls.append(dl.find("resources").find("url").text) 614 | 615 | if len(dl_urls) > 0: 616 | return dl_urls 617 | else: 618 | return None 619 | 620 | # Get download urls from a csv file 621 | def process_csv(self, csv_file): 622 | print("Processing csv file: {0}".format(csv_file)) 623 | 624 | dl_urls = [] 625 | with open(csv_file, "r") as csvf: 626 | try: 627 | csvr = csv.DictReader(csvf) 628 | for row in csvr: 629 | dl_urls.append(row["URL"]) 630 | except csv.Error as e: 631 | print( 632 | "WARNING: Could not parse file %s, line %d: %s. Skipping." 633 | % (csv_file, csvr.line_num, e) 634 | ) 635 | return None 636 | except KeyError as e: 637 | print( 638 | "WARNING: Could not find URL column in file %s. Skipping." 639 | % (csv_file) 640 | ) 641 | 642 | if len(dl_urls) > 0: 643 | return dl_urls 644 | else: 645 | return None 646 | 647 | # Download all the files in the list 648 | def download_files(self): 649 | for file_name in self.files: 650 | 651 | # make sure we haven't ctrl+c'd or some other abort trap 652 | if abort == True: 653 | raise SystemExit 654 | 655 | # download counter 656 | self.cnt += 1 657 | 658 | # set a timer 659 | start = time.time() 660 | 661 | # run download 662 | size, total_size = self.download_file_with_cookiejar( 663 | file_name, self.cnt, len(self.files) 664 | ) 665 | 666 | # calculte rate 667 | end = time.time() 668 | 669 | # stats: 670 | if size is None: 671 | self.skipped.append(file_name) 672 | # Check to see that the download didn't error and is the correct size 673 | elif size is not False and ( 674 | total_size < (size + (size * 0.01)) 675 | and total_size > (size - (size * 0.01)) 676 | ): 677 | # Download was good! 678 | elapsed = end - start 679 | elapsed = 1.0 if elapsed < 1 else elapsed 680 | rate = (size / 1024**2) / elapsed 681 | 682 | print( 683 | "Downloaded {0}b in {1:.2f}secs, Average Rate: {2:.2f}MB/sec".format( 684 | size, elapsed, rate 685 | ) 686 | ) 687 | 688 | # add up metrics 689 | self.total_bytes += size 690 | self.total_time += elapsed 691 | self.success.append({"file": file_name, "size": size}) 692 | 693 | else: 694 | print("There was a problem downloading {0}".format(file_name)) 695 | self.failed.append(file_name) 696 | 697 | def print_summary(self): 698 | # Print summary: 699 | print("\n\nDownload Summary ") 700 | print( 701 | "--------------------------------------------------------------------------------" 702 | ) 703 | print( 704 | " Successes: {0} files, {1} bytes ".format( 705 | len(self.success), self.total_bytes 706 | ) 707 | ) 708 | for success_file in self.success: 709 | print( 710 | " - {0} {1:.2f}MB".format( 711 | success_file["file"], (success_file["size"] / 1024.0**2) 712 | ) 713 | ) 714 | if len(self.failed) > 0: 715 | print(" Failures: {0} files".format(len(self.failed))) 716 | for failed_file in self.failed: 717 | print(" - {0}".format(failed_file)) 718 | if len(self.skipped) > 0: 719 | print(" Skipped: {0} files".format(len(self.skipped))) 720 | for skipped_file in self.skipped: 721 | print(" - {0}".format(skipped_file)) 722 | if len(self.success) > 0: 723 | print( 724 | " Average Rate: {0:.2f}MB/sec".format( 725 | (self.total_bytes / 1024.0**2) / self.total_time 726 | ) 727 | ) 728 | print( 729 | "--------------------------------------------------------------------------------" 730 | ) 731 | 732 | 733 | #!/usr/bin/python 734 | 735 | # Usage: 736 | # 737 | # In a terminal/command line, cd to the directory where this file lives. Then... 738 | # 739 | # With embedded urls: ( download the hardcoded list of files in the 'files =' block below) 740 | # 741 | # python ./download-all-2022-02-16_10-55-15.py 742 | # 743 | # Download all files in a Metalink/CSV: (downloaded from ASF Vertex) 744 | # 745 | # python ./download-all-2022-02-16_10-55-15.py /path/to/downloads.metalink localmetalink.metalink localcsv.csv 746 | # 747 | # Compatibility: python >= 2.6.5, 2.7.5, 3.0 748 | # 749 | # If downloading from a trusted source with invalid SSL Certs, use --insecure to ignore 750 | # 751 | # For more information on bulk downloads, navigate to: 752 | # https://asf.alaska.edu/how-to/data-tools/data-tools/#bulk_download 753 | # 754 | # 755 | # 756 | # This script was generated by the Alaska Satellite Facility's bulk download service. 757 | # For more information on the service, navigate to: 758 | # http://bulk-download.asf.alaska.edu/help 759 | # 760 | 761 | import base64 762 | import csv 763 | import getpass 764 | import os 765 | import os.path 766 | import re 767 | import shutil 768 | import signal 769 | import socket 770 | import ssl 771 | import sys 772 | import tempfile 773 | import time 774 | import xml.etree.ElementTree as ET 775 | 776 | ############# 777 | # This next block is a bunch of Python 2/3 compatability 778 | 779 | try: 780 | # Python 2.x Libs 781 | from cookielib import MozillaCookieJar 782 | from StringIO import StringIO 783 | from urllib2 import ( 784 | HTTPCookieProcessor, 785 | HTTPError, 786 | HTTPHandler, 787 | HTTPSHandler, 788 | Request, 789 | URLError, 790 | build_opener, 791 | install_opener, 792 | urlopen, 793 | ) 794 | 795 | except ImportError as e: 796 | 797 | # Python 3.x Libs 798 | from http.cookiejar import MozillaCookieJar 799 | from io import StringIO 800 | from urllib.error import HTTPError, URLError 801 | from urllib.request import ( 802 | HTTPCookieProcessor, 803 | HTTPHandler, 804 | HTTPSHandler, 805 | Request, 806 | build_opener, 807 | install_opener, 808 | urlopen, 809 | ) 810 | 811 | ### 812 | # Global variables intended for cross-thread modification 813 | abort = False 814 | 815 | ### 816 | # A routine that handles trapped signals 817 | def signal_handler(sig, frame): 818 | global abort 819 | sys.stderr.output("\n > Caught Signal. Exiting!\n") 820 | abort = True # necessary to cause the program to stop 821 | raise SystemExit # this will only abort the thread that the ctrl+c was caught in 822 | 823 | 824 | class xml_bulk_downloader: 825 | """ 826 | Class to download Sentinel-1 files... Currectly, only GRD_HD files can be downloaded.. It takes like 10 min to add the other features, but I dont wanna now.. 827 | 828 | 829 | """ 830 | 831 | def __init__(self, gdf): 832 | # List of files to download 833 | signal.signal(signal.SIGINT, signal_handler) 834 | self.gdf = gdf 835 | download_files = [] 836 | for i in range(len(self.gdf)): 837 | name = self.gdf.iloc[i].dwl_link.split("/") 838 | download_files.append( 839 | f"https://datapool.asf.alaska.edu/METADATA_{name[3]}_HD/S{name[-2][2]}/{name[-2]}.iso.xml" 840 | ) 841 | 842 | # self.files = [ "https://datapool.asf.alaska.edu/SLC/SA/S1A_IW_SLC__1SDV_20190708T142448_20190708T142515_028027_032A4E_E948.zip" ] 843 | self.files = download_files 844 | 845 | # Local stash of cookies so we don't always have to ask 846 | self.cookie_jar_path = os.path.join( 847 | os.path.expanduser("~"), ".bulk_download_cookiejar.txt" 848 | ) 849 | self.cookie_jar = None 850 | 851 | self.asf_urs4 = { 852 | "url": "https://urs.earthdata.nasa.gov/oauth/authorize", 853 | "client": "BO_n7nTIlMljdvU6kRRB3g", 854 | "redir": "https://auth.asf.alaska.edu/login", 855 | } 856 | 857 | # Make sure we can write it our current directory 858 | if os.access(os.getcwd(), os.W_OK) is False: 859 | print( 860 | "WARNING: Cannot write to current path! Check permissions for {0}".format( 861 | os.getcwd() 862 | ) 863 | ) 864 | exit(-1) 865 | 866 | # For SSL 867 | self.context = {} 868 | 869 | # Check if user handed in a Metalink or CSV: 870 | if len(sys.argv) > 0: 871 | download_files = [] 872 | input_files = [] 873 | for arg in sys.argv[1:]: 874 | if arg == "--insecure": 875 | try: 876 | ctx = ssl.create_default_context() 877 | ctx.check_hostname = False 878 | ctx.verify_mode = ssl.CERT_NONE 879 | self.context["context"] = ctx 880 | except AttributeError: 881 | # Python 2.6 won't complain about SSL Validation 882 | pass 883 | 884 | elif arg.endswith(".metalink") or arg.endswith(".csv"): 885 | if os.path.isfile(arg): 886 | input_files.append(arg) 887 | if arg.endswith(".metalink"): 888 | new_files = self.process_metalink(arg) 889 | else: 890 | new_files = self.process_csv(arg) 891 | if new_files is not None: 892 | for file_url in new_files: 893 | download_files.append(file_url) 894 | else: 895 | print( 896 | " > I cannot find the input file you specified: {0}".format( 897 | arg 898 | ) 899 | ) 900 | else: 901 | print( 902 | " > Command line argument '{0}' makes no sense, ignoring.".format( 903 | arg 904 | ) 905 | ) 906 | 907 | if len(input_files) > 0: 908 | if len(download_files) > 0: 909 | print( 910 | " > Processing {0} downloads from {1} input files. ".format( 911 | len(download_files), len(input_files) 912 | ) 913 | ) 914 | self.files = download_files 915 | else: 916 | print( 917 | " > I see you asked me to download files from {0} input files, but they had no downloads!".format( 918 | len(input_files) 919 | ) 920 | ) 921 | print(" > I'm super confused and exiting.") 922 | exit(-1) 923 | 924 | # Make sure cookie_jar is good to go! 925 | self.get_cookie() 926 | 927 | # summary 928 | self.total_bytes = 0 929 | self.total_time = 0 930 | self.cnt = 0 931 | self.success = [] 932 | self.failed = [] 933 | self.skipped = [] 934 | 935 | # Get and validate a cookie 936 | def get_cookie(self): 937 | if os.path.isfile(self.cookie_jar_path): 938 | self.cookie_jar = MozillaCookieJar() 939 | self.cookie_jar.load(self.cookie_jar_path) 940 | 941 | # make sure cookie is still valid 942 | if self.check_cookie(): 943 | print(" > Reusing previous cookie jar.") 944 | return True 945 | else: 946 | print(" > Could not validate old cookie Jar") 947 | 948 | # We don't have a valid cookie, prompt user or creds 949 | print( 950 | "No existing URS cookie found, please enter Earthdata username & password:" 951 | ) 952 | print("(Credentials will not be stored, saved or logged anywhere)") 953 | 954 | # Keep trying 'till user gets the right U:P 955 | while self.check_cookie() is False: 956 | self.get_new_cookie() 957 | 958 | return True 959 | 960 | # Validate cookie before we begin 961 | def check_cookie(self): 962 | 963 | if self.cookie_jar is None: 964 | print(" > Cookiejar is bunk: {0}".format(self.cookie_jar)) 965 | return False 966 | 967 | # File we know is valid, used to validate cookie 968 | file_check = "https://urs.earthdata.nasa.gov/profile" 969 | 970 | # Apply custom Redirect Hanlder 971 | opener = build_opener( 972 | HTTPCookieProcessor(self.cookie_jar), 973 | HTTPHandler(), 974 | HTTPSHandler(**self.context), 975 | ) 976 | install_opener(opener) 977 | 978 | # Attempt a HEAD request 979 | request = Request(file_check) 980 | request.get_method = lambda: "HEAD" 981 | try: 982 | print(" > attempting to download {0}".format(file_check)) 983 | response = urlopen(request, timeout=30) 984 | resp_code = response.getcode() 985 | # Make sure we're logged in 986 | if not self.check_cookie_is_logged_in(self.cookie_jar): 987 | return False 988 | 989 | # Save cookiejar 990 | self.cookie_jar.save(self.cookie_jar_path) 991 | 992 | except HTTPError: 993 | # If we ge this error, again, it likely means the user has not agreed to current EULA 994 | print("\nIMPORTANT: ") 995 | print( 996 | "Your user appears to lack permissions to download data from the ASF Datapool." 997 | ) 998 | print( 999 | "\n\nNew users: you must first log into Vertex and accept the EULA. In addition, your Study Area must be set at Earthdata https://urs.earthdata.nasa.gov" 1000 | ) 1001 | exit(-1) 1002 | 1003 | # This return codes indicate the USER has not been approved to download the data 1004 | if resp_code in (300, 301, 302, 303): 1005 | try: 1006 | redir_url = response.info().getheader("Location") 1007 | except AttributeError: 1008 | redir_url = response.getheader("Location") 1009 | 1010 | # Funky Test env: 1011 | if ( 1012 | "vertex-retired.daac.asf.alaska.edu" in redir_url 1013 | and "test" in self.asf_urs4["redir"] 1014 | ): 1015 | print("Cough, cough. It's dusty in this test env!") 1016 | return True 1017 | 1018 | print("Redirect ({0}) occured, invalid cookie value!".format(resp_code)) 1019 | return False 1020 | 1021 | # These are successes! 1022 | if resp_code in (200, 307): 1023 | return True 1024 | 1025 | return False 1026 | 1027 | def get_new_cookie(self): 1028 | # Start by prompting user to input their credentials 1029 | 1030 | # Another Python2/3 workaround 1031 | try: 1032 | new_username = raw_input("Username: ") 1033 | except NameError: 1034 | new_username = input("Username: ") 1035 | new_password = getpass.getpass(prompt="Password (will not be displayed): ") 1036 | 1037 | # Build URS4 Cookie request 1038 | auth_cookie_url = ( 1039 | self.asf_urs4["url"] 1040 | + "?client_id=" 1041 | + self.asf_urs4["client"] 1042 | + "&redirect_uri=" 1043 | + self.asf_urs4["redir"] 1044 | + "&response_type=code&state=" 1045 | ) 1046 | 1047 | try: 1048 | # python2 1049 | user_pass = base64.b64encode(bytes(new_username + ":" + new_password)) 1050 | except TypeError: 1051 | # python3 1052 | user_pass = base64.b64encode( 1053 | bytes(new_username + ":" + new_password, "utf-8") 1054 | ) 1055 | user_pass = user_pass.decode("utf-8") 1056 | 1057 | # Authenticate against URS, grab all the cookies 1058 | self.cookie_jar = MozillaCookieJar() 1059 | opener = build_opener( 1060 | HTTPCookieProcessor(self.cookie_jar), 1061 | HTTPHandler(), 1062 | HTTPSHandler(**self.context), 1063 | ) 1064 | request = Request( 1065 | auth_cookie_url, headers={"Authorization": "Basic {0}".format(user_pass)} 1066 | ) 1067 | 1068 | # Watch out cookie rejection! 1069 | try: 1070 | response = opener.open(request) 1071 | except HTTPError as e: 1072 | if ( 1073 | "WWW-Authenticate" in e.headers 1074 | and "Please enter your Earthdata Login credentials" 1075 | in e.headers["WWW-Authenticate"] 1076 | ): 1077 | print( 1078 | " > Username and Password combo was not successful. Please try again." 1079 | ) 1080 | return False 1081 | else: 1082 | # If an error happens here, the user most likely has not confirmed EULA. 1083 | print("\nIMPORTANT: There was an error obtaining a download cookie!") 1084 | print( 1085 | "Your user appears to lack permission to download data from the ASF Datapool." 1086 | ) 1087 | print( 1088 | "\n\nNew users: you must first log into Vertex and accept the EULA. In addition, your Study Area must be set at Earthdata https://urs.earthdata.nasa.gov" 1089 | ) 1090 | exit(-1) 1091 | except URLError as e: 1092 | print( 1093 | "\nIMPORTANT: There was a problem communicating with URS, unable to obtain cookie. " 1094 | ) 1095 | print("Try cookie generation later.") 1096 | exit(-1) 1097 | 1098 | # Did we get a cookie? 1099 | if self.check_cookie_is_logged_in(self.cookie_jar): 1100 | # COOKIE SUCCESS! 1101 | self.cookie_jar.save(self.cookie_jar_path) 1102 | return True 1103 | 1104 | # if we aren't successful generating the cookie, nothing will work. Stop here! 1105 | print( 1106 | "WARNING: Could not generate new cookie! Cannot proceed. Please try Username and Password again." 1107 | ) 1108 | print("Response was {0}.".format(response.getcode())) 1109 | print( 1110 | "\n\nNew users: you must first log into Vertex and accept the EULA. In addition, your Study Area must be set at Earthdata https://urs.earthdata.nasa.gov" 1111 | ) 1112 | exit(-1) 1113 | 1114 | # make sure we're logged into URS 1115 | def check_cookie_is_logged_in(self, cj): 1116 | for cookie in cj: 1117 | if cookie.name == "urs_user_already_logged": 1118 | # Only get this cookie if we logged in successfully! 1119 | return True 1120 | 1121 | return False 1122 | 1123 | # Download the file 1124 | def download_file_with_cookiejar(self, url, file_count, total, recursion=False): 1125 | # see if we've already download this file and if it is that it is the correct size 1126 | download_file = os.path.basename(url).split("?")[0] 1127 | if os.path.isfile(download_file): 1128 | try: 1129 | request = Request(url) 1130 | request.get_method = lambda: "HEAD" 1131 | response = urlopen(request, timeout=30) 1132 | remote_size = self.get_total_size(response) 1133 | # Check that we were able to derive a size. 1134 | if remote_size: 1135 | local_size = os.path.getsize(download_file) 1136 | if remote_size < ( 1137 | local_size + (local_size * 0.01) 1138 | ) and remote_size > (local_size - (local_size * 0.01)): 1139 | print( 1140 | " > Download file {0} exists! \n > Skipping download of {1}. ".format( 1141 | download_file, url 1142 | ) 1143 | ) 1144 | return None, None 1145 | # partial file size wasn't full file size, lets blow away the chunk and start again 1146 | print( 1147 | " > Found {0} but it wasn't fully downloaded. Removing file and downloading again.".format( 1148 | download_file 1149 | ) 1150 | ) 1151 | os.remove(download_file) 1152 | 1153 | except ssl.CertificateError as e: 1154 | print(" > ERROR: {0}".format(e)) 1155 | print( 1156 | " > Could not validate SSL Cert. You may be able to overcome this using the --insecure flag" 1157 | ) 1158 | return False, None 1159 | 1160 | except HTTPError as e: 1161 | if e.code == 401: 1162 | print( 1163 | " > IMPORTANT: Your user may not have permission to download this type of data!" 1164 | ) 1165 | else: 1166 | print(" > Unknown Error, Could not get file HEAD: {0}".format(e)) 1167 | 1168 | except URLError as e: 1169 | print("URL Error (from HEAD): {0}, {1}".format(e.reason, url)) 1170 | if "ssl.c" in "{0}".format(e.reason): 1171 | print( 1172 | "IMPORTANT: Remote location may not be accepting your SSL configuration. This is a terminal error." 1173 | ) 1174 | return False, None 1175 | 1176 | # attempt https connection 1177 | try: 1178 | request = Request(url) 1179 | response = urlopen(request, timeout=30) 1180 | 1181 | # Watch for redirect 1182 | if response.geturl() != url: 1183 | 1184 | # See if we were redirect BACK to URS for re-auth. 1185 | if ( 1186 | "https://urs.earthdata.nasa.gov/oauth/authorize" 1187 | in response.geturl() 1188 | ): 1189 | 1190 | if recursion: 1191 | print(" > Entering seemingly endless auth loop. Aborting. ") 1192 | return False, None 1193 | 1194 | # make this easier. If there is no app_type=401, add it 1195 | new_auth_url = response.geturl() 1196 | if "app_type" not in new_auth_url: 1197 | new_auth_url += "&app_type=401" 1198 | 1199 | print(" > While attempting to download {0}....".format(url)) 1200 | print(" > Need to obtain new cookie from {0}".format(new_auth_url)) 1201 | old_cookies = [cookie.name for cookie in self.cookie_jar] 1202 | opener = build_opener( 1203 | HTTPCookieProcessor(self.cookie_jar), 1204 | HTTPHandler(), 1205 | HTTPSHandler(**self.context), 1206 | ) 1207 | request = Request(new_auth_url) 1208 | try: 1209 | response = opener.open(request) 1210 | for cookie in self.cookie_jar: 1211 | if cookie.name not in old_cookies: 1212 | print(" > Saved new cookie: {0}".format(cookie.name)) 1213 | 1214 | # A little hack to save session cookies 1215 | if cookie.discard: 1216 | cookie.expires = ( 1217 | int(time.time()) + 60 * 60 * 24 * 30 1218 | ) 1219 | print( 1220 | " > Saving session Cookie that should have been discarded! " 1221 | ) 1222 | 1223 | self.cookie_jar.save( 1224 | self.cookie_jar_path, 1225 | ignore_discard=True, 1226 | ignore_expires=True, 1227 | ) 1228 | except HTTPError as e: 1229 | print("HTTP Error: {0}, {1}".format(e.code, url)) 1230 | return False, None 1231 | 1232 | # Okay, now we have more cookies! Lets try again, recursively! 1233 | print(" > Attempting download again with new cookies!") 1234 | return self.download_file_with_cookiejar( 1235 | url, file_count, total, recursion=True 1236 | ) 1237 | 1238 | print( 1239 | " > 'Temporary' Redirect download @ Remote archive:\n > {0}".format( 1240 | response.geturl() 1241 | ) 1242 | ) 1243 | 1244 | # seems to be working 1245 | print("({0}/{1}) Downloading {2}".format(file_count, total, url)) 1246 | 1247 | # Open our local file for writing and build status bar 1248 | tf = tempfile.NamedTemporaryFile(mode="w+b", delete=False, dir=".") 1249 | self.chunk_read(response, tf, report_hook=self.chunk_report) 1250 | 1251 | # Reset download status 1252 | sys.stdout.write("\n") 1253 | 1254 | tempfile_name = tf.name 1255 | tf.close() 1256 | 1257 | # handle errors 1258 | except HTTPError as e: 1259 | print("HTTP Error: {0}, {1}".format(e.code, url)) 1260 | 1261 | if e.code == 401: 1262 | print( 1263 | " > IMPORTANT: Your user does not have permission to download this type of data!" 1264 | ) 1265 | 1266 | if e.code == 403: 1267 | print(" > Got a 403 Error trying to download this file. ") 1268 | print(" > You MAY need to log in this app and agree to a EULA. ") 1269 | 1270 | return False, None 1271 | 1272 | except URLError as e: 1273 | print("URL Error (from GET): {0}, {1}, {2}".format(e, e.reason, url)) 1274 | if "ssl.c" in "{0}".format(e.reason): 1275 | print( 1276 | "IMPORTANT: Remote location may not be accepting your SSL configuration. This is a terminal error." 1277 | ) 1278 | return False, None 1279 | 1280 | except socket.timeout as e: 1281 | print(" > timeout requesting: {0}; {1}".format(url, e)) 1282 | return False, None 1283 | 1284 | except ssl.CertificateError as e: 1285 | print(" > ERROR: {0}".format(e)) 1286 | print( 1287 | " > Could not validate SSL Cert. You may be able to overcome this using the --insecure flag" 1288 | ) 1289 | return False, None 1290 | 1291 | # Return the file size 1292 | shutil.copy(tempfile_name, download_file) 1293 | os.remove(tempfile_name) 1294 | file_size = self.get_total_size(response) 1295 | actual_size = os.path.getsize(download_file) 1296 | if file_size is None: 1297 | # We were unable to calculate file size. 1298 | file_size = actual_size 1299 | return actual_size, file_size 1300 | 1301 | def get_redirect_url_from_error(self, error): 1302 | find_redirect = re.compile(r"id=\"redir_link\"\s+href=\"(\S+)\"") 1303 | print("error file was: {}".format(error)) 1304 | redirect_url = find_redirect.search(error) 1305 | if redirect_url: 1306 | print("Found: {0}".format(redirect_url.group(0))) 1307 | return redirect_url.group(0) 1308 | 1309 | return None 1310 | 1311 | # chunk_report taken from http://stackoverflow.com/questions/2028517/python-urllib2-progress-hook 1312 | def chunk_report(self, bytes_so_far, file_size): 1313 | if file_size is not None: 1314 | percent = float(bytes_so_far) / file_size 1315 | percent = round(percent * 100, 2) 1316 | sys.stdout.write( 1317 | " > Downloaded %d of %d bytes (%0.2f%%)\r" 1318 | % (bytes_so_far, file_size, percent) 1319 | ) 1320 | else: 1321 | # We couldn't figure out the size. 1322 | sys.stdout.write(" > Downloaded %d of unknown Size\r" % (bytes_so_far)) 1323 | 1324 | # chunk_read modified from http://stackoverflow.com/questions/2028517/python-urllib2-progress-hook 1325 | def chunk_read(self, response, local_file, chunk_size=8192, report_hook=None): 1326 | file_size = self.get_total_size(response) 1327 | bytes_so_far = 0 1328 | 1329 | while 1: 1330 | try: 1331 | chunk = response.read(chunk_size) 1332 | except: 1333 | sys.stdout.write("\n > There was an error reading data. \n") 1334 | break 1335 | 1336 | try: 1337 | local_file.write(chunk) 1338 | except TypeError: 1339 | local_file.write(chunk.decode(local_file.encoding)) 1340 | bytes_so_far += len(chunk) 1341 | 1342 | if not chunk: 1343 | break 1344 | 1345 | if report_hook: 1346 | report_hook(bytes_so_far, file_size) 1347 | 1348 | return bytes_so_far 1349 | 1350 | def get_total_size(self, response): 1351 | try: 1352 | file_size = response.info().getheader("Content-Length").strip() 1353 | except AttributeError: 1354 | try: 1355 | file_size = response.getheader("Content-Length").strip() 1356 | except AttributeError: 1357 | print("> Problem getting size") 1358 | return None 1359 | 1360 | return int(file_size) 1361 | 1362 | # Get download urls from a metalink file 1363 | def process_metalink(self, ml_file): 1364 | print("Processing metalink file: {0}".format(ml_file)) 1365 | with open(ml_file, "r") as ml: 1366 | xml = ml.read() 1367 | 1368 | # Hack to remove annoying namespace 1369 | it = ET.iterparse(StringIO(xml)) 1370 | for _, el in it: 1371 | if "}" in el.tag: 1372 | el.tag = el.tag.split("}", 1)[1] # strip all namespaces 1373 | root = it.root 1374 | 1375 | dl_urls = [] 1376 | ml_files = root.find("files") 1377 | for dl in ml_files: 1378 | dl_urls.append(dl.find("resources").find("url").text) 1379 | 1380 | if len(dl_urls) > 0: 1381 | return dl_urls 1382 | else: 1383 | return None 1384 | 1385 | # Get download urls from a csv file 1386 | def process_csv(self, csv_file): 1387 | print("Processing csv file: {0}".format(csv_file)) 1388 | 1389 | dl_urls = [] 1390 | with open(csv_file, "r") as csvf: 1391 | try: 1392 | csvr = csv.DictReader(csvf) 1393 | for row in csvr: 1394 | dl_urls.append(row["URL"]) 1395 | except csv.Error as e: 1396 | print( 1397 | "WARNING: Could not parse file %s, line %d: %s. Skipping." 1398 | % (csv_file, csvr.line_num, e) 1399 | ) 1400 | return None 1401 | except KeyError as e: 1402 | print( 1403 | "WARNING: Could not find URL column in file %s. Skipping." 1404 | % (csv_file) 1405 | ) 1406 | 1407 | if len(dl_urls) > 0: 1408 | return dl_urls 1409 | else: 1410 | return None 1411 | 1412 | # Download all the files in the list 1413 | def download_files(self): 1414 | for file_name in self.files: 1415 | 1416 | # make sure we haven't ctrl+c'd or some other abort trap 1417 | if abort == True: 1418 | raise SystemExit 1419 | 1420 | # download counter 1421 | self.cnt += 1 1422 | 1423 | # set a timer 1424 | start = time.time() 1425 | 1426 | # run download 1427 | size, total_size = self.download_file_with_cookiejar( 1428 | file_name, self.cnt, len(self.files) 1429 | ) 1430 | 1431 | # calculte rate 1432 | end = time.time() 1433 | 1434 | # stats: 1435 | if size is None: 1436 | self.skipped.append(file_name) 1437 | # Check to see that the download didn't error and is the correct size 1438 | elif size is not False and ( 1439 | total_size < (size + (size * 0.01)) 1440 | and total_size > (size - (size * 0.01)) 1441 | ): 1442 | # Download was good! 1443 | elapsed = end - start 1444 | elapsed = 1.0 if elapsed < 1 else elapsed 1445 | rate = (size / 1024**2) / elapsed 1446 | 1447 | print( 1448 | "Downloaded {0}b in {1:.2f}secs, Average Rate: {2:.2f}MB/sec".format( 1449 | size, elapsed, rate 1450 | ) 1451 | ) 1452 | 1453 | # add up metrics 1454 | self.total_bytes += size 1455 | self.total_time += elapsed 1456 | self.success.append({"file": file_name, "size": size}) 1457 | 1458 | else: 1459 | print("There was a problem downloading {0}".format(file_name)) 1460 | self.failed.append(file_name) 1461 | 1462 | def print_summary(self): 1463 | # Print summary: 1464 | print("\n\nDownload Summary ") 1465 | print( 1466 | "--------------------------------------------------------------------------------" 1467 | ) 1468 | print( 1469 | " Successes: {0} files, {1} bytes ".format( 1470 | len(self.success), self.total_bytes 1471 | ) 1472 | ) 1473 | for success_file in self.success: 1474 | print( 1475 | " - {0} {1:.2f}MB".format( 1476 | success_file["file"], (success_file["size"] / 1024.0**2) 1477 | ) 1478 | ) 1479 | if len(self.failed) > 0: 1480 | print(" Failures: {0} files".format(len(self.failed))) 1481 | for failed_file in self.failed: 1482 | print(" - {0}".format(failed_file)) 1483 | if len(self.skipped) > 0: 1484 | print(" Skipped: {0} files".format(len(self.skipped))) 1485 | for skipped_file in self.skipped: 1486 | print(" - {0}".format(skipped_file)) 1487 | if len(self.success) > 0: 1488 | print( 1489 | " Average Rate: {0:.2f}MB/sec".format( 1490 | (self.total_bytes / 1024.0**2) / self.total_time 1491 | ) 1492 | ) 1493 | print( 1494 | "--------------------------------------------------------------------------------" 1495 | ) 1496 | -------------------------------------------------------------------------------- /src/sentinel_1_python/download/satellite_download.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from ._utilities_dwl import * 4 | from .download_s1grd import * 5 | 6 | 7 | class Satellite_download: 8 | def __init__( 9 | self, 10 | metadata=None, 11 | password=os.getenv("COPERNICUS_HUP_PASSWORD"), 12 | username=os.getenv("COPERNICUS_HUP_USERNAME"), 13 | ): 14 | super(Satellite_download, self).__init__() 15 | 16 | self.PASSWORD = password 17 | self.USERNAME = username 18 | self.products_df = metadata 19 | 20 | def __enter__(self): 21 | return self 22 | 23 | def __exit__(self, exception_type, exception_value, traceback): 24 | pass 25 | 26 | def download_sentinel_1(self, data_folder: str = "sentinel_images"): 27 | 28 | if self.products_df.shape[0] > 0: 29 | download_sentinel_1_function(self.products_df, data_folder) 30 | else: 31 | print("\n(note): No products") 32 | 33 | def download_thumbnails(self, index: list = None, folder="s1_thumnails"): 34 | download_thumbnails_function( 35 | self.products_df, 36 | index, 37 | folder, 38 | username=self.USERNAME, 39 | password=self.PASSWORD, 40 | ) 41 | -------------------------------------------------------------------------------- /src/sentinel_1_python/metadata/__init__.py: -------------------------------------------------------------------------------- 1 | from ._masking import * 2 | from ._utilities import * 3 | from .query_data import * 4 | from .sentinel_metadata import * -------------------------------------------------------------------------------- /src/sentinel_1_python/metadata/_masking.py: -------------------------------------------------------------------------------- 1 | import geopandas as gpd 2 | 3 | 4 | def inwater(products_df, landmask_file: str = "landmask/simplified_land_polygons.shp"): 5 | landmasks = get_mask(landmask_file) 6 | products_df = gpd.overlay( 7 | products_df, landmasks, how="difference", keep_geom_type=False 8 | ) 9 | return products_df 10 | 11 | 12 | def inland(products_df, landmask_file: str = "landmask/simplified_land_polygons.shp"): 13 | landmasks = get_mask(landmask_file) 14 | products_df = gpd.overlay( 15 | products_df, landmasks, how="intersection", keep_geom_type=False 16 | ) 17 | return products_df 18 | 19 | 20 | def get_mask(mask: str = None): 21 | """ 22 | Fetching the landmask shape file. This is turned into a geopanda dataframe.. 23 | """ 24 | try: 25 | mask = gpd.read_file(mask) 26 | mask = mask.set_crs("epsg:3857", allow_override=True) # 32634 27 | 28 | except: 29 | mask = gpd.read_file(mask) 30 | pass 31 | mask = mask.to_crs(crs=4326) 32 | return mask 33 | 34 | 35 | def landmask_df(gdf, mask): 36 | """ 37 | Removing land from the original gdf using the landmask gdf. 38 | If youre looking for habours, dont use this.. Lookin for ships, use this. 39 | 40 | """ 41 | gdf_landmask = gpd.overlay(gdf, mask, how="difference", keep_geom_type=False) 42 | 43 | return gdf_landmask 44 | -------------------------------------------------------------------------------- /src/sentinel_1_python/metadata/_utilities.py: -------------------------------------------------------------------------------- 1 | import mgrs 2 | import numpy as np 3 | from sentinelsat import geojson_to_wkt 4 | 5 | 6 | def intersection(lst1, lst2): 7 | 8 | # Use of hybrid method 9 | temp = set(lst2) 10 | lst3 = [value for value in lst1 if value in temp] 11 | return lst3 12 | 13 | 14 | def intersect_ids(products_df): 15 | names = np.array( 16 | [ 17 | name.split("_")[0] 18 | + name.split("_")[1] 19 | + name.split("_")[-3] 20 | + name.split("_")[-2] 21 | for name in products_df.title.values 22 | ] 23 | ) 24 | missionid = products_df.missiondatatakeid.values 25 | names = [name + "_" + str(missionid[ix]) for ix, name in enumerate(names)] 26 | return names 27 | 28 | 29 | def add_mgcs(df): 30 | try: 31 | del df["mgcs"] 32 | except: 33 | pass 34 | m = mgrs.MGRS() 35 | lon = df.geometry.to_crs(4326).centroid.x.values 36 | lat = df.geometry.to_crs(4326).centroid.y.values 37 | mgc = [] 38 | for i in range(len(lat)): 39 | mg = (m.toMGRS(lat[i], lon[i]))[0:] 40 | mgc.append(mg[0:5]) 41 | mgc = np.array(mgc) 42 | df["mgcs"] = mgc 43 | 44 | return df 45 | 46 | 47 | def get_area(bbox: list = []): 48 | # getting area func 49 | area = { 50 | "coordinates": [ 51 | [ 52 | [bbox[0], bbox[2]], 53 | [bbox[0], bbox[3]], 54 | [bbox[1], bbox[3]], 55 | [bbox[1], bbox[2]], 56 | [bbox[0], bbox[2]], 57 | ] 58 | ], 59 | "type": "Polygon", 60 | } 61 | footprint = geojson_to_wkt(area) 62 | return footprint 63 | 64 | 65 | def original_metadata(self): 66 | self.products_df = self.org_products_df 67 | 68 | return self 69 | -------------------------------------------------------------------------------- /src/sentinel_1_python/metadata/query_data.py: -------------------------------------------------------------------------------- 1 | from sentinelsat import SentinelAPI 2 | 3 | from ._utilities import add_mgcs 4 | 5 | 6 | def get_s2_metadata_area( 7 | footprint, 8 | username, 9 | password, 10 | start_data: str = "20151023", 11 | end_date: str = "20151025", 12 | ): 13 | # getting metadata s2 func 14 | api = api = SentinelAPI(username, password, "https://scihub.copernicus.eu/dhus/") 15 | products = api.query( 16 | footprint, date=(start_data, end_date), platformname="Sentinel-2" 17 | ) 18 | products_df = api.to_geodataframe(products) 19 | # products_df = Satellite_metadata.add_mgcs(products_df) 20 | return products_df 21 | 22 | 23 | def get_s1_slc_metadata_area( 24 | footprint, 25 | username, 26 | password, 27 | start_data: str = "20151023", 28 | end_date: str = "20151025", 29 | ): 30 | # getting metadata s1 slc func 31 | api = api = SentinelAPI(username, password, "https://scihub.copernicus.eu/dhus/") 32 | products = api.query( 33 | footprint, 34 | date=(start_data, end_date), 35 | producttype="SLC", 36 | platformname="Sentinel-1", 37 | ) 38 | products_df = api.to_geodataframe(products) 39 | # products_df = Satellite_metadata.add_mgcs(products_df) 40 | return products_df 41 | 42 | 43 | def get_s1_raw_metadata_area( 44 | footprint, 45 | username, 46 | password, 47 | start_data: str = "20151023", 48 | end_date: str = "20151025", 49 | ): 50 | # getting metadata s1 slc func 51 | api = api = SentinelAPI(username, password, "https://scihub.copernicus.eu/dhus/") 52 | products = api.query( 53 | footprint, 54 | date=(start_data, end_date), 55 | producttype="RAW", 56 | platformname="Sentinel-1", 57 | ) 58 | products_df = api.to_geodataframe(products) 59 | 60 | return products_df 61 | 62 | 63 | def get_s1_grd_metadata_area( 64 | footprint, 65 | username, 66 | password, 67 | start_data: str = "20151023", 68 | end_date: str = "20151025", 69 | ): 70 | # getting metadata s1 grd func 71 | api = api = SentinelAPI(username, password, "https://scihub.copernicus.eu/dhus/") 72 | products = api.query( 73 | footprint, 74 | date=(start_data, end_date), 75 | producttype="GRD", 76 | platformname="Sentinel-1", 77 | ) 78 | products_df = api.to_geodataframe(products) 79 | products_df = add_mgcs(products_df) 80 | return products_df 81 | 82 | 83 | 84 | def get_s2_l2a_metadata_area( 85 | footprint, 86 | username, 87 | password, 88 | start_data: str = "20151023", 89 | end_date: str = "20151025", 90 | ): 91 | # getting metadata s1 slc func 92 | api = api = SentinelAPI(username, password, "https://scihub.copernicus.eu/dhus/") 93 | products = api.query( 94 | footprint, 95 | date=(start_data, end_date), 96 | producttype="Level-2A", 97 | platformname="Sentinel-2", 98 | ) 99 | products_df = api.to_geodataframe(products) 100 | 101 | return products_df 102 | -------------------------------------------------------------------------------- /src/sentinel_1_python/metadata/sentinel_metadata.py: -------------------------------------------------------------------------------- 1 | from ._masking import * 2 | from ..visualize import show 3 | from ._utilities import * 4 | from .query_data import * 5 | import getpass 6 | 7 | 8 | class Sentinel_metadata: 9 | """'""" 10 | 11 | def __init__(self, fontsize: int = 32): 12 | super(Sentinel_metadata, self).__init__() 13 | 14 | try: 15 | new_username = raw_input("Username for Copernicus Hub: ") 16 | except NameError: 17 | new_username = input("Username for Copernicus Hub: ") 18 | new_password = getpass.getpass(prompt="Password (will not be displayed): ") 19 | 20 | self.USERNAME = new_username 21 | self.PASSWORD = new_password 22 | 23 | show.initi(fontsize) 24 | 25 | def __enter__(self): 26 | return self 27 | 28 | def __exit__(self, exception_type, exception_value, traceback): 29 | pass 30 | 31 | def get_metadata( 32 | self, sensor: str = "s1_slc", start_data: str = "", end_date: str = "" 33 | ): 34 | 35 | # getting metadata 36 | 37 | if sensor.lower() in ["s1_slc", "sentinel1_slc", "sentinel-1_slc"]: 38 | self.products_df = get_s1_slc_metadata_area( 39 | self.bbox, self.USERNAME, self.PASSWORD, start_data, end_date 40 | ) 41 | self.sensor = "sentinel_1_slc" 42 | 43 | if sensor.lower() in ["s1_raw", "sentinel1_level0", "s1_l0", "raw"]: 44 | try: 45 | self.products_df = get_s1_raw_metadata_area( 46 | self.bbox, self.USERNAME, self.PASSWORD, start_data, end_date 47 | ) 48 | except: 49 | pass 50 | self.sensor = "sentinel_1_raw" 51 | 52 | elif sensor.lower() in ["s1_grd", "sentinel1_grd", "sentinel-1_grd"]: 53 | self.products_df = get_s1_grd_metadata_area( 54 | self.bbox, self.USERNAME, self.PASSWORD, start_data, end_date 55 | ) 56 | self.sensor = "sentinel_1_grd" 57 | elif sensor.lower() in ["s2_l1c", "sentinel2_l1c", "sentinel-2_l1c"]: 58 | self.products_df = get_s2_metadata_area( 59 | self.bbox, self.USERNAME, self.PASSWORD, start_data, end_date 60 | ) 61 | self.sensor = "sentinel_2_l1c" 62 | 63 | elif sensor.lower() in ["s2_l2a", "sentinel2_l2a", "sentinel-2_l2a"]: 64 | self.products_df = get_s2_l2a_metadata_area( 65 | self.bbox, self.USERNAME, self.PASSWORD, start_data, end_date 66 | ) 67 | self.sensor = "sentinel_2_l2a" 68 | 69 | self.org_products_df = self.products_df 70 | 71 | return self 72 | 73 | def area(self, bbox: list = []): 74 | # getting area 75 | """lonmin, lonmax, latmin,latmax""" 76 | self.bbox = get_area(bbox) 77 | 78 | return self 79 | 80 | def show_thumnails(self, amount=10): 81 | show.show_thumbnail_function( 82 | self.products_df, 83 | amount=amount, 84 | username=self.USERNAME, 85 | password=self.PASSWORD, 86 | ) 87 | return None 88 | 89 | def show_cross_pol(self, amount=10): 90 | if self.sensor in ["sentinel_1_grd", "sentinel_1_slc", "sentinel_1_raw"]: 91 | show.show_cross_pol_function( 92 | self.products_df, 93 | amount=amount, 94 | username=self.USERNAME, 95 | password=self.PASSWORD, 96 | ) 97 | return None 98 | 99 | def show_co_pol(self, amount=10): 100 | if self.sensor in ["sentinel_1_grd", "sentinel_1_slc", "sentinel_1_raw"]: 101 | show.show_co_pol_function( 102 | self.products_df, 103 | amount=amount, 104 | username=self.USERNAME, 105 | password=self.PASSWORD, 106 | ) 107 | return None 108 | 109 | def land(self): 110 | self.products_df = inland(self.products_df) 111 | 112 | def water(self): 113 | self.products_df = inwater(self.products_df) 114 | 115 | def plot_image_areas(self): 116 | show.plot_polygon(self.products_df) 117 | 118 | def cloud_cover(self, cloud_cover: float = 1): 119 | if self.sensor in ["sentinel_2_l1c", "sentinel_2_l2a"]: 120 | self.products_df = self.products_df[ 121 | self.products_df.cloudcoverpercentage < cloud_cover 122 | ] 123 | 124 | def vv(self): 125 | if self.sensor in ["sentinel_1_grd", "sentinel_1_slc", "sentinel_1_raw"]: 126 | self.products_df = self.products_df[ 127 | self.products_df.polarisationmode == "VV VH" 128 | ] 129 | 130 | def iw(self): 131 | if self.sensor in ["sentinel_1_grd", "sentinel_1_slc", "sentinel_1_raw"]: 132 | self.products_df = self.products_df[ 133 | self.products_df.sensoroperationalmode == "IW" 134 | ] 135 | 136 | def ew(self): 137 | if self.sensor in ["sentinel_1_grd", "sentinel_1_slc", "sentinel_1_raw"]: 138 | self.products_df = self.products_df[ 139 | self.products_df.sensoroperationalmode == "EW" 140 | ] 141 | 142 | def hh(self): 143 | if self.sensor in ["sentinel_1_grd", "sentinel_1_slc", "sentinel_1_raw"]: 144 | self.products_df = self.products_df[ 145 | self.products_df.polarisationmode == "HH HV" 146 | ] 147 | -------------------------------------------------------------------------------- /src/sentinel_1_python/pre_process_grd/Process.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import copy 3 | import os 4 | import pickle 5 | import warnings 6 | import matplotlib.pyplot as plt 7 | from . import _get_functions as get_functions 8 | from . import _proces_tools as tools 9 | from . import filters 10 | 11 | # TODO: Decide the amount of checking and control in the class 12 | 13 | 14 | class SarImage: 15 | """Class to contain SAR image, relevant meta data and methods. 16 | Attributes: 17 | bands(list of numpy arrays): The measurements. 18 | mission(str): Mission name: 19 | time(datetime): start time of acquisition 20 | footprint(dict): dictionary with footprint of image 21 | footprint = {'latitude': np.array 22 | 'longitude': np.array} 23 | product_meta(dict): Dictionary with meta data. 24 | band_names(list of str): Names of the band. Normally the polarisation. 25 | calibration_tables(list of dict): Dictionary with calibration_tables information for each band. 26 | geo_tie_point(list of dict): Dictionary with geo tie point for each band. 27 | band_meta(list of dict): Dictionary with meta data for each band. 28 | """ 29 | 30 | def __init__( 31 | self, 32 | bands, 33 | mission=None, 34 | time=None, 35 | footprint=None, 36 | product_meta=None, 37 | band_names=None, 38 | calibration_tables=None, 39 | geo_tie_point=None, 40 | band_meta=None, 41 | unit=None, 42 | ): 43 | 44 | # assign values 45 | self.bands = bands 46 | self.mission = mission 47 | self.time = time 48 | self.footprint = footprint 49 | self.product_meta = product_meta 50 | self.band_names = band_names 51 | self.calibration_tables = calibration_tables 52 | self.geo_tie_point = geo_tie_point 53 | self.band_meta = band_meta 54 | self.unit = unit 55 | # Note that SlC is in strips. Maybe load as list of images 56 | 57 | def __repr__(self): 58 | return "Mission: %s \n Bands: %s" % (self.mission, str(self.band_names)) 59 | 60 | def __getitem__(self, key): 61 | # Overload the get and slicing function a[2,4] a[:10,3:34] 62 | 63 | # Check i 2 dimension are given 64 | if len(key) != 2: 65 | raise ValueError("Need to slice both column and row like test_image[:,:]") 66 | 67 | # Get values as array 68 | if isinstance(key[0], int) & isinstance(key[1], int): 69 | return [band[key] for band in self.bands] 70 | 71 | if not isinstance(key[0], slice) & isinstance(key[1], slice): 72 | raise ValueError("Only get at slice is supported: a[2,4] a[:10,3:34]") 73 | 74 | # Else. Try to slice the image 75 | slice_row = key[0] 76 | slice_column = key[1] 77 | 78 | row_start = slice_row.start 79 | row_step = slice_row.step 80 | row_stop = slice_row.stop 81 | 82 | column_start = slice_column.start 83 | column_step = slice_column.step 84 | column_stop = slice_column.stop 85 | 86 | if row_start is None: 87 | row_start = 0 88 | if row_step is None: 89 | row_step = 1 90 | if row_stop is None: 91 | row_stop = self.bands[0].shape[0] 92 | if column_start is None: 93 | column_start = 0 94 | if column_step is None: 95 | column_step = 1 96 | if column_stop is None: 97 | column_stop = self.bands[0].shape[1] 98 | 99 | # Adjust footprint to window 100 | footprint_lat = np.zeros(4) 101 | footprint_long = np.zeros(4) 102 | 103 | window = ((row_start, row_stop), (column_start, column_stop)) 104 | 105 | for i in range(2): 106 | for j in range(2): 107 | lat_i, long_i = self.get_coordinate(window[0][i], window[1][j]) 108 | footprint_lat[2 * i + j] = lat_i 109 | footprint_long[2 * i + j] = long_i 110 | 111 | footprint = {"latitude": footprint_lat, "longitude": footprint_long} 112 | 113 | # Adjust geo_tie_point, calibration_tables 114 | n_bands = len(self.bands) 115 | geo_tie_point = copy.deepcopy(self.geo_tie_point) 116 | calibration_tables = copy.deepcopy(self.calibration_tables) 117 | for i in range(n_bands): 118 | geo_tie_point[i]["row"] = (geo_tie_point[i]["row"] - row_start) / row_step 119 | geo_tie_point[i]["column"] = ( 120 | geo_tie_point[i]["column"] - column_start 121 | ) / column_step 122 | 123 | calibration_tables[i]["row"] = ( 124 | calibration_tables[i]["row"] - row_start 125 | ) / row_step 126 | calibration_tables[i]["column"] = ( 127 | calibration_tables[i]["column"] - column_start 128 | ) / column_step 129 | 130 | # slice the bands 131 | bands = [band[key] for band in self.bands] 132 | 133 | return SarImage( 134 | bands, 135 | mission=self.mission, 136 | time=self.time, 137 | footprint=footprint, 138 | product_meta=self.product_meta, 139 | band_names=self.band_names, 140 | calibration_tables=calibration_tables, 141 | geo_tie_point=geo_tie_point, 142 | band_meta=self.band_meta, 143 | ) 144 | 145 | def get_index(self, lat, long): 146 | """Get index of a location by interpolating grid-points 147 | Args: 148 | lat(number): Latitude of the location 149 | long(number): Longitude of location 150 | Returns: 151 | row(int): The row index of the location 152 | column(int): The column index of the location 153 | Raises: 154 | """ 155 | geo_tie_point = self.geo_tie_point 156 | row = np.zeros(len(geo_tie_point), dtype=int) 157 | column = np.zeros(len(geo_tie_point), dtype=int) 158 | 159 | # find index for each band 160 | for i in range(len(geo_tie_point)): 161 | lat_grid = geo_tie_point[i]["latitude"] 162 | long_grid = geo_tie_point[i]["longitude"] 163 | row_grid = geo_tie_point[i]["row"] 164 | column_grid = geo_tie_point[i]["column"] 165 | row[i], column[i] = get_functions.get_index_v2( 166 | lat, long, lat_grid, long_grid, row_grid, column_grid 167 | ) 168 | 169 | # check that the results are the same 170 | if (abs(row.max() - row.min()) > 0.5) or ( 171 | abs(column.max() - column.min()) > 0.5 172 | ): 173 | warnings.warn( 174 | "Warning different index found for each band. First index returned" 175 | ) 176 | 177 | return row[0], column[0] 178 | 179 | def get_coordinate(self, row, column): 180 | """Get coordinate from index by interpolating grid-points 181 | Args: 182 | row(number): index of the row of interest position 183 | column(number): index of the column of interest position 184 | Returns: 185 | lat(float): Latitude of the position 186 | long(float): longitude of the position 187 | Raises: 188 | """ 189 | 190 | geo_tie_point = self.geo_tie_point 191 | lat = np.zeros(len(geo_tie_point), dtype=float) 192 | long = np.zeros(len(geo_tie_point), dtype=float) 193 | 194 | # find index for each band 195 | for i in range(len(geo_tie_point)): 196 | lat_grid = geo_tie_point[i]["latitude"] 197 | long_grid = geo_tie_point[i]["longitude"] 198 | row_grid = geo_tie_point[i]["row"] 199 | column_grid = geo_tie_point[i]["column"] 200 | lat[i], long[i] = get_functions.get_coordinate( 201 | row, column, lat_grid, long_grid, row_grid, column_grid 202 | ) 203 | 204 | # check that the results are the same 205 | if (abs(lat.max() - lat.min()) > 0.001) or ( 206 | abs(long.max() - long.min()) > 0.001 207 | ): 208 | warnings.warn( 209 | "Warning different coordinates found for each band. Mean returned" 210 | ) 211 | 212 | return lat.mean(), long.mean() 213 | 214 | def simple_plot(self, band_index=0, q_max=0.95, stride=1, **kwargs): 215 | """Makes a simple image of band and a color bar. 216 | Args: 217 | band_index(int): index of the band to plot. 218 | q_max(number): q_max is the quantile used to set the max of the color range for example 219 | q_max = 0.95 shows the lowest 95 percent of pixel values in the color range 220 | stride(int): Used to skip pixels when showing. Good for large images. 221 | **kwargs: Passed on to matplotlib imshow 222 | Returns: 223 | Raises: 224 | """ 225 | v_max = np.quantile(self.bands[band_index].reshape(-1), q_max) 226 | 227 | plt.imshow( 228 | self.bands[band_index][::stride, ::stride], 229 | cmap="gray", 230 | vmax=v_max, 231 | **kwargs 232 | ) 233 | plt.colorbar() 234 | plt.show() 235 | 236 | return 237 | 238 | def calibrate(self, mode="gamma", tiles=4): 239 | """Get coordinate from index by interpolating grid-points 240 | Args: 241 | mode(string): 'sigma_0', 'beta' or 'gamma' 242 | tiles(int): number of tiles the image is divided into. This saves memory but reduce speed a bit 243 | Returns: 244 | Calibrated image as (SarImage) 245 | Raises: 246 | """ 247 | if "raw" not in self.unit: 248 | warnings.warn( 249 | "Raw is not in units. The image have all ready been calibrated" 250 | ) 251 | 252 | calibrated_bands = [] 253 | for i, band in enumerate(self.bands): 254 | row = self.calibration_tables[i]["row"] 255 | column = self.calibration_tables[i]["column"] 256 | calibration_values = self.calibration_tables[i][mode] 257 | calibrated_bands.append( 258 | tools.calibration(band, row, column, calibration_values, tiles=tiles) 259 | ) 260 | 261 | return SarImage( 262 | calibrated_bands, 263 | mission=self.mission, 264 | time=self.time, 265 | footprint=self.footprint, 266 | product_meta=self.product_meta, 267 | band_names=self.band_names, 268 | calibration_tables=self.calibration_tables, 269 | geo_tie_point=self.geo_tie_point, 270 | band_meta=self.band_meta, 271 | unit=mode, 272 | ) 273 | 274 | def to_db(self): 275 | """Convert to decibel""" 276 | db_bands = [] 277 | for band in self.bands: 278 | if "amplitude" in self.unit: 279 | db_bands.append(20 * np.log(band)) 280 | else: 281 | db_bands.append(10 * np.log(band)) 282 | 283 | return SarImage( 284 | db_bands, 285 | mission=self.mission, 286 | time=self.time, 287 | footprint=self.footprint, 288 | product_meta=self.product_meta, 289 | band_names=self.band_names, 290 | calibration_tables=self.calibration_tables, 291 | geo_tie_point=self.geo_tie_point, 292 | band_meta=self.band_meta, 293 | unit=(self.unit + " dB"), 294 | ) 295 | 296 | def boxcar(self, kernel_size, **kwargs): 297 | """Simple (kernel_size x kernel_size) boxcar filter. 298 | Args: 299 | kernel_size(int): size of kernel 300 | **kwargs: Additional arguments passed to scipy.ndimage.convolve 301 | Returns: 302 | Filtered image 303 | """ 304 | 305 | filter_bands = [] 306 | for band in self.bands: 307 | filter_bands.append(filters.boxcar(band, kernel_size, **kwargs)) 308 | 309 | return SarImage( 310 | filter_bands, 311 | mission=self.mission, 312 | time=self.time, 313 | footprint=self.footprint, 314 | product_meta=self.product_meta, 315 | band_names=self.band_names, 316 | calibration_tables=self.calibration_tables, 317 | geo_tie_point=self.geo_tie_point, 318 | band_meta=self.band_meta, 319 | unit=self.unit, 320 | ) 321 | 322 | def save(self, path): 323 | """Save the SarImage object in a folder at path. 324 | Args: 325 | path(str): Path of the folder where the the SarImage is saved. 326 | Note that the folder is created and must not exist in advance 327 | Raises: 328 | ValueError: There already exist a folder at path 329 | """ 330 | 331 | # Check if folder exists 332 | if os.path.exists(path): 333 | print("please give a path that is not used") 334 | raise ValueError 335 | 336 | # make folder 337 | os.makedirs(path) 338 | 339 | # save elements in separate files 340 | 341 | # product_meta 342 | file_path = os.path.join(path, "product_meta.pkl") 343 | pickle.dump(self.product_meta, open(file_path, "wb")) 344 | 345 | # unit 346 | file_path = os.path.join(path, "unit.pkl") 347 | pickle.dump(self.unit, open(file_path, "wb")) 348 | 349 | # footprint 350 | file_path = os.path.join(path, "footprint.pkl") 351 | pickle.dump(self.footprint, open(file_path, "wb")) 352 | 353 | # geo_tie_point 354 | file_path = os.path.join(path, "geo_tie_point.pkl") 355 | pickle.dump(self.geo_tie_point, open(file_path, "wb")) 356 | 357 | # band_names 358 | file_path = os.path.join(path, "band_names.pkl") 359 | pickle.dump(self.band_names, open(file_path, "wb")) 360 | 361 | # band_meta 362 | file_path = os.path.join(path, "band_meta.pkl") 363 | pickle.dump(self.band_meta, open(file_path, "wb")) 364 | 365 | # bands 366 | file_path = os.path.join(path, "bands.pkl") 367 | pickle.dump(self.bands, open(file_path, "wb")) 368 | 369 | # reduce size of calibration_tables list 370 | reduced_calibration = [] 371 | for i in range(len(self.bands)): 372 | cal = self.calibration_tables[i] 373 | 374 | # Get mask of rows in the image. 375 | index_row = (0 < cal["row"]) & (cal["row"] < self.bands[i].shape[0]) 376 | # Include one extra row on each side of the image to ensure interpolation 377 | index_row[1:] = index_row[1:] + index_row[:-1] 378 | index_row[:-1] = index_row[:-1] + index_row[1:] 379 | 380 | # Get mask of column in the image 381 | index_column = (0 < cal["column"]) & ( 382 | cal["column"] < self.bands[i].shape[1] 383 | ) 384 | # Include one extra column on each side of the image to ensure interpolation 385 | index_column[1:] = index_column[1:] + index_column[:-1] 386 | index_column[:-1] = index_column[:-1] + index_column[1:] 387 | 388 | # Get the relevant calibration_tables values 389 | reduced_cal_i = { 390 | "abs_calibration_const": cal["abs_calibration_const"], 391 | "row": cal["row"][index_row], 392 | "column": cal["column"][index_column], 393 | "azimuth_time": cal["azimuth_time"][index_row, :][:, index_column], 394 | "sigma_0": cal["sigma_0"][index_row, :][:, index_column], 395 | "beta_0": cal["beta_0"][index_row, :][:, index_column], 396 | "gamma": cal["gamma"][index_row, :][:, index_column], 397 | "dn": cal["dn"][index_row, :][:, index_column], 398 | } 399 | 400 | reduced_calibration.append(reduced_cal_i) 401 | 402 | # calibration_tables 403 | file_path = os.path.join(path, "calibration_tables.pkl") 404 | pickle.dump(reduced_calibration, open(file_path, "wb")) 405 | 406 | return 407 | 408 | def pop(self, index=-1): 409 | """ 410 | Remove and return band at index (default last). 411 | Raises IndexError if list is empty or index is out of range. 412 | """ 413 | 414 | band = self.bands.pop(index) 415 | name = self.band_names.pop(index) 416 | calibration_tables = self.calibration_tables.pop(index) 417 | geo_tie_point = self.geo_tie_point.pop(index) 418 | band_meta = self.band_meta.pop() 419 | 420 | return SarImage( 421 | [band], 422 | mission=self.mission, 423 | time=self.time, 424 | footprint=self.footprint, 425 | product_meta=self.product_meta, 426 | band_names=[name], 427 | calibration_tables=[calibration_tables], 428 | geo_tie_point=[geo_tie_point], 429 | band_meta=[band_meta], 430 | unit=self.unit, 431 | ) 432 | 433 | def get_band(self, index): 434 | """ 435 | Return SarImage of band at index (default last). 436 | """ 437 | 438 | band = self.bands[index] 439 | name = self.band_names[index] 440 | calibration_tables = self.calibration_tables[index] 441 | geo_tie_point = self.geo_tie_point[index] 442 | band_meta = self.band_meta[index] 443 | 444 | return SarImage( 445 | [band], 446 | mission=self.mission, 447 | time=self.time, 448 | footprint=self.footprint, 449 | product_meta=self.product_meta, 450 | band_names=[name], 451 | calibration_tables=[calibration_tables], 452 | geo_tie_point=[geo_tie_point], 453 | band_meta=[band_meta], 454 | unit=self.unit, 455 | ) 456 | -------------------------------------------------------------------------------- /src/sentinel_1_python/pre_process_grd/README.md: -------------------------------------------------------------------------------- 1 | 2 | # Pre-process Sentine-1 images.. 3 | 4 | This module is implemented based on the https://github.com/eyhl/sarpy repo. Credits goes mostly to them. 5 | 6 | (need waay for description here...) -------------------------------------------------------------------------------- /src/sentinel_1_python/pre_process_grd/__init__.py: -------------------------------------------------------------------------------- 1 | from ._get_functions import * 2 | from .load_data import * 3 | from ._load_image import * 4 | from ._proces_tools import * 5 | from .filters import * 6 | from .Process import * 7 | 8 | #### 9 | # The pre_process module is HEAVILY dependant on the help from Eigil Lippert (DTU Space) and Simon Lupembda (EUMETSAT) 10 | # https://github.com/eyhl/sarpy/ 11 | # 12 | ### -------------------------------------------------------------------------------- /src/sentinel_1_python/pre_process_grd/_get_functions.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from scipy import interpolate 3 | from scipy.optimize import minimize 4 | 5 | 6 | def get_coordinate( 7 | row, column, lat_gridpoints, long_gridpoints, row_gridpoints, column_gridpoints 8 | ): 9 | """Get coordinate from index by interpolating grid-points 10 | Args: 11 | row(number): index of the row of interest position 12 | column(number): index of the column of interest position 13 | lat_gridpoints(numpy array of length n): Latitude of grid-points 14 | long_gridpoints(numpy array of length n): Longitude of grid-points 15 | row_gridpoints(numpy array of length n): row of grid-points 16 | column_gridpoints(numpy array of length n): column of grid-points 17 | Returns: 18 | lat(float): Latitude of the position 19 | long(float): longitude of the position 20 | Raises: 21 | """ 22 | 23 | # Create interpolate functions 24 | points = np.vstack([row_gridpoints, column_gridpoints]).transpose() 25 | lat = float(interpolate.griddata(points, lat_gridpoints, (row, column))) 26 | long = float(interpolate.griddata(points, long_gridpoints, (row, column))) 27 | return lat, long 28 | 29 | 30 | def get_index_v1( 31 | lat, long, lat_gridpoints, long_gridpoints, row_gridpoints, column_gridpoints 32 | ): 33 | """Get index of a location by interpolating grid-points 34 | Args: 35 | lat(number): Latitude of the location 36 | long(number): Longitude of location 37 | lat_gridpoints(numpy array of length n): Latitude of grid-points 38 | long_gridpoints(numpy array of length n): Longitude of grid-points 39 | row_gridpoints(numpy array of length n): row of grid-points 40 | column_gridpoints(numpy array of length n): column of grid-points 41 | Returns: 42 | row(int): The row index of the location 43 | column(int): The column index of the location 44 | Raises: 45 | """ 46 | 47 | points = np.vstack([lat_gridpoints, long_gridpoints]).transpose() 48 | row = int(np.round(interpolate.griddata(points, row_gridpoints, (lat, long)))) 49 | column = int(np.round(interpolate.griddata(points, column_gridpoints, (lat, long)))) 50 | return row, column 51 | 52 | 53 | def get_index_v2( 54 | lat, long, lat_gridpoints, long_gridpoints, row_gridpoints, column_gridpoints 55 | ): 56 | """ 57 | Same as "get_index_v1" but consistent with "get_coordinate". Drawback is that it is slower 58 | """ 59 | 60 | # Get an initial guess 61 | row_i, column_i = get_index_v1( 62 | lat, long, lat_gridpoints, long_gridpoints, row_gridpoints, column_gridpoints 63 | ) 64 | 65 | # Define a loss function 66 | def loss_function(index): 67 | lat_res, long_res = get_coordinate( 68 | index[0], 69 | index[1], 70 | lat_gridpoints, 71 | long_gridpoints, 72 | row_gridpoints, 73 | column_gridpoints, 74 | ) 75 | return ((lat - lat_res) * 100) ** 2 + ((long - long_res) * 100) ** 2 76 | 77 | # Find the index where "get_coordinate" gives the closest coordinates 78 | res = minimize(loss_function, [row_i, column_i]) 79 | 80 | return int(round(res.x[0])), int(round(res.x[1])) 81 | -------------------------------------------------------------------------------- /src/sentinel_1_python/pre_process_grd/_load_image.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pickle 3 | import warnings 4 | from itertools import compress 5 | import matplotlib 6 | import numpy as np 7 | from . import load_data 8 | from ._get_functions import get_coordinate, get_index_v2 9 | from .Process import SarImage 10 | 11 | 12 | def s1_load(path, polarisation="all", location=None, size=None): 13 | """Function to load SAR image into SarImage python object. 14 | Currently supports: unzipped Sentinel 1 GRDH products 15 | Args: 16 | path(number): Path to the folder containing the SAR image 17 | as retrieved from: https://scihub.copernicus.eu/ 18 | polarisation(list of str): List of polarisations to load. 19 | location(array/list): [latitude,longitude] Location to center the image. 20 | If None the entire Image is loaded 21 | size(array/list): [width, height] Extend of image to load. 22 | If None the entire Image is loaded 23 | Returns: 24 | SarImage: object with the SAR measurements and meta data from path. Meta data index 25 | and foot print are adjusted to the window 26 | Raises: 27 | ValueError: Location not in image 28 | """ 29 | # manifest.safe 30 | path_safe = os.path.join(path, "manifest.safe") 31 | meta, error = load_data._load_meta(path_safe) 32 | 33 | # annotation 34 | ls_annotation = os.listdir(os.path.join(path, "annotation")) 35 | xml_files = [file[-3:] == "xml" for file in ls_annotation] 36 | xml_files = list(compress(ls_annotation, xml_files)) 37 | annotation_temp = [ 38 | load_data._load_annotation(os.path.join(path, "annotation", file)) 39 | for file in xml_files 40 | ] 41 | 42 | # calibration_tables 43 | path_cal = os.path.join(path, "annotation", "calibration") 44 | ls_cal = os.listdir(path_cal) 45 | cal_files = [file[:11] == "calibration" for file in ls_cal] 46 | cal_files = list(compress(ls_cal, cal_files)) 47 | calibration_temp = [ 48 | load_data._load_calibration(os.path.join(path_cal, file)) for file in cal_files 49 | ] 50 | 51 | # measurement 52 | measurement_path = os.path.join(path, "measurement") 53 | ls_meas = os.listdir(measurement_path) 54 | tiff_files = [file[-4:] == "tiff" for file in ls_meas] 55 | tiff_files = list(compress(ls_meas, tiff_files)) 56 | with warnings.catch_warnings(): # Ignore the "NotGeoreferencedWarning" when opening the tiff 57 | warnings.simplefilter("ignore") 58 | measurement_temp = [ 59 | matplotlib.open(os.path.join(measurement_path, file)) for file in tiff_files 60 | ] 61 | 62 | # Check if polarisation is given 63 | if polarisation == "all": 64 | polarisation = meta["polarisation"] 65 | else: 66 | polarisation = [elem.upper() for elem in polarisation] 67 | 68 | # only take bands of interest and sort 69 | n_bands = len(polarisation) 70 | calibration_tables = [None] * n_bands 71 | geo_tie_point = [None] * n_bands 72 | band_meta = [None] * n_bands 73 | measurement = [None] * n_bands 74 | 75 | for i in range(n_bands): 76 | 77 | for idx, file in enumerate(tiff_files): 78 | if file.split("-")[3].upper() == polarisation[i]: 79 | measurement[i] = measurement_temp[idx] 80 | 81 | for band in calibration_temp: 82 | if band[1]["polarisation"] == polarisation[i]: 83 | calibration_tables[i] = band[0] 84 | 85 | for band in annotation_temp: 86 | if band[1]["polarisation"] == polarisation[i]: 87 | geo_tie_point[i] = band[0] 88 | band_meta[i] = band[1] 89 | 90 | # Check that there is one band in each tiff 91 | for i in range(n_bands): 92 | if measurement[i].count != 1: 93 | warnings.warn( 94 | "Warning tiff file contains several bands. First band read from each tiff file" 95 | ) 96 | 97 | if (location is None) or (size is None): 98 | bands = [image.read(1) for image in measurement] 99 | else: 100 | # Check location is in foot print 101 | maxlat = meta["footprint"]["latitude"].max() 102 | minlat = meta["footprint"]["latitude"].min() 103 | maxlong = meta["footprint"]["longitude"].max() 104 | minlong = meta["footprint"]["longitude"].min() 105 | 106 | if not (minlat < location[0] < maxlat) & (minlong < location[1] < maxlong): 107 | raise ValueError("Location not inside the footprint") 108 | 109 | # get the index 110 | row = np.zeros(len(geo_tie_point), dtype=int) 111 | column = np.zeros(len(geo_tie_point), dtype=int) 112 | for i in range(len(geo_tie_point)): 113 | lat_grid = geo_tie_point[i]["latitude"] 114 | long_grid = geo_tie_point[i]["longitude"] 115 | row_grid = geo_tie_point[i]["row"] 116 | column_grid = geo_tie_point[i]["column"] 117 | row[i], column[i] = get_index_v2( 118 | location[0], location[1], lat_grid, long_grid, row_grid, column_grid 119 | ) 120 | # check if index are the same for all bands 121 | if (abs(row.max() - row.min()) > 0.5) or ( 122 | abs(column.max() - column.min()) > 0.5 123 | ): 124 | warnings.warn( 125 | "Warning different index found for each band. First index returned" 126 | ) 127 | 128 | # Find the window 129 | row_index_min = row[0] - int(size[0] / 2) 130 | row_index_max = row[0] + int(size[0] / 2) 131 | 132 | column_index_min = column[0] - int(size[1] / 2) 133 | column_index_max = column[0] + int(size[1] / 2) 134 | 135 | # Check if window is in image 136 | if row_index_max < 0 or column_index_max < 0: 137 | raise ValueError("Error window not in image ") 138 | 139 | if row_index_min < 0: 140 | warnings.warn("Extend out of image. Window constrained ") 141 | row_index_min = 0 142 | 143 | if column_index_min < 0: 144 | warnings.warn("Extend out of image. Window constrained ") 145 | column_index_min = 0 146 | 147 | for image in measurement: 148 | if row_index_min > image.height or column_index_min > image.width: 149 | raise ValueError("Error window not in image") 150 | 151 | if row_index_max > image.height: 152 | warnings.warn("Extend out of image. Window constrained ") 153 | row_index_max = image.height 154 | 155 | if column_index_max > image.width: 156 | warnings.warn("Extend out of image. Window constrained ") 157 | column_index_max = image.width 158 | 159 | # Adjust footprint to window 160 | footprint_lat = np.zeros(4) 161 | footprint_long = np.zeros(4) 162 | window = ((row_index_min, row_index_max), (column_index_min, column_index_max)) 163 | 164 | for i in range(2): 165 | for j in range(2): 166 | lat_i, long_i = get_coordinate( 167 | window[0][i], 168 | window[1][j], 169 | geo_tie_point[0]["latitude"], 170 | geo_tie_point[0]["longitude"], 171 | geo_tie_point[0]["row"], 172 | geo_tie_point[0]["column"], 173 | ) 174 | footprint_lat[2 * i + j] = lat_i 175 | footprint_long[2 * i + j] = long_i 176 | 177 | meta["footprint"]["latitude"] = footprint_lat 178 | meta["footprint"]["longitude"] = footprint_long 179 | 180 | # Adjust geo_tie_point, calibration_tables 181 | for i in range(n_bands): 182 | geo_tie_point[i]["row"] = geo_tie_point[i]["row"] - row_index_min 183 | geo_tie_point[i]["column"] = geo_tie_point[i]["column"] - column_index_min 184 | 185 | calibration_tables[i]["row"] = calibration_tables[i]["row"] - row_index_min 186 | calibration_tables[i]["column"] = ( 187 | calibration_tables[i]["column"] - column_index_min 188 | ) 189 | 190 | # load the data window 191 | bands = [image.read(1, window=window) for image in measurement] 192 | 193 | return SarImage( 194 | bands, 195 | mission=meta["mission"], 196 | time=meta["start_time"], 197 | footprint=meta["footprint"], 198 | product_meta=meta, 199 | band_names=polarisation, 200 | calibration_tables=calibration_tables, 201 | geo_tie_point=geo_tie_point, 202 | band_meta=band_meta, 203 | unit="raw amplitude", 204 | ) 205 | 206 | 207 | def load(path): 208 | """Load SarImage saved with the SarImage save method (img.save(path)). 209 | Args: 210 | path(str): Path to the folder where SarImage is saved. 211 | Returns: 212 | SarImage 213 | """ 214 | 215 | # product_meta 216 | file_path = os.path.join(path, "product_meta.pkl") 217 | product_meta = pickle.load(open(file_path, "rb")) 218 | 219 | # unit 220 | file_path = os.path.join(path, "unit.pkl") 221 | unit = pickle.load(open(file_path, "rb")) 222 | 223 | # footprint 224 | file_path = os.path.join(path, "footprint.pkl") 225 | footprint = pickle.load(open(file_path, "rb")) 226 | 227 | # geo_tie_point 228 | file_path = os.path.join(path, "geo_tie_point.pkl") 229 | geo_tie_point = pickle.load(open(file_path, "rb")) 230 | 231 | # band_names 232 | file_path = os.path.join(path, "band_names.pkl") 233 | band_names = pickle.load(open(file_path, "rb")) 234 | 235 | # band_meta 236 | file_path = os.path.join(path, "band_meta.pkl") 237 | band_meta = pickle.load(open(file_path, "rb")) 238 | 239 | # bands 240 | file_path = os.path.join(path, "bands.pkl") 241 | bands = pickle.load(open(file_path, "rb")) 242 | 243 | # calibration_tables 244 | file_path = os.path.join(path, "calibration_tables.pkl") 245 | calibration_tables = pickle.load(open(file_path, "rb")) 246 | 247 | return SarImage( 248 | bands, 249 | mission=product_meta["mission"], 250 | time=product_meta["start_time"], 251 | footprint=footprint, 252 | product_meta=product_meta, 253 | band_names=band_names, 254 | calibration_tables=calibration_tables, 255 | geo_tie_point=geo_tie_point, 256 | band_meta=band_meta, 257 | unit=unit, 258 | ) 259 | -------------------------------------------------------------------------------- /src/sentinel_1_python/pre_process_grd/_proces_tools.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from scipy.interpolate import RegularGridInterpolator 3 | 4 | 5 | def calibration(band, rows, columns, calibration_values, tiles=4): 6 | """Calibrates image using linear interpolation. 7 | See https://sentinel.esa.int/documents/247904/685163/S1-Radiometric-Calibration-V1.0.pdf 8 | Args: 9 | band(2d numpy array): The non calibrated image 10 | rows(number): rows of calibration point 11 | columns(number): columns of calibration point 12 | calibration_values(2d numpy array): grid of calibration values 13 | tiles(int): number of tiles the image is divided into. This saves memory but reduce speed a bit 14 | Returns: 15 | calibrated image (2d numpy array) 16 | Raises: 17 | """ 18 | 19 | # Create interpolation function 20 | f = RegularGridInterpolator((rows, columns), calibration_values) 21 | 22 | result = np.zeros(band.shape) 23 | # Calibrate one tile at the time 24 | column_start = 0 25 | column_max = band.shape[1] 26 | for i in range(tiles): 27 | column_end = int(column_max / tiles * (i + 1)) 28 | # Create array of point where calibration is needed 29 | column_mesh, row_mesh = np.meshgrid( 30 | np.array(range(column_start, column_end)), np.array(range(band.shape[0])) 31 | ) 32 | points = np.array([row_mesh.reshape(-1), column_mesh.reshape(-1)]).T 33 | # Get the image tile and the calibration values for it 34 | img_tile = band[:, column_start:column_end] 35 | img_cal = f(points).reshape(img_tile.shape) 36 | # Set in result 37 | result[:, column_start:column_end] = img_tile / img_cal 38 | 39 | column_start = column_end 40 | return result**2 41 | -------------------------------------------------------------------------------- /src/sentinel_1_python/pre_process_grd/filters.py: -------------------------------------------------------------------------------- 1 | from scipy import ndimage 2 | import numpy as np 3 | # 4 | 5 | def boxcar(img, kernel_size, **kwargs): 6 | """Simple (kernel_size x kernel_size) boxcar filter. 7 | Args: 8 | img(2d numpy array): image 9 | kernel_size(int): size of kernel 10 | **kwargs: Additional arguments passed to scipy.ndimage.convolve 11 | Returns: 12 | Filtered image 13 | Raises: 14 | """ 15 | # For small kernels simple convolution 16 | if kernel_size < 8: 17 | kernel = np.ones([kernel_size, kernel_size]) 18 | box_img = ndimage.convolve(img, kernel, **kwargs) / kernel_size**2 19 | 20 | # For large kernels use Separable Filters. (https://www.youtube.com/watch?v=SiJpkucGa1o) 21 | else: 22 | kernel1 = np.ones([kernel_size, 1]) 23 | kernel2 = np.ones([1, kernel_size]) 24 | box_img = ndimage.convolve(img, kernel1, **kwargs) / kernel_size 25 | box_img = ndimage.convolve(box_img, kernel2, **kwargs) / kernel_size 26 | 27 | return box_img 28 | -------------------------------------------------------------------------------- /src/sentinel_1_python/pre_process_grd/load_data.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import warnings 3 | import xml.etree.ElementTree 4 | 5 | import lxml.etree 6 | import numpy as np 7 | 8 | 9 | def _load_calibration(path): 10 | """Load sentinel 1 calibration_table file as dictionary from PATH. 11 | The calibration_table file should be as included in .SAFE format 12 | retrieved from: https://scihub.copernicus.eu/ 13 | Args: 14 | path: The path to the calibration_table file 15 | Returns: 16 | calibration_table: A dictionary with calibration_table constants 17 | {"abs_calibration_const": float(), 18 | "row": np.array(int), 19 | "column": np.array(int), 20 | "azimuth_time": np.array(datetime64[us]), 21 | "sigma_0": np.array(float), 22 | "beta_0": np.array(float), 23 | "gamma": np.array(float), 24 | "dn": np.array(float),} 25 | info: A dictionary with the meta data given in 'adsHeader' 26 | {child[0].tag: child[0].text, 27 | child[1].tag: child[1].text, 28 | ...} 29 | """ 30 | # open xml file 31 | tree = xml.etree.ElementTree.parse(path) 32 | root = tree.getroot() 33 | 34 | # Find info 35 | info_xml = root.findall("adsHeader") 36 | if len(info_xml) == 1: 37 | info = {} 38 | for child in info_xml[0]: 39 | info[child.tag] = child.text 40 | else: 41 | warnings.warn("Warning adsHeader not found") 42 | info = None 43 | 44 | # Find calibration_table list 45 | cal_vectors = root.findall("calibrationVectorList") 46 | if len(cal_vectors) == 1: 47 | cal_vectors = cal_vectors[0] 48 | else: 49 | warnings.warn("Error loading calibration_table list") 50 | return None, info 51 | 52 | # get pixels from first vector 53 | pixel = np.array(list(map(int, cal_vectors[0][2].text.split()))) 54 | # initialize arrays 55 | azimuth_time = np.empty([len(cal_vectors), len(pixel)], dtype="datetime64[us]") 56 | line = np.empty([len(cal_vectors)], dtype=int) 57 | sigma_0 = np.empty([len(cal_vectors), len(pixel)], dtype=float) 58 | beta_0 = np.empty([len(cal_vectors), len(pixel)], dtype=float) 59 | gamma = np.empty([len(cal_vectors), len(pixel)], dtype=float) 60 | dn = np.empty([len(cal_vectors), len(pixel)], dtype=float) 61 | 62 | # get data 63 | for i, cal_vec in enumerate(cal_vectors): 64 | pixel_i = np.array(list(map(int, cal_vec[2].text.split()))) 65 | if not np.array_equal(pixel, pixel_i): 66 | warnings.warn( 67 | "Warning in _load_calibration. The calibration_table data is not on a proper grid" 68 | ) 69 | azimuth_time[i, :] = np.datetime64(cal_vec[0].text) 70 | line[i] = int(cal_vec[1].text) 71 | sigma_0[i, :] = np.array(list(map(float, cal_vec[3].text.split()))) 72 | beta_0[i, :] = np.array(list(map(float, cal_vec[4].text.split()))) 73 | gamma[i, :] = np.array(list(map(float, cal_vec[5].text.split()))) 74 | dn[i, :] = np.array(list(map(float, cal_vec[6].text.split()))) 75 | 76 | # Combine calibration_table info 77 | calibration_table = { 78 | "abs_calibration_const": float(root[1][0].text), 79 | "row": line, 80 | "column": pixel, 81 | "azimuth_time": azimuth_time, 82 | "sigma_0": sigma_0, 83 | "beta_0": beta_0, 84 | "gamma": gamma, 85 | "dn": dn, 86 | } 87 | 88 | return calibration_table, info 89 | 90 | 91 | def _load_meta(SAFE_path): 92 | """Load manifest.safe as dictionary from SAFE_path. 93 | The manifest.safe file should be as included in .SAFE format 94 | retrieved from: https://scihub.copernicus.eu/ 95 | Args: 96 | path: The path to the manifest.safe file 97 | Returns: 98 | metadata: A dictionary with meta_data 99 | example: 100 | {'mode': 'EW', 101 | 'swath': ['EW'], 102 | 'instrument_config': 1, 103 | 'mission_data_ID': '110917', 104 | 'polarisation': ['HH', 'HV'], 105 | 'product_class': 'S', 106 | 'product_composition': 'Slice', 107 | 'product_type': 'GRD', 108 | 'product_timeliness': 'Fast-24h', 109 | 'slice_product_flag': 'true', 110 | 'segment_start_time': datetime.datetime(2019, 1, 17, 19, 12, 32, 164986), 111 | 'slice_number': 4, 112 | 'total_slices': 4, 113 | 'footprint': {'latitude': array([69.219566, 69.219566, 69.219566, 69.219566]), 114 | 'longitude': array([-35.149223, -35.149223, -35.149223, -35.149223])}, 115 | 'nssdc_identifier': '2016-025A', 116 | 'mission': 'SENTINEL-1B', 117 | 'orbit_number': array([14538, 14538]), 118 | 'relative_orbit_number': array([162, 162]), 119 | 'cycle_number': 89, 120 | 'phase_identifier': 1, 121 | 'start_time': datetime.datetime(2019, 1, 17, 19, 15, 36, 268585), 122 | 'stop_time': datetime.datetime(2019, 1, 17, 19, 16, 25, 598196), 123 | 'pass': 'ASCENDING', 124 | 'ascending_node_time': datetime.datetime(2019, 1, 17, 18, 57, 16, 851007), 125 | 'start_time_ANX': 1099418.0, 126 | 'stop_time_ANX': 1148747.0} 127 | error: List of dictionary keys that was not found. 128 | """ 129 | # Sorry the code look like shit but I do not like the file format 130 | # and I do not trust that ESA will keep the structure. 131 | # This is the reason for all the if statements and the error list 132 | 133 | # Open the xml like file 134 | with open(SAFE_path) as f: 135 | safe_test = f.read() 136 | safe_string = safe_test.encode(errors="ignore") 137 | safe_xml = lxml.etree.fromstring(safe_string) 138 | 139 | # Initialize results 140 | metadata = {} 141 | error = [] 142 | 143 | # Prefixes used in the tag of the file. Do not ask me why the use them 144 | prefix1 = "{http://www.esa.int/safe/sentinel-1.0}" 145 | prefix2 = "{http://www.esa.int/safe/sentinel-1.0/sentinel-1}" 146 | prefix3 = "{http://www.esa.int/safe/sentinel-1.0/sentinel-1/sar/level-1}" 147 | 148 | # Put the data into the metadata 149 | 150 | # Get nssdc_identifier 151 | values = [elem for elem in safe_xml.iterfind(".//" + prefix1 + "nssdcIdentifier")] 152 | if len(values) == 1: 153 | metadata["nssdc_identifier"] = values[0].text 154 | else: 155 | error.append("nssdcIdentifier") 156 | 157 | # Get mission 158 | values = [elem for elem in safe_xml.iterfind(".//" + prefix1 + "familyName")] 159 | values2 = [elem for elem in safe_xml.iterfind(".//" + prefix1 + "number")] 160 | if (len(values) > 0) & (len(values2) == 1): 161 | metadata["mission"] = values[0].text + values2[0].text 162 | else: 163 | error.append("mission") 164 | 165 | # get orbit_number 166 | values = [elem for elem in safe_xml.iterfind(".//" + prefix1 + "orbitNumber")] 167 | if len(values) == 2: 168 | metadata["orbit_number"] = np.array([int(values[0].text), int(values[1].text)]) 169 | else: 170 | error.append("orbit_number") 171 | 172 | # get relative_orbit_number 173 | values = [ 174 | elem for elem in safe_xml.iterfind(".//" + prefix1 + "relativeOrbitNumber") 175 | ] 176 | if len(values) == 2: 177 | metadata["relative_orbit_number"] = np.array( 178 | [int(values[0].text), int(values[1].text)] 179 | ) 180 | else: 181 | error.append("relative_orbit_number") 182 | 183 | # get cycle_number 184 | values = [elem for elem in safe_xml.iterfind(".//" + prefix1 + "cycleNumber")] 185 | if len(values) == 1: 186 | metadata["cycle_number"] = int(values[0].text) 187 | else: 188 | error.append("cycle_number") 189 | 190 | # get phase_identifier 191 | values = [elem for elem in safe_xml.iterfind(".//" + prefix1 + "phaseIdentifier")] 192 | if len(values) == 1: 193 | metadata["phase_identifier"] = int(values[0].text) 194 | else: 195 | error.append("phase_identifier") 196 | 197 | # get start_time 198 | values = [elem for elem in safe_xml.iterfind(".//" + prefix1 + "startTime")] 199 | if len(values) == 1: 200 | t = values[0].text 201 | metadata["start_time"] = datetime.datetime( 202 | int(t[:4]), 203 | int(t[5:7]), 204 | int(t[8:10]), 205 | int(t[11:13]), 206 | int(t[14:16]), 207 | int(t[17:19]), 208 | int(float(t[19:]) * 10**6), 209 | ) 210 | else: 211 | error.append("start_time") 212 | 213 | # get stop_time 214 | values = [elem for elem in safe_xml.iterfind(".//" + prefix1 + "stopTime")] 215 | if len(values) == 1: 216 | t = values[0].text 217 | metadata["stop_time"] = datetime.datetime( 218 | int(t[:4]), 219 | int(t[5:7]), 220 | int(t[8:10]), 221 | int(t[11:13]), 222 | int(t[14:16]), 223 | int(t[17:19]), 224 | int(float(t[19:]) * 10**6), 225 | ) 226 | else: 227 | error.append("stop_time") 228 | 229 | # get pass 230 | values = [elem for elem in safe_xml.iterfind(".//" + prefix2 + "pass")] 231 | if len(values) == 1: 232 | metadata["pass"] = values[0].text 233 | else: 234 | error.append("pass") 235 | 236 | # get ascending_node_time 237 | values = [elem for elem in safe_xml.iterfind(".//" + prefix2 + "ascendingNodeTime")] 238 | if len(values) == 1: 239 | t = values[0].text 240 | metadata["ascending_node_time"] = datetime.datetime( 241 | int(t[:4]), 242 | int(t[5:7]), 243 | int(t[8:10]), 244 | int(t[11:13]), 245 | int(t[14:16]), 246 | int(t[17:19]), 247 | int(float(t[19:]) * 10**6), 248 | ) 249 | else: 250 | error.append("ascending_node_time") 251 | 252 | # get start_time_ANX 253 | values = [elem for elem in safe_xml.iterfind(".//" + prefix2 + "startTimeANX")] 254 | if len(values) == 1: 255 | metadata["start_time_ANX"] = float(values[0].text) 256 | else: 257 | error.append("start_time_ANX") 258 | 259 | # get stop_time_ANX 260 | values = [elem for elem in safe_xml.iterfind(".//" + prefix2 + "stopTimeANX")] 261 | if len(values) == 1: 262 | metadata["stop_time_ANX"] = float(values[0].text) 263 | else: 264 | error.append("stop_time_ANX") 265 | 266 | # get mode 267 | values = [elem for elem in safe_xml.iterfind(".//" + prefix3 + "mode")] 268 | if len(values) == 1: 269 | metadata["mode"] = values[0].text 270 | else: 271 | error.append("mode") 272 | 273 | # get swath 274 | values = [elem for elem in safe_xml.iterfind(".//" + prefix3 + "swath")] 275 | if len(values) > 0: 276 | metadata["swath"] = [child.text for child in values] 277 | else: 278 | error.append("swath") 279 | 280 | # get instrument_config 281 | values = [ 282 | elem 283 | for elem in safe_xml.iterfind(".//" + prefix3 + "instrumentConfigurationID") 284 | ] 285 | if len(values) == 1: 286 | metadata["instrument_config"] = int(values[0].text) 287 | else: 288 | error.append("instrument_config") 289 | 290 | # get mission_data_ID 291 | values = [elem for elem in safe_xml.iterfind(".//" + prefix3 + "missionDataTakeID")] 292 | if len(values) == 1: 293 | metadata["mission_data_ID"] = values[0].text 294 | else: 295 | error.append("mission_data_ID") 296 | 297 | # get polarisation 298 | values = [ 299 | elem 300 | for elem in safe_xml.iterfind( 301 | ".//" + prefix3 + "transmitterReceiverPolarisation" 302 | ) 303 | ] 304 | if len(values) > 0: 305 | metadata["polarisation"] = [child.text for child in values] 306 | else: 307 | error.append("polarisation") 308 | 309 | # get product_class 310 | values = [elem for elem in safe_xml.iterfind(".//" + prefix3 + "productClass")] 311 | if len(values) == 1: 312 | metadata["product_class"] = values[0].text 313 | else: 314 | error.append("product_class") 315 | 316 | # get product_composition 317 | values = [ 318 | elem for elem in safe_xml.iterfind(".//" + prefix3 + "productComposition") 319 | ] 320 | if len(values) == 1: 321 | metadata["product_composition"] = values[0].text 322 | else: 323 | error.append("product_composition") 324 | 325 | # get product_type 326 | values = [elem for elem in safe_xml.iterfind(".//" + prefix3 + "productType")] 327 | if len(values) == 1: 328 | metadata["product_type"] = values[0].text 329 | else: 330 | error.append("product_type") 331 | 332 | # get product_timeliness 333 | values = [ 334 | elem 335 | for elem in safe_xml.iterfind(".//" + prefix3 + "productTimelinessCategory") 336 | ] 337 | if len(values) == 1: 338 | metadata["product_timeliness"] = values[0].text 339 | else: 340 | error.append("product_timeliness") 341 | 342 | # get slice_product_flag 343 | values = [elem for elem in safe_xml.iterfind(".//" + prefix3 + "sliceProductFlag")] 344 | if len(values) == 1: 345 | metadata["slice_product_flag"] = values[0].text 346 | else: 347 | error.append("slice_product_flag") 348 | 349 | # get segment_start_time 350 | values = [elem for elem in safe_xml.iterfind(".//" + prefix3 + "segmentStartTime")] 351 | if len(values) == 1: 352 | t = values[0].text 353 | metadata["segment_start_time"] = datetime.datetime( 354 | int(t[:4]), 355 | int(t[5:7]), 356 | int(t[8:10]), 357 | int(t[11:13]), 358 | int(t[14:16]), 359 | int(t[17:19]), 360 | int(float(t[19:]) * 10**6), 361 | ) 362 | else: 363 | error.append("segment_start_time") 364 | 365 | # get slice_number 366 | values = [elem for elem in safe_xml.iterfind(".//" + prefix3 + "sliceNumber")] 367 | if len(values) == 1: 368 | metadata["slice_number"] = int(values[0].text) 369 | else: 370 | error.append("slice_number") 371 | 372 | # get total_slices 373 | values = [elem for elem in safe_xml.iterfind(".//" + prefix3 + "totalSlices")] 374 | if len(values) == 1: 375 | metadata["total_slices"] = int(values[0].text) 376 | else: 377 | error.append("total_slices") 378 | 379 | # get footprint 380 | values = [ 381 | elem 382 | for elem in safe_xml.iterfind(".//" + "{http://www.opengis.net/gml}coordinates") 383 | ] 384 | if len(values) == 1: 385 | coordinates = values[0].text.split() 386 | lat = np.zeros(4) 387 | lon = np.zeros(4) 388 | for i in range(0, len(coordinates)): 389 | coord_i = coordinates[i].split(",") 390 | lat[i] = float(coord_i[0]) 391 | lon[i] = float(coord_i[1]) 392 | footprint = {"latitude": lat, "longitude": lon} 393 | metadata["footprint"] = footprint 394 | else: 395 | error.append("footprint") 396 | 397 | return metadata, error 398 | 399 | 400 | def _load_annotation(path): 401 | """Load sentinel 1 annotation file as dictionary from PATH. 402 | The annotation file should be as included in .SAFE format 403 | retrieved from: https://scihub.copernicus.eu/ 404 | Note that the file contains more information. Only the relevant have been chosen 405 | Args: 406 | path: The path to the annotation file 407 | Returns: 408 | geo_locations: A dictionary with geo location tie-points 409 | {'azimuth_time': np.array(datetime64[us]), 410 | 'slant_range_time': np.array(float), 411 | 'row': np.array(int), 412 | 'column': np.array(int), 413 | 'latitude': np.array(float), 414 | 'longitude': np.array(float), 415 | 'height': np.array(float), 416 | 'incidence_angle': np.array(float), 417 | 'elevation_angle': np.array(float)} 418 | info: A dictionary with the meta data given in 'adsHeader' 419 | {child[0].tag: child[0].text, 420 | child[1].tag: child[1].text, 421 | ...} 422 | """ 423 | 424 | # open xml file 425 | tree = xml.etree.ElementTree.parse(path) 426 | root = tree.getroot() 427 | 428 | # Find info 429 | info_xml = root.findall("adsHeader") 430 | if len(info_xml) == 1: 431 | info = {} 432 | for child in info_xml[0]: 433 | info[child.tag] = child.text 434 | else: 435 | warnings.warn("Warning adsHeader not found") 436 | info = None 437 | 438 | # Find geo location list 439 | geo_points = root.findall("geolocationGrid") 440 | if len(geo_points) == 1: 441 | geo_points = geo_points[0][0] 442 | else: 443 | warnings.warn("Warning geolocationGrid not found") 444 | return None, None 445 | 446 | # initialize arrays 447 | n_points = len(geo_points) 448 | azimuth_time = np.empty(n_points, dtype="datetime64[us]") 449 | slant_range_time = np.zeros(n_points, dtype=float) 450 | line = np.zeros(n_points, dtype=int) 451 | pixel = np.zeros(n_points, dtype=int) 452 | latitude = np.zeros(n_points, dtype=float) 453 | longitude = np.zeros(n_points, dtype=float) 454 | height = np.zeros(n_points, dtype=float) 455 | incidence_angle = np.zeros(n_points, dtype=float) 456 | elevation_angle = np.zeros(n_points, dtype=float) 457 | 458 | # get the data 459 | for i in range(0, n_points): 460 | point = geo_points[i] 461 | 462 | azimuth_time[i] = np.datetime64(point[0].text) 463 | slant_range_time[i] = float(point[1].text) 464 | line[i] = int(point[2].text) 465 | pixel[i] = int(point[3].text) 466 | latitude[i] = float(point[4].text) 467 | longitude[i] = float(point[5].text) 468 | height[i] = float(point[6].text) 469 | incidence_angle[i] = float(point[7].text) 470 | elevation_angle[i] = float(point[8].text) 471 | 472 | # Combine geo_locations info 473 | geo_locations = { 474 | "azimuth_time": azimuth_time, 475 | "slant_range_time": slant_range_time, 476 | "row": line, 477 | "column": pixel, 478 | "latitude": latitude, 479 | "longitude": longitude, 480 | "height": height, 481 | "incidence_angle": incidence_angle, 482 | "elevation_angle": elevation_angle, 483 | } 484 | 485 | return geo_locations, info 486 | -------------------------------------------------------------------------------- /src/sentinel_1_python/visualize/__init__.py: -------------------------------------------------------------------------------- 1 | from .show import * -------------------------------------------------------------------------------- /src/sentinel_1_python/visualize/show.py: -------------------------------------------------------------------------------- 1 | import cartopy 2 | import matplotlib 3 | import matplotlib.pyplot as plt 4 | import numpy as np 5 | import requests 6 | from matplotlib import rc 7 | from PIL import Image 8 | 9 | 10 | def show_thumbnail_function( 11 | gpd, amount: int = 60, username: str = "", password: str = "" 12 | ): 13 | """ """ 14 | if (len(gpd)) < amount: 15 | amount = len(gpd) 16 | 17 | fig, axs = plt.subplots( 18 | int(amount / 5) + 1, 19 | 5, 20 | figsize=(25, int(amount + 5)), 21 | facecolor="w", 22 | edgecolor="k", 23 | ) 24 | fig.subplots_adjust(hspace=0.5, wspace=0.001) 25 | axs = axs.ravel() 26 | 27 | for i in range(amount): 28 | try: 29 | link = gpd.link_icon.iloc[i] 30 | im = Image.open( 31 | requests.get(link, stream=True, auth=(username, password)).raw 32 | ) 33 | axs[i].imshow(im) 34 | axs[i].set_title(f"Img {i}") 35 | except: 36 | pass 37 | 38 | plt.show() 39 | 40 | 41 | def show_co_pol_function(gpd, amount: int = 60, username: str = "", password: str = ""): 42 | """ """ 43 | if (len(gpd)) < amount: 44 | amount = len(gpd) 45 | 46 | fig, axs = plt.subplots( 47 | int(amount / 5) + 1, 48 | 5, 49 | figsize=(25, int(amount + 5)), 50 | facecolor="w", 51 | edgecolor="k", 52 | ) 53 | fig.subplots_adjust(hspace=0.5, wspace=0.001) 54 | axs = axs.ravel() 55 | 56 | for i in range(amount): 57 | try: 58 | link = gpd.link_icon.iloc[i] 59 | im = np.array( 60 | Image.open( 61 | requests.get(link, stream=True, auth=(username, password)).raw 62 | ) 63 | ) 64 | 65 | axs[i].imshow(im[:, :, 0], cmap="gray") 66 | axs[i].set_title(f"Img {i}") 67 | except: 68 | pass 69 | 70 | plt.show() 71 | 72 | 73 | def show_cross_pol_function( 74 | gpd, amount: int = 60, username: str = "", password: str = "" 75 | ): 76 | """ """ 77 | if (len(gpd)) < amount: 78 | amount = len(gpd) 79 | 80 | fig, axs = plt.subplots( 81 | int(amount / 5) + 1, 82 | 5, 83 | figsize=(25, int(amount + 5)), 84 | facecolor="w", 85 | edgecolor="k", 86 | ) 87 | fig.subplots_adjust(hspace=0.5, wspace=0.001) 88 | axs = axs.ravel() 89 | 90 | for i in range(amount): 91 | try: 92 | link = gpd.link_icon.iloc[i] 93 | im = np.array( 94 | Image.open( 95 | requests.get(link, stream=True, auth=(username, password)).raw 96 | ) 97 | ) 98 | 99 | axs[i].imshow(im[:, :, 1], cmap="gray") 100 | axs[i].set_title(f"Img {i}") 101 | except: 102 | pass 103 | 104 | plt.show() 105 | 106 | 107 | def plot_polygon(gdf): 108 | """ 109 | Plotting polygons from a gdf. 110 | """ 111 | projections = [ 112 | cartopy.crs.PlateCarree(), 113 | cartopy.crs.Robinson(), 114 | cartopy.crs.Mercator(), 115 | cartopy.crs.Orthographic(), 116 | cartopy.crs.InterruptedGoodeHomolosine(), 117 | ] 118 | 119 | plt.figure(figsize=(20, 10)) 120 | ax = plt.axes(projection=projections[0]) 121 | try: 122 | extent = [ 123 | gdf.bounds.minx.min() - 3, 124 | gdf.bounds.maxx.max() + 3, 125 | gdf.bounds.miny.min() - 3, 126 | gdf.bounds.maxy.max() + 3, 127 | ] 128 | ax.set_extent(extent) 129 | except: 130 | pass 131 | 132 | ax.stock_img() 133 | ax.coastlines(resolution="10m") 134 | ax.add_feature(cartopy.feature.OCEAN, zorder=0) 135 | ax.add_feature(cartopy.feature.LAND, zorder=0, edgecolor="black") 136 | ax.gridlines() 137 | try: 138 | ax.add_geometries(gdf.geometry, alpha=0.7, crs=projections[0], ec="red") 139 | except: 140 | pass 141 | gl3 = ax.gridlines( 142 | draw_labels=True, linewidth=1.0, color="black", alpha=0.75, linestyle="--" 143 | ) 144 | gl3.left_labels = False 145 | gl3.top_labels = False 146 | gl3.xlabel_style = {"size": 10.0, "color": "gray", "weight": "bold"} 147 | gl3.ylabel_style = {"size": 10.0, "color": "gray", "weight": "bold"} 148 | gl3.xlabel_style = {"rotation": 45} 149 | gl3.ylabel_style = {"rotation": 45} 150 | 151 | return None 152 | 153 | 154 | def initi(size: int = 32): 155 | matplotlib.rcParams.update({"font.size": size}) 156 | rc("font", **{"family": "sans-serif", "sans-serif": ["Helvetica"]}) 157 | rc("font", **{"family": "serif", "serif": ["Palatino"]}) 158 | rc("text", usetex=True) 159 | -------------------------------------------------------------------------------- /test_environment.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | REQUIRED_PYTHON = "python3" 4 | 5 | 6 | def main(): 7 | system_major = sys.version_info.major 8 | if REQUIRED_PYTHON == "python": 9 | required_major = 2 10 | elif REQUIRED_PYTHON == "python3": 11 | required_major = 3 12 | else: 13 | raise ValueError("Unrecognized python interpreter: {}".format(REQUIRED_PYTHON)) 14 | 15 | if system_major != required_major: 16 | raise TypeError( 17 | "This project requires Python {}. Found: Python {}".format( 18 | required_major, sys.version 19 | ) 20 | ) 21 | else: 22 | print(">>> Development environment passes all tests!") 23 | 24 | 25 | if __name__ == "__main__": 26 | main() 27 | --------------------------------------------------------------------------------