├── img ├── landing.jpg └── open_file_qgis_copy_link.gif ├── metadata ├── .~lock.cog_list.csv# └── cog_list.csv ├── parametrization ├── equi7_tiles ├── scaling.csv ├── 004.retile_parameters.py ├── 002.generate_tiles.ipynb ├── 001.coarse_res_parameterization.ipynb └── 003.fine_res_parameterization.ipynb ├── LICENSE ├── .gitignore ├── README.md └── modeling └── tile_example.ipynb /img/landing.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openlandmap/GEDTM30/HEAD/img/landing.jpg -------------------------------------------------------------------------------- /metadata/.~lock.cog_list.csv#: -------------------------------------------------------------------------------- 1 | ,faen,faen-super-pc,26.03.2025 12:25,file:///home/faen/.config/libreoffice/4; -------------------------------------------------------------------------------- /parametrization/equi7_tiles: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openlandmap/GEDTM30/HEAD/parametrization/equi7_tiles -------------------------------------------------------------------------------- /img/open_file_qgis_copy_link.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openlandmap/GEDTM30/HEAD/img/open_file_qgis_copy_link.gif -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2025 OpenLandMap 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /parametrization/scaling.csv: -------------------------------------------------------------------------------- 1 | parameters,old_data_type,ori_min,ori_max ,multiplier,new_data_type,final_min,final_max ,no_data 2 | geomorphon,Int16,1,10,1,Byte,1,10,255 3 | hillshade,Float32,0,28357,1,UInt16,0,28357,65535 4 | slope.in.degree,Float32,0,15.29,100,UInt16,0,1529,65535 5 | ls.factor,Float32,0,13,1000,UInt16,0,13000,65535 6 | pro.curv,Float32,-8.210329056,8.011730194,1000,Int16,-8210.329056,8011.730194,32767 7 | ring.curv,Float32,-8.15170002,7.971476078,10000,Int16,-81517.0002,79714.76078,32767 8 | shpindx,Float32,-1,1,1000,Int16,-1000,1000,32767 9 | tan.curv,Float32,-8.210329056,8.011730194,1000,Int16,-8210.329056,8011.730194,32767 10 | maxic,Float32,-0.01404332742,0.05662668124,1000,Int16,-14.04332742,56.62668124,32767 11 | minic,Float32,-0.05388308689,0.02065591887,1000,Int16,-53.88308689,20.65591887,32767 12 | neg.openness,Float32,11.25551224,163.3332672,100,UInt16,1125.551224,16333.32672,65535 13 | dfme,Float32,-50,50,100,Int16,-5000,5000,32767 14 | pos.openness,Float32,13.74073124,167.3908234,100,UInt16,1374.073124,16739.08234,65535 15 | spec.catch,Float32,0,15.29,1000,UInt16,0,15290,65535 16 | nodepress.dtm,Float32,-400,8888,10,Int32,-4000,88880,2147483647 17 | filtered.dtm,Float32,-400,8888,10,Int32,-4000,88880,2147483647 18 | ssdon,Float32,0,100,100,Int16,0,10000,32767 19 | twi,Float32,0,100,100,Int16,0,10000,32767 -------------------------------------------------------------------------------- /parametrization/004.retile_parameters.py: -------------------------------------------------------------------------------- 1 | import geopandas as gpd 2 | import numpy as np 3 | from minio import Minio 4 | import os 5 | import rasterio 6 | from shapely import geometry 7 | import pandas as pd 8 | from tqdm import tqdm 9 | import warnings 10 | import requests 11 | warnings.filterwarnings("ignore") 12 | import random 13 | from joblib import Parallel, delayed 14 | 15 | ## read the global_input that contains all the files in all the tiles 16 | df_total_tiles=gpd.read_file('one_degree_tiles_equi7/global_input.geojson') 17 | ## shuffle the geopandas table 18 | df_total_tiles=df_total_tiles.sample(frac=1) 19 | # Convert selected columns to the desired list format, create the argument 20 | selected_columns = ['file_name','file_path','bbox','EQUI7_TILE','TILE'] # Specify the columns you want 21 | args = list(zip(*(df_total_tiles[col] for col in selected_columns))) 22 | server_name='apollo' 23 | def worker(info): 24 | # print(info) 25 | root_dir=f'/mnt/{server_name}/equi_tiling' 26 | file_name,file_path,bbox,equi7_tile,tile = info[0],info[1],info[2],info[3],info[4] 27 | var = file_name.split('_')[0] 28 | 29 | url=f"http://192.168.49.30:8333/tmp-global-geomorpho/latlon/v6/{tile}_{equi7_tile.lower().replace('_','.').replace('extracont','aa')}/{var}_edtm_m_30m_s_20060101_20151231_go_epsg.4326.3855_v20250619.tif" 30 | # print(url) 31 | r = requests.head(url) 32 | if r.status_code == 200: 33 | print(f'{url} has been processed') 34 | return 35 | 36 | #bbox=' '.join([str(i) for i in bbox_arr]) 37 | 38 | s3_config = { 39 | 'access_key': 'iwum9G1fEQ920lYV4ol9', 40 | 'secret_access_key': 'GMBME3Wsm8S7mBXw3U4CNWurkzWMqGZ0n2rXHggS0', 41 | 'host': '192.168.49.30:8333', 42 | 'bucket': 'tmp-global-geomorpho'} 43 | client = Minio(s3_config['host'], s3_config['access_key'], s3_config['secret_access_key'], secure=False) 44 | 45 | #file_name = file_name.replace('_go_',f"_{equi7_tile.lower().replace('_','.')}_") 46 | if var == 'geomorphon': 47 | resample_method = 'mode' 48 | else : 49 | resample_method = 'cubicspline' 50 | 51 | gdal_cmd = f'gdalwarp -overwrite -t_srs "+proj=longlat +datum=WGS84 +no_defs +type=crs" -tr 0.00025 0.00025 \ 52 | -r {resample_method} -te {bbox} --config GDAL_CACHEMAX 9216 \ 53 | -co BLOCKXSIZE=1024 -co BLOCKYSIZE=1024 -co BIGTIFF=YES -co COMPRESS=DEFLATE \ 54 | -co PREDICTOR=2 -co NUM_THREADS=8 -co SPARSE_OK=TRUE' 55 | out_dir=f"{root_dir}/{tile}_{equi7_tile.lower().replace('_','.').replace('extracont','aa')}" 56 | 57 | 58 | os.makedirs(out_dir,exist_ok=True) 59 | cmd=f"{gdal_cmd} /vsicurl/{file_path} {out_dir}/{file_name}" 60 | os.system(cmd) 61 | 62 | 63 | s3_path = f"latlon/v6/{tile}_{equi7_tile.lower().replace('_','.').replace('extracont','aa')}/{file_name}" 64 | client.fput_object(s3_config['bucket'], s3_path, f'{out_dir}/{file_name}') 65 | print(f'http://192.168.1.30:8333/tmp-global-geomorpho/{s3_path} on S3') 66 | os.remove(f'{out_dir}/{file_name}') 67 | #for i in args: 68 | # worker(i) 69 | 70 | Parallel(n_jobs=70)(delayed(worker)(i) for i in args) -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | share/python-wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | MANIFEST 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .nox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | *.py,cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | cover/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | .pybuilder/ 76 | target/ 77 | 78 | # Jupyter Notebook 79 | .ipynb_checkpoints 80 | 81 | # IPython 82 | profile_default/ 83 | ipython_config.py 84 | 85 | # pyenv 86 | # For a library or package, you might want to ignore these files since the code is 87 | # intended to run in multiple environments; otherwise, check them in: 88 | # .python-version 89 | 90 | # pipenv 91 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 92 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 93 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 94 | # install all needed dependencies. 95 | #Pipfile.lock 96 | 97 | # UV 98 | # Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control. 99 | # This is especially recommended for binary packages to ensure reproducibility, and is more 100 | # commonly ignored for libraries. 101 | #uv.lock 102 | 103 | # poetry 104 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 105 | # This is especially recommended for binary packages to ensure reproducibility, and is more 106 | # commonly ignored for libraries. 107 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 108 | #poetry.lock 109 | 110 | # pdm 111 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 112 | #pdm.lock 113 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 114 | # in version control. 115 | # https://pdm.fming.dev/latest/usage/project/#working-with-version-control 116 | .pdm.toml 117 | .pdm-python 118 | .pdm-build/ 119 | 120 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 121 | __pypackages__/ 122 | 123 | # Celery stuff 124 | celerybeat-schedule 125 | celerybeat.pid 126 | 127 | # SageMath parsed files 128 | *.sage.py 129 | 130 | # Environments 131 | .env 132 | .venv 133 | env/ 134 | venv/ 135 | ENV/ 136 | env.bak/ 137 | venv.bak/ 138 | 139 | # Spyder project settings 140 | .spyderproject 141 | .spyproject 142 | 143 | # Rope project settings 144 | .ropeproject 145 | 146 | # mkdocs documentation 147 | /site 148 | 149 | # mypy 150 | .mypy_cache/ 151 | .dmypy.json 152 | dmypy.json 153 | 154 | # Pyre type checker 155 | .pyre/ 156 | 157 | # pytype static type analyzer 158 | .pytype/ 159 | 160 | # Cython debug symbols 161 | cython_debug/ 162 | 163 | # PyCharm 164 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 165 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 166 | # and can be added to the global gitignore or merged into this file. For a more nuclear 167 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 168 | #.idea/ 169 | 170 | # PyPI configuration file 171 | .pypirc 172 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # GEDTM30 2 | ## Overview 3 | 4 | [GEDTM30](https://doi.org/10.7717/peerj.19673) is a **Global 1-arc-second (~30m) Digital Terrain Model (DTM)** built using a machine-learning-based data fusion approach. This dataset was generated using a global-to-local random forest model trained on ICEsat-2 and GEDI data, leveraging almost 30 billion of the highest-quality elevation points. 5 | 6 | [GEDTM30](https://doi.org/10.7717/peerj.19673) is also used to generate 15 land surface parameters at six scales (30, 60, 120, 240, 480 and 960m), covering aspects of topographic position, light and shadow, landform characteristics, and hydrology. A publication describing methods used has been submitted to PeerJ and is in review. The repository demonstrates the modeling and parametrization. 7 | 8 | 9 | ***Importatn Note*** 10 | 11 | (Update 27-08-2025): We computed DTM and Terrain variables in 0.00025 (~0.9 arc sec), which could be resampled by cubicspline to 0.000277777777778 (1 arc sec). We are also preparing 1 arc second of GEDTM30, that can be accessed from here:* 12 | 13 | - [https://s3.opengeohub.org/global/dtm/v1.2/gedtm_rf_m_30m_s_20060101_20151231_go_epsg.4326.3855_v1.2.tif](https://s3.opengeohub.org/global/dtm/v1.2/gedtm_rf_m_30m_s_20060101_20151231_go_epsg.4326.3855_v1.2.tif) 14 | 15 | 16 | (Update 13-08-2025): currently the dataset was produced by resampling to 0.00025 degree, which is closer to 0.9 arc sec. We are curretly working on produce a 0.000277778 degree GEDTM30 to resolve the confusion. See the issue detail [here](https://github.com/openlandmap/GEDTM30/issues/8).* 17 | 18 | 19 | 20 | ## Data Components 21 | 22 | 1. **GEDTM30: Terrain Height Prediction** 23 | 24 | Represents the predicted terrain height. 25 | 26 | 2. **Uncertainty Map of Terrain Prediction** 27 | 28 | Provides an uncertainty map of the terrain prediction, derived from the standard deviation of individual tree predictions in the Random Forest model. 29 | 30 | ![Alt text](img/landing.jpg) 31 | 32 | 3. **15 land surface parameters** 33 | 34 | Produced by DTM parametrization, representing different terrain features. Metadata of each parameter is currently stored at [scale.csv](parametrization/scaling.csv). The optimized [Equi7](https://github.com/TUW-GEO/Equi7Grid) tiling system for parameterization is currently stored at [equi7_tiles](parametrization/equi7_tiles). 35 | 36 | 37 | - Landform: 38 | [Slope in Degree](https://zenodo.org/records/14920379), [Geomorphons](https://zenodo.org/records/14920357) 39 | - Light and Shadow: 40 | [Positive Openness](https://zenodo.org/records/14920371), [Negative Openness](https://zenodo.org/records/14920369), [Hillshade](https://zenodo.org/records/14920359) 41 | - Curvature: 42 | [Minimal Curvature](https://zenodo.org/records/14920365), [Maximal Curvature](https://zenodo.org/records/14920363), [Profile Curvature](https://zenodo.org/records/14920373), [Tangential Curvature](https://zenodo.org/records/14920385), [Ring Curvature](https://zenodo.org/records/14920375), [Shape Index](https://zenodo.org/records/14920377) 43 | - Local Topographic Position: 44 | [Difference from Mean Elevation](https://zenodo.org/records/14919451), [Spherical Standard Deviation of the Normals](https://zenodo.org/records/14920383) 45 | - Hydrology: 46 | [Specific Catchment Area](https://zenodo.org/records/14920381), [LS Factor](https://zenodo.org/records/14920361), [Topographic Wetness Index](https://zenodo.org/records/14920387) 47 | 48 | ## Metadata and Scaling 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 | 142 | 143 | 144 | 145 | 146 | 147 | 148 | 149 | 150 | 151 | 152 | 153 | 154 | 155 | 156 | 157 | 158 | 159 | 160 | 161 | 162 | 163 | 164 | 165 | 166 | 167 | 168 |
LayerScaleData TypeNo Data
Ensemble Digital Terrain Model10Int32-2,147,483,647
Standard Deviation EDTM100UInt1665,535
Difference from Mean Elevation100Int1632,767
Geomorphons1Byte255
Hillshade1UInt1665,535
LS Factor1,000UInt1665,535
Maximal Curvature1,000Int1632,767
Minimal Curvature1,000Int1632,767
Negative Openness100UInt1665,535
Positive Openness100UInt1665,535
Profile Curvature1,000Int1632,767
Ring Curvature10,000Int1632,767
Shape Index1,000Int1632,767
Slope in Degree100UInt1665,535
Specific Catchment Area1,000UInt1665,535
Spherical Standard Deviation of the Normals100Int1632,767
Tangential Curvature1,000Int1632,767
Topographic Wetness Index100Int1632,767
169 | 170 | ## Usage 171 | 172 | This dataset is designed for researchers, developers, and professionals working in earth sciences, GIS, and remote sensing. It can be integrated into various geospatial analysis workflows to enhance terrain representation and modeling accuracy. This dataset covers the entire world and is well-suited for applications in: 173 | 174 | - Topography 175 | 176 | - Hydrology 177 | 178 | - Geomorphometry 179 | 180 | - Others 181 | 182 | ## Getting Started 183 | 184 | - Access and test the model and parametrization, please clone this repository: 185 | 186 | ``` 187 | git clone https://github.com/openlandmap/GEDTM30.git 188 | ``` 189 | - COG urls of GEDTM30 and Land surface parameters can be found in this csv here: [metadata/cog_list.csv](metadata/cog_list.csv) 190 | 191 | - Download the data set from Zenodo ([10.5281/zenodo.15689805](https://zenodo.org/records/15689805)) 192 | 193 | - Access and visualize COGs in QGIS. 194 | 195 | Please follow the step in the GIF below. 196 | 197 | ![Alt text](img/open_file_qgis_copy_link.gif) 198 | 199 | **Instruction**: 200 | `right click` --> `copy link` --> `paste to QGIS Layer >> Add Layer >> Add Raster Layer >> select Protocol:HTTP(S), cloud, etc. >> paste the url and you can visualize in QGIS`. 201 | 202 | ## Citation 203 | 204 | If you use this dataset in your research or application, please cite as: 205 | 206 | ``` 207 | @dataset{ho_2025_14900181, 208 | author = {Ho, Yufeng and Hengl, Tomislav}, 209 | title = {Global Ensemble Digital Terrain Model 30m (GEDTM30)}, 210 | month = feb, 211 | year = 2025, 212 | publisher = {Zenodo}, 213 | version = {v20250130}, 214 | doi = {10.5281/zenodo.14900181}, 215 | url = {https://doi.org/10.5281/zenodo.14900181}, 216 | } 217 | ``` 218 | 219 | [Technical documentation](https://doi.org/10.7717/peerj.19673) can be cited as: 220 | 221 | ``` 222 | @Article{yufengho2025GEDTM30, 223 | AUTHOR = {Ho, Yu-Feng and Grohmann, Carlos H and Lindsay, John and Reuter, Hannes I and Parente, Leandro and Witjes, Martijn and Hengl, Tomislav}, 224 | TITLE = {{Global Ensemble Digital Terrain modeling and parametrization at 30 m resolution (GEDTM30): a data fusion approach based on ICESat-2, GEDI and multisource data}}, 225 | JOURNAL = {PeerJ}, 226 | VOLUME = {13}, 227 | YEAR = {2025?}, 228 | PAGES = {e19673}, 229 | DOI = {10.7717/peerj.19673} 230 | } 231 | ``` 232 | 233 | ## License 234 | 235 | This dataset is released under fully open license [CC-BY 4.0](https://creativecommons.org/licenses/by/4.0/deed.en). 236 | 237 | ## Acknowledgements & Funding 238 | 239 | This work is supported by [OpenGeoHub Foundation](https://opengeohub.org/) and has received funding from the European Commission (EC) through the projects: 240 | 241 | - [Open-Earth-Monitor Cyberinfrastructure](https://earthmonitor.org/): Environmental information to support EU’s Green Deal (1 Jun. 2022 – 31 May 2026 - [101059548](https://cordis.europa.eu/project/id/101059548)) 242 | 243 | 244 | ## Contact 245 | 246 | For any questions or contributions, feel free to open an issue or reach out via [yu-feng.ho@opengeohub.org]. 247 | -------------------------------------------------------------------------------- /metadata/cog_list.csv: -------------------------------------------------------------------------------- 1 | name,doi,scale,data_type,no_data,measurement unit,url,,,,,, 2 | Ensemble Digital Terrain Model v1.1,https://zenodo.org/records/15689805,10,Int32,-2147483647,meter,https://s3.opengeohub.org/global/edtm/gedtm_rf_m_30m_s_20060101_20151231_go_epsg.4326.3855_v20250611.tif,,,,,, 3 | Standard Deviation EDTM v1.1,https://zenodo.org/records/15689805,100,UInt16,65535,meter,https://s3.opengeohub.org/global/edtm/gedtm_rf_std_30m_s_20060101_20151231_go_epsg.4326.3855_v20250611.tif,,,,,, 4 | Global-to-local mask v1.1,https://zenodo.org/records/15689805,1,Byte,255,,https://s3.opengeohub.org/global/edtm/gedtm_mask_c_120m_s_20060101_20151231_go_epsg.4326.3855_v20250611.tif,,,,,, 5 | Ensemble Digital Terrain Model v1.0,https://zenodo.org/records/14900181,10,Int32,-2147483647,meter,https://s3.opengeohub.org/global/edtm/legendtm_rf_30m_m_s_20000101_20231231_go_epsg.4326_v20250130.tif,,,,,, 6 | Standard Deviation EDTM v1.0,https://zenodo.org/records/14900181,100,UInt16,65535,meter,https://s3.opengeohub.org/global/edtm/gendtm_rf_30m_std_s_20000101_20231231_go_epsg.4326_v20250209.tif,,,,,, 7 | Difference from Mean Elevation,https://zenodo.org/records/14919451,100,Int16,32767,,https://s3.opengeohub.org/global/dtm/v3/dfme_edtm_m_30m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/dfme_edtm_m_30m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/dfme_edtm_m_60m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/dfme_edtm_m_120m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/dfme_edtm_m_240m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/dfme_edtm_m_480m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/dfme_edtm_m_960m_s_20000101_20221231_go_epsg.4326_v20241230.tif 8 | Geomorphons,https://zenodo.org/records/14920357,1,Byte,255,,https://s3.opengeohub.org/global/dtm/v3/geomorphon_edtm_m_30m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/geomorphon_edtm_m_30m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/geomorphon_edtm_m_60m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/geomorphon_edtm_m_120m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/geomorphon_edtm_m_240m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/geomorphon_edtm_m_480m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/geomorphon_edtm_m_960m_s_20000101_20221231_go_epsg.4326_v20241230.tif 9 | Hillshade,https://zenodo.org/records/14920359,1,UInt16,65535,,https://s3.opengeohub.org/global/dtm/v3/hillshade_edtm_m_30m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/hillshade_edtm_m_30m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/hillshade_edtm_m_60m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/hillshade_edtm_m_120m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/hillshade_edtm_m_240m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/hillshade_edtm_m_480m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/hillshade_edtm_m_960m_s_20000101_20221231_go_epsg.4326_v20241230.tif 10 | LS Factor,https://zenodo.org/records/14920361,1000,UInt16,65535,,https://s3.opengeohub.org/global/dtm/v3/ls.factor_edtm_m_30m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/ls.factor_edtm_m_30m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/ls.factor_edtm_m_60m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/ls.factor_edtm_m_120m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/ls.factor_edtm_m_240m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/ls.factor_edtm_m_480m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/ls.factor_edtm_m_960m_s_20000101_20221231_go_epsg.4326_v20241230.tif 11 | Maximal Curvature,https://zenodo.org/records/14920363,1000,Int16,32767,m-1,https://s3.opengeohub.org/global/dtm/v3/maxic_edtm_m_30m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/maxic_edtm_m_30m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/maxic_edtm_m_60m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/maxic_edtm_m_120m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/maxic_edtm_m_240m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/maxic_edtm_m_480m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/maxic_edtm_m_960m_s_20000101_20221231_go_epsg.4326_v20241230.tif 12 | Minimal Curvature,https://zenodo.org/records/14920365,1000,Int16,32767,m-1,https://s3.opengeohub.org/global/dtm/v3/minic_edtm_m_30m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/minic_edtm_m_30m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/minic_edtm_m_60m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/minic_edtm_m_120m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/minic_edtm_m_240m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/minic_edtm_m_480m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/minic_edtm_m_960m_s_20000101_20221231_go_epsg.4326_v20241230.tif 13 | Negative Openness,https://zenodo.org/records/14920369,100,UInt16,65535,,https://s3.opengeohub.org/global/dtm/v3/neg.openness_edtm_m_30m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/neg.openness_edtm_m_30m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/neg.openness_edtm_m_60m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/neg.openness_edtm_m_120m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/neg.openness_edtm_m_240m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/neg.openness_edtm_m_480m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/neg.openness_edtm_m_960m_s_20000101_20221231_go_epsg.4326_v20241230.tif 14 | Positive Openness,https://zenodo.org/records/14920371,100,UInt16,65535,,https://s3.opengeohub.org/global/dtm/v3/pos.openness_edtm_m_30m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/pos.openness_edtm_m_30m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/pos.openness_edtm_m_60m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/pos.openness_edtm_m_120m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/pos.openness_edtm_m_240m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/pos.openness_edtm_m_480m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/pos.openness_edtm_m_960m_s_20000101_20221231_go_epsg.4326_v20241230.tif 15 | Profile Curvature,https://zenodo.org/records/14920373,1000,Int16,32767,m-1,https://s3.opengeohub.org/global/dtm/v3/pro.curv_edtm_m_30m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/pro.curv_edtm_m_30m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/pro.curv_edtm_m_60m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/pro.curv_edtm_m_120m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/pro.curv_edtm_m_240m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/pro.curv_edtm_m_480m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/pro.curv_edtm_m_960m_s_20000101_20221231_go_epsg.4326_v20241230.tif 16 | Ring Curvature,https://zenodo.org/records/14920375,10000,Int16,32767,m-2,https://s3.opengeohub.org/global/dtm/v3/ring.curv_edtm_m_30m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/ring.curv_edtm_m_30m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/ring.curv_edtm_m_60m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/ring.curv_edtm_m_120m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/ring.curv_edtm_m_240m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/ring.curv_edtm_m_480m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/ring.curv_edtm_m_960m_s_20000101_20221231_go_epsg.4326_v20241230.tif 17 | Shape Index,https://zenodo.org/records/14920377,1000,Int16,32767,,https://s3.opengeohub.org/global/dtm/v3/shpindx_edtm_m_30m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/shpindx_edtm_m_30m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/shpindx_edtm_m_60m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/shpindx_edtm_m_120m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/shpindx_edtm_m_240m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/shpindx_edtm_m_480m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/shpindx_edtm_m_960m_s_20000101_20221231_go_epsg.4326_v20241230.tif 18 | Slope in Degree,https://zenodo.org/records/14920379,100,UInt16,65535,,https://s3.opengeohub.org/global/dtm/v3/slope.in.degree_edtm_m_30m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/slope.in.degree_edtm_m_30m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/slope.in.degree_edtm_m_60m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/slope.in.degree_edtm_m_120m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/slope.in.degree_edtm_m_240m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/slope.in.degree_edtm_m_480m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/slope.in.degree_edtm_m_960m_s_20000101_20221231_go_epsg.4326_v20241230.tif 19 | Specific Catchment Area,https://zenodo.org/records/14920381,1000,UInt16,65535,,https://s3.opengeohub.org/global/dtm/v3/spec.catch_edtm_m_30m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/spec.catch_edtm_edtm_m_30m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/spec.catch_edtm_edtm_m_60m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/spec.catch_edtm_edtm_m_120m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/spec.catch_edtm_edtm_m_240m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/spec.catch_edtm_edtm_m_480m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/spec.catch_edtm_edtm_m_960m_s_20000101_20221231_go_epsg.4326_v20241230.tif 20 | Spherical Standard Deviation of the Normals,https://zenodo.org/records/14920383,100,Int16,32767,,https://s3.opengeohub.org/global/dtm/v3/ssdon_edtm_m_30m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/ssdon_edtm_m_30m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/ssdon_edtm_m_60m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/ssdon_edtm_m_120m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/ssdon_edtm_m_240m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/ssdon_edtm_m_480m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/ssdon_edtm_m_960m_s_20000101_20221231_go_epsg.4326_v20241230.tif 21 | Tangential Curvature,https://zenodo.org/records/14920385,1000,Int16,32767,m-1,https://s3.opengeohub.org/global/dtm/v3/tan.curv_edtm_m_30m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/tan.curv_edtm_m_30m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/tan.curv_edtm_m_60m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/tan.curv_edtm_m_120m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/tan.curv_edtm_m_240m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/tan.curv_edtm_m_480m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/tan.curv_edtm_m_960m_s_20000101_20221231_go_epsg.4326_v20241230.tif 22 | Topographic Wetness Index,https://zenodo.org/records/14920387,100,Int16,32767,,https://s3.opengeohub.org/global/dtm/v3/twi_edtm_m_30m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/twi_edtm_m_30m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/twi_edtm_m_60m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/twi_edtm_m_120m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/twi_edtm_m_240m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/twi_edtm_m_480m_s_20000101_20221231_go_epsg.4326_v20241230.tif,https://s3.opengeohub.org/global/dtm/v3/twi_edtm_m_960m_s_20000101_20221231_go_epsg.4326_v20241230.tif -------------------------------------------------------------------------------- /parametrization/002.generate_tiles.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "4e3387f7-dcfc-48a3-8dba-6edda332cc38", 6 | "metadata": {}, 7 | "source": [ 8 | "# Generate tiles for fine resolution land relief parameterization\n", 9 | "\n", 10 | "This script is dedicated to generating tile that can be used for fine resolution land relief parameterization. In general, they are three types of tile.\n", 11 | "\n", 12 | "1. regional tile\n", 13 | "\n", 14 | "Regional tile has an extend distance of 3392*30=101,760km (pixel x resolution). Due to the factor of 2, which in 960m resolution the extend pixels would be 106. \n", 15 | "\n", 16 | "2. local tile\n", 17 | "\n", 18 | "Local tile has an extend distance of 64*30=1.92km (pixel x resolution). Due to the factor of 2, which in 240m resolution the extend pixels would be 8\n", 19 | "\n", 20 | "3. final tile\n", 21 | "final tile is used to crop the parameters before saving. It has an extend distance of 960m. It removes the border effect and the tile would be 4 pixels overlap to make sure there'd be no line in the map. \n", 22 | "\n", 23 | "- irregular tile\n", 24 | "\n", 25 | "There is a set of tile which is classified as irregular tiles. The irregular tiles were either missing out from the equi7 tiling system or across 180th meridians. The detail is explained in the paper. Due to its characteristics, the tile has the same boundary of regional, local and final tile. The reason is these irregular tiles would service at the bottom of the tile so the border effect will be covered by the regular tiles." 26 | ] 27 | }, 28 | { 29 | "cell_type": "markdown", 30 | "id": "b5ce6c94-d1a9-4374-9a81-5d878b848890", 31 | "metadata": {}, 32 | "source": [ 33 | "## Part 1: Setup" 34 | ] 35 | }, 36 | { 37 | "cell_type": "code", 38 | "execution_count": null, 39 | "id": "a5a4603e-6485-4aeb-b6ba-35b34dd6c8c6", 40 | "metadata": { 41 | "tags": [] 42 | }, 43 | "outputs": [], 44 | "source": [ 45 | "import os\n", 46 | "import geopandas as gpd\n", 47 | "import numpy as np\n", 48 | "from shapely.geometry import Polygon,mapping,box\n", 49 | "from shapely import segmentize\n", 50 | "import pickle\n", 51 | "\n", 52 | "def find_equi_7_proj(epsg4326_bound,equi_7_proj):\n", 53 | " # Extract coordinates from bounding box\n", 54 | " xmin, ymin, xmax, ymax = epsg4326_bound\n", 55 | "\n", 56 | " # Define coordinates for the polygon\n", 57 | " polygon_coords = [(xmin, ymin), (xmax, ymin), (xmax, ymax),(xmin, ymax)]\n", 58 | "\n", 59 | " # Create polygon\n", 60 | " polygon = Polygon(polygon_coords)\n", 61 | "\n", 62 | " # Create a GeoDataFrame with a single row representing the bounding box\n", 63 | " gdf = gpd.GeoDataFrame(geometry=[Polygon(polygon_coords)])\n", 64 | " # Set CRS for the GeoDataFrame\n", 65 | " gdf.crs = 'EPSG:4326'\n", 66 | " # Check the resulting GeoDataFrame\n", 67 | " gdf.geometry = segmentize(gdf.geometry,max_segment_length=0.000277778)\n", 68 | "\n", 69 | " return gdf.to_crs(equi_7_proj).geometry.bounds.values[0]\n", 70 | "\n", 71 | "def find_epsg4326_proj(equi7_bound,equi_7_proj):\n", 72 | " # Extract coordinates from bounding box\n", 73 | " xmin, ymin, xmax, ymax = equi7_bound\n", 74 | "\n", 75 | " # Define coordinates for the polygon\n", 76 | " polygon_coords = [(xmin, ymin), (xmax, ymin), (xmax, ymax),(xmin, ymax)]\n", 77 | "\n", 78 | " # Create polygon\n", 79 | " polygon = Polygon(polygon_coords)\n", 80 | "\n", 81 | " # Create a GeoDataFrame with a single row representing the bounding box\n", 82 | " gdf = gpd.GeoDataFrame(geometry=[Polygon(polygon_coords)])\n", 83 | " # Set CRS for the GeoDataFrame\n", 84 | " gdf.crs = equi_7_proj\n", 85 | " gdf.geometry = segmentize(gdf.geometry,max_segment_length=30)\n", 86 | " return gdf.to_crs('EPSG:4326')\n", 87 | "\n", 88 | "equi7_ind={'AF':'+proj=aeqd +lat_0=8.5 +lon_0=21.5 +x_0=5621452.01998 +y_0=5990638.42298 +datum=WGS84 +units=m +no_defs',\n", 89 | "'AN':'+proj=aeqd +lat_0=-90 +lon_0=0 +x_0=3714266.97719 +y_0=3402016.50625 +datum=WGS84 +units=m +no_defs',\n", 90 | "'AS':'+proj=aeqd +lat_0=47 +lon_0=94 +x_0=4340913.84808 +y_0=4812712.92347 +datum=WGS84 +units=m +no_defs',\n", 91 | "'EU':'+proj=aeqd +lat_0=53 +lon_0=24 +x_0=5837287.81977 +y_0=2121415.69617 +datum=WGS84 +units=m +no_defs',\n", 92 | "'NA':'+proj=aeqd +lat_0=52 +lon_0=-97.5 +x_0=8264722.17686 +y_0=4867518.35323 +datum=WGS84 +units=m +no_defs',\n", 93 | "'OC':'+proj=aeqd +lat_0=-19.5 +lon_0=131.5 +x_0=6988408.5356 +y_0=7654884.53733 +datum=WGS84 +units=m +no_defs',\n", 94 | "'SA':'+proj=aeqd +lat_0=-14 +lon_0=-60.5 +x_0=7257179.23559 +y_0=5592024.44605 +datum=WGS84 +units=m +no_defs'}\n", 95 | "\n", 96 | "conts = ['AF',\n", 97 | "'AN',\n", 98 | "'AS',\n", 99 | "'EU',\n", 100 | "'NA',\n", 101 | "'OC',\n", 102 | "'SA']\n", 103 | "\n" 104 | ] 105 | }, 106 | { 107 | "cell_type": "markdown", 108 | "id": "db88b99c-1158-40e5-8da3-16cc5ff836eb", 109 | "metadata": { 110 | "tags": [] 111 | }, 112 | "source": [ 113 | "## Part 2: Generate tiles in a for-loop\n", 114 | "\n", 115 | "- loop1: continent\n", 116 | "- loop2: tiles\n", 117 | "\n", 118 | "The irregular tiles have their own loops and at the final the two lists will be merged into one.\n", 119 | "\n", 120 | "The Equi7Grid tile can be downloaded from this GitHub repository (https://github.com/TUW-GEO/Equi7Grid)" 121 | ] 122 | }, 123 | { 124 | "cell_type": "code", 125 | "execution_count": null, 126 | "id": "c332b300-05d1-494b-a8f1-1485358f36f6", 127 | "metadata": { 128 | "tags": [] 129 | }, 130 | "outputs": [], 131 | "source": [ 132 | "args=[]\n", 133 | "rls_padding=3392 #3392*30=101760 is equivalent to 106 pixel in 960m resolution\n", 134 | "ls_padding=64 #64*30=960 is equivalent to 1 pixel in 960m resolution\n", 135 | "for cont in conts:\n", 136 | " equi_7_proj=equi7_ind[cont]\n", 137 | " proj_dir = f'Equi7Grid/src/equi7grid/grids/{cont}/PROJ/'\n", 138 | " for i in [i for i in os.scandir(proj_dir) if 'T6' in i.name and 'shp' in i.name]:\n", 139 | " equi7_grid = gpd.read_file(i.path)\n", 140 | " equi7_grid = equi7_grid[equi7_grid['COVERSLAND']==1]\n", 141 | " epsg4326_grid = equi7_grid.to_crs(epsg=4326)\n", 142 | " for tile_id in range(len(equi7_grid)):\n", 143 | " equi7_bound = np.array(equi7_grid.bounds.iloc[tile_id])\n", 144 | " epsg4326_bound = np.array(epsg4326_grid.bounds.iloc[tile_id])\n", 145 | " tile_name = equi7_grid.SHORTNAME.iloc[tile_id].split('_')[-1] + '_' + equi7_grid.TILE.iloc[tile_id] \n", 146 | " tile = equi7_grid.TILE.iloc[tile_id]\n", 147 | " if tile_name.startswith('AN'):\n", 148 | " continue\n", 149 | " if abs(epsg4326_bound[0] - epsg4326_bound[2]) >150 : ##AS\n", 150 | " continue\n", 151 | "\n", 152 | " equi7_bounds = equi7_grid[equi7_grid['TILE'] == tile].geometry.bounds\n", 153 | " equi7_bounds = np.array(equi7_bounds)[0]\n", 154 | " equi7_bounds_final = np.array([equi7_bounds[0]-960\n", 155 | " ,equi7_bounds[1]-960 ,equi7_bounds[2]+960 ,equi7_bounds[3]+960])\n", 156 | " equi7_bounds_rls = np.array([equi7_bounds[0]-rls_padding*30\n", 157 | " ,equi7_bounds[1]-rls_padding*30 ,equi7_bounds[2]+rls_padding*30 ,equi7_bounds[3]+rls_padding*30])\n", 158 | " equi7_bounds_ls = np.array([equi7_bounds[0]-ls_padding*30\n", 159 | " ,equi7_bounds[1]-ls_padding*30 ,equi7_bounds[2]+ls_padding*30 ,equi7_bounds[3]+ls_padding*30])\n", 160 | "\n", 161 | " epsg4326_bounds_final = find_epsg4326_proj(equi7_bounds_final,equi_7_proj).geometry.bounds.values[0]\n", 162 | " epsg4326_bounds_rls = find_epsg4326_proj(equi7_bounds_rls,equi_7_proj).geometry.bounds.values[0]\n", 163 | " epsg4326_bounds_ls = find_epsg4326_proj(equi7_bounds_ls,equi_7_proj).geometry.bounds.values[0]\n", 164 | " args.append((equi7_bounds_final,equi7_bounds_rls,epsg4326_bounds_rls,equi7_bounds_ls,epsg4326_bounds_ls,tile_name,equi_7_proj))\n", 165 | "\n", 166 | "# russia-us\n", 167 | "epsg4326_bounds_chuckotka = np.array([-180,50.696,-168.3,83.3])\n", 168 | "epsg4326_bounds_chuckotka2 = np.array([159.657,58.148,180,73.442])\n", 169 | "epsg4326_bounds_agattu = np.array([172.3292,53.2292,176.3044,52.0630])\n", 170 | "# pacific islands\n", 171 | "epsg4326_bounds_pukapuka = np.array([-163.621,-13.863,-162.621,-12.863])\n", 172 | "epsg4326_bounds_plamerston = np.array([-163.6532,-18.5411,-162.6532,-17.5411])\n", 173 | "epsg4326_bounds_futuna = np.array([-178.60509,-14.91163,-177.60509,-13.91163])\n", 174 | "epsg4326_bounds_vahen = np.array([-176.13766,-16.10176,-175.13766,-15.10176])\n", 175 | "epsg4326_bounds_uvea = np.array([-175.70106,-12.78761,-176.70106,-13.78761])\n", 176 | "epsg4326_bounds_tonga = np.array([-175.6306,-21.0206,-173.6306,-19.0206])\n", 177 | "epsg4326_bounds_hunga = np.array([-175.99074,-21.14193,-174.99074,-20.14193])\n", 178 | "epsg4326_bounds_tuvana = np.array([-179.29184,-21.52716,-178.29184,-20.52716])\n", 179 | "epsg4326_bounds_tonga_atoll = np.array([-179.5027,-24.2846,-178.5027,-23.2846])\n", 180 | "epsg4326_bounds_kermadec = np.array([-179.2019,-30.7827,-177.2019,-28.7827])\n", 181 | "epsg4326_bounds_moutere = np.array([168.66513,-53.04048,169.66513,-52.04048])\n", 182 | "epsg4326_bounds_newzealand_north = np.array([172.122,-41.794,180,-33.825])\n", 183 | "epsg4326_bounds_okinotori = np.array([136.177838,20.322660,135.977838,20.522660])\n", 184 | "epsg4326_bounds_yap = np.array([140.88595,7.63592,139.88595,8.63592])\n", 185 | "epsg4326_bounds_yap2 = np.array([142.6412,6.4175,148.079,9.494])\n", 186 | "epsg4326_bounds_luf = np.array([144.56444,-2.02959,145.56444,-1.02959])\n", 187 | "epsg4326_bounds_anewetak = np.array([161.7339,11.0093,162.7339,12.0093])\n", 188 | "epsg4326_bounds_tuvalu = np.array([176.7941,-8.6144,180,-6.9762])\n", 189 | "epsg4326_bounds_niulao = np.array([177.24250,-6.20757,177.44250,-6.00757])\n", 190 | "epsg4326_bounds_viti_levu=np.array([176.303,-18.991, 180,-16.026])\n", 191 | "epsg4326_bounds_taomaroa = np.array([176.72487,-2.73982,176.92487,-2.53982])\n", 192 | "epsg4326_bounds_nouvelle_caledonie=np.array([157.8054,-22.2446,160.0386,-18.5696])\n", 193 | "epsg4326_bounds_nikunau = np.array([175.93294,-1.45133,176.54619,-1.26088])\n", 194 | "epsg4326_bounds_majel = np.array([170.00179,12.13888,170.20179,12.33888])\n", 195 | "\n", 196 | "edges = {'chuckotka' : np.array([-180,50.696,-168.3,83.3]),\n", 197 | "'chuckotka2' : np.array([159.657,58.148,180,73.442]),\n", 198 | "'agattu' : np.array([172.3292,52.0630,176.3044,53.2292]),\n", 199 | "'pukapuka' : np.array([-163.621,-13.863,-162.621,-12.863]),\n", 200 | "'plamerston' : np.array([-163.6532,-18.5411,-162.6532,-17.5411]),\n", 201 | "'futuna' : np.array([-178.60509,-14.91163,-177.60509,-13.91163]),\n", 202 | "'vahen' : np.array([-176.13766,-16.10176,-175.13766,-15.10176]),\n", 203 | "'uvea' : np.array([-176.70106,-13.78761,-175.70106,-12.78761]),\n", 204 | "'tonga' : np.array([-175.6306,-21.0206,-173.6306,-19.0206]),\n", 205 | "'hunga' : np.array([-175.99074,-21.14193,-174.99074,-20.14193]),\n", 206 | "'tuvana' : np.array([-179.29184,-21.52716,-178.29184,-20.52716]),\n", 207 | "'tonga.atoll' : np.array([-179.5027,-24.2846,-178.5027,-23.2846]),\n", 208 | "'kermadec' : np.array([-179.2019,-30.7827,-177.2019,-28.7827]),\n", 209 | "'moutere' : np.array([168.66513,-53.04048,169.66513,-52.04048]),\n", 210 | "'newzealand.north' : np.array([172.122,-41.794,180,-33.825]),\n", 211 | "'okinotori' : np.array([135.977838,20.322660,136.177838,20.522660]),\n", 212 | "'yap' : np.array([139.88595,7.63592,140.88595,8.63592]),\n", 213 | "'yap2' : np.array([142.6412,6.4175,148.079,9.494]),\n", 214 | "'luf' : np.array([144.56444,-2.02959,145.56444,-1.02959]),\n", 215 | "'anewetak' : np.array([161.7339,11.0093,162.7339,12.0093]),\n", 216 | "'tuvalu' : np.array([176.7941,-8.6144,180,-6.9762]),\n", 217 | "'niulao' : np.array([177.24250,-6.20757,177.44250,-6.00757]),\n", 218 | "'viti.levu' :np.array([176.303,-18.991, 180,-16.026]),\n", 219 | "'taomaroa' : np.array([176.72487,-2.73982,176.92487,-2.53982]),\n", 220 | "'nouvelle.caledonie':np.array([157.8054,-22.2446,160.0386,-18.5696]),\n", 221 | "'nikunau' : np.array([175.93294,-1.45133,176.54619,-1.26088]),\n", 222 | "'majel' : np.array([170.00179,12.13888,170.20179,12.33888]),\n", 223 | "'pukapuka': np.array([-163.621,-13.863,-162.621,-12.863]),\n", 224 | "'muru' : np.array([169.750032,-11.711507,169.950032,-11.511507]),\n", 225 | "'fututaka': np.array([170.089185,-12.009242,170.289185,-11.809242])}\n", 226 | "\n", 227 | "args_special=[]\n", 228 | "for i in ['chuckotka','chuckotka2','agattu']:\n", 229 | " epsg4326_bounds = edges[i]\n", 230 | " equi7_bounds = find_equi_7_proj(epsg4326_bounds, equi7_ind['AS'])\n", 231 | " args_special.append((equi7_bounds,equi7_bounds,epsg4326_bounds,equi7_bounds,epsg4326_bounds,f'extracont_{i}',equi7_ind['AS']))\n", 232 | " \n", 233 | "for i in ['pukapuka','plamerston','futuna','vahen','uvea','tonga','hunga','tuvana','tonga.atoll','kermadec','moutere','newzealand.north','okinotori','yap','yap2','luf','anewetak','tuvalu','niulao','viti.levu','taomaroa','nouvelle.caledonie','nikunau','majel','muru','fututaka']:\n", 234 | " epsg4326_bounds = edges[i]\n", 235 | " equi7_bounds = find_equi_7_proj(epsg4326_bounds, equi7_ind['OC'])\n", 236 | " args_special.append((equi7_bounds,equi7_bounds,epsg4326_bounds,equi7_bounds,epsg4326_bounds,f'extracont_{i}',equi7_ind['OC']))" 237 | ] 238 | }, 239 | { 240 | "cell_type": "code", 241 | "execution_count": null, 242 | "id": "19eb1c73-d5ab-4716-bca9-c8c874ae57a3", 243 | "metadata": { 244 | "tags": [] 245 | }, 246 | "outputs": [], 247 | "source": [ 248 | "# save the tile list in pickle\n", 249 | "with open('equi7_tiles', 'wb') as f:\n", 250 | " pickle.dump(args+args_special, f)" 251 | ] 252 | }, 253 | { 254 | "cell_type": "code", 255 | "execution_count": null, 256 | "id": "fc3e2c95-43f1-4cdd-bf21-a6fee1e607a2", 257 | "metadata": { 258 | "tags": [] 259 | }, 260 | "outputs": [], 261 | "source": [ 262 | "# total number of tiles\n", 263 | "len(args+args_special)" 264 | ] 265 | } 266 | ], 267 | "metadata": { 268 | "kernelspec": { 269 | "display_name": "Python 3 (ipykernel)", 270 | "language": "python", 271 | "name": "python3" 272 | }, 273 | "language_info": { 274 | "codemirror_mode": { 275 | "name": "ipython", 276 | "version": 3 277 | }, 278 | "file_extension": ".py", 279 | "mimetype": "text/x-python", 280 | "name": "python", 281 | "nbconvert_exporter": "python", 282 | "pygments_lexer": "ipython3", 283 | "version": "3.8.16" 284 | } 285 | }, 286 | "nbformat": 4, 287 | "nbformat_minor": 5 288 | } 289 | -------------------------------------------------------------------------------- /modeling/tile_example.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "06af7998-4098-46ef-923e-c7ae15a88dbb", 6 | "metadata": {}, 7 | "source": [ 8 | "# Local Enhanced Global Ensemble Modeling (LEGEN-DTM): local modeling notebook\n", 9 | "\n", 10 | "The notebook describes a global-to-local model example in the research of GEDTM30 (https://github.com/openlandmap/GEDTM30). A 1 degree x 1 degree tile is predicted through\n", 11 | "\n", 12 | "1. Global model:\n", 13 | "\n", 14 | "GEDTM30 global model is trained by globally stratified ICESat-2 and GEDI terrain height samples.\n", 15 | "\n", 16 | "2. Local model:\n", 17 | "\n", 18 | "Additional samples (samples.csv) from a 5 degree x 5 degree ICESat-2 and GEDI samples that covers this 1 degree tile are used to enhance the global model to a local model with 100 additional trees upon the 100 trees from the global model.\n", 19 | "\n", 20 | "We will go through a detail in reproducing the locally enhanced modeling process. That includes:\n", 21 | "\n", 22 | "(1) Rewrite \"RandomForestRegressor\" from sklearn to return individual trees output so as to derive terrain prediction and uncertainty (stanrd deviation)\n", 23 | "\n", 24 | "(2) Load covariates from online Zenodo bucket via HTTP range request\n", 25 | "\n", 26 | "(3) Predict a global model \n", 27 | "\n", 28 | "(4) Build additional trees to the global model in order to obtain a local-enhanced model (global-to-local modeling)\n", 29 | "\n", 30 | "(5) Predict a local model\n", 31 | "\n", 32 | "This notebook is the reproducible script powered by Zenodo, from the repository (https://zenodo.org/records/14914836) which stores the covariates in COG format and the global model (file_name). The script is designed to run in local independently. \n", 33 | "\n" 34 | ] 35 | }, 36 | { 37 | "cell_type": "code", 38 | "execution_count": null, 39 | "id": "6ad60db2-454e-4e67-846a-c605416c8ea0", 40 | "metadata": { 41 | "tags": [] 42 | }, 43 | "outputs": [], 44 | "source": [ 45 | "# import libraries\n", 46 | "from sklearn.ensemble import RandomForestRegressor\n", 47 | "from sklearn.utils.validation import check_is_fitted\n", 48 | "import rasterio\n", 49 | "import numpy as np\n", 50 | "import joblib\n", 51 | "import pandas as pd\n", 52 | "import bottleneck as bn\n", 53 | "import threading\n", 54 | "from joblib import Parallel, delayed\n", 55 | "import matplotlib.pyplot as plt\n", 56 | "from tqdm import tqdm" 57 | ] 58 | }, 59 | { 60 | "cell_type": "markdown", 61 | "id": "5f7699b6-286b-49c0-ad3b-33adc99a55aa", 62 | "metadata": {}, 63 | "source": [ 64 | "## Part 1: Rewrite \"RandomForestRegressor\" from sklearn to return individual trees output so as to derive terrain prediction and uncertainty (stanrd deviation)" 65 | ] 66 | }, 67 | { 68 | "cell_type": "code", 69 | "execution_count": null, 70 | "id": "bc495c24-f47b-4de8-9351-ab74f49cf76b", 71 | "metadata": { 72 | "tags": [] 73 | }, 74 | "outputs": [], 75 | "source": [ 76 | "# define functions \n", 77 | "def _single_prediction(predict, X, out, i, lock):\n", 78 | " prediction = predict(X, check_input=False)\n", 79 | " with lock:\n", 80 | " out[i, :] = prediction\n", 81 | "\n", 82 | "def cast_tree_rf(model):\n", 83 | " model.__class__ = TreesRandomForestRegressor\n", 84 | " return model\n", 85 | "\n", 86 | "class TreesRandomForestRegressor(RandomForestRegressor):\n", 87 | " def predict(self, X):\n", 88 | " \"\"\"\n", 89 | " Predict regression target for X.\n", 90 | "\n", 91 | " The predicted regression target of an input sample is computed according\n", 92 | " to a list of functions that receives the predicted regression targets of each \n", 93 | " single tree in the forest.\n", 94 | "\n", 95 | " Parameters\n", 96 | " ----------\n", 97 | " X : {array-like, sparse matrix} of shape (n_samples, n_features)\n", 98 | " The input samples. Internally, its dtype will be converted to\n", 99 | " ``dtype=np.float32``. If a sparse matrix is provided, it will be\n", 100 | " converted into a sparse ``csr_matrix``.\n", 101 | "\n", 102 | " Returns\n", 103 | " -------\n", 104 | " s : an ndarray of shape (n_estimators, n_samples)\n", 105 | " The predicted values for each single tree.\n", 106 | " \"\"\"\n", 107 | " check_is_fitted(self)\n", 108 | " # Check data\n", 109 | " X = self._validate_X_predict(X)\n", 110 | "\n", 111 | " # store the output of every estimator\n", 112 | " assert(self.n_outputs_ == 1)\n", 113 | " pred_t = np.empty((len(self.estimators_), X.shape[0]), dtype=np.float32)\n", 114 | " # Assign chunk of trees to jobs\n", 115 | " n_jobs = min(self.n_estimators, self.n_jobs)\n", 116 | " # Parallel loop prediction\n", 117 | " lock = threading.Lock()\n", 118 | " Parallel(n_jobs=n_jobs, verbose=self.verbose, require=\"sharedmem\")(\n", 119 | " delayed(_single_prediction)(self.estimators_[i].predict, X, pred_t, i, lock)\n", 120 | " for i in range(len(self.estimators_))\n", 121 | " )\n", 122 | " return pred_t" 123 | ] 124 | }, 125 | { 126 | "cell_type": "markdown", 127 | "id": "16723101-bf09-4a03-bd7c-908b40db7508", 128 | "metadata": {}, 129 | "source": [ 130 | "## Part 2: Load covariates from online Zenodo bucket via HTTP range request" 131 | ] 132 | }, 133 | { 134 | "cell_type": "code", 135 | "execution_count": null, 136 | "id": "3ff1307a-3631-4f38-a089-0bd27b8e400d", 137 | "metadata": { 138 | "tags": [] 139 | }, 140 | "outputs": [], 141 | "source": [ 142 | "files=['https://zenodo.org/records/14914836/files/slope_etopo2022.tiff?download=1',\n", 143 | "'https://zenodo.org/records/14914836/files/canopy.height_glad.tiff?download=1',\n", 144 | "'https://zenodo.org/records/14914836/files/canopy.height_eth.tiff?download=1',\n", 145 | "'https://zenodo.org/records/14914836/files/dsm_aw3d30.tiff?download=1',\n", 146 | "'https://zenodo.org/records/14914836/files/dsm_glo30.tiff?download=1',\n", 147 | "'https://zenodo.org/records/14914836/files/edge.canopy.height_glad.tiff?download=1',\n", 148 | "'https://zenodo.org/records/14914836/files/edge.canopy.height_eth.tiff?download=1',\n", 149 | "'https://zenodo.org/records/14914836/files/ndvi.p025_2006.2010tiff?download=1',\n", 150 | "'https://zenodo.org/records/14914836/files/ndvi.p025_2011.2015.tiff?download=1',\n", 151 | "'https://zenodo.org/records/14914836/files/ndvi.p50_2006.2010.tiff?download=1',\n", 152 | "'https://zenodo.org/records/14914836/files/ndvi.p50_2011.2015.tiff?download=1',\n", 153 | "'https://zenodo.org/records/14914836/files/ndvi.p975_2006.2010.tiff?download=1',\n", 154 | "'https://zenodo.org/records/14914836/files/ndvi.p975_2011.2015.tiff?download=1',\n", 155 | "'https://zenodo.org/records/14914836/files/ndwi.p025_2006.2010.tiff?download=1',\n", 156 | "'https://zenodo.org/records/14914836/files/ndwi.p025_2011.2015.tiff?download=1',\n", 157 | "'https://zenodo.org/records/14914836/files/ndwi.p50_2006.2010.tiff?download=1',\n", 158 | "'https://zenodo.org/records/14914836/files/ndwi.p50_2011.2015.tiff?download=1',\n", 159 | "'https://zenodo.org/records/14914836/files/ndwi.p975_2006.2010.tiff?download=1',\n", 160 | "'https://zenodo.org/records/14914836/files/ndwi.p975_2011.2015.tiff?download=1',\n", 161 | "'https://zenodo.org/records/14914836/files/nir.p025_2006.2010.tiff?download=1',\n", 162 | "'https://zenodo.org/records/14914836/files/nir.p025_2011.2015.tiff?download=1',\n", 163 | "'https://zenodo.org/records/14914836/files/nir.p50_2006.2010.tiff?download=1',\n", 164 | "'https://zenodo.org/records/14914836/files/nir.p50_2011.2015.tiff?download=1',\n", 165 | "'https://zenodo.org/records/14914836/files/nir.p975_2006_2010.tiff?download=1',\n", 166 | "'https://zenodo.org/records/14914836/files/nir.p975_2011.2015.tiff?download=1',\n", 167 | "'https://zenodo.org/records/14914836/files/tree.cover_glad.tiff?download=1',\n", 168 | "'https://zenodo.org/records/14914836/files/building.height_3dglobfp.tiff?download=1',\n", 169 | "'https://zenodo.org/records/14914836/files/building.height_ghsbuilth.tiff?download=1',\n", 170 | "'https://zenodo.org/records/14914836/files/building.extent_wsf2019.tiff?download=1',\n", 171 | "'https://zenodo.org/records/14914836/files/lcluc.change_glad.tiff?download=1']" 172 | ] 173 | }, 174 | { 175 | "cell_type": "code", 176 | "execution_count": null, 177 | "id": "b31cfaa0-8438-4ea9-8dab-72452f74abe7", 178 | "metadata": { 179 | "tags": [] 180 | }, 181 | "outputs": [], 182 | "source": [ 183 | "data_list=[]\n", 184 | "for file in tqdm(files):\n", 185 | " data=rasterio.open(file).read(1)\n", 186 | " data_list.append(data)" 187 | ] 188 | }, 189 | { 190 | "cell_type": "code", 191 | "execution_count": null, 192 | "id": "18addb02-bd03-464f-8bfd-3c2c984b0501", 193 | "metadata": { 194 | "tags": [] 195 | }, 196 | "outputs": [], 197 | "source": [ 198 | "input_data=np.dstack(data_list)" 199 | ] 200 | }, 201 | { 202 | "cell_type": "code", 203 | "execution_count": null, 204 | "id": "f532791e-454b-42db-9ec4-e5438dc7dec3", 205 | "metadata": { 206 | "tags": [] 207 | }, 208 | "outputs": [], 209 | "source": [ 210 | "x_size,y_size,layers=input_data.shape" 211 | ] 212 | }, 213 | { 214 | "cell_type": "code", 215 | "execution_count": null, 216 | "id": "c59aa1ce-5c6a-4466-a980-535b21f90d74", 217 | "metadata": { 218 | "tags": [] 219 | }, 220 | "outputs": [], 221 | "source": [ 222 | "input_data.transpose(2,0,1).shape" 223 | ] 224 | }, 225 | { 226 | "cell_type": "code", 227 | "execution_count": null, 228 | "id": "fb936d6f-eaa4-4505-9480-a7ed9d67dbf6", 229 | "metadata": { 230 | "tags": [] 231 | }, 232 | "outputs": [], 233 | "source": [ 234 | "data = input_data.transpose(2,0,1).reshape(layers,-1)" 235 | ] 236 | }, 237 | { 238 | "cell_type": "code", 239 | "execution_count": null, 240 | "id": "fa1536eb-5c29-414f-9c5d-6ebd2e380125", 241 | "metadata": { 242 | "tags": [] 243 | }, 244 | "outputs": [], 245 | "source": [ 246 | "# extract the permanent ice from lc_glad.glcluc.change\n", 247 | "lulc=data[-1,:]\n", 248 | "permanent_ice=lulc==241\n", 249 | "data=data[:-1,:]\n", 250 | "\n", 251 | "# assign aw3d30 value on glo30 in azerbaijan_mask\n", 252 | "azerbaijan_mask=np.isnan(data[4,:])\n", 253 | "data[4,azerbaijan_mask]=data[3,azerbaijan_mask]\n", 254 | "build_mask=np.isnan(data[-2,:])\n", 255 | "data[-2,build_mask]=data[-3,build_mask]\n", 256 | "\n", 257 | "# give canopy to 0 if it is on permanent ice pixel\n", 258 | "data[1,permanent_ice]=0\n", 259 | "data[2,permanent_ice]=0\n", 260 | "\n", 261 | "# clean up huge discrepancy from DSMs\n", 262 | "d=10 # threshold for terrain difference\n", 263 | "alos_sub_mask=abs(data[3,:]-data[4,:])>d\n", 264 | "data[3,alos_sub_mask]=data[4,alos_sub_mask]\n", 265 | "\n", 266 | "# set nan data to 0 (no builing, no canopy, etc...)\n", 267 | "data[np.isnan(data)] = 0\n", 268 | "data=data.transpose(1,0)" 269 | ] 270 | }, 271 | { 272 | "cell_type": "markdown", 273 | "id": "669e3986-6d71-4027-b7a1-af2f0ec19328", 274 | "metadata": {}, 275 | "source": [ 276 | "## Part 3: Predict a global model" 277 | ] 278 | }, 279 | { 280 | "cell_type": "code", 281 | "execution_count": null, 282 | "id": "ce606ba6-3623-42e3-a9b4-f36db832feaf", 283 | "metadata": { 284 | "tags": [] 285 | }, 286 | "outputs": [], 287 | "source": [ 288 | "# download the global model\n", 289 | "!wget https://zenodo.org/records/14914777/files/global.model_gedtm30.lz4?download=1 -O global.model_gedtm30.lz4" 290 | ] 291 | }, 292 | { 293 | "cell_type": "code", 294 | "execution_count": null, 295 | "id": "1a9c4cfd-333e-4475-b3ef-bb32ef807886", 296 | "metadata": { 297 | "tags": [] 298 | }, 299 | "outputs": [], 300 | "source": [ 301 | "# load the model\n", 302 | "m = joblib.load('global.model_gedtm30.lz4')['model_rf']\n", 303 | "# change the class to return individual trees\n", 304 | "m.__class__ = TreesRandomForestRegressor\n", 305 | "m" 306 | ] 307 | }, 308 | { 309 | "cell_type": "code", 310 | "execution_count": null, 311 | "id": "5a3c5bc2-785b-45b2-95d6-2b18ab96821e", 312 | "metadata": { 313 | "tags": [] 314 | }, 315 | "outputs": [], 316 | "source": [ 317 | "# predict the terrain height and obtain standard devation through the global model\n", 318 | "y_rf = m.predict(data) \n", 319 | "print(y_rf.shape) # (tree, pixel)\n", 320 | "gpredictions = bn.nanmean(y_rf, axis=0)*10 # scaling to decimeter\n", 321 | "gstd = bn.nanstd(y_rf, axis=0)*100 # scaling to millimeter" 322 | ] 323 | }, 324 | { 325 | "cell_type": "code", 326 | "execution_count": null, 327 | "id": "fd702519-d11f-433e-8a95-12f46a9c6b50", 328 | "metadata": { 329 | "tags": [] 330 | }, 331 | "outputs": [], 332 | "source": [ 333 | "# visualize the result\n", 334 | "plt.imshow(gpredictions.reshape(1,x_size,y_size)[0,:,:])" 335 | ] 336 | }, 337 | { 338 | "cell_type": "code", 339 | "execution_count": null, 340 | "id": "87558fda-ef53-4149-ae2a-35adb215bd8a", 341 | "metadata": { 342 | "tags": [] 343 | }, 344 | "outputs": [], 345 | "source": [ 346 | "# save the result\n", 347 | "\n", 348 | "template = rasterio.open(files[5])\n", 349 | "kwargs = template.meta\n", 350 | "kwargs['compress'] = 'deflate'\n", 351 | "\n", 352 | "kwargs['dtype']= rasterio.int32\n", 353 | "kwargs['nodata']=2147483647\n", 354 | "with rasterio.open('gedtm_pred.tif', 'w', **kwargs) as dst:\n", 355 | " dst.write(gpredictions.reshape(1,x_size,y_size)[0,:,:],1)\n", 356 | " \n", 357 | "kwargs['dtype']= rasterio.int16\n", 358 | "kwargs['nodata']=32767\n", 359 | "with rasterio.open('gedtm_pred_std.tif', 'w', **kwargs) as dst:\n", 360 | " dst.write(gstd.reshape(1,x_size,y_size)[0,:,:],1)" 361 | ] 362 | }, 363 | { 364 | "cell_type": "markdown", 365 | "id": "411b61b8-cb7a-4d96-b56b-dadb598fb790", 366 | "metadata": { 367 | "tags": [] 368 | }, 369 | "source": [ 370 | "## Part 4: Build additional trees to the global model in order to obtain a local-enhanced model (global-to-local modeling)\n" 371 | ] 372 | }, 373 | { 374 | "cell_type": "code", 375 | "execution_count": null, 376 | "id": "97866497-a4be-492d-8de4-0eee2c59b770", 377 | "metadata": { 378 | "tags": [] 379 | }, 380 | "outputs": [], 381 | "source": [ 382 | "# download the additional local samples\n", 383 | "!wget https://zenodo.org/records/14914777/files/local_samples.csv?download=1 -O local_samples.csv" 384 | ] 385 | }, 386 | { 387 | "cell_type": "code", 388 | "execution_count": null, 389 | "id": "758c1114-81d1-44b3-9b0d-1f827b88aacd", 390 | "metadata": { 391 | "tags": [] 392 | }, 393 | "outputs": [], 394 | "source": [ 395 | "# load the local samples\n", 396 | "samples = pd.read_csv('local_samples.csv',index_col=0)" 397 | ] 398 | }, 399 | { 400 | "cell_type": "code", 401 | "execution_count": null, 402 | "id": "cd2b4d45-b267-46f6-846c-4a923c15d00c", 403 | "metadata": { 404 | "tags": [] 405 | }, 406 | "outputs": [], 407 | "source": [ 408 | "# compare with global model feature importance\n", 409 | "feature_importances = pd.Series(m.feature_importances_, index=samples.columns[:-1])\n", 410 | "print(feature_importances.sort_values(ascending=False))" 411 | ] 412 | }, 413 | { 414 | "cell_type": "code", 415 | "execution_count": null, 416 | "id": "16ed09ba-fdfd-4797-8bfc-23035f3ce772", 417 | "metadata": { 418 | "tags": [] 419 | }, 420 | "outputs": [], 421 | "source": [ 422 | "# add 100 estimators (total 200) for local fine tuning\n", 423 | "m.set_params(n_estimators=200, warm_start=True)\n", 424 | "m.fit(samples[samples.columns[:-1]], samples['dtm_y'])" 425 | ] 426 | }, 427 | { 428 | "cell_type": "code", 429 | "execution_count": null, 430 | "id": "972fa25d-9dcb-4439-bb03-9529cda948c0", 431 | "metadata": { 432 | "tags": [] 433 | }, 434 | "outputs": [], 435 | "source": [ 436 | "# local enhanced global model feature importance\n", 437 | "feature_importances = pd.Series(m.feature_importances_, index=samples.columns[:-1])\n", 438 | "print(feature_importances.sort_values(ascending=False))" 439 | ] 440 | }, 441 | { 442 | "cell_type": "code", 443 | "execution_count": null, 444 | "id": "255d6a0b-0b59-4b4e-b6e7-394634e1bc13", 445 | "metadata": { 446 | "tags": [] 447 | }, 448 | "outputs": [], 449 | "source": [ 450 | "# predict the terrain height and obtain standard devation through the locally enhanced model\n", 451 | "y_rf = m.predict(data) \n", 452 | "print(y_rf.shape) # (tree, pixel)\n", 453 | "lpredictions = bn.nanmean(y_rf, axis=0)*10 # scaling to decimeter\n", 454 | "lstd = bn.nanstd(y_rf, axis=0)*100 # scaling to millimeter" 455 | ] 456 | }, 457 | { 458 | "cell_type": "code", 459 | "execution_count": null, 460 | "id": "89f4552e-f7c6-4c60-aabd-1a2b5357c210", 461 | "metadata": { 462 | "tags": [] 463 | }, 464 | "outputs": [], 465 | "source": [ 466 | "# visualize the result\n", 467 | "plt.imshow(lpredictions.reshape(1,x_size,y_size)[0,:,:])" 468 | ] 469 | }, 470 | { 471 | "cell_type": "code", 472 | "execution_count": null, 473 | "id": "64364a32-efe1-4827-8096-375d60ba7706", 474 | "metadata": { 475 | "tags": [] 476 | }, 477 | "outputs": [], 478 | "source": [ 479 | "# save the result\n", 480 | "kwargs['dtype']= rasterio.int32\n", 481 | "kwargs['nodata']=2147483647\n", 482 | "with rasterio.open('legdtm_pred.tif', 'w', **kwargs) as dst:\n", 483 | " dst.write(lpredictions.reshape(1,x_size,y_size)[0,:,:],1)\n", 484 | "\n", 485 | "kwargs['dtype']= rasterio.int32\n", 486 | "kwargs['nodata']=32767\n", 487 | "with rasterio.open('legdtm_pred_std.tif', 'w', **kwargs) as dst:\n", 488 | " dst.write(lstd.reshape(1,x_size,y_size)[0,:,:],1)" 489 | ] 490 | } 491 | ], 492 | "metadata": { 493 | "kernelspec": { 494 | "display_name": "Python 3 (ipykernel)", 495 | "language": "python", 496 | "name": "python3" 497 | }, 498 | "language_info": { 499 | "codemirror_mode": { 500 | "name": "ipython", 501 | "version": 3 502 | }, 503 | "file_extension": ".py", 504 | "mimetype": "text/x-python", 505 | "name": "python", 506 | "nbconvert_exporter": "python", 507 | "pygments_lexer": "ipython3", 508 | "version": "3.8.16" 509 | } 510 | }, 511 | "nbformat": 4, 512 | "nbformat_minor": 5 513 | } 514 | -------------------------------------------------------------------------------- /parametrization/001.coarse_res_parameterization.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "9f169da3-e46a-4a5b-8b61-6044f3e42644", 6 | "metadata": {}, 7 | "source": [ 8 | "# Land relief parameterization in coarser resolution \n", 9 | "\n", 10 | "This scr{ip}t is used to generate the coarse resolution land relief parameters. The main difference from higher resolution is that the data can fit in RAM of the whole each continent. There the parameters can generate in one shot." 11 | ] 12 | }, 13 | { 14 | "cell_type": "markdown", 15 | "id": "118b90ac-5a61-4ae4-b343-1a99643bb0ff", 16 | "metadata": {}, 17 | "source": [ 18 | "## Part 1: setup and functions" 19 | ] 20 | }, 21 | { 22 | "cell_type": "code", 23 | "execution_count": 2, 24 | "id": "9d16e8d4-25d8-4876-be76-976bef9121a2", 25 | "metadata": { 26 | "tags": [] 27 | }, 28 | "outputs": [], 29 | "source": [ 30 | "import os\n", 31 | "os.environ['USE_PYGEOS'] = '0'\n", 32 | "import geopandas as gpd\n", 33 | "import numpy as np\n", 34 | "from shapely.geometry import Polygon,mapping,box\n", 35 | "from shapely import segmentize\n", 36 | "import time\n", 37 | "import sys\n", 38 | "from joblib import Parallel, delayed\n", 39 | "from minio import Minio\n", 40 | "from eumap.misc import ttprint\n", 41 | "import requests\n", 42 | "import pandas as pd\n", 43 | "import pickle\n", 44 | "\n", 45 | "def find_equi_7_proj(epsg4326_bound,equi_7_proj):\n", 46 | " # Extract coordinates from bounding box\n", 47 | " xmin, ymin, xmax, ymax = epsg4326_bound\n", 48 | "\n", 49 | " # Define coordinates for the polygon\n", 50 | " polygon_coords = [(xmin, ymin), (xmax, ymin), (xmax, ymax),(xmin, ymax)]\n", 51 | "\n", 52 | " # Create polygon\n", 53 | " polygon = Polygon(polygon_coords)\n", 54 | "\n", 55 | " # Create a GeoDataFrame with a single row representing the bounding box\n", 56 | " gdf = gpd.GeoDataFrame(geometry=[Polygon(polygon_coords)])\n", 57 | " # Set CRS for the GeoDataFrame\n", 58 | " gdf.crs = 'EPSG:4326'\n", 59 | " # Check the resulting GeoDataFrame\n", 60 | " gdf.geometry = segmentize(gdf.geometry,max_segment_length=0.000277778)\n", 61 | "\n", 62 | " return gdf.to_crs(equi_7_proj).geometry.bounds.values[0]\n", 63 | "\n", 64 | "def find_epsg4326_proj(equi7_bound,equi_7_proj):\n", 65 | " # Extract coordinates from bounding box\n", 66 | " xmin, ymin, xmax, ymax = equi7_bound\n", 67 | "\n", 68 | " # Define coordinates for the polygon\n", 69 | " polygon_coords = [(xmin, ymin), (xmax, ymin), (xmax, ymax),(xmin, ymax)]\n", 70 | "\n", 71 | " # Create polygon\n", 72 | " polygon = Polygon(polygon_coords)\n", 73 | "\n", 74 | " # Create a GeoDataFrame with a single row representing the bounding box\n", 75 | " gdf = gpd.GeoDataFrame(geometry=[Polygon(polygon_coords)])\n", 76 | " # Set CRS for the GeoDataFrame\n", 77 | " gdf.crs = equi_7_proj\n", 78 | " gdf.geometry = segmentize(gdf.geometry,max_segment_length=30)\n", 79 | " return gdf.to_crs('EPSG:4326')\n", 80 | "\n", 81 | "\n", 82 | "\n", 83 | "equi7_ind={'AF':'+proj=aeqd +lat_0=8.5 +lon_0=21.5 +x_0=5621452.01998 +y_0=5990638.42298 +datum=WGS84 +units=m +no_defs',\n", 84 | "'AN':'+proj=aeqd +lat_0=-90 +lon_0=0 +x_0=3714266.97719 +y_0=3402016.50625 +datum=WGS84 +units=m +no_defs',\n", 85 | "'AS':'+proj=aeqd +lat_0=47 +lon_0=94 +x_0=4340913.84808 +y_0=4812712.92347 +datum=WGS84 +units=m +no_defs',\n", 86 | "'EU':'+proj=aeqd +lat_0=53 +lon_0=24 +x_0=5837287.81977 +y_0=2121415.69617 +datum=WGS84 +units=m +no_defs',\n", 87 | "'NA':'+proj=aeqd +lat_0=52 +lon_0=-97.5 +x_0=8264722.17686 +y_0=4867518.35323 +datum=WGS84 +units=m +no_defs',\n", 88 | "'OC':'+proj=aeqd +lat_0=-19.5 +lon_0=131.5 +x_0=6988408.5356 +y_0=7654884.53733 +datum=WGS84 +units=m +no_defs',\n", 89 | "'SA':'+proj=aeqd +lat_0=-14 +lon_0=-60.5 +x_0=7257179.23559 +y_0=5592024.44605 +datum=WGS84 +units=m +no_defs'}\n", 90 | "\n", 91 | "conts = ['AF',\n", 92 | "'AS',\n", 93 | "'EU',\n", 94 | "'NA',\n", 95 | "'OC',\n", 96 | "'SA']\n", 97 | "\n", 98 | "\n" 99 | ] 100 | }, 101 | { 102 | "cell_type": "code", 103 | "execution_count": 3, 104 | "id": "15fde784-1977-4179-b93e-454fa8ccd3ea", 105 | "metadata": { 106 | "tags": [] 107 | }, 108 | "outputs": [ 109 | { 110 | "data": { 111 | "text/html": [ 112 | "
\n", 113 | "\n", 126 | "\n", 127 | " \n", 128 | " \n", 129 | " \n", 130 | " \n", 131 | " \n", 132 | " \n", 133 | " \n", 134 | " \n", 135 | " \n", 136 | " \n", 137 | " \n", 138 | " \n", 139 | " \n", 140 | " \n", 141 | " \n", 142 | " \n", 143 | " \n", 144 | " \n", 145 | " \n", 146 | " \n", 147 | " \n", 148 | " \n", 149 | " \n", 150 | " \n", 151 | " \n", 152 | " \n", 153 | " \n", 154 | " \n", 155 | " \n", 156 | " \n", 157 | " \n", 158 | " \n", 159 | " \n", 160 | " \n", 161 | " \n", 162 | " \n", 163 | " \n", 164 | " \n", 165 | " \n", 166 | " \n", 167 | " \n", 168 | " \n", 169 | " \n", 170 | " \n", 171 | " \n", 172 | " \n", 173 | " \n", 174 | " \n", 175 | " \n", 176 | " \n", 177 | " \n", 178 | " \n", 179 | " \n", 180 | " \n", 181 | " \n", 182 | " \n", 183 | " \n", 184 | " \n", 185 | " \n", 186 | " \n", 187 | " \n", 188 | " \n", 189 | " \n", 190 | " \n", 191 | " \n", 192 | " \n", 193 | " \n", 194 | " \n", 195 | " \n", 196 | " \n", 197 | " \n", 198 | " \n", 199 | " \n", 200 | " \n", 201 | " \n", 202 | " \n", 203 | "
parametersold_data_typeori_minori_maxmultipliernew_data_typefinal_minfinal_maxno_data
0geomorphonInt161.00000010.000001Byte1.00000010.000000255
1hillshadeFloat320.00000028357.000001UInt160.00000028357.00000065535
2slope.in.degreeFloat320.00000015.29000100UInt160.0000001529.00000065535
3ls.factorFloat320.00000013.000001000UInt160.00000013000.00000065535
4pro.curvFloat32-8.2103298.011731000Int16-8210.3290568011.73019432767
\n", 204 | "
" 205 | ], 206 | "text/plain": [ 207 | " parameters old_data_type ori_min ori_max multiplier \\\n", 208 | "0 geomorphon Int16 1.000000 10.00000 1 \n", 209 | "1 hillshade Float32 0.000000 28357.00000 1 \n", 210 | "2 slope.in.degree Float32 0.000000 15.29000 100 \n", 211 | "3 ls.factor Float32 0.000000 13.00000 1000 \n", 212 | "4 pro.curv Float32 -8.210329 8.01173 1000 \n", 213 | "\n", 214 | " new_data_type final_min final_max no_data \n", 215 | "0 Byte 1.000000 10.000000 255 \n", 216 | "1 UInt16 0.000000 28357.000000 65535 \n", 217 | "2 UInt16 0.000000 1529.000000 65535 \n", 218 | "3 UInt16 0.000000 13000.000000 65535 \n", 219 | "4 Int16 -8210.329056 8011.730194 32767 " 220 | ] 221 | }, 222 | "execution_count": 3, 223 | "metadata": {}, 224 | "output_type": "execute_result" 225 | } 226 | ], 227 | "source": [ 228 | "# scale factors for each parameters\n", 229 | "p_table=pd.read_csv('scaling.csv')\n", 230 | "p_table.head()" 231 | ] 232 | }, 233 | { 234 | "cell_type": "markdown", 235 | "id": "6eb2106e-965e-4bc9-8e84-9197e5d4fcf1", 236 | "metadata": {}, 237 | "source": [ 238 | "## Part 2: parameterization in the for-loop by continents" 239 | ] 240 | }, 241 | { 242 | "cell_type": "code", 243 | "execution_count": null, 244 | "id": "b77e9f45-d294-44d2-b6c8-81ad04c41daf", 245 | "metadata": {}, 246 | "outputs": [], 247 | "source": [ 248 | "#def worker_cont(cont,p_table):\n", 249 | "for cont in conts:\n", 250 | " s3_config = {\n", 251 | " 'access_key': 's3_key',\n", 252 | " 'secret_access_key': 's3_secret_key',\n", 253 | " 'host': '{ip}',\n", 254 | " 'bucket': 'tmp-global-geomorpho'}\n", 255 | " client = Minio(s3_config['host'], s3_config['access_key'], s3_config['secret_access_key'], secure=False) \n", 256 | "\n", 257 | " for resolution in [960,480]:\n", 258 | " url_cont=f'http://{ip}/tmp-global-geomorpho/v4/{cont}/tan.curv_edtm_m_{resolution}m_s_20000101_20221231_{cont.lower().replace(\"_\",\".\")}_equi7_v20241230.tif'\n", 259 | " r = requests.head(url_cont)\n", 260 | " if r.status_code == 200:\n", 261 | " ttprint(f'{cont} has been process')\n", 262 | " continue\n", 263 | " equi_7_proj=equi7_ind[cont]\n", 264 | " gpd_cont=gpd.read_file(f'Equi7Grid/src/equi7grid/grids/{cont}/PROJ/EQUI7_V14_{cont}_PROJ_ZONE.shp')\n", 265 | " equi7_bound = np.array(gpd_cont.bounds)[0]\n", 266 | " gdalwarp_bbox = ' '.join([str(i) for i in equi7_bound])\n", 267 | " tmp_outdir=f'/tmp/tmp-global-geomorpho/{cont}'\n", 268 | " os.makedirs(tmp_outdir,exist_ok=True)\n", 269 | " gdal_cmd_reproject = f'gdalwarp -t_srs \"{equi_7_proj}\" -tr {resolution} {resolution} -r average \\\n", 270 | " -co TILED=YES -co BIGTIFF=YES -co COMPRESS=DEFLATE -co ZLEVEL=9 -co BLOCKXSIZE=1024 \\\n", 271 | " -co BLOCKYSIZE=1024 -co NUM_THREADS=8 -co SPARSE_OK=TRUE -of GTiff -overwrite'\n", 272 | " url_cont=f'http://{ip}/tmp-global-geomorpho/v4/{cont}/dtm_edtm_m_{resolution}m_s_20000101_20221231_{cont.lower().replace(\"_\",\".\")}_equi7_v20241230.tif'\n", 273 | " r = requests.head(url_cont)\n", 274 | " if r.status_code == 200:\n", 275 | " ttprint(f'{cont} dtm is on S3')\n", 276 | " else:\n", 277 | "\n", 278 | " out_ori_file=f'dtm_edtm_m_{resolution}m_s_20000101_20221231_{cont.lower().replace(\"_\",\".\")}_equi7_v20241230.tif'\n", 279 | " rn_file = f'{tmp_outdir}/{out_ori_file}'\n", 280 | " filepath = '/vsicurl/{ip}/global/edtm/legendtm_rf_30m_m_s_20000101_20231231_go_epsg.4326_v20250130.tif'\n", 281 | "\n", 282 | " os.system(f'{gdal_cmd_reproject} -te {gdalwarp_bbox} {filepath} {tmp_outdir}/scaled_dtm_tmp.tif')\n", 283 | " os.system(f'gdal_calc.py --overwrite -A {tmp_outdir}/scaled_dtm_tmp.tif \\\n", 284 | " --outfile={rn_file} --calc=\"A * 0.1\" \\\n", 285 | " --type=Float32 --co=\"COMPRESS=DEFLATE\" --co=\"BLOCKXSIZE=2048\" --co=\"BLOCKYSIZE=2048\"')\n", 286 | " s3_path = f\"v4/{cont}/{out_ori_file}\"\n", 287 | " client.fput_object(s3_config['bucket'], s3_path, rn_file)\n", 288 | " ttprint(f'{ip}/tmp-global-geomorpho/{s3_path} on S3')\n", 289 | " \n", 290 | " gdal_cmd = f'gdalwarp \\\n", 291 | " -co TILED=YES -co BIGTIFF=YES -co COMPRESS=DEFLATE -co ZLEVEL=9 -co BLOCKXSIZE=1024 \\\n", 292 | " -co BLOCKYSIZE=1024 -co NUM_THREADS=8 -co SPARSE_OK=TRUE -of GTiff -overwrite'\n", 293 | " input_file = f'{tmp_outdir}/dtm_cont.tif'\n", 294 | " os.system(f'{gdal_cmd} \\\n", 295 | " /vsicurl/http://{ip}/tmp-global-geomorpho/v4/{cont}/dtm_edtm_m_{resolution}m_s_20000101_20221231_{cont.lower().replace(\"_\",\".\")}_equi7_v20241230.tif \\\n", 296 | " {input_file}')\n", 297 | "\n", 298 | " import whitebox_workflows\n", 299 | " from whitebox_workflows import download_sample_data, show, WbEnvironment\n", 300 | "\n", 301 | " # Test your license by setting up the WbW environment\n", 302 | " wbe = whitebox_workflows.WbEnvironment()\n", 303 | " start_time = time.time()\n", 304 | " #tmp_dtm_file=f'vsicrul/http://{ip}/tmp-global-geomorpho/{s3_path}'\n", 305 | " # Reading raster data\n", 306 | " dtm = wbe.read_raster(input_file)\n", 307 | " ttprint(\"read_raster--- %s seconds ---\" % (time.time() - start_time))\n", 308 | " outdir=f'/mnt/{server_name}/tmp-global-geomorpho/{cont}'\n", 309 | " #outdir=f'/mnt/landmark/tmp-global-geomorpho/{cont}'\n", 310 | " os.makedirs(outdir,exist_ok=True)\n", 311 | " file_list=[]\n", 312 | "\n", 313 | "\n", 314 | " global_landmask_file='http://{ip}/global/dsm.landmask_ensemble_m_30m_s_20000101_20221231_go_epsg.4326_v4.1.tif'\n", 315 | " #tmp_ori_landmask_file = f'{tmp_outdir}/tmp_landmask.tif'\n", 316 | " tmp_landmask_file = f'{tmp_outdir}/landmask.tif'\n", 317 | " #os.system(f'{gdal_cmd_reproject} -r min {global_landmask_file} {tmp_landmask_file}')\n", 318 | " os.system(f'{gdal_cmd_reproject} -te {gdalwarp_bbox} -r min {global_landmask_file} {tmp_landmask_file}')\n", 319 | "\n", 320 | " start_time = time.time()\n", 321 | " # Reading raster data\n", 322 | " ttprint(f\"{cont} read_raster--- %s seconds ---\" % (time.time() - start_time))\n", 323 | "\n", 324 | " # geomorphon\n", 325 | " tmp_geomorphon_file=input_file.replace('dtm','geomorphon')\n", 326 | " scale=p_table[p_table['parameters']=='geomorphon'].multiplier.iloc[0]\n", 327 | "\n", 328 | " start_time = time.time()\n", 329 | " geomorphon=wbe.geomorphons(dtm, search_distance=3, \n", 330 | " output_forms=True, analyze_residuals=False)\n", 331 | " wbe.write_raster(geomorphon*scale, tmp_geomorphon_file, compress=True)#, compress=False) # Compression is good, but it is a bit slower so here we won't use it.\n", 332 | " ttprint(f\"{cont} calculate geomporphon--- %s seconds ---\" % (time.time() - start_time)) \n", 333 | " file_list.append(tmp_geomorphon_file)\n", 334 | "\n", 335 | "\n", 336 | " # fill depression for hydrological analysis\n", 337 | " tmp_flled_dtm_file=input_file.replace('dtm','nodepress.dtm')\n", 338 | " scale=p_table[p_table['parameters']=='nodepress.dtm'].multiplier.iloc[0]\n", 339 | "\n", 340 | " start_time = time.time()\n", 341 | " dtm_no_deps = wbe.breach_depressions_least_cost(dtm, fill_deps=True, flat_increment=0.001)\n", 342 | " wbe.write_raster(dtm_no_deps*scale, tmp_flled_dtm_file, compress=True)#, compress=False) # Compression is good, but it is a bit slower so here we won't use it.\n", 343 | " ttprint(f\"{cont} fill depressions--- %s seconds ---\" % (time.time() - start_time)) \n", 344 | " file_list.append(tmp_flled_dtm_file)\n", 345 | "\n", 346 | "\n", 347 | " # slope for hydrology\n", 348 | " tmp_slope_file=input_file.replace('dtm','slope.in.degree')\n", 349 | " scale=p_table[p_table['parameters']=='slope.in.degree'].multiplier.iloc[0]\n", 350 | "\n", 351 | " start_time = time.time()\n", 352 | " slope_in_degree = wbe.slope(dtm)\n", 353 | " wbe.write_raster(slope_in_degree*scale, tmp_slope_file, compress=True)#, compress=False) # Compression is good, but it is a bit slower so here we won't use it.\n", 354 | " ttprint(f\"{cont} calculate slope--- %s seconds ---\" % (time.time() - start_time))\n", 355 | " file_list.append(tmp_slope_file)\n", 356 | "\n", 357 | " # SCA\n", 358 | " tmp_sca_file=input_file.replace('dtm','spec.catch')\n", 359 | " scale=p_table[p_table['parameters']=='spec.catch'].multiplier.iloc[0]\n", 360 | "\n", 361 | " start_time = time.time()\n", 362 | " sca = wbe.qin_flow_accumulation(dtm_no_deps, out_type='sca', log_transform=True)\n", 363 | " wbe.write_raster(sca*scale, tmp_sca_file, compress=True)#, compress=False) # Compression is good, but it is a bit slower so here we won't use it.\n", 364 | " ttprint(f\"{cont} specific catchment area--- %s seconds ---\" % (time.time() - start_time))\n", 365 | " file_list.append(tmp_sca_file)\n", 366 | "\n", 367 | " # ls factor\n", 368 | " tmp_lsfactor_file=input_file.replace('dtm','ls.factor')\n", 369 | " start_time = time.time()\n", 370 | " scale=p_table[p_table['parameters']=='ls.factor'].multiplier.iloc[0]\n", 371 | "\n", 372 | " ls_factor=wbe.sediment_transport_index(sca, slope_in_degree, sca_exponent=0.4, slope_exponent=1.3)\n", 373 | " wbe.write_raster(ls_factor*scale, tmp_lsfactor_file, compress=True)#, compress=False) # Compression is good, but it is a bit slower so here we won't use it.\n", 374 | " ttprint(f\"{cont} calculate ls factor--- %s seconds ---\" % (time.time() - start_time)) \n", 375 | " file_list.append(tmp_lsfactor_file)\n", 376 | "\n", 377 | " #twi\n", 378 | " start_time = time.time()\n", 379 | " tmp_twi_file=input_file.replace('dtm','twi')\n", 380 | " scale=p_table[p_table['parameters']=='twi'].multiplier.iloc[0]\n", 381 | "\n", 382 | " twi = wbe.wetness_index(specific_catchment_area=sca, slope=slope_in_degree)\n", 383 | " #twi_filled = wbe.fill_missing_data(twi, exclude_edge_nodata=True)\n", 384 | " ttprint(f\"{cont} topographic wetness index--- %s seconds ---\" % (time.time() - start_time))\n", 385 | " wbe.write_raster(twi*scale, tmp_twi_file, compress=True) # Compression is good, but it is a bit slower so here we won't use it.\n", 386 | " file_list.append(tmp_twi_file)\n", 387 | "\n", 388 | " # Reading raster data\n", 389 | " start_time = time.time()\n", 390 | " # Reading raster data\n", 391 | " ttprint(f\"{cont} read local surface raster--- %s seconds ---\" % (time.time() - start_time))\n", 392 | "\n", 393 | " # diff from mean elev\n", 394 | " start_time = time.time()\n", 395 | " tmp_dfme_file=input_file.replace('dtm','dfme')\n", 396 | " scale=p_table[p_table['parameters']=='dfme'].multiplier.iloc[0]\n", 397 | "\n", 398 | " dfme=wbe.difference_from_mean_elevation(\n", 399 | " dtm, \n", 400 | " filter_size_x=3, \n", 401 | " filter_size_y=3)\n", 402 | "\n", 403 | " wbe.write_raster(dfme*scale, tmp_dfme_file, compress=True) # Compression is good, but it is a bit slower so here we won't use it.\n", 404 | " ttprint(f\"{cont} diff from mean elev--- %s seconds ---\" % (time.time() - start_time))\n", 405 | " file_list.append(tmp_dfme_file)\n", 406 | "\n", 407 | " #Spherical Std Dev Of Normals\n", 408 | " start_time = time.time()\n", 409 | " tmp_ssdon_file=input_file.replace('dtm','ssdon')\n", 410 | " scale=p_table[p_table['parameters']=='ssdon'].multiplier.iloc[0]\n", 411 | "\n", 412 | " start_time = time.time()\n", 413 | " ssdon=wbe.spherical_std_dev_of_normals(\n", 414 | " dtm, \n", 415 | " filter_size=3 \n", 416 | " )\n", 417 | "\n", 418 | " wbe.write_raster(ssdon*scale, tmp_ssdon_file, compress=True) # Compression is good, but it is a bit slower so here we won't use it.\n", 419 | " ttprint(f\"{cont} spherical std dev of normals--- %s seconds ---\" % (time.time() - start_time))\n", 420 | " file_list.append(tmp_ssdon_file)\n", 421 | "\n", 422 | "\n", 423 | " # Hillshade\n", 424 | " tmp_hillshade_file=input_file.replace('dtm','hillshade')\n", 425 | " scale=p_table[p_table['parameters']=='hillshade'].multiplier.iloc[0]\n", 426 | "\n", 427 | " start_time = time.time()\n", 428 | " hs = wbe.multidirectional_hillshade(dtm)\n", 429 | " wbe.write_raster(hs*scale, tmp_hillshade_file, compress=True)#, compress=False) # Compression is good, but it is a bit slower so here we won't use it.\n", 430 | " ttprint(f\"{cont} calculate hillshade--- %s seconds ---\" % (time.time() - start_time)) \n", 431 | " file_list.append(tmp_hillshade_file)\n", 432 | "\n", 433 | " # Minic\n", 434 | " tmp_minic_file=input_file.replace('dtm','minic')\n", 435 | " scale=p_table[p_table['parameters']=='minic'].multiplier.iloc[0]\n", 436 | "\n", 437 | " start_time = time.time()\n", 438 | " minic = wbe.minimal_curvature(dtm, log_transform=True)\n", 439 | " wbe.write_raster(minic*scale, tmp_minic_file, compress=True)#, compress=False) # Compression is good, but it is a bit slower so here we won't use it.\n", 440 | " ttprint(f\"{cont} calculate minic--- %s seconds ---\" % (time.time() - start_time))\n", 441 | " file_list.append(tmp_minic_file)\n", 442 | "\n", 443 | "\n", 444 | " # Maxic\n", 445 | " tmp_maxic_file=input_file.replace('dtm','maxic')\n", 446 | " scale=p_table[p_table['parameters']=='maxic'].multiplier.iloc[0]\n", 447 | "\n", 448 | " start_time = time.time()\n", 449 | " maxic = wbe.maximal_curvature(dtm, log_transform=True)\n", 450 | " wbe.write_raster(maxic*scale, tmp_maxic_file, compress=True)#, compress=False) # Compression is good, but it is a bit slower so here we won't use it.\n", 451 | " ttprint(f\"{cont} calculate maxic--- %s seconds ---\" % (time.time() - start_time)) \n", 452 | " file_list.append(tmp_maxic_file)\n", 453 | "\n", 454 | " # Openness\n", 455 | " tmp_pos_file=input_file.replace('dtm','pos.openness')\n", 456 | " tmp_neg_file=input_file.replace('dtm','neg.openness')\n", 457 | " start_time = time.time()\n", 458 | " pos,neg = wbe.openness(dtm,dist=3)\n", 459 | " scale=p_table[p_table['parameters']=='pos.openness'].multiplier.iloc[0]\n", 460 | " wbe.write_raster(pos*scale, tmp_pos_file, compress=True)#, compress=False) # Compression is good, but it is a bit slower so here we won't use it.\n", 461 | " scale=p_table[p_table['parameters']=='neg.openness'].multiplier.iloc[0]\n", 462 | "\n", 463 | " wbe.write_raster(neg*scale, tmp_neg_file, compress=True)#, compress=False) # Compression is good, but it is a bit slower so here we won't use it.\n", 464 | "\n", 465 | " ttprint(f\"{cont} calculate openness--- %s seconds ---\" % (time.time() - start_time)) \n", 466 | " file_list.append(tmp_pos_file)\n", 467 | " file_list.append(tmp_neg_file)\n", 468 | "\n", 469 | "\n", 470 | " # profile curve\n", 471 | " start_time = time.time()\n", 472 | " tmp_procurv_file=input_file.replace('dtm','pro.curv')\n", 473 | " scale=p_table[p_table['parameters']=='pro.curv'].multiplier.iloc[0]\n", 474 | " procurv = wbe.profile_curvature(dtm, log_transform=True)\n", 475 | "\n", 476 | " wbe.write_raster(procurv*scale, tmp_procurv_file, compress=True) # Compression is good, but it is a bit slower so here we won't use it.\n", 477 | "\n", 478 | " ttprint(f\"{cont} profile curve --- %s seconds ---\" % (time.time() - start_time))\n", 479 | " file_list.append(tmp_procurv_file)\n", 480 | "\n", 481 | "\n", 482 | " # shape index\n", 483 | " start_time = time.time()\n", 484 | " tmp_shpindx_file=input_file.replace('dtm','shpindx')\n", 485 | " scale=p_table[p_table['parameters']=='shpindx'].multiplier.iloc[0]\n", 486 | "\n", 487 | "\n", 488 | " shpindx=wbe.shape_index(dtm)\n", 489 | " wbe.write_raster(shpindx*scale, tmp_shpindx_file, compress=True)\n", 490 | " ttprint(f\"{cont} shape index --- %s seconds ---\" % (time.time() - start_time))\n", 491 | " file_list.append(tmp_shpindx_file)\n", 492 | "\n", 493 | " # ring curvature\n", 494 | " start_time = time.time()\n", 495 | " tmp_ring_curv_file=input_file.replace('dtm','ring.curv')\n", 496 | " scale=p_table[p_table['parameters']=='ring.curv'].multiplier.iloc[0]\n", 497 | "\n", 498 | "\n", 499 | " ring_curv=wbe.ring_curvature(dtm, log_transform=True)\n", 500 | " wbe.write_raster(ring_curv*scale, tmp_ring_curv_file, compress=True) # Compression is good, but it is a bit slower so here we won't use it.\n", 501 | "\n", 502 | " ttprint(f\"{cont} ring curvature --- %s seconds ---\" % (time.time() - start_time))\n", 503 | " file_list.append(tmp_ring_curv_file)\n", 504 | "\n", 505 | " # tangential curvatures\n", 506 | " start_time = time.time()\n", 507 | " tmp_tan_curv_file=input_file.replace('dtm','tan.curv')\n", 508 | " scale=p_table[p_table['parameters']=='tan.curv'].multiplier.iloc[0]\n", 509 | "\n", 510 | " tan_curv=wbe.tangential_curvature(dtm, log_transform=True)\n", 511 | " wbe.write_raster(tan_curv*scale, tmp_tan_curv_file, compress=True) # Compression is good, but it is a bit slower so here we won't use it.\n", 512 | "\n", 513 | " ttprint(f\"{cont} tangential curvature --- %s seconds ---\" % (time.time() - start_time))\n", 514 | " file_list.append(tmp_tan_curv_file)\n", 515 | "\n", 516 | "\n", 517 | " start_time = time.time()\n", 518 | " def para_gdal_warp(file_path,cont,bbox,p_table,tmp_landmask_file):\n", 519 | " file_name = file_path.split('/')[-1]\n", 520 | " parameter = file_name.split('_')[0]\n", 521 | " dtype=p_table[p_table['parameters']==parameter].new_data_type.iloc[0]\n", 522 | " no_data=p_table[p_table['parameters']==parameter].no_data.iloc[0]\n", 523 | "\n", 524 | " gdalcmd = f'gdalwarp -overwrite -ot {dtype} -tr {resolution} {resolution} -te {bbox} -co TILED=YES -co BIGTIFF=YES -co COMPRESS=DEFLATE -co ZLEVEL=9 -co BLOCKXSIZE=2048 -co BLOCKYSIZE=2048 -co NUM_THREADS=8 -co SPARSE_OK=TRUE'\n", 525 | "\n", 526 | " file_name = parameter + '_edtm' + '_m' + f'_{resolution}m' + '_s' + '_20000101_20221231' + f'_{cont.lower()}' + '_equi7' + '_v20241230' + '.tif'\n", 527 | " out_path = f'{outdir}/{file_name}' \n", 528 | " tmp_out_path = f'{outdir}/tmp_{file_name}'\n", 529 | " os.system(f'{gdalcmd} {file_path} {tmp_out_path}')\n", 530 | " # landmasking\n", 531 | " os.system(f'gdal_calc.py -A {tmp_out_path} -B {tmp_landmask_file} --overwrite --outfile={out_path} \\\n", 532 | " --calc=\"(B==100)*A + (B!=100)*{no_data}\" --type={dtype} --co=\"ZLEVEL=9\" --co=\"COMPRESS=DEFLATE\" \\\n", 533 | " --co=\"BLOCKXSIZE=2048\" --NoDataValue={no_data} --co=\"BLOCKYSIZE=2048\" \\\n", 534 | " --co=\"NUM_THREADS=8\" --co=\"SPARSE_OK=TRUE\"')\n", 535 | " os.remove(file_path)\n", 536 | " return out_path,file_name\n", 537 | "\n", 538 | " args = [(i,cont,gdalwarp_bbox,p_table,tmp_landmask_file) for i in file_list]\n", 539 | " for arg in args:\n", 540 | " out_file,rn_file=para_gdal_warp(arg[0],arg[1],arg[2],arg[3],arg[4])\n", 541 | " s3_path = f\"v4/{cont}/{rn_file}\"\n", 542 | " client.fput_object(s3_config['bucket'], s3_path, out_file)\n", 543 | " os.remove(out_file)\n", 544 | " ttprint(f'http://{ip}/tmp-global-geomorpho/{s3_path} on S3')\n", 545 | " #os.remove(rn_file)\n", 546 | " os.system(f'rm -r {tmp_outdir}/*')\n", 547 | " ttprint(f\"{cont} crop and save to local--- %s seconds ---\" % (time.time() - start_time))\n", 548 | "\n", 549 | "\n", 550 | "Parallel(n_jobs=6)(delayed(worker_cont)(i,p_table) for i in conts)" 551 | ] 552 | } 553 | ], 554 | "metadata": { 555 | "kernelspec": { 556 | "display_name": "Python 3 (ipykernel)", 557 | "language": "python", 558 | "name": "python3" 559 | }, 560 | "language_info": { 561 | "codemirror_mode": { 562 | "name": "ipython", 563 | "version": 3 564 | }, 565 | "file_extension": ".py", 566 | "mimetype": "text/x-python", 567 | "name": "python", 568 | "nbconvert_exporter": "python", 569 | "pygments_lexer": "ipython3", 570 | "version": "3.8.16" 571 | } 572 | }, 573 | "nbformat": 4, 574 | "nbformat_minor": 5 575 | } 576 | -------------------------------------------------------------------------------- /parametrization/003.fine_res_parameterization.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "b959b3d0-a0d4-4da7-9f27-f915530d0871", 6 | "metadata": { 7 | "tags": [] 8 | }, 9 | "source": [ 10 | "# Land relief parameterization in coarser resolution \n", 11 | "\n", 12 | "This script is used to generate the fine resolution land relief parameters. The data would be cropped into tiles in 6 continents, Africa, Asia, Europe, South America, and North America. Each tile would be extended to have overlap to prevent border effect. The tiles has been generated by `002.generate_tiles.ipynb`, and stored in `equi7_tiles`. \n", 13 | "\n", 14 | "Each tile runs in its individual process with a stream of parameterization. A landmask is used to mask out the ocean pixels. " 15 | ] 16 | }, 17 | { 18 | "cell_type": "markdown", 19 | "id": "06431dbb-e8bd-4a8e-816c-0795d2d99289", 20 | "metadata": {}, 21 | "source": [ 22 | "## Part 1: Set up" 23 | ] 24 | }, 25 | { 26 | "cell_type": "code", 27 | "execution_count": 1, 28 | "id": "fed3be49-cd90-45c7-bf38-3837b3a101d4", 29 | "metadata": { 30 | "tags": [] 31 | }, 32 | "outputs": [], 33 | "source": [ 34 | "import os\n", 35 | "os.environ['USE_PYGEOS'] = '0'\n", 36 | "import geopandas as gpd\n", 37 | "import numpy as np\n", 38 | "from shapely.geometry import Polygon,mapping,box\n", 39 | "from shapely import segmentize\n", 40 | "import time\n", 41 | "import sys\n", 42 | "from joblib import Parallel, delayed\n", 43 | "from minio import Minio\n", 44 | "from eumap.misc import ttprint\n", 45 | "import requests\n", 46 | "import pandas as pd\n", 47 | "import pickle\n", 48 | "# set up whiteboxworkflow environment\n", 49 | "import whitebox_workflows\n", 50 | "from whitebox_workflows import download_sample_data, show, WbEnvironment\n", 51 | "wbe = whitebox_workflows.WbEnvironment()" 52 | ] 53 | }, 54 | { 55 | "cell_type": "code", 56 | "execution_count": 5, 57 | "id": "a833526b-cccb-491a-b9d4-862cdf321b3b", 58 | "metadata": { 59 | "tags": [] 60 | }, 61 | "outputs": [ 62 | { 63 | "data": { 64 | "text/html": [ 65 | "
\n", 66 | "\n", 79 | "\n", 80 | " \n", 81 | " \n", 82 | " \n", 83 | " \n", 84 | " \n", 85 | " \n", 86 | " \n", 87 | " \n", 88 | " \n", 89 | " \n", 90 | " \n", 91 | " \n", 92 | " \n", 93 | " \n", 94 | " \n", 95 | " \n", 96 | " \n", 97 | " \n", 98 | " \n", 99 | " \n", 100 | " \n", 101 | " \n", 102 | " \n", 103 | " \n", 104 | " \n", 105 | " \n", 106 | " \n", 107 | " \n", 108 | " \n", 109 | " \n", 110 | " \n", 111 | " \n", 112 | " \n", 113 | " \n", 114 | " \n", 115 | " \n", 116 | " \n", 117 | " \n", 118 | " \n", 119 | " \n", 120 | " \n", 121 | " \n", 122 | " \n", 123 | " \n", 124 | " \n", 125 | " \n", 126 | " \n", 127 | " \n", 128 | " \n", 129 | " \n", 130 | " \n", 131 | " \n", 132 | " \n", 133 | " \n", 134 | " \n", 135 | " \n", 136 | " \n", 137 | " \n", 138 | " \n", 139 | " \n", 140 | " \n", 141 | " \n", 142 | " \n", 143 | " \n", 144 | " \n", 145 | " \n", 146 | " \n", 147 | " \n", 148 | " \n", 149 | " \n", 150 | " \n", 151 | " \n", 152 | " \n", 153 | " \n", 154 | " \n", 155 | " \n", 156 | "
parametersold_data_typeori_minori_maxmultipliernew_data_typefinal_minfinal_maxno_data
0geomorphonInt161.00000010.000001Byte1.00000010.000000255
1hillshadeFloat320.00000028357.000001UInt160.00000028357.00000065535
2slope.in.degreeFloat320.00000015.29000100UInt160.0000001529.00000065535
3ls.factorFloat320.00000013.000001000UInt160.00000013000.00000065535
4pro.curvFloat32-8.2103298.011731000Int16-8210.3290568011.73019432767
\n", 157 | "
" 158 | ], 159 | "text/plain": [ 160 | " parameters old_data_type ori_min ori_max multiplier \\\n", 161 | "0 geomorphon Int16 1.000000 10.00000 1 \n", 162 | "1 hillshade Float32 0.000000 28357.00000 1 \n", 163 | "2 slope.in.degree Float32 0.000000 15.29000 100 \n", 164 | "3 ls.factor Float32 0.000000 13.00000 1000 \n", 165 | "4 pro.curv Float32 -8.210329 8.01173 1000 \n", 166 | "\n", 167 | " new_data_type final_min final_max no_data \n", 168 | "0 Byte 1.000000 10.000000 255 \n", 169 | "1 UInt16 0.000000 28357.000000 65535 \n", 170 | "2 UInt16 0.000000 1529.000000 65535 \n", 171 | "3 UInt16 0.000000 13000.000000 65535 \n", 172 | "4 Int16 -8210.329056 8011.730194 32767 " 173 | ] 174 | }, 175 | "execution_count": 5, 176 | "metadata": {}, 177 | "output_type": "execute_result" 178 | } 179 | ], 180 | "source": [ 181 | "## scale factors for each parameters\n", 182 | "p_table=pd.read_csv('scaling.csv')\n", 183 | "p_table.head()" 184 | ] 185 | }, 186 | { 187 | "cell_type": "code", 188 | "execution_count": 2, 189 | "id": "e60dbb6d-7967-4119-8006-04ec27db0e68", 190 | "metadata": { 191 | "tags": [] 192 | }, 193 | "outputs": [], 194 | "source": [ 195 | "#os.system(f'rm -r tmp-global-geomorpho/*')\n", 196 | "with open('shuf.txt', 'r') as file:\n", 197 | " shuf = [int(line.strip()) for line in file]\n", 198 | " \n", 199 | "with open(f'equi7_tiles', \"rb\") as fp: # Unpickling\n", 200 | " args_whole = pickle.load(fp)\n", 201 | " \n", 202 | "start_tile=0\n", 203 | "end_tile=790\n", 204 | "args = args_whole[start_tile:end_tile]" 205 | ] 206 | }, 207 | { 208 | "cell_type": "code", 209 | "execution_count": 18, 210 | "id": "3f5ccddf-927c-46d0-b937-9ebada6cb185", 211 | "metadata": { 212 | "tags": [] 213 | }, 214 | "outputs": [], 215 | "source": [ 216 | "import os\n", 217 | "os.environ['USE_PYGEOS'] = '0'\n", 218 | "import geopandas as gpd\n", 219 | "import numpy as np\n", 220 | "from shapely.geometry import Polygon,mapping,box\n", 221 | "from shapely import segmentize\n", 222 | "import time\n", 223 | "import sys\n", 224 | "from joblib import Parallel, delayed\n", 225 | "from minio import Minio\n", 226 | "from eumap.misc import ttprint\n", 227 | "import requests\n", 228 | "import pandas as pd\n", 229 | "import pickle\n", 230 | "import pandas as pd\n", 231 | "p_table=pd.read_csv('scaling.csv')\n", 232 | "#scaling={name:'geomorphon',dtype}\n", 233 | "\n", 234 | "\n", 235 | "\n", 236 | "def find_equi_7_proj(epsg4326_bound,equi_7_proj):\n", 237 | " # Extract coordinates from bounding box\n", 238 | " xmin, ymin, xmax, ymax = epsg4326_bound\n", 239 | "\n", 240 | " # Define coordinates for the polygon\n", 241 | " polygon_coords = [(xmin, ymin), (xmax, ymin), (xmax, ymax),(xmin, ymax)]\n", 242 | "\n", 243 | " # Create polygon\n", 244 | " polygon = Polygon(polygon_coords)\n", 245 | "\n", 246 | " # Create a GeoDataFrame with a single row representing the bounding box\n", 247 | " gdf = gpd.GeoDataFrame(geometry=[Polygon(polygon_coords)])\n", 248 | " # Set CRS for the GeoDataFrame\n", 249 | " gdf.crs = 'EPSG:4326'\n", 250 | " # Check the resulting GeoDataFrame\n", 251 | " gdf.geometry = segmentize(gdf.geometry,max_segment_length=0.000277778)\n", 252 | "\n", 253 | " return gdf.to_crs(equi_7_proj).geometry.bounds.values[0]\n", 254 | "\n", 255 | "def find_epsg4326_proj(equi7_bound,equi_7_proj):\n", 256 | " # Extract coordinates from bounding box\n", 257 | " xmin, ymin, xmax, ymax = equi7_bound\n", 258 | "\n", 259 | " # Define coordinates for the polygon\n", 260 | " polygon_coords = [(xmin, ymin), (xmax, ymin), (xmax, ymax),(xmin, ymax)]\n", 261 | "\n", 262 | " # Create polygon\n", 263 | " polygon = Polygon(polygon_coords)\n", 264 | "\n", 265 | " # Create a GeoDataFrame with a single row representing the bounding box\n", 266 | " gdf = gpd.GeoDataFrame(geometry=[Polygon(polygon_coords)])\n", 267 | " # Set CRS for the GeoDataFrame\n", 268 | " gdf.crs = equi_7_proj\n", 269 | " gdf.geometry = segmentize(gdf.geometry,max_segment_length=30)\n", 270 | " return gdf.to_crs('EPSG:4326')\n", 271 | "\n", 272 | "\n", 273 | "\n", 274 | "equi7_ind={'AF':'+proj=aeqd +lat_0=8.5 +lon_0=21.5 +x_0=5621452.01998 +y_0=5990638.42298 +datum=WGS84 +units=m +no_defs',\n", 275 | "'AN':'+proj=aeqd +lat_0=-90 +lon_0=0 +x_0=3714266.97719 +y_0=3402016.50625 +datum=WGS84 +units=m +no_defs',\n", 276 | "'AS':'+proj=aeqd +lat_0=47 +lon_0=94 +x_0=4340913.84808 +y_0=4812712.92347 +datum=WGS84 +units=m +no_defs',\n", 277 | "'EU':'+proj=aeqd +lat_0=53 +lon_0=24 +x_0=5837287.81977 +y_0=2121415.69617 +datum=WGS84 +units=m +no_defs',\n", 278 | "'NA':'+proj=aeqd +lat_0=52 +lon_0=-97.5 +x_0=8264722.17686 +y_0=4867518.35323 +datum=WGS84 +units=m +no_defs',\n", 279 | "'OC':'+proj=aeqd +lat_0=-19.5 +lon_0=131.5 +x_0=6988408.5356 +y_0=7654884.53733 +datum=WGS84 +units=m +no_defs',\n", 280 | "'SA':'+proj=aeqd +lat_0=-14 +lon_0=-60.5 +x_0=7257179.23559 +y_0=5592024.44605 +datum=WGS84 +units=m +no_defs'}\n", 281 | "\n", 282 | "conts = ['AF',\n", 283 | "'AN',\n", 284 | "'AS',\n", 285 | "'EU',\n", 286 | "'NA',\n", 287 | "'OC',\n", 288 | "'SA']\n", 289 | "\n", 290 | "\n", 291 | "\n", 292 | "#args_whole = args+args_special \n", 293 | "#resolution=240\n", 294 | "#start_tile=int(sys.argv[1])\n", 295 | "#end_tile=int(sys.argv[2])\n", 296 | "#server_name=sys.argv[3]\n", 297 | "#args = args_whole[start_tile:end_tile]\n", 298 | "\n", 299 | "\n", 300 | "#tiles=pd.read_csv('/mnt/slurm/jobs/global_geomorphometric-whiteboxtool/equi7_tiles.csv')\n", 301 | "#start_tile=1\n", 302 | "#end_tile=10\n", 303 | "#with open('/mnt/slurm/jobs/global_geomorphometric-whiteboxtool/shuf.txt', 'r') as file:\n", 304 | "# shuf = [int(line.strip()) for line in file]\n", 305 | " \n", 306 | "#with open(f'/mnt/slurm/jobs/global_geomorphometric-whiteboxtool/equi7_tiles', \"rb\") as fp: # Unpickling\n", 307 | "# args_whole = pickle.load(fp)\n", 308 | "\n", 309 | "\n", 310 | " \n", 311 | "\n", 312 | "#args=[]\n", 313 | "#for i in shuf[start_tile:end_tile]:\n", 314 | "# args.append(args_special[i])\n", 315 | "#print(len(args))\n", 316 | "#args_whole = args+args_special \n", 317 | "#resolution=240\n", 318 | "#start_tile=int(sys.argv[1])\n", 319 | "#end_tile=int(sys.argv[2])\n", 320 | "#server_name=sys.argv[3]\n", 321 | "#args = args_whole[start_tile:end_tile]\n", 322 | "\n", 323 | "#os.system(f'rm -r /mnt/{server_name}/tmp-global-geomorpho/*')\n", 324 | "#tiles=pd.read_csv('/mnt/slurm/jobs/global_geomorphometric-whiteboxtool/equi7_tiles.csv')\n", 325 | "#start_tile=1\n", 326 | "#end_tile=10\n", 327 | "with open('/mnt/slurm/jobs/global_geomorphometric-whiteboxtool/shuf.txt', 'r') as file:\n", 328 | " shuf = [int(line.strip()) for line in file]\n", 329 | " \n", 330 | "with open(f'/mnt/slurm/jobs/global_geomorphometric-whiteboxtool/equi7_tiles', \"rb\") as fp: # Unpickling\n", 331 | " args_whole = pickle.load(fp)\n", 332 | "\n", 333 | "args=[]\n", 334 | "for i in shuf[start_tile:end_tile]:\n", 335 | " args.append(args_whole[i])" 336 | ] 337 | }, 338 | { 339 | "cell_type": "markdown", 340 | "id": "acaf9554-c391-4a43-ac29-64a26f14886f", 341 | "metadata": {}, 342 | "source": [ 343 | "## Part 2: parametrization by tiles\n", 344 | "\n", 345 | "The parametrization using pyramid representation. The steps include:\n", 346 | "\n", 347 | "1. create local tile and convert DTM value from decimeter to meter and save it as float\n", 348 | "2. crop the landmask to the given tile.\n", 349 | "3. apply guassian filter at 30m \n", 350 | "4. apply paramtrization \n", 351 | "5. crop and save the parameters raster into local tile\n", 352 | "6. push to back the S3 server" 353 | ] 354 | }, 355 | { 356 | "cell_type": "code", 357 | "execution_count": null, 358 | "id": "808b8507-f983-4214-96e6-cc1bf29f3ea9", 359 | "metadata": {}, 360 | "outputs": [], 361 | "source": [ 362 | "for info in args:\n", 363 | " equi7_bounds_final,equi7_bounds_rls,epsg4326_bounds_rls,equi7_bounds_ls,epsg4326_bounds_ls,tile_name,equi_7_proj = info[0],info[1],info[2],info[3],info[4],info[5],info[6]\n", 364 | "\n", 365 | "\n", 366 | " outdir=f'tmp-global-geomorpho/{tile_name}'\n", 367 | " os.makedirs(outdir,exist_ok=True)\n", 368 | " gdalwarp_bbox_rls = ' '.join([str(i) for i in equi7_bounds_rls])\n", 369 | " gdalwarp_bbox_ls = ' '.join([str(i) for i in equi7_bounds_ls])\n", 370 | " gdalwarp_bbox_final = ' '.join([str(i) for i in equi7_bounds_final])\n", 371 | " filepath = 'legendtm_rf_30m_m_s_20000101_20231231_go_epsg.4326_v20250130.tif'\n", 372 | "\n", 373 | " start_time = time.time()\n", 374 | " tmp_outdir=f'/tmp/tmp-global-geomorpho/{tile_name}'\n", 375 | " os.makedirs(tmp_outdir,exist_ok=True)\n", 376 | " gdal_cmd = f'gdalwarp -co TILED=YES -co BIGTIFF=YES -co COMPRESS=DEFLATE \\\n", 377 | " -co ZLEVEL=9 -co BLOCKXSIZE=1024 -co BLOCKYSIZE=1024 -co NUM_THREADS=8 \\\n", 378 | " -co SPARSE_OK=TRUE -of GTiff -overwrite'\n", 379 | " out_ori_file=f'dtm_edtm_m_30m_s_20000101_20221231_{tile_name.lower().replace(\"_\",\".\")}.rls_equi7_v20241230.tif'\n", 380 | " url_rls=f'{ip}://tmp-global-geomorpho/{tile_name}/{out_ori_file}'\n", 381 | "\n", 382 | " r = requests.head(url_rls)\n", 383 | " if r.status_code == 200:\n", 384 | " ttprint(f'{url_rls} exists')\n", 385 | " else:\n", 386 | " rn_file = f'{tmp_outdir}/{out_ori_file}'\n", 387 | " os.system(f'{gdal_cmd} -t_srs \"{equi_7_proj}\" \\\n", 388 | " -te {gdalwarp_bbox_rls} -tr 30 30 -r bilinear {filepath} {tmp_outdir}/scaled_dtm_tmp_rls.tif')\n", 389 | " os.system(f'gdal_calc.py --overwrite -A {tmp_outdir}/scaled_dtm_tmp_rls.tif \\\n", 390 | " --outfile={rn_file} --calc=\"A * 0.1\" \\\n", 391 | " --type=Float32 --co=\"COMPRESS=DEFLATE\" --co=\"BLOCKXSIZE=2048\" --co=\"BLOCKYSIZE=2048\"')\n", 392 | " \n", 393 | " for resolution in [30,60,120,240]:\n", 394 | " url=f'{ip}://tmp-global-geomorpho/{tile_name}/tan.curv_edtm_m_{resolution}m_s_20000101_20221231_go_epsg.4326_v20241230.tif'\n", 395 | " r = requests.head(url)\n", 396 | " if r.status_code == 200:\n", 397 | " ttprint(f'{tile_name} has been process')\n", 398 | " return\n", 399 | "\n", 400 | " if resolution==30:\n", 401 | " tmp_dtm_rls_file = f'{tmp_outdir}/dtm_tmp_rls_{resolution}.tif'\n", 402 | " os.system(f'{gdal_cmd} /vsicurl/{url_rls} {tmp_dtm_rls_file}')\n", 403 | " # crop to local land surface tiff\n", 404 | " tmp_dtm_ls_file = f'{tmp_outdir}/dtm_tmp_ls.tif'\n", 405 | " os.system(f'{gdal_cmd} -te {gdalwarp_bbox_ls} /vsicurl/{url_rls} {tmp_dtm_ls_file}')\n", 406 | "\n", 407 | " else: \n", 408 | " # crop to regional land surface tiff\n", 409 | " tmp_dtm_rls_file = f'{tmp_outdir}/dtm_tmp_rls_{resolution}.tif'\n", 410 | " os.system(f'{gdal_cmd} -r average -tr {resolution} {resolution} -te {gdalwarp_bbox_rls} /vsicurl/{url_rls} {tmp_dtm_rls_file}')\n", 411 | " \n", 412 | " # crop to local land surface tiff\n", 413 | " tmp_dtm_ls_file = f'{tmp_outdir}/dtm_tmp_ls_{resolution}.tif'\n", 414 | " os.system(f'{gdal_cmd} -r average -tr {resolution} {resolution} -te {gdalwarp_bbox_ls} /vsicurl/{url_rls} {tmp_dtm_ls_file}')\n", 415 | " \n", 416 | " # crop the landmask\n", 417 | " global_landmask_file=f'{ip}://global/dsm.landmask_ensemble_m_30m_s_20000101_20221231_go_epsg.4326_v4.1.tif'\n", 418 | " tmp_landmask_file = f'{tmp_outdir}/landmask_{resolution}.tif'\n", 419 | " os.system(f'{gdal_cmd} -t_srs \"{equi_7_proj}\" -r min -tr {resolution} {resolution} -te {gdalwarp_bbox_final} {global_landmask_file} {tmp_landmask_file}')\n", 420 | "\n", 421 | "\n", 422 | " start_time = time.time()\n", 423 | " # Reading raster data\n", 424 | " dtm = wbe.read_raster(tmp_dtm_rls_file)\n", 425 | " ttprint(f\"{tile_name} read_raster--- %s seconds ---\" % (time.time() - start_time))\n", 426 | "\n", 427 | " file_list=[]\n", 428 | " if resolution == 30:\n", 429 | " start_time = time.time()\n", 430 | " dtm = wbe.gaussian_filter(dtm)\n", 431 | " ttprint(f\"{tile_name} calculate gaussian filter--- %s seconds ---\" % (time.time() - start_time)) \n", 432 | "\n", 433 | "\n", 434 | " # geomorphon\n", 435 | " #tmp_geomorphon_file=tmp_dtm_rls_file.replace('dtm','geomorphon')\n", 436 | " #scale=p_table[p_table['parameters']=='geomorphon'].multiplier.iloc[0]\n", 437 | "\n", 438 | " #start_time = time.time()\n", 439 | " #geomorphon=wbe.geomorphons(dtm, search_distance=3, \n", 440 | " # output_forms=True, analyze_residuals=False)\n", 441 | " #wbe.write_raster(geomorphon*scale, tmp_geomorphon_file, compress=True)#, compress=False) # Compression is good, but it is a bit slower so here we won't use it.\n", 442 | " #ttprint(f\"{tile_name} calculate geomporphon--- %s seconds ---\" % (time.time() - start_time)) \n", 443 | " #file_list.append(tmp_geomorphon_file)\n", 444 | "\n", 445 | "\n", 446 | " # fill depression for hydrological analysis\n", 447 | " tmp_flled_dtm_file=tmp_dtm_rls_file.replace('dtm','nodepress.dtm')\n", 448 | " scale=p_table[p_table['parameters']=='nodepress.dtm'].multiplier.iloc[0]\n", 449 | "\n", 450 | " start_time = time.time()\n", 451 | " dtm_no_deps = wbe.breach_depressions_least_cost(dtm, fill_deps=True, flat_increment=0.001)\n", 452 | " wbe.write_raster(dtm_no_deps*scale, tmp_flled_dtm_file, compress=True)#, compress=False) # Compression is good, but it is a bit slower so here we won't use it.\n", 453 | " ttprint(f\"{tile_name} fill depressions--- %s seconds ---\" % (time.time() - start_time)) \n", 454 | " file_list.append(tmp_flled_dtm_file)\n", 455 | "\n", 456 | "\n", 457 | " # slope for hydrology\n", 458 | " tmp_slope_file=tmp_dtm_rls_file.replace('dtm','slope.in.degree')\n", 459 | " scale=p_table[p_table['parameters']=='slope.in.degree'].multiplier.iloc[0]\n", 460 | "\n", 461 | " start_time = time.time()\n", 462 | " slope_in_degree = wbe.slope(dtm)\n", 463 | " wbe.write_raster(slope_in_degree*scale, tmp_slope_file, compress=True)#, compress=False) # Compression is good, but it is a bit slower so here we won't use it.\n", 464 | " ttprint(f\"{tile_name} calculate slope--- %s seconds ---\" % (time.time() - start_time))\n", 465 | " file_list.append(tmp_slope_file)\n", 466 | "\n", 467 | " # SCA\n", 468 | " tmp_sca_file=tmp_dtm_rls_file.replace('dtm','spec.catch')\n", 469 | " scale=p_table[p_table['parameters']=='spec.catch'].multiplier.iloc[0]\n", 470 | "\n", 471 | " start_time = time.time()\n", 472 | " sca = wbe.qin_flow_accumulation(dtm_no_deps, out_type='sca', log_transform=True)\n", 473 | " wbe.write_raster(sca*scale, tmp_sca_file, compress=True)#, compress=False) # Compression is good, but it is a bit slower so here we won't use it.\n", 474 | " ttprint(f\"{tile_name} specific catchment area--- %s seconds ---\" % (time.time() - start_time))\n", 475 | " file_list.append(tmp_sca_file)\n", 476 | "\n", 477 | " # ls factor\n", 478 | " tmp_lsfactor_file=tmp_dtm_rls_file.replace('dtm','ls.factor')\n", 479 | " start_time = time.time()\n", 480 | " scale=p_table[p_table['parameters']=='ls.factor'].multiplier.iloc[0]\n", 481 | "\n", 482 | " ls_factor=wbe.sediment_transport_index(sca, slope_in_degree, sca_exponent=0.4, slope_exponent=1.3)\n", 483 | " wbe.write_raster(ls_factor*scale, tmp_lsfactor_file, compress=True)#, compress=False) # Compression is good, but it is a bit slower so here we won't use it.\n", 484 | " ttprint(f\"{tile_name} calculate ls factor--- %s seconds ---\" % (time.time() - start_time)) \n", 485 | " file_list.append(tmp_lsfactor_file)\n", 486 | "\n", 487 | " #twi\n", 488 | " start_time = time.time()\n", 489 | " tmp_twi_file=tmp_dtm_rls_file.replace('dtm','twi')\n", 490 | " scale=p_table[p_table['parameters']=='twi'].multiplier.iloc[0]\n", 491 | "\n", 492 | " twi = wbe.wetness_index(specific_catchment_area=sca, slope=slope_in_degree)\n", 493 | " #twi_filled = wbe.fill_missing_data(twi, exclude_edge_nodata=True)\n", 494 | " ttprint(f\"{tile_name} topographic wetness index--- %s seconds ---\" % (time.time() - start_time))\n", 495 | " wbe.write_raster(twi*scale, tmp_twi_file, compress=True) # Compression is good, but it is a bit slower so here we won't use it.\n", 496 | " file_list.append(tmp_twi_file)\n", 497 | "\n", 498 | " # Reading raster data\n", 499 | " start_time = time.time()\n", 500 | " # Reading raster data\n", 501 | " dtm = wbe.read_raster(tmp_dtm_ls_file)\n", 502 | " ttprint(f\"{tile_name} read local surface raster--- %s seconds ---\" % (time.time() - start_time))\n", 503 | "\n", 504 | " # diff from mean elev\n", 505 | " start_time = time.time()\n", 506 | " tmp_dfme_file=tmp_dtm_ls_file.replace('dtm','dfme')\n", 507 | " scale=p_table[p_table['parameters']=='dfme'].multiplier.iloc[0]\n", 508 | "\n", 509 | " dfme=wbe.difference_from_mean_elevation(\n", 510 | " dtm, \n", 511 | " filter_size_x=3, \n", 512 | " filter_size_y=3)\n", 513 | "\n", 514 | " wbe.write_raster(dfme*scale, tmp_dfme_file, compress=True) # Compression is good, but it is a bit slower so here we won't use it.\n", 515 | " ttprint(f\"{tile_name} diff from mean elev--- %s seconds ---\" % (time.time() - start_time))\n", 516 | " file_list.append(tmp_dfme_file)\n", 517 | "\n", 518 | " #Spherical Std Dev Of Normals\n", 519 | " start_time = time.time()\n", 520 | " tmp_ssdon_file=tmp_dtm_ls_file.replace('dtm','ssdon')\n", 521 | " scale=p_table[p_table['parameters']=='ssdon'].multiplier.iloc[0]\n", 522 | "\n", 523 | " start_time = time.time()\n", 524 | " ssdon=wbe.spherical_std_dev_of_normals(\n", 525 | " dtm, \n", 526 | " filter_size=3 \n", 527 | " )\n", 528 | "\n", 529 | " wbe.write_raster(ssdon*scale, tmp_ssdon_file, compress=True) # Compression is good, but it is a bit slower so here we won't use it.\n", 530 | " ttprint(f\"{tile_name} spherical std dev of normals--- %s seconds ---\" % (time.time() - start_time))\n", 531 | " file_list.append(tmp_ssdon_file)\n", 532 | "\n", 533 | " if resolution == 30:\n", 534 | " start_time = time.time()\n", 535 | " dtm_g = wbe.gaussian_filter(dtm)\n", 536 | " ttprint(f\"{tile_name} calculate gaussian filter--- %s seconds ---\" % (time.time() - start_time))\n", 537 | " tmp_dtm_gaus_file=tmp_dtm_rls_file.replace('dtm','filtered.dtm')\n", 538 | " scale=p_table[p_table['parameters']=='filtered.dtm'].multiplier.iloc[0]\n", 539 | " wbe.write_raster(dtm_g*scale, tmp_dtm_gaus_file, compress=True)#, compress=False) # Compression is good, but it \n", 540 | " del dtm\n", 541 | " dtm = wbe.read_raster(tmp_dtm_gaus_file)\n", 542 | " file_list.append(tmp_dtm_gaus_file)\n", 543 | "\n", 544 | " # Hillshade\n", 545 | " tmp_hillshade_file=tmp_dtm_ls_file.replace('dtm','hillshade')\n", 546 | " scale=p_table[p_table['parameters']=='hillshade'].multiplier.iloc[0]\n", 547 | "\n", 548 | " start_time = time.time()\n", 549 | " hs = wbe.multidirectional_hillshade(dtm)\n", 550 | " wbe.write_raster(hs*scale, tmp_hillshade_file, compress=True)#, compress=False) # Compression is good, but it is a bit slower so here we won't use it.\n", 551 | " ttprint(f\"{tile_name} calculate hillshade--- %s seconds ---\" % (time.time() - start_time)) \n", 552 | " file_list.append(tmp_hillshade_file)\n", 553 | "\n", 554 | " # Minic\n", 555 | " tmp_minic_file=tmp_dtm_ls_file.replace('dtm','minic')\n", 556 | " scale=p_table[p_table['parameters']=='minic'].multiplier.iloc[0]\n", 557 | "\n", 558 | " start_time = time.time()\n", 559 | " minic = wbe.minimal_curvature(dtm, log_transform=True)\n", 560 | " wbe.write_raster(minic*scale, tmp_minic_file, compress=True)#, compress=False) # Compression is good, but it is a bit slower so here we won't use it.\n", 561 | " ttprint(f\"{tile_name} calculate minic--- %s seconds ---\" % (time.time() - start_time))\n", 562 | " file_list.append(tmp_minic_file)\n", 563 | "\n", 564 | "\n", 565 | " # Maxic\n", 566 | " tmp_maxic_file=tmp_dtm_ls_file.replace('dtm','maxic')\n", 567 | " scale=p_table[p_table['parameters']=='maxic'].multiplier.iloc[0]\n", 568 | "\n", 569 | " start_time = time.time()\n", 570 | " maxic = wbe.maximal_curvature(dtm, log_transform=True)\n", 571 | " wbe.write_raster(maxic*scale, tmp_maxic_file, compress=True)#, compress=False) # Compression is good, but it is a bit slower so here we won't use it.\n", 572 | " ttprint(f\"{tile_name} calculate maxic--- %s seconds ---\" % (time.time() - start_time)) \n", 573 | " file_list.append(tmp_maxic_file)\n", 574 | "\n", 575 | " # Openness\n", 576 | " tmp_pos_file=tmp_dtm_ls_file.replace('dtm','pos.openness')\n", 577 | " tmp_neg_file=tmp_dtm_ls_file.replace('dtm','neg.openness')\n", 578 | " start_time = time.time()\n", 579 | " pos,neg = wbe.openness(dtm,dist=3)\n", 580 | " scale=p_table[p_table['parameters']=='pos.openness'].multiplier.iloc[0]\n", 581 | " wbe.write_raster(pos*scale, tmp_pos_file, compress=True)#, compress=False) # Compression is good, but it is a bit slower so here we won't use it.\n", 582 | " scale=p_table[p_table['parameters']=='neg.openness'].multiplier.iloc[0]\n", 583 | "\n", 584 | " wbe.write_raster(neg*scale, tmp_neg_file, compress=True)#, compress=False) # Compression is good, but it is a bit slower so here we won't use it.\n", 585 | "\n", 586 | " ttprint(f\"{tile_name} calculate openness--- %s seconds ---\" % (time.time() - start_time)) \n", 587 | " file_list.append(tmp_pos_file)\n", 588 | " file_list.append(tmp_neg_file)\n", 589 | "\n", 590 | "\n", 591 | " # profile curve\n", 592 | " start_time = time.time()\n", 593 | " tmp_procurv_file=tmp_dtm_ls_file.replace('dtm','pro.curv')\n", 594 | " scale=p_table[p_table['parameters']=='pro.curv'].multiplier.iloc[0]\n", 595 | " procurv = wbe.profile_curvature(dtm, log_transform=True)\n", 596 | " wbe.write_raster(procurv*scale, tmp_procurv_file, compress=True) # Compression is good, but it is a bit slower so here we won't use it.\n", 597 | " ttprint(f\"{tile_name} profile curve --- %s seconds ---\" % (time.time() - start_time))\n", 598 | " file_list.append(tmp_procurv_file)\n", 599 | "\n", 600 | "\n", 601 | " # shape index\n", 602 | " start_time = time.time()\n", 603 | " tmp_shpindx_file=tmp_dtm_ls_file.replace('dtm','shpindx')\n", 604 | " scale=p_table[p_table['parameters']=='shpindx'].multiplier.iloc[0]\n", 605 | " shpindx=wbe.shape_index(dtm)\n", 606 | " wbe.write_raster(shpindx*scale, tmp_shpindx_file, compress=True)\n", 607 | " ttprint(f\"{tile_name} shape index --- %s seconds ---\" % (time.time() - start_time))\n", 608 | " file_list.append(tmp_shpindx_file)\n", 609 | "\n", 610 | " # ring curvature\n", 611 | " start_time = time.time()\n", 612 | " tmp_ring_curv_file=tmp_dtm_ls_file.replace('dtm','ring.curv')\n", 613 | " scale=p_table[p_table['parameters']=='ring.curv'].multiplier.iloc[0]\n", 614 | " ring_curv=wbe.ring_curvature(dtm, log_transform=True)\n", 615 | "\n", 616 | " ttprint(f\"{tile_name} ring curvature --- %s seconds ---\" % (time.time() - start_time))\n", 617 | " file_list.append(tmp_ring_curv_file)\n", 618 | "\n", 619 | " # tangential curvatures\n", 620 | " start_time = time.time()\n", 621 | " tmp_tan_curv_file=tmp_dtm_ls_file.replace('dtm','tan.curv')\n", 622 | " scale=p_table[p_table['parameters']=='tan.curv'].multiplier.iloc[0]\n", 623 | "\n", 624 | " tan_curv=wbe.tangential_curvature(dtm, log_transform=True)\n", 625 | " wbe.write_raster(tan_curv*scale, tmp_tan_curv_file, compress=True) # Compression is good, but it is a bit slower so here we won't use it.\n", 626 | " ttprint(f\"{tile_name} tangential curvature --- %s seconds ---\" % (time.time() - start_time))\n", 627 | " file_list.append(tmp_tan_curv_file)\n", 628 | "\n", 629 | "\n", 630 | " start_time = time.time()\n", 631 | " def para_gdal_warp(file_path,tile_name,bbox,p_table,tmp_landmask_file):\n", 632 | " file_name = file_path.split('/')[-1]\n", 633 | " parameter = file_name.split('_')[0]\n", 634 | " dtype=p_table[p_table['parameters']==parameter].new_data_type.iloc[0]\n", 635 | " no_data=p_table[p_table['parameters']==parameter].no_data.iloc[0]\n", 636 | "\n", 637 | " gdalcmd = f'gdalwarp -overwrite -ot {dtype} -tr {resolution} {resolution} -te {bbox} -co TILED=YES -co BIGTIFF=YES -co COMPRESS=DEFLATE -co ZLEVEL=9 -co BLOCKXSIZE=2048 -co BLOCKYSIZE=2048 -co NUM_THREADS=8 -co SPARSE_OK=TRUE'\n", 638 | "\n", 639 | " file_name = parameter + '_edtm' + '_m' + f'_{resolution}m' + '_s' + '_20000101_20221231' + '_go' + '_epsg.4326' + '_v20241230' + '.tif'\n", 640 | " out_path = f'{outdir}/{file_name}'\n", 641 | " tmp_out_path = f'{outdir}/tmp_{file_name}'\n", 642 | " os.system(f'{gdalcmd} {file_path} {tmp_out_path}')\n", 643 | " # landmasking\n", 644 | " os.system(f'gdal_calc.py -A {tmp_out_path} -B {tmp_landmask_file} --overwrite --outfile={out_path} \\\n", 645 | " --calc=\"(B==100)*A + (B!=100)*{no_data}\" --type={dtype} --co=\"ZLEVEL=9\" --co=\"COMPRESS=DEFLATE\" \\\n", 646 | " --co=\"BLOCKXSIZE=2048\" --NoDataValue={no_data} --co=\"BLOCKYSIZE=2048\" \\\n", 647 | " --co=\"NUM_THREADS=8\" --co=\"SPARSE_OK=TRUE\"')\n", 648 | " os.remove(file_path)\n", 649 | " return out_path,file_name\n", 650 | "\n", 651 | " args = [(i,tile_name,gdalwarp_bbox_final,p_table,tmp_landmask_file) for i in file_list]\n", 652 | " for arg in args:\n", 653 | " out_file,rn_file=para_gdal_warp(arg[0],arg[1],arg[2],arg[3],arg[4])\n", 654 | " s3_path = f\"{tile_name}/{rn_file}\"\n", 655 | " client.fput_object(s3_config['bucket'], s3_path, out_file)\n", 656 | " os.remove(out_file)\n", 657 | " ttprint(f'{ip}://tmp-global-geomorpho/{s3_path} on S3')\n", 658 | " os.remove(tmp_dtm_ls_file)\n", 659 | " os.remove(tmp_dtm_rls_file)\n", 660 | " os.system(f'rm -r {tmp_outdir}/*')\n", 661 | " ttprint(f\"{tile_name} crop and save to local--- %s seconds ---\" % (time.time() - start_time))\n", 662 | "\n", 663 | "Parallel(n_jobs=10)(delayed(worker)(i,p_table) for i in args)" 664 | ] 665 | } 666 | ], 667 | "metadata": { 668 | "kernelspec": { 669 | "display_name": "Python 3 (ipykernel)", 670 | "language": "python", 671 | "name": "python3" 672 | }, 673 | "language_info": { 674 | "codemirror_mode": { 675 | "name": "ipython", 676 | "version": 3 677 | }, 678 | "file_extension": ".py", 679 | "mimetype": "text/x-python", 680 | "name": "python", 681 | "nbconvert_exporter": "python", 682 | "pygments_lexer": "ipython3", 683 | "version": "3.8.16" 684 | } 685 | }, 686 | "nbformat": 4, 687 | "nbformat_minor": 5 688 | } 689 | --------------------------------------------------------------------------------