├── .devcontainer └── devcontainer.json ├── LICENSE ├── README.md ├── assets ├── asp_aster_output_plot.jpg └── images │ └── asp_aster_output_plot.jpg ├── development └── explore_earthaccess.ipynb └── tutorials ├── asp_binder_utils.py ├── example-aster_stereo_reconstruction.ipynb ├── example-dem_altimetry_coregistration.ipynb ├── example-dem_coregistration.ipynb ├── providence_mountains.geojson └── providence_mountains_small.geojson /.devcontainer/devcontainer.json: -------------------------------------------------------------------------------- 1 | { 2 | "image": "docker.io/uwcryo/asp-binder:9e07dd1ff9db", 3 | "hostRequirements": { 4 | "cpus": 4 5 | }, 6 | "postCreateCommand": "", 7 | "remoteEnv": { 8 | "PATH": "${localEnv:PATH}:/srv/StereoPipeline/bin" 9 | }, 10 | "customizations": { 11 | "codespaces": { 12 | "openFiles": [] 13 | }, 14 | "vscode": { 15 | "extensions": [ 16 | "ms-toolsai.jupyter", 17 | "ms-python.python" 18 | ], 19 | "settings": { 20 | "python.defaultInterpreterPath": "/srv/conda/envs/notebook/bin/python" 21 | } 22 | } 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 UW CEE Terrain Analysis and Cryosphere Observation Lab 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # asp_tutorials 2 | Interactive tutorials for [Ames Stereo Pipeline (ASP)](https://stereopipeline.readthedocs.io/en/latest/introduction.html) processing. 3 | 4 | ## Launch on GitHub Codespaces 5 | [![Open in GitHub Codespaces](https://github.com/codespaces/badge.svg)](https://codespaces.new/uw-cryo/asp_tutorials?quickstart=1) 6 | 7 | ☝️ this button will launch an Cloud-hosted computer on Microsoft Azure with ASP and related software pre-installed. 8 | 9 | ### Usage 10 | Once the codespace is launched, navigate to the tutorials folder from the file Explorer on the left of your codespace screen, and open the `example-aster_stereo_reconstruction.ipynb` notebook and run the cells to execute the interactive tutorial. 11 | ## Table of Contents 12 | * [Using ASP to process ASTER Stereo Imagery](https://nbviewer.org/github/uw-cryo/asp_tutorials/blob/master/tutorials/example-aster_stereo_reconstruction.ipynb) 13 | * [Aligning two Digital Elevation Models or point clouds](https://nbviewer.org/github/uw-cryo/asp_tutorials/blob/master/tutorials/example-dem_coregistration.ipynb) 14 | * [Aligning a Digital Elevation Model to sparse altimetry point-cloud](https://nbviewer.org/github/uw-cryo/asp_tutorials/blob/master/tutorials/example-dem_altimetry_coregistration.ipynb) 15 | 16 | #### Sample L1A ASTER stereo images 17 | [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.7972223.svg)](https://doi.org/10.5281/zenodo.7972223) 18 | * The tutorial uses sample L1A stereo images acquired by the ASTER instrument over Mt. Rainier, WA on July 31, 2017 (AST_L1A_00307312017190728_20200218153629_19952.zip) 19 | * More details can be found on the data product page: https://lpdaac.usgs.gov/products/ast_l1av003/ 20 | * The sample data were downloaded from the [NASA EarthData website](https://www.earthdata.nasa.gov/). We are rehosting on Zenodo to enable on-demand access to the sample images when running the tutorial. 21 | 22 | #### Example output 23 | ![Example DEM produced from the ASTEER tutorial](./assets/images/asp_aster_output_plot.jpg) 24 | Figure: Example output DEM produced from ASTER imagery acquired over Mt. Rainier. Top Row: Orthorectified A) left and B) right stereo images. Middle Row: Disparity in C) x (E-W) and D) y (N-S) direction. Bottom Row: E) Intersection error and F) Digital Elevation Model. 25 | 26 | ## Development Status 27 | This repository is under active development; we will be adding tutorials for different ASP processing capabilities, and for different Earth and Planetary datasets. Stay tuned!! Community feedback is welcomed through github issues :D 28 | ## Additional details on Github CodeSpace and usage 29 | GitHub currently gives every user [120 vCPU hours per month for free](https://docs.github.com/en/billing/managing-billing-for-github-codespaces/about-billing-for-github-codespaces#monthly-included-storage-and-core-hours-for-personal-accounts), beyond that you must pay. **So be sure to explicitly stop or shut down your codespace when you are done by going to this page (https://github.com/codespaces/).** 30 | 31 | * More details on codespace lifecycle is explained [here](https://docs.github.com/en/codespaces/getting-started/the-codespace-lifecycle#). 32 | -------------------------------------------------------------------------------- /assets/asp_aster_output_plot.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uw-cryo/asp_tutorials/233f0f1203bc4194b0f32bd04b2ae5bf153cf333/assets/asp_aster_output_plot.jpg -------------------------------------------------------------------------------- /assets/images/asp_aster_output_plot.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uw-cryo/asp_tutorials/233f0f1203bc4194b0f32bd04b2ae5bf153cf333/assets/images/asp_aster_output_plot.jpg -------------------------------------------------------------------------------- /tutorials/asp_binder_utils.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pandas as pd 3 | from scipy import stats 4 | import os,sys,glob 5 | import matplotlib.pyplot as plt 6 | from mpl_toolkits.axes_grid1 import make_axes_locatable 7 | import rasterio 8 | from osgeo import gdal 9 | import geopandas as gpd 10 | import contextily as ctx 11 | from pyproj import Proj, transform 12 | 13 | import subprocess 14 | 15 | 16 | def get_clim(ar): 17 | try: 18 | clim = np.percentile(ar.compressed(),(2,98)) 19 | except: 20 | clim = np.percentile(ar,(2,98)) 21 | return clim 22 | def find_common_clim(im1,im2): 23 | perc1 = get_clim(im1) 24 | perc2 = get_clim(im2) 25 | perc = (np.min([perc1[0],perc2[0]]),np.max([perc1[1],perc2[1]])) 26 | abs_max = np.max(np.abs(perc)) 27 | perc = (-abs_max,abs_max) 28 | return perc 29 | def fn_2_ma(fn,b=1): 30 | ds = rasterio.open(fn) 31 | ar = ds.read(b) 32 | ndv = get_ndv(ds) 33 | ma_ar = np.ma.masked_equal(ar,ndv) 34 | return ma_ar 35 | def get_ndv(ds): 36 | no_data = ds.nodatavals[0] 37 | if no_data == None: 38 | #this means no data is not set in tif tag, nead to cheat it from raster 39 | ndv = ds.read(1)[0,0] 40 | else: 41 | ndv = no_data 42 | return ndv 43 | 44 | def plot_stereo_results(out_folder,ax): 45 | dem = glob.glob(out_folder+'*-DEM.tif')[0] 46 | l_img = glob.glob(out_folder+'*L.tif')[0] 47 | r_img = glob.glob(out_folder+'*R.tif')[0] 48 | disp = glob.glob(out_folder+'*-F.tif')[0] 49 | error_fn = glob.glob(out_folder+'*In*.tif')[0] 50 | print("Found files {}\n {}\n {}\n {}\n {}\n".format(l_img,r_img,disp,error_fn,dem)) 51 | dem_ma = fn_2_ma(dem,1) 52 | l_img_ma = fn_2_ma(l_img) 53 | r_img_ma = fn_2_ma(r_img) 54 | dx_ma = fn_2_ma(disp,1) 55 | dy_ma = fn_2_ma(disp,2) 56 | error_ma = fn_2_ma(error_fn,1) 57 | dem_ds = gdal.Open(dem) 58 | producttype = 'hillshade' 59 | hs_ds = gdal.DEMProcessing('',dem_ds,producttype,format='MEM') 60 | hs = hs_ds.ReadAsArray() 61 | #fig,ax = plt.subplots(3,2,figsize=(6,5)) 62 | axa = ax.ravel() 63 | cmap_img = 'gray' 64 | cmap_disp = 'RdBu' 65 | cmap_error = 'inferno' 66 | plot_ar(l_img_ma,ax=axa[0],clim=get_clim(l_img_ma),cbar=False,cmap=cmap_img) 67 | #print(get_clim(l_img_ma)) 68 | plot_ar(l_img_ma,ax=axa[1],clim=get_clim(r_img_ma),cbar=False,cmap=cmap_img) 69 | disp_clim = find_common_clim(dx_ma,dy_ma) 70 | plot_ar(dx_ma,ax=axa[2],clim=disp_clim,cmap=cmap_disp,label='dx (px)') 71 | plot_ar(dy_ma,ax=axa[3],clim=disp_clim,cmap=cmap_disp,label='dy (px)') 72 | #plt.colorbar(dx_im,axa[2]) 73 | #plt.colorbar(dy_im,axa[3]) 74 | #error_im = axa[4].imshow(error_ma,cmap=cmap_error,clim=get_clim(error_ma),interpolation='none') 75 | plot_ar(error_ma,ax=axa[4],clim=get_clim(error_ma),cmap=cmap_error,label='Intersection Error(m)') 76 | #plt.colorbar(error_im,ax=axa[4]) 77 | axa[5].imshow(hs,cmap=cmap_img,clim=get_clim(hs),interpolation='none') 78 | plot_ar(dem_ma,ax=axa[5],clim=get_clim(dem_ma),label='HAE (m WGS84)',alpha=0.6) 79 | #print(get_clim(dem_ma)) 80 | #plt.colorbar(dem_im,axa[5]) 81 | plt.tight_layout() 82 | #print(type(dem_ma)) 83 | for idx,axa in enumerate(ax.ravel()): 84 | if idx == 4: 85 | axa.set_facecolor('gray') 86 | else: 87 | axa.set_facecolor('k') 88 | 89 | def plot_ar(im,ax,clim,cmap=None,label=None,cbar=True,alpha=1): 90 | if cmap: 91 | img = ax.imshow(im,cmap=cmap,clim=clim,alpha=alpha,interpolation='None') 92 | else: 93 | img = ax.imshow(im,clim=clim,alpha=alpha,interpolation='None') 94 | if cbar: 95 | divider = make_axes_locatable(ax) 96 | #cax = divider.append_axes("right", size="5%", pad=0.05) 97 | #cax = divider.append_axes("right", size="5%", pad="2%") 98 | cax = divider.append_axes("right", size="2%", pad="1.5%") 99 | cb = plt.colorbar(img,cax=cax,ax=ax,extend='both') 100 | cax.set_ylabel(label) 101 | ax.set_xticks([]) 102 | ax.set_yticks([]) 103 | 104 | def subsetBBox(rast,proj_out): 105 | #originally written by Justing Pflug 106 | # rasterio open data 107 | rB = rasterio.open(rast) 108 | proj_in = rB.crs 109 | # rasterio get bounding box 110 | [L,B,R,T] = rB.bounds 111 | 112 | if proj_in == proj_out: 113 | return L, R, T, B 114 | else: 115 | incord = Proj(init=proj_in) 116 | outcord = Proj(init=proj_out) 117 | 118 | [Left,Bottom] = transform(incord,outcord,L,B) 119 | [Right,Top] = transform(incord,outcord,R,T) 120 | return Left, Bottom, Right, Top 121 | 122 | def run_bash_command(cmd,verbose=False): 123 | #written by Scott Henderson 124 | # move to asp_binder_utils 125 | """Call a system command through the subprocess python module.""" 126 | print(cmd) 127 | try: 128 | if verbose: 129 | retcode = subprocess.call(cmd,shell=True) 130 | else: 131 | retcode = subprocess.call(cmd, stdout=subprocess.DEVNULL, 132 | stderr=subprocess.STDOUT,shell=True) 133 | if retcode < 0: 134 | print("Child was terminated by signal", -retcode, file=sys.stderr) 135 | else: 136 | print("Child returned", retcode, file=sys.stderr) 137 | except OSError as e: 138 | print("Execution failed:", e, file=sys.stderr) 139 | 140 | def plot_alignment_maps(refdem,src_dem,initial_elevation_difference_fn,aligned_elevation_difference_fn): 141 | f,ax = plt.subplots(2,2,figsize=(8,6)) 142 | axa = ax.ravel() 143 | 144 | refdem_ma = fn_2_ma(refdem,1) 145 | src_dem_ma = fn_2_ma(src_dem) 146 | initial_diff = fn_2_ma(initial_elevation_difference_fn) 147 | final_diff = fn_2_ma(aligned_elevation_difference_fn) 148 | 149 | refdem_ds = gdal.Open(refdem) 150 | producttype = 'hillshade' 151 | ref_hs_ds = gdal.DEMProcessing('',refdem_ds,producttype,format='MEM') 152 | ref_hs = ref_hs_ds.ReadAsArray() 153 | 154 | src_dem_ds = gdal.Open(src_dem) 155 | producttype = 'hillshade' 156 | src_hs_ds = gdal.DEMProcessing('',src_dem_ds,producttype,format='MEM') 157 | src_hs = src_hs_ds.ReadAsArray() 158 | #fig,ax = plt.subplots(3,2,figsize=(6,5)) 159 | axa = ax.ravel() 160 | cmap_diff = 'RdBu' 161 | cmap_hs = 'gray' 162 | axa[0].imshow(ref_hs,cmap=cmap_hs,clim=get_clim(ref_hs),interpolation='none') 163 | plot_ar(refdem_ma,ax=axa[0],clim=get_clim(refdem_ma),label='Elevation (m WGS84)',alpha=0.6) 164 | axa[0].set_title('Reference DEM') 165 | 166 | axa[1].imshow(src_hs,cmap=cmap_hs,clim=get_clim(src_hs),interpolation='none') 167 | plot_ar(src_dem_ma,ax=axa[1],clim=get_clim(src_dem_ma),label='Elevation (m WGS84)',alpha=0.6) 168 | axa[1].set_title('Source DEM') 169 | 170 | diff_clim = find_common_clim(initial_diff,final_diff) 171 | plot_ar(initial_diff,ax=axa[2],clim=diff_clim,cmap=cmap_diff,label=' Elevation difference (m)') 172 | axa[2].set_title("Before alignment") 173 | 174 | plot_ar(final_diff,ax=axa[3],clim=diff_clim,cmap=cmap_diff,label='Elevation difference (m)') 175 | axa[3].set_title("After alignment") 176 | plt.tight_layout() 177 | 178 | f2,ax2 = plt.subplots() 179 | bins = np.linspace(diff_clim[0],diff_clim[1], 128) 180 | initial_mad = stats.median_abs_deviation(initial_diff.compressed()) 181 | initial_med = np.median(initial_diff.compressed()) 182 | 183 | final_mad = stats.median_abs_deviation(final_diff.compressed()) 184 | final_med = np.median(final_diff.compressed()) 185 | 186 | title = f" Pre-alignment elev. diff. median: {initial_med: .2f} m, mad: {initial_mad: .2f} m\nPost-alignment elev. diff. median: {final_med: .2f} m, mad: {final_mad: .2f} m" 187 | 188 | ax2.hist(initial_diff.compressed(),bins=bins,color='blue',alpha=0.5,label='Initial') 189 | ax2.hist(final_diff.compressed(),bins=bins,color='green',alpha=0.5,label='Final') 190 | ax2.axvline(x=0,linestyle='--',linewidth=1,color='k') 191 | ax2.legend() 192 | ax2.set_xlabel('Elevation difference') 193 | ax2.set_ylabel('#pixels') 194 | ax2.set_title(title) 195 | 196 | def read_geodiff(csv_fn): 197 | #from David Shean 198 | resid_cols=['lon', 'lat', 'diff'] 199 | resid_df = pd.read_csv(csv_fn, comment='#', names=resid_cols) 200 | resid_gdf = gpd.GeoDataFrame(resid_df, geometry=gpd.points_from_xy(resid_df['lon'], resid_df['lat'], crs='EPSG:4326')) 201 | return resid_gdf 202 | 203 | def plot_alignment_maps_altimetry(reference_altimetry,src_dem,initial_elevation_difference_fn, 204 | aligned_elevation_difference_fn,plot_crs,provider=ctx.providers.Esri.WorldImagery,diff_clim=(-5,5)): 205 | f,ax = plt.subplots(2,2,figsize=(8,6)) 206 | axa = ax.ravel() 207 | markersize = 1 208 | #ref_alitmetry_gdf = gpd.read_file(reference_altimetry) 209 | src_dem_ma = fn_2_ma(src_dem) 210 | initial_diff = read_geodiff(initial_elevation_difference_fn) 211 | final_diff = read_geodiff(aligned_elevation_difference_fn) 212 | 213 | # change point file crs 214 | initial_diff = initial_diff.to_crs(plot_crs) 215 | final_diff = final_diff.to_crs(plot_crs) 216 | reference_altimetry = reference_altimetry.to_crs(plot_crs) 217 | 218 | src_dem_ds = gdal.Open(src_dem) 219 | producttype = 'hillshade' 220 | src_hs_ds = gdal.DEMProcessing('',src_dem_ds,producttype,format='MEM') 221 | src_hs = src_hs_ds.ReadAsArray() 222 | #fig,ax = plt.subplots(3,2,figsize=(6,5)) 223 | axa = ax.ravel() 224 | cmap_diff = 'RdBu' 225 | cmap_hs = 'gray' 226 | if len(reference_altimetry)>10000: 227 | reference_altimetry.sample(10000).plot('h_mean',ax=axa[0]) 228 | ctx.add_basemap(ax=axa[0],crs="EPSG:4326",attribution=False,source=provider) 229 | else: 230 | reference_altimetry.plot(column='h_mean',ax=axa[0],markersize=markersize) 231 | ctx.add_basemap(ax=axa[0],crs=plot_crs,attribution=False,source=provider) 232 | axa[0].set_title('Reference ICESat-2 altimetry points') 233 | 234 | axa[1].imshow(src_hs,cmap=cmap_hs,clim=get_clim(src_hs),interpolation='none') 235 | plot_ar(src_dem_ma,ax=axa[1],clim=get_clim(src_dem_ma),label='Elevation (m WGS84)',alpha=0.6) 236 | axa[1].set_title('Source DEM') 237 | 238 | ## Difference maps 239 | 240 | initial_diff.plot(column='diff',ax=axa[2],cmap='RdBu',vmin=diff_clim[0],vmax=diff_clim[1],markersize=markersize) 241 | final_diff.plot(column='diff',ax=axa[3],cmap='RdBu',vmin=diff_clim[0],vmax=diff_clim[1],markersize=markersize) 242 | axa[2].set_title("Before alignment") 243 | axa[3].set_title("After alignment") 244 | ctx.add_basemap(ax=axa[2],crs=plot_crs,attribution=False, 245 | source = provider) 246 | ctx.add_basemap(ax=axa[3],crs=plot_crs,attribution=False, 247 | source = provider) 248 | axa[0].set_xticks([]) 249 | axa[2].set_xticks([]) 250 | axa[3].set_xticks([]) 251 | 252 | axa[0].set_yticks([]) 253 | axa[2].set_yticks([]) 254 | axa[3].set_yticks([]) 255 | 256 | 257 | plt.tight_layout() 258 | 259 | f2,ax2 = plt.subplots() 260 | bins = np.linspace(diff_clim[0],diff_clim[1], 128) 261 | initial_mad = stats.median_abs_deviation(initial_diff['diff'].values) 262 | initial_med = np.median(initial_diff['diff'].values) 263 | 264 | final_mad = stats.median_abs_deviation(final_diff['diff'].values) 265 | final_med = np.median(final_diff['diff'].values) 266 | 267 | title = f" Pre-alignment elev. diff. median: {initial_med: .2f} m, mad: {initial_mad: .2f} m\nPost-alignment elev. diff. median: {final_med: .2f} m, mad: {final_mad: .2f} m" 268 | 269 | ax2.hist(initial_diff['diff'].values,bins=bins,color='blue',alpha=0.5,label='Initial') 270 | ax2.hist(final_diff['diff'].values,bins=bins,color='green',alpha=0.5,label='Final') 271 | ax2.axvline(x=0,linestyle='--',linewidth=1,color='k') 272 | ax2.legend() 273 | ax2.set_xlabel('Elevation difference (m)') 274 | ax2.set_ylabel('#pixels') 275 | ax2.set_title(title) -------------------------------------------------------------------------------- /tutorials/providence_mountains.geojson: -------------------------------------------------------------------------------- 1 | { 2 | "type": "FeatureCollection", 3 | "features": [ 4 | { 5 | "type": "Feature", 6 | "properties": {}, 7 | "geometry": { 8 | "coordinates": [ 9 | [ 10 | [ 11 | -116.48587141088018, 12 | 35.082632314855246 13 | ], 14 | [ 15 | -116.48587141088018, 16 | 33.59766751364852 17 | ], 18 | [ 19 | -114.62857444034678, 20 | 33.59766751364852 21 | ], 22 | [ 23 | -114.62857444034678, 24 | 35.082632314855246 25 | ], 26 | [ 27 | -116.48587141088018, 28 | 35.082632314855246 29 | ] 30 | ] 31 | ], 32 | "type": "Polygon" 33 | } 34 | } 35 | ] 36 | } -------------------------------------------------------------------------------- /tutorials/providence_mountains_small.geojson: -------------------------------------------------------------------------------- 1 | { 2 | "type": "FeatureCollection", 3 | "features": [ 4 | { 5 | "type": "Feature", 6 | "properties": {}, 7 | "geometry": { 8 | "coordinates": [ 9 | [ 10 | [ 11 | -115.52704215217467, 12 | 34.9235421982971 13 | ], 14 | [ 15 | -115.52704215217467, 16 | 34.87494635467863 17 | ], 18 | [ 19 | -115.49273343095588, 20 | 34.87494635467863 21 | ], 22 | [ 23 | -115.49273343095588, 24 | 34.9235421982971 25 | ], 26 | [ 27 | -115.52704215217467, 28 | 34.9235421982971 29 | ] 30 | ] 31 | ], 32 | "type": "Polygon" 33 | } 34 | } 35 | ] 36 | } --------------------------------------------------------------------------------