├── examples └── Coweeta │ ├── output_data │ └── .empty │ └── input_data │ ├── coweeta_simplified.cpg │ ├── soil_structure │ ├── GLHYMPS │ │ ├── GLHYMPS.cpg │ │ ├── GLHYMPS.dbf │ │ ├── GLHYMPS.shp │ │ ├── GLHYMPS.shx │ │ ├── GLHYMPS.prj │ │ └── GLHYMPS.qpj │ └── DTB │ │ └── DTB.tif │ ├── coweeta_basin.sbn │ ├── coweeta_basin.sbx │ ├── coweeta_basin.shp │ ├── coweeta_basin.shx │ ├── coweeta_simplified.shp │ ├── coweeta_simplified.shx │ ├── land_cover │ ├── land_cover.tif │ └── MODIS │ │ ├── modis_LAI_08-01-2010_08-01-2011_35.0782x-83.4826_35.0237x-83.4178.nc │ │ └── modis_LULC_08-01-2010_08-01-2011_35.0782x-83.4826_35.0237x-83.4178.nc │ ├── coweeta_simplified.prj │ ├── coweeta_basin.dbf │ ├── coweeta_basin.prj │ └── coweeta_simplified.dbf ├── watershed_workflow ├── test │ ├── fixture_data │ │ ├── hucs.cpg │ │ ├── river_060102020103.cpg │ │ ├── hucs.shp │ │ ├── hucs.shx │ │ ├── river_060102020103.dbf │ │ ├── river_060102020103.shp │ │ ├── river_060102020103.shx │ │ ├── hucs.prj │ │ ├── river_060102020103.prj │ │ └── hucs.dbf │ ├── test_14_find_huc │ │ ├── copper_creek.cpg │ │ ├── test_shapefile.cpg │ │ ├── test_shapefile.dbf │ │ ├── copper_creek.shp │ │ ├── copper_creek.shx │ │ ├── test_polygon.sbn │ │ ├── test_polygon.sbx │ │ ├── test_polygon.shp │ │ ├── test_polygon.shx │ │ ├── copper_creek.dbf │ │ ├── test_shapefile.shp │ │ ├── test_shapefile.shx │ │ ├── test_shapefile.prj │ │ ├── test_polygon.dbf │ │ ├── copper_creek.prj │ │ └── test_polygon.prj │ ├── test_plot_gold.pickle │ ├── data_test_streamlight │ │ └── data_test_streamlight_parameters.csv │ ├── source_fixtures.py │ ├── test_13_land_cover_properties.py │ ├── test_01_crs.py │ ├── FAIL_test_dem_interp.py │ ├── crs_fixtures.py │ ├── test_12_soil_properties.py │ ├── test_08_triangulate.py │ ├── FAIL_test_hilev.py │ ├── test_14_find_huc.py │ ├── test_00_shapely.py │ ├── source_fixture_helpers.py │ ├── test_09_mesh.py │ └── shapes.py ├── sources │ ├── test │ │ ├── fixtures.py │ │ ├── test_source_names.py │ │ ├── test_manager_glhymps.py │ │ ├── test_manager_nrcs.py │ │ ├── test_manager_soilgrids.py │ │ ├── test_manager_basin3d.py │ │ └── test_manager_wbd.py │ ├── standard_names.py │ ├── filenames.py │ ├── manager_3dep.py │ ├── manager_pelletier_dtb.py │ ├── manager_wbd.py │ ├── manager_glhymps.py │ ├── utils.py │ ├── manager_hyriver.py │ ├── manager_raster.py │ └── __init__.py ├── config.py ├── io.py └── bin_utils.py ├── .gitattributes ├── docs ├── source │ ├── examples │ │ ├── get_MODIS_LAI.ipynb │ │ ├── coweeta_ats.ipynb │ │ ├── get_AORC_met_data.ipynb │ │ ├── comparison_of_soil_structure.ipynb │ │ ├── coweeta_stream_aligned_mesh.ipynb │ │ └── toy_problem_stream_aligned_mesh.ipynb │ ├── _templates │ │ └── version.html │ ├── hilev.rst │ ├── _static │ │ ├── images │ │ │ ├── gallery_sources_nhd.png │ │ │ ├── gallery_sources_wbd.png │ │ │ ├── watershed_workflow.png │ │ │ └── gallery_sources_nhdplus.png │ │ ├── main.js │ │ ├── styles │ │ │ └── custom_theme.css │ │ └── versions.json │ ├── api.rst │ ├── data.rst │ ├── mesh.rst │ ├── examples.rst │ ├── utilities.rst │ ├── gallery.rst │ ├── geometry.rst │ ├── index.rst │ ├── concepts.rst │ ├── conf.py │ └── intro.rst ├── build │ └── doctrees │ │ └── gallery.doctree └── Makefile ├── MANIFEST.in ├── INSTALL.md ├── setup.cfg ├── docker ├── build_user_container.sh ├── Create-Envs.Dockerfile ├── CI.Dockerfile ├── configure-seacas.sh ├── ATS-User-Env.Dockerfile ├── CI-Env.Dockerfile ├── CI-Env-FromFile.Dockerfile └── User-Env.Dockerfile ├── environments ├── recover_docker_envs.sh ├── create_linux_envs.sh └── create_osx_envs.sh ├── .gitignore ├── requirements.txt ├── pyproject.toml ├── AUTHORS.rst ├── watershed_workflowrc ├── bin ├── condition_dem.py ├── extrude_mesh.py ├── plot_hucs.py ├── plot_shape.py ├── mesh_hucs.py ├── mesh_shape.py └── run_ww_lab.py ├── LICENSE ├── setup.py ├── .github └── workflows │ ├── user_container.yml │ ├── env.yml │ ├── main.yml │ └── ats_user_container.yml └── README.md /examples/Coweeta/output_data/.empty: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /examples/Coweeta/input_data/coweeta_simplified.cpg: -------------------------------------------------------------------------------- 1 | UTF-8 -------------------------------------------------------------------------------- /watershed_workflow/test/fixture_data/hucs.cpg: -------------------------------------------------------------------------------- 1 | ISO-8859-1 -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | watershed_workflow/_version.py export-subst 2 | -------------------------------------------------------------------------------- /examples/Coweeta/input_data/soil_structure/GLHYMPS/GLHYMPS.cpg: -------------------------------------------------------------------------------- 1 | UTF-8 -------------------------------------------------------------------------------- /watershed_workflow/test/test_14_find_huc/copper_creek.cpg: -------------------------------------------------------------------------------- 1 | ISO-8859-1 -------------------------------------------------------------------------------- /watershed_workflow/test/fixture_data/river_060102020103.cpg: -------------------------------------------------------------------------------- 1 | ISO-8859-1 -------------------------------------------------------------------------------- /watershed_workflow/test/test_14_find_huc/test_shapefile.cpg: -------------------------------------------------------------------------------- 1 | ISO-8859-1 -------------------------------------------------------------------------------- /docs/source/examples/get_MODIS_LAI.ipynb: -------------------------------------------------------------------------------- 1 | ../../../examples/get_MODIS_LAI.ipynb -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include versioneer.py 2 | include watershed_workflow/_version.py 3 | -------------------------------------------------------------------------------- /docs/source/examples/coweeta_ats.ipynb: -------------------------------------------------------------------------------- 1 | ../../../examples/Coweeta/coweeta_ats.ipynb -------------------------------------------------------------------------------- /docs/source/examples/get_AORC_met_data.ipynb: -------------------------------------------------------------------------------- 1 | ../../../examples/get_AORC_met_data.ipynb -------------------------------------------------------------------------------- /docs/source/examples/comparison_of_soil_structure.ipynb: -------------------------------------------------------------------------------- 1 | ../../../examples/comparison_of_soil_structure.ipynb -------------------------------------------------------------------------------- /docs/source/examples/coweeta_stream_aligned_mesh.ipynb: -------------------------------------------------------------------------------- 1 | ../../../examples/coweeta_stream_aligned_mesh.ipynb -------------------------------------------------------------------------------- /INSTALL.md: -------------------------------------------------------------------------------- 1 | See https://environmental-modeling-workflows.github.io/watershed-workflow/build/html/install.html 2 | -------------------------------------------------------------------------------- /docs/source/examples/toy_problem_stream_aligned_mesh.ipynb: -------------------------------------------------------------------------------- 1 | ../../../examples/toy_problem_stream_aligned_mesh.ipynb -------------------------------------------------------------------------------- /docs/source/_templates/version.html: -------------------------------------------------------------------------------- 1 | 2 | Version {{ version }} 3 | -------------------------------------------------------------------------------- /docs/source/hilev.rst: -------------------------------------------------------------------------------- 1 | High Level API 2 | ~~~~~~~~~~~~~~ 3 | .. automodule:: watershed_workflow 4 | :members: 5 | 6 | 7 | -------------------------------------------------------------------------------- /watershed_workflow/test/test_14_find_huc/test_shapefile.dbf: -------------------------------------------------------------------------------- 1 | wA FIDN 0 -------------------------------------------------------------------------------- /docs/build/doctrees/gallery.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/environmental-modeling-workflows/watershed-workflow/HEAD/docs/build/doctrees/gallery.doctree -------------------------------------------------------------------------------- /examples/Coweeta/input_data/coweeta_basin.sbn: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/environmental-modeling-workflows/watershed-workflow/HEAD/examples/Coweeta/input_data/coweeta_basin.sbn -------------------------------------------------------------------------------- /examples/Coweeta/input_data/coweeta_basin.sbx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/environmental-modeling-workflows/watershed-workflow/HEAD/examples/Coweeta/input_data/coweeta_basin.sbx -------------------------------------------------------------------------------- /examples/Coweeta/input_data/coweeta_basin.shp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/environmental-modeling-workflows/watershed-workflow/HEAD/examples/Coweeta/input_data/coweeta_basin.shp -------------------------------------------------------------------------------- /examples/Coweeta/input_data/coweeta_basin.shx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/environmental-modeling-workflows/watershed-workflow/HEAD/examples/Coweeta/input_data/coweeta_basin.shx -------------------------------------------------------------------------------- /watershed_workflow/test/fixture_data/hucs.shp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/environmental-modeling-workflows/watershed-workflow/HEAD/watershed_workflow/test/fixture_data/hucs.shp -------------------------------------------------------------------------------- /watershed_workflow/test/fixture_data/hucs.shx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/environmental-modeling-workflows/watershed-workflow/HEAD/watershed_workflow/test/fixture_data/hucs.shx -------------------------------------------------------------------------------- /watershed_workflow/test/test_plot_gold.pickle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/environmental-modeling-workflows/watershed-workflow/HEAD/watershed_workflow/test/test_plot_gold.pickle -------------------------------------------------------------------------------- /docs/source/_static/images/gallery_sources_nhd.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/environmental-modeling-workflows/watershed-workflow/HEAD/docs/source/_static/images/gallery_sources_nhd.png -------------------------------------------------------------------------------- /docs/source/_static/images/gallery_sources_wbd.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/environmental-modeling-workflows/watershed-workflow/HEAD/docs/source/_static/images/gallery_sources_wbd.png -------------------------------------------------------------------------------- /docs/source/_static/images/watershed_workflow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/environmental-modeling-workflows/watershed-workflow/HEAD/docs/source/_static/images/watershed_workflow.png -------------------------------------------------------------------------------- /examples/Coweeta/input_data/coweeta_simplified.shp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/environmental-modeling-workflows/watershed-workflow/HEAD/examples/Coweeta/input_data/coweeta_simplified.shp -------------------------------------------------------------------------------- /examples/Coweeta/input_data/coweeta_simplified.shx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/environmental-modeling-workflows/watershed-workflow/HEAD/examples/Coweeta/input_data/coweeta_simplified.shx -------------------------------------------------------------------------------- /examples/Coweeta/input_data/land_cover/land_cover.tif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/environmental-modeling-workflows/watershed-workflow/HEAD/examples/Coweeta/input_data/land_cover/land_cover.tif -------------------------------------------------------------------------------- /docs/source/_static/images/gallery_sources_nhdplus.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/environmental-modeling-workflows/watershed-workflow/HEAD/docs/source/_static/images/gallery_sources_nhdplus.png -------------------------------------------------------------------------------- /examples/Coweeta/input_data/soil_structure/DTB/DTB.tif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/environmental-modeling-workflows/watershed-workflow/HEAD/examples/Coweeta/input_data/soil_structure/DTB/DTB.tif -------------------------------------------------------------------------------- /watershed_workflow/test/test_14_find_huc/copper_creek.shp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/environmental-modeling-workflows/watershed-workflow/HEAD/watershed_workflow/test/test_14_find_huc/copper_creek.shp -------------------------------------------------------------------------------- /watershed_workflow/test/test_14_find_huc/copper_creek.shx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/environmental-modeling-workflows/watershed-workflow/HEAD/watershed_workflow/test/test_14_find_huc/copper_creek.shx -------------------------------------------------------------------------------- /watershed_workflow/test/test_14_find_huc/test_polygon.sbn: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/environmental-modeling-workflows/watershed-workflow/HEAD/watershed_workflow/test/test_14_find_huc/test_polygon.sbn -------------------------------------------------------------------------------- /watershed_workflow/test/test_14_find_huc/test_polygon.sbx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/environmental-modeling-workflows/watershed-workflow/HEAD/watershed_workflow/test/test_14_find_huc/test_polygon.sbx -------------------------------------------------------------------------------- /watershed_workflow/test/test_14_find_huc/test_polygon.shp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/environmental-modeling-workflows/watershed-workflow/HEAD/watershed_workflow/test/test_14_find_huc/test_polygon.shp -------------------------------------------------------------------------------- /watershed_workflow/test/test_14_find_huc/test_polygon.shx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/environmental-modeling-workflows/watershed-workflow/HEAD/watershed_workflow/test/test_14_find_huc/test_polygon.shx -------------------------------------------------------------------------------- /watershed_workflow/test/fixture_data/river_060102020103.dbf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/environmental-modeling-workflows/watershed-workflow/HEAD/watershed_workflow/test/fixture_data/river_060102020103.dbf -------------------------------------------------------------------------------- /watershed_workflow/test/fixture_data/river_060102020103.shp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/environmental-modeling-workflows/watershed-workflow/HEAD/watershed_workflow/test/fixture_data/river_060102020103.shp -------------------------------------------------------------------------------- /watershed_workflow/test/fixture_data/river_060102020103.shx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/environmental-modeling-workflows/watershed-workflow/HEAD/watershed_workflow/test/fixture_data/river_060102020103.shx -------------------------------------------------------------------------------- /watershed_workflow/test/test_14_find_huc/copper_creek.dbf: -------------------------------------------------------------------------------- 1 | xAQnameCP copper creek  -------------------------------------------------------------------------------- /watershed_workflow/test/test_14_find_huc/test_shapefile.shp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/environmental-modeling-workflows/watershed-workflow/HEAD/watershed_workflow/test/test_14_find_huc/test_shapefile.shp -------------------------------------------------------------------------------- /watershed_workflow/test/test_14_find_huc/test_shapefile.shx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/environmental-modeling-workflows/watershed-workflow/HEAD/watershed_workflow/test/test_14_find_huc/test_shapefile.shx -------------------------------------------------------------------------------- /examples/Coweeta/input_data/soil_structure/GLHYMPS/GLHYMPS.dbf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/environmental-modeling-workflows/watershed-workflow/HEAD/examples/Coweeta/input_data/soil_structure/GLHYMPS/GLHYMPS.dbf -------------------------------------------------------------------------------- /examples/Coweeta/input_data/soil_structure/GLHYMPS/GLHYMPS.shp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/environmental-modeling-workflows/watershed-workflow/HEAD/examples/Coweeta/input_data/soil_structure/GLHYMPS/GLHYMPS.shp -------------------------------------------------------------------------------- /examples/Coweeta/input_data/soil_structure/GLHYMPS/GLHYMPS.shx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/environmental-modeling-workflows/watershed-workflow/HEAD/examples/Coweeta/input_data/soil_structure/GLHYMPS/GLHYMPS.shx -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [versioneer] 2 | VCS = git 3 | style = pep440-post 4 | versionfile_source = watershed_workflow/_version.py 5 | versionfile_build = watershed_workflow/_version.py 6 | tag_prefix = watershed-workflow- 7 | -------------------------------------------------------------------------------- /watershed_workflow/test/fixture_data/hucs.prj: -------------------------------------------------------------------------------- 1 | GEOGCS["GCS_North_American_1983",DATUM["D_North_American_1983",SPHEROID["GRS_1980",6378137.0,298.257222101]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.0174532925199433]] -------------------------------------------------------------------------------- /examples/Coweeta/input_data/coweeta_simplified.prj: -------------------------------------------------------------------------------- 1 | GEOGCS["GCS_North_American_1983",DATUM["D_North_American_1983",SPHEROID["GRS_1980",6378137.0,298.257222101]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.0174532925199433]] -------------------------------------------------------------------------------- /watershed_workflow/test/fixture_data/river_060102020103.prj: -------------------------------------------------------------------------------- 1 | GEOGCS["GCS_North_American_1983",DATUM["D_North_American_1983",SPHEROID["GRS_1980",6378137.0,298.257222101]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.0174532925199433]] -------------------------------------------------------------------------------- /watershed_workflow/test/test_14_find_huc/test_shapefile.prj: -------------------------------------------------------------------------------- 1 | GEOGCS["GCS_North_American_1983",DATUM["D_North_American_1983",SPHEROID["GRS_1980",6378137,298.257222101]],PRIMEM["Greenwich",0],UNIT["Degree",0.017453292519943295]] -------------------------------------------------------------------------------- /watershed_workflow/test/data_test_streamlight/data_test_streamlight_parameters.csv: -------------------------------------------------------------------------------- 1 | "lat","lon","channel_azimuth","bottom_width","bh","bs","wl","th","overhang","overhang_height","x_LAD" 2 | 35.9925,-79.046,330,18.9,0.1,100,0.05,23,2.3,2.3,1 3 | -------------------------------------------------------------------------------- /docker/build_user_container.sh: -------------------------------------------------------------------------------- 1 | docker build --progress=plain -f docker/User-Env.Dockerfile -t ecoon/watershed_workflow:master . && \ 2 | docker build --progress=plain -f docker/ATS-User-Env.Dockerfile -t ecoon/watershed_workflow-ats:master . 3 | 4 | 5 | -------------------------------------------------------------------------------- /docs/source/_static/main.js: -------------------------------------------------------------------------------- 1 | $(document).ready( function () { 2 | $('table.datatable').DataTable({ 3 | paging: false, 4 | scrollCollapse: true, 5 | scrollY: '800px', 6 | order: [[4,'asc'],[0,'asc']], 7 | }).columns.align(); 8 | }); 9 | -------------------------------------------------------------------------------- /environments/recover_docker_envs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # copy back the environment.yml files from the docker container 3 | docker create -it --name watershed_workflow_env-ci-linux bash 4 | docker cp watershed_workflow_env-ci-linux:/environment.yml ./ 5 | docker rm -f watershed_workflow_env-ci-linux 6 | -------------------------------------------------------------------------------- /examples/Coweeta/input_data/land_cover/MODIS/modis_LAI_08-01-2010_08-01-2011_35.0782x-83.4826_35.0237x-83.4178.nc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/environmental-modeling-workflows/watershed-workflow/HEAD/examples/Coweeta/input_data/land_cover/MODIS/modis_LAI_08-01-2010_08-01-2011_35.0782x-83.4826_35.0237x-83.4178.nc -------------------------------------------------------------------------------- /examples/Coweeta/input_data/land_cover/MODIS/modis_LULC_08-01-2010_08-01-2011_35.0782x-83.4826_35.0237x-83.4178.nc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/environmental-modeling-workflows/watershed-workflow/HEAD/examples/Coweeta/input_data/land_cover/MODIS/modis_LULC_08-01-2010_08-01-2011_35.0782x-83.4826_35.0237x-83.4178.nc -------------------------------------------------------------------------------- /docs/source/api.rst: -------------------------------------------------------------------------------- 1 | API Documentation 2 | ================= 3 | 4 | .. toctree:: 5 | :maxdepth: 2 6 | :caption: Contents: 7 | 8 | High Level API 9 | Data Sources 10 | Geometry and Shape Manipulation 11 | Meshes 12 | Data Manipulation 13 | Utilities 14 | 15 | 16 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | __pycache__ 3 | examples/Coweeta/output_data 4 | *.js 5 | watershed_workflow.egg-info/PKG-INFO 6 | *.txt 7 | .DS_Store 8 | *checkpoint.ipynb 9 | docs/build 10 | docs/deploy 11 | .ipynb_checkpoints 12 | examples/**/*.parquet 13 | examples/**/*.pickle 14 | examples/**/*.h5 15 | examples/**/*.nc 16 | examples/**/*.xml 17 | examples/**/*.csv 18 | examples/**/*.exo 19 | CLAUDE* 20 | .claude 21 | cache -------------------------------------------------------------------------------- /examples/Coweeta/input_data/coweeta_basin.dbf: -------------------------------------------------------------------------------- 1 | j}WAREANPERIMETERNCWTBASINNAN CWTBASIN_1N BASIN_CODEN SPOTN LABELC2 16260198.383 17521.768 2 1 1 -9999Coweeta Hydrologic Lab  -------------------------------------------------------------------------------- /examples/Coweeta/input_data/soil_structure/GLHYMPS/GLHYMPS.prj: -------------------------------------------------------------------------------- 1 | PROJCS["World_Cylindrical_Equal_Area",GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137,298.257223563]],PRIMEM["Greenwich",0],UNIT["Degree",0.017453292519943295]],PROJECTION["Cylindrical_Equal_Area"],PARAMETER["False_Easting",0.0],PARAMETER["False_Northing",0.0],PARAMETER["Central_Meridian",0.0],PARAMETER["Standard_Parallel_1",0.0],UNIT["Meter",1.0]] -------------------------------------------------------------------------------- /watershed_workflow/test/test_14_find_huc/test_polygon.dbf: -------------------------------------------------------------------------------- 1 | j}WAREANPERIMETERNCWTBASINNAN CWTBASIN_1N BASIN_CODEN SPOTN LABELC2 16260198.383 17521.768 2 1 1 -9999Coweeta Hydrologic Lab  -------------------------------------------------------------------------------- /watershed_workflow/sources/test/fixtures.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import os 3 | 4 | from watershed_workflow.sources.manager_shapefile import ManagerShapefile 5 | import watershed_workflow.crs 6 | 7 | @pytest.fixture 8 | def coweeta(): 9 | ms = ManagerShapefile(os.path.join('examples', 'Coweeta', 'input_data', 'coweeta_simplified.shp')) 10 | shp = ms.getShapes() 11 | return shp.to_crs(watershed_workflow.crs.from_epsg(4269)) 12 | -------------------------------------------------------------------------------- /examples/Coweeta/input_data/coweeta_basin.prj: -------------------------------------------------------------------------------- 1 | PROJCS["NAD_1983_UTM_Zone_17N",GEOGCS["GCS_North_American_1983",DATUM["D_North_American_1983",SPHEROID["GRS_1980",6378137.0,298.257222101]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.0174532925199433]],PROJECTION["Transverse_Mercator"],PARAMETER["False_Easting",500000.0],PARAMETER["False_Northing",0.0],PARAMETER["Central_Meridian",-81.0],PARAMETER["Scale_Factor",0.9996],PARAMETER["Latitude_Of_Origin",0.0],UNIT["Meter",1.0]] -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | # NOTE: the requirements.txt listed here is VERY incomplete. This is 2 | # intentional -- most of the pip-based GIS packages don't correctly 3 | # deal with dependencies on GIS libraries. Instead, the majority of 4 | # packages here MUST be installed via Anaconda or done manually. 5 | # However, there are a few requirements that cannot be provided via 6 | # conda, but CAN be provided via pip, so we get those here... 7 | rosetta-soil 8 | meshpy 9 | -------------------------------------------------------------------------------- /watershed_workflow/test/test_14_find_huc/copper_creek.prj: -------------------------------------------------------------------------------- 1 | PROJCS["NAD_1983_UTM_Zone_13N",GEOGCS["GCS_North_American_1983",DATUM["D_North_American_1983",SPHEROID["GRS_1980",6378137,298.257222101]],PRIMEM["Greenwich",0],UNIT["Degree",0.017453292519943295]],PROJECTION["Transverse_Mercator"],PARAMETER["latitude_of_origin",0],PARAMETER["central_meridian",-105],PARAMETER["scale_factor",0.9996],PARAMETER["false_easting",500000],PARAMETER["false_northing",0],UNIT["Meter",1]] -------------------------------------------------------------------------------- /watershed_workflow/test/test_14_find_huc/test_polygon.prj: -------------------------------------------------------------------------------- 1 | PROJCS["NAD_1983_UTM_Zone_17N",GEOGCS["GCS_North_American_1983",DATUM["D_North_American_1983",SPHEROID["GRS_1980",6378137.0,298.257222101]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.0174532925199433]],PROJECTION["Transverse_Mercator"],PARAMETER["False_Easting",500000.0],PARAMETER["False_Northing",0.0],PARAMETER["Central_Meridian",-81.0],PARAMETER["Scale_Factor",0.9996],PARAMETER["Latitude_Of_Origin",0.0],UNIT["Meter",1.0]] -------------------------------------------------------------------------------- /docs/source/data.rst: -------------------------------------------------------------------------------- 1 | Data Manipulation 2 | ~~~~~~~~~~~~~~~~~ 3 | 4 | Dataset manipulation 5 | ++++++++++++++++++++ 6 | .. automodule:: watershed_workflow.data 7 | :members: 8 | 9 | Soil properties data manipulation 10 | +++++++++++++++++++++++++++++++++ 11 | .. automodule:: watershed_workflow.soil_properties 12 | :members: 13 | 14 | Meteorology data manipulation 15 | +++++++++++++++++++++++++++++++++ 16 | .. automodule:: watershed_workflow.meteorology 17 | :members: 18 | 19 | -------------------------------------------------------------------------------- /docs/source/_static/styles/custom_theme.css: -------------------------------------------------------------------------------- 1 | div.pst-scrollable-table-container { 2 | width : 100%; 3 | } 4 | 5 | .bd-main .bd-content .bd-article-container { 6 | max-width: 100%; 7 | } 8 | /* .section { */ 9 | /* max-width: 60em; */ 10 | /* } */ 11 | 12 | /* .ats-native-xml-input-specification-v-dev { */ 13 | /* max-width: 60em; */ 14 | /* } */ 15 | 16 | 17 | /* .name-and-symbol-index { */ 18 | /* max-widht: 100%; */ 19 | /* } */ 20 | 21 | 22 | 23 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.pytest.ini_options] 2 | log_cli = true 3 | log_cli_level = "INFO" 4 | 5 | [tool.yapf] 6 | based_on_style = "pep8" 7 | arithmetic_precedence_indication = true 8 | coalesce_brackets = false 9 | disable_ending_comma_heuristic = true 10 | spaces_around_dict_delimiters = true 11 | split_before_arithmetic_operator = true 12 | column_limit = 100 13 | blank_line_before_nested_class_or_def = false 14 | 15 | [tool.mypy] 16 | disable_error_code = ['import-untyped', 'import-not-found'] 17 | 18 | 19 | -------------------------------------------------------------------------------- /docs/source/_static/versions.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "2.0 (stable)", 4 | "version": "2.0", 5 | "url": "https://environmental-modeling-workflows.github.io/watershed-workflow/stable/", 6 | "preferred": "2.0" 7 | }, 8 | { 9 | "version": "1.6", 10 | "url": "https://environmental-modeling-workflows.github.io/watershed-workflow/1.6/" 11 | }, 12 | { 13 | "version": "dev", 14 | "url": "https://environmental-modeling-workflows.github.io/watershed-workflow/dev/" 15 | } 16 | ] 17 | -------------------------------------------------------------------------------- /examples/Coweeta/input_data/soil_structure/GLHYMPS/GLHYMPS.qpj: -------------------------------------------------------------------------------- 1 | PROJCS["World_Cylindrical_Equal_Area",GEOGCS["WGS 84",DATUM["WGS_1984",SPHEROID["WGS 84",6378137,298.257223563,AUTHORITY["EPSG","7030"]],AUTHORITY["EPSG","6326"]],PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],UNIT["degree",0.0174532925199433,AUTHORITY["EPSG","9122"]],AUTHORITY["EPSG","4326"]],PROJECTION["Cylindrical_Equal_Area"],PARAMETER["False_Easting",0.0],PARAMETER["False_Northing",0.0],PARAMETER["Central_Meridian",0.0],PARAMETER["Standard_Parallel_1",0.0],UNIT["Meter",1.0],AUTHORITY["Esri","54034"]] 2 | -------------------------------------------------------------------------------- /docs/source/mesh.rst: -------------------------------------------------------------------------------- 1 | Meshing 2 | ~~~~~~~ 3 | 4 | Mesh 5 | ++++ 6 | .. automodule:: watershed_workflow.mesh 7 | :members: 8 | 9 | River-aligned Mesh 10 | ++++++++++++++++++ 11 | .. automodule:: watershed_workflow.river_mesh 12 | :members: 13 | 14 | Triangulation 15 | +++++++++++++ 16 | .. automodule:: watershed_workflow.triangulation 17 | :members: 18 | 19 | Condition 20 | +++++++++ 21 | .. automodule:: watershed_workflow.condition 22 | :members: 23 | 24 | Regions 25 | +++++++ 26 | .. automodule:: watershed_workflow.regions 27 | :members: 28 | 29 | -------------------------------------------------------------------------------- /watershed_workflow/test/source_fixtures.py: -------------------------------------------------------------------------------- 1 | import os 2 | from distutils import dir_util 3 | import pytest 4 | 5 | @pytest.fixture 6 | def datadir(tmpdir, request): 7 | """Fixture responsible for searching a folder with the same name of test 8 | module and, if available, moving all contents to a temporary directory so 9 | tests can use them freely. 10 | """ 11 | filename = request.module.__file__ 12 | test_dir, _ = os.path.splitext(filename) 13 | 14 | if os.path.isdir(test_dir): 15 | dir_util.copy_tree(test_dir, str(tmpdir)) 16 | return tmpdir 17 | -------------------------------------------------------------------------------- /docs/source/examples.rst: -------------------------------------------------------------------------------- 1 | Examples 2 | ========== 3 | 4 | The best way to learn Watershed Workflow is to look at our various 5 | examples, each of which focus on one aspect of the process toward 6 | parameterizing hydrologic models. 7 | 8 | .. toctree:: 9 | :maxdepth: 1 10 | :caption: Examples: 11 | :titlesonly: 12 | 13 | examples/toy_problem_stream_aligned_mesh.ipynb 14 | examples/comparison_of_soil_structure.ipynb 15 | examples/get_AORC_met_data.ipynb 16 | examples/get_MODIS_LAI.ipynb 17 | examples/coweeta_stream_aligned_mesh.ipynb 18 | examples/coweeta_ats.ipynb 19 | -------------------------------------------------------------------------------- /docs/source/utilities.rst: -------------------------------------------------------------------------------- 1 | Utilities 2 | ~~~~~~~~~ 3 | 4 | CRS 5 | +++ 6 | .. automodule:: watershed_workflow.crs 7 | :members: 8 | 9 | Package Configuration 10 | +++++++++++++++++++++ 11 | .. automodule:: watershed_workflow.config 12 | :members: 13 | 14 | Plotting 15 | ++++++++ 16 | .. automodule:: watershed_workflow.colors 17 | :members: 18 | 19 | Generic utilities 20 | +++++++++++++++++ 21 | .. automodule:: watershed_workflow.utils 22 | :members: 23 | 24 | Warping 25 | +++++++ 26 | .. automodule:: watershed_workflow.warp 27 | :members: 28 | 29 | -------------------------------------------------------------------------------- /AUTHORS.rst: -------------------------------------------------------------------------------- 1 | :Author: `Ethan Coon `_ 2 | :Contributors: * `Pin Shuai `_, 3 | * `Saubhagya Rathore `_ 4 | * `Bo Gao `_ 5 | * `Soumendra Bhanja `_ 6 | * `Jesus Velez-Gomez `_ 7 | * ... your name here! 8 | :License: This work, including code, images, and documentation, unless 9 | otherwise specified, is copyright UT Battelle/Oak Ridge National 10 | Laboratory, and is licensed under the 3-clause BSD license. 11 | -------------------------------------------------------------------------------- /examples/Coweeta/input_data/coweeta_simplified.dbf: -------------------------------------------------------------------------------- 1 | }aNAREANPERIMETERNCWTBASINNAN CWTBASIN_1N BASIN_CODEN SPOTN LABELCPIDN nameCPoutletCP 16260198.382999999448657 17521.768000000000029 2 1 1 -9999Coweeta Hydrologic Lab 11 POINT (1446635.6183921252 -646208.5446217591)  -------------------------------------------------------------------------------- /docs/source/gallery.rst: -------------------------------------------------------------------------------- 1 | Gallery 2 | ======= 3 | 4 | This is a simple gallery of images generated with this workflow. Each 5 | provides the source required to generate the image -- often this is 6 | running a plotting script from `bin`, other times it is a custom 7 | workflow. 8 | 9 | Data Sources 10 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 11 | 12 | National Hydrologic Data across resolutions 13 | ------------------------------------------- 14 | 15 | .. code-block:: console 16 | 17 | python bin/plot_hucs.py --source-huc=NHD --source-hydro=NHD --output-filename=docs/_static/images/gallery_sources_nhd.png 060102020103 18 | 19 | .. image:: _static/images/gallery_sources_nhd.png 20 | 21 | 22 | 23 | 24 | -------------------------------------------------------------------------------- /environments/create_linux_envs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker build --progress=plain -f docker/Create-Envs.Dockerfile -t ww_linux_envs . 3 | docker create -it --name ww_linux_envs_tmp ww_linux_envs 4 | docker cp ww_linux_envs_tmp:/ww/environments/environment-Linux.yml ./environments/environment-Linux.yml 5 | docker cp ww_linux_envs_tmp:/ww/environments/environment-CI-Linux.yml ./environments/environment-CI-Linux.yml 6 | docker cp ww_linux_envs_tmp:/ww/environments/environment-DEV-Linux.yml ./environments/environment-DEV-Linux.yml 7 | 8 | # definitely want to remove the container 9 | docker container rm -f ww_linux_envs_tmp 10 | 11 | # also remove the image -- we have to rebuild to get new repos, and we 12 | # only use this image once to generate the files 13 | docker image rm -f ww_linux_envs 14 | -------------------------------------------------------------------------------- /watershed_workflow/test/test_13_land_cover_properties.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import numpy as np 3 | import pandas as pd 4 | import xarray as xr 5 | import watershed_workflow 6 | 7 | import watershed_workflow.land_cover_properties 8 | 9 | 10 | def test_crosswalk(): 11 | modis = np.array([[1, 2], [2, 2], [2, 3]]) 12 | nlcd = np.array([[4, 6], [4, 5], [4, 7]]) 13 | x = np.array([1,2,3]) 14 | y = np.array([1,2]) 15 | 16 | modis_da = xr.DataArray(name='modis', data=modis, coords={'x':x, 'y':y}) 17 | nlcd_da = xr.DataArray(name='nlcd', data=nlcd, coords={'x':x, 'y':y}) 18 | 19 | crosswalk = watershed_workflow.land_cover_properties.computeCrosswalk( 20 | modis_da, nlcd_da, plot=False, warp=False) 21 | assert (crosswalk[4][0][0] == 2) 22 | assert (crosswalk[5][0][0] == 2) 23 | assert (crosswalk[6][0][0] == 2) 24 | assert (crosswalk[7][0][0] == 3) 25 | -------------------------------------------------------------------------------- /watershed_workflowrc: -------------------------------------------------------------------------------- 1 | # Currently only a few parameters are possible 2 | 3 | [DEFAULT] 4 | 5 | # Watershed workflow stores all data in a common library for shared 6 | # use across the machine. This directory defaults to: 7 | # 8 | # data_directory : ${WATERSHED_WORKFLOW_DIR}/data 9 | 10 | # We rely extensively on downloaded files, which are certified via the 11 | # SSL library. Sometimes certificates are tricky, and if your 12 | # certificates aren't working, it can be useful to try other 13 | # certificate files, such as your system ones (likely installed 14 | # somewhere like /etc/ssl/cert.pem ). If you're really desparate and 15 | # not worried about getting hacked, you can set this to False to not 16 | # verify certificates, but that is not recommended. Likely if you 17 | # have trouble, the first thing to try is to update your certifi 18 | # package in anaconda (conda update certifi) 19 | # 20 | # ssl_cert : True 21 | -------------------------------------------------------------------------------- /docker/Create-Envs.Dockerfile: -------------------------------------------------------------------------------- 1 | # Simply creates envs and dumps them to disk, so that I can save them 2 | # for the repo. 3 | # 4 | # To be used by WW maintainer only... 5 | 6 | FROM condaforge/mambaforge:4.12.0-0 7 | WORKDIR /ww 8 | COPY environments/create_envs.py /ww/create_envs.py 9 | RUN mkdir environments 10 | ENV CONDA_BIN=mamba 11 | 12 | RUN ${CONDA_BIN} install -n base -y -c conda-forge python=3 13 | 14 | RUN --mount=type=cache,target=/opt/conda/pkgs \ 15 | /opt/conda/bin/python create_envs.py --manager=${CONDA_BIN} --env-type=CI --OS=Linux watershed_workflow_ci 16 | 17 | RUN --mount=type=cache,target=/opt/conda/pkgs \ 18 | /opt/conda/bin/python create_envs.py --manager=${CONDA_BIN} --env-type=STANDARD --OS=Linux watershed_workflow 19 | 20 | RUN --mount=type=cache,target=/opt/conda/pkgs \ 21 | /opt/conda/bin/python create_envs.py --manager=${CONDA_BIN} --env-type=DEV --OS=Linux watershed_workflow_dev 22 | 23 | 24 | -------------------------------------------------------------------------------- /docs/source/geometry.rst: -------------------------------------------------------------------------------- 1 | Data Structures and Shape Manipulation 2 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 3 | Several custom data structures are used in the manipulation of 4 | geometry to form a consistent mesh. Two are the most important: the 5 | SplitHUC object defines a set of polygons that partition the domain 6 | into sub-catchments of the full domain (e.g. HUC 12s in a a HUC8 7 | domain, or differential contributing areas to a series of gages). The 8 | RiverTree object defines a tree data structure defined by a 9 | child-parent relationship where children of a reach are all reaches 10 | that flow into that reach. 11 | 12 | SplitHUCs 13 | +++++++++ 14 | .. automodule:: watershed_workflow.split_hucs 15 | :members: 16 | 17 | RiverTree 18 | +++++++++ 19 | .. automodule:: watershed_workflow.river_tree 20 | :members: 21 | 22 | Hydrography 23 | +++++++++++ 24 | .. automodule:: watershed_workflow.hydrography 25 | :members: 26 | 27 | -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- 1 | ****************** 2 | Watershed Workflow 3 | ****************** 4 | 5 | .. image:: _static/images/watershed_workflow.png 6 | 7 | Watershed Workflow is a python-based, open source chain of tools for 8 | generating meshes and other data inputs for hyper-resolution 9 | hydrology, anywhere in the (conterminous + Alaska?) US. 10 | 11 | Browse the code at: https://github.com/environmental-modeling-workflow/watershed-workflow/ 12 | 13 | .. toctree:: 14 | :maxdepth: 2 15 | :caption: Contents: 16 | 17 | Introduction 18 | Examples 19 | Installation 20 | Concepts 21 | API Documentation 22 | Gallery 23 | 24 | 25 | Indices and tables 26 | ===================== 27 | 28 | * :ref:`genindex` 29 | * :ref:`modindex` 30 | * :ref:`search` 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | -------------------------------------------------------------------------------- /watershed_workflow/sources/test/test_source_names.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | import watershed_workflow.sources.filenames 4 | import watershed_workflow.config 5 | 6 | 7 | def test_names(): 8 | ddir = watershed_workflow.config.rcParams['DEFAULT']['data_directory'] 9 | watershed_workflow.config.rcParams['DEFAULT']['data_directory'] = '/my' 10 | 11 | names = watershed_workflow.sources.filenames.Names('mynames', 'hydrography', 'rivers_{}', 12 | 'rivers_{}.gdb') 13 | assert ('rivers_0102.gdb' == names.file_name_base('0102')) 14 | assert ('/my/hydrography' == names.data_dir()) 15 | assert ('/my/hydrography/rivers_0102' == names.folder_name('0102')) 16 | assert ('/my/hydrography/rivers_0102/raw' == names.raw_folder_name('0102')) 17 | assert ('/my/hydrography/rivers_0102/rivers_0102.gdb' == names.file_name('0102')) 18 | watershed_workflow.config.rcParams['DEFAULT']['data_directory'] = ddir 19 | -------------------------------------------------------------------------------- /watershed_workflow/sources/standard_names.py: -------------------------------------------------------------------------------- 1 | """Namespace defining a bunch of standard names for properties of hydrologic units or reaches.""" 2 | 3 | # generic property names used everywhere 4 | ID = 'ID' 5 | NAME = 'name' 6 | 7 | # watershed/polygon property names 8 | HUC = 'huc' 9 | AREA = 'area' 10 | OUTLET = 'outlet' 11 | 12 | # reach property names 13 | TARGET_SEGMENT_WIDTH = 'target_width' 14 | TARGET_SEGMENT_LENGTH = 'target_length' 15 | ORDER = 'stream_order' 16 | DRAINAGE_AREA = 'drainage_area_sqkm' 17 | HYDROSEQ = 'hydroseq' 18 | UPSTREAM_HYDROSEQ = 'uphydroseq' 19 | DOWNSTREAM_HYDROSEQ = 'dnhydroseq' 20 | DIVERGENCE = 'divergence' 21 | CATCHMENT = 'catchment' 22 | CATCHMENT_AREA = 'catchment_area' 23 | LENGTH = 'length' 24 | 25 | # reach property names used in conditioning 26 | PROFILE_ELEVATION = 'elev_profile' 27 | 28 | 29 | # internal reach properties, not set by user 30 | ELEMS = 'elems' 31 | ELEMS_GID_START = 'elems_gid_start' 32 | PARENT = 'parent_ID' 33 | CHILDREN = 'children_IDs' 34 | -------------------------------------------------------------------------------- /docker/CI.Dockerfile: -------------------------------------------------------------------------------- 1 | # 2 | # Stage 2 -- clone repo run 3 | # 4 | ARG CI_ENV_DOCKER_TAG=master 5 | 6 | FROM ecoon/watershed_workflow-ci_env:${CI_ENV_DOCKER_TAG} AS watershed_workflow 7 | 8 | WORKDIR /ww 9 | 10 | # copy over source code 11 | COPY . /ww 12 | RUN conda run -n watershed_workflow_CI python -m pip install -e . 13 | 14 | # create a watershed_workflowrc that will be picked up 15 | RUN cp watershed_workflowrc .watershed_workflowrc 16 | RUN cat .watershed_workflowrc 17 | RUN echo "data_directory : /ww/examples/Coweeta/input_data" >> .watershed_workflowrc 18 | RUN cat .watershed_workflowrc 19 | 20 | # note it ALSO needs to be in the examples directory to be picked up there, and with the right path 21 | RUN cp watershed_workflowrc examples/.watershed_workflowrc 22 | RUN echo "data_directory : /ww/examples/Coweeta/input_data" >> examples/.watershed_workflowrc 23 | 24 | # run the library tests 25 | RUN conda run -n watershed_workflow_CI python -m pytest watershed_workflow/test 26 | 27 | # run the notebook examples 28 | RUN conda run -n watershed_workflow_CI pytest --nbmake --nbmake-kernel=python3 examples/coweeta_stream_aligned_mesh.ipynb 29 | 30 | 31 | -------------------------------------------------------------------------------- /environments/create_osx_envs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # how to (re-)create the local environments 4 | ENV_NAME="watershed_workflow-`date +%F`" 5 | python environments/create_envs.py --manager=mamba --OS=OSX --with-user-env=watershed_workflow_user --with-tools-env=watershed_workflow_tools ${ENV_NAME} 6 | conda run -n ${ENV_NAME} python -m ipykernel install \ 7 | --name ${ENV_NAME} --display-name "Python3 ${ENV_NAME}" 8 | conda env export -n ${ENV_NAME} --no-builds > environments/environment-OSX.yml 9 | 10 | CI_ENV_NAME="watershed_workflow_CI-`date +%F`" 11 | python environments/create_envs.py --manager=mamba --OS=OSX --env-type=CI ${CI_ENV_NAME} 12 | conda run -n ${CI_ENV_NAME} python -m ipykernel install \ 13 | --name ${CI_ENV_NAME} --display-name "Python3 ${CI_ENV_NAME}" 14 | conda env export -n ${CI_ENV_NAME} --no-builds > environments/environment-CI-OSX.yml 15 | 16 | DEV_ENV_NAME="watershed_workflow_DEV-`date +%F`" 17 | python environments/create_envs.py --manager=mamba --OS=OSX --env-type=DEV ${DEV_ENV_NAME} 18 | conda run -n ${DEV_ENV_NAME} python -m ipykernel install \ 19 | --name ${DEV_ENV_NAME} --display-name "Python3 ${DEV_ENV_NAME}" 20 | conda env export -n ${DEV_ENV_NAME} --no-builds > environments/environment-DEV-OSX.yml 21 | 22 | -------------------------------------------------------------------------------- /bin/condition_dem.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """Conditions unstructured DEMs from VTK.""" 3 | 4 | 5 | import argparse 6 | import logging 7 | 8 | import watershed_workflow.ui 9 | import watershed_workflow.condition 10 | import watershed_workflow.extrude 11 | 12 | def get_args(): 13 | parser = watershed_workflow.ui.get_basic_argparse(__doc__) 14 | parser.add_argument('input_file', 15 | type=watershed_workflow.ui.vtkfile, help='Input VTK file') 16 | parser.add_argument('output_file', 17 | type=str, help='Output VTK file') 18 | parser.add_argument('--outlet_node', type=int, 19 | help='Outlet node index (default searches boundary for lowest point).') 20 | 21 | # parse args, log 22 | return parser.parse_args() 23 | 24 | if __name__ == '__main__': 25 | args = get_args() 26 | watershed_workflow.ui.setup_logging(args.verbosity, args.logfile) 27 | 28 | logging.info("Reading file: {}".format(args.input_file)) 29 | m2 = watershed_workflow.extrude.Mesh2D.read_VTK(args.input_file) 30 | 31 | coords_old = m2.coords[:] 32 | watershed_workflow.condition.condition(m2, args.outlet_node) 33 | coords_new = m2.coords 34 | 35 | logging.info("max/min coordinate difference: {} / {}".format((coords_new[:,2] - coords_old[:,2]).max(), (coords_new[:,2] - coords_old[:,2]).min())) 36 | 37 | logging.info("Writing file: {}".format(args.output_file)) 38 | m2.write_VTK(args.output_file) 39 | -------------------------------------------------------------------------------- /watershed_workflow/test/test_01_crs.py: -------------------------------------------------------------------------------- 1 | """Not really sure what the best way to test these are. For now, we 2 | simply try converting things that we think should be the same and 3 | making sure they are the same. 4 | """ 5 | 6 | import pytest 7 | import pyproj 8 | 9 | import watershed_workflow.crs 10 | 11 | 12 | def epsg_harness(epsg): 13 | gold = watershed_workflow.crs.from_epsg(epsg) 14 | 15 | ppcrs2 = watershed_workflow.crs.from_proj(pyproj.crs.CRS('EPSG:{}'.format(epsg))) 16 | assert (watershed_workflow.crs.isEqual(gold, ppcrs2)) 17 | 18 | try: 19 | import fiona 20 | except ImportError: 21 | pass 22 | else: 23 | fcrs = watershed_workflow.crs.from_fiona(fiona.crs.CRS.from_epsg(epsg)) 24 | assert (watershed_workflow.crs.isEqual(gold, fcrs)) 25 | 26 | try: 27 | import rasterio 28 | except ImportError: 29 | pass 30 | else: 31 | rcrs = watershed_workflow.crs.from_rasterio(rasterio.crs.CRS.from_string( 32 | 'EPSG:{}'.format(epsg))) 33 | assert (watershed_workflow.crs.isEqual(gold, rcrs)) 34 | 35 | try: 36 | import cartopy 37 | except ImportError: 38 | pass 39 | else: 40 | cartopy_crs = cartopy.crs.epsg(epsg) 41 | ccrs = watershed_workflow.crs.from_cartopy(cartopy_crs) 42 | assert(watershed_workflow.crs.isEqual(gold, ccrs)) 43 | 44 | 45 | def test_default(): 46 | epsg_harness(5070) 47 | 48 | 49 | def test_alaska(): 50 | epsg_harness(3338) 51 | 52 | 53 | -------------------------------------------------------------------------------- /docker/configure-seacas.sh: -------------------------------------------------------------------------------- 1 | # NOTE: this requires that you have defined CONDA_PREFIX to point to 2 | # your Anaconda environment's installation, but this is set by default 3 | # on conda activate. 4 | 5 | CC=${COMPILERS}/bin/gcc 6 | CXX=${COMPILERS}/bin/g++ 7 | FC=${COMPILERS}/bin/gfortran 8 | 9 | SEACAS_SRC_DIR=${SEACAS_DIR}/src/seacas 10 | 11 | echo "Building SEACAS:" 12 | echo " at: ${SEACAS_DIR}" 13 | echo " with conda: ${CONDA_PREFIX}" 14 | echo " with CC: ${CC}" 15 | echo " with CXX: ${CXX}" 16 | echo " with FC: ${FC}" 17 | 18 | cmake \ 19 | -D Seacas_ENABLE_ALL_PACKAGES:BOOL=OFF \ 20 | -D Seacas_ENABLE_SEACASExodus:BOOL=ON \ 21 | -D CMAKE_INSTALL_PREFIX:PATH=${SEACAS_DIR} \ 22 | -D CMAKE_BUILD_TYPE=Debug \ 23 | -D BUILD_SHARED_LIBS:BOOL=ON \ 24 | \ 25 | -D CMAKE_CXX_COMPILER:FILEPATH=${CXX} \ 26 | -D CMAKE_C_COMPILER:FILEPATH=${CC} \ 27 | -D CMAKE_Fortran_COMPILER:FILEPATH=${FC} \ 28 | -D CMAKE_AR:FILEPATH=${COMPILERS}/bin/gcc-ar \ 29 | -D CMAKE_RANLIB:FILEPATH=${COMPILERS}/bin/gcc-ranlib \ 30 | -D Seacas_SKIP_FORTRANCINTERFACE_VERIFY_TEST:BOOL=ON \ 31 | -D TPL_ENABLE_Netcdf:BOOL=ON \ 32 | -D TPL_ENABLE_HDF5:BOOL=ON \ 33 | -D TPL_ENABLE_Matio:BOOL=OFF \ 34 | -D TPL_ENABLE_MPI=OFF \ 35 | -D TPL_ENABLE_CGNS:BOOL=OFF \ 36 | \ 37 | -D Netcdf_LIBRARY_DIRS:PATH=${CONDA_PREFIX}/lib \ 38 | -D Netcdf_INCLUDE_DIRS:PATH=${CONDA_PREFIX}/include \ 39 | -D HDF5_ROOT:PATH=${CONDA_PREFIX} \ 40 | -D HDF5_NO_SYSTEM_PATHS=ON \ 41 | ${SEACAS_SRC_DIR} 42 | 43 | 44 | -------------------------------------------------------------------------------- /docker/ATS-User-Env.Dockerfile: -------------------------------------------------------------------------------- 1 | # 2 | # In addition to that from User-Env.Dockerfile, this adds layers for 3 | # Amanzi-ATS source code (not an executable), ats_input_spec python 4 | # package, and amanzi_xml python package. 5 | # 6 | 7 | # 8 | # Stage 1 -- setup base CI environment 9 | # 10 | ARG USER_ENV_DOCKER_TAG 11 | FROM ecoon/watershed_workflow:${USER_ENV_DOCKER_TAG} 12 | LABEL Description="ATS layers on top of WW" 13 | 14 | ARG env_name=watershed_workflow 15 | ARG user=jovyan 16 | ARG ats_version=1.6 17 | ENV CONDA_BIN=mamba 18 | 19 | # get Amanzi-ATS source 20 | RUN mkdir /home/${user}/ats 21 | WORKDIR /home/${user}/ats 22 | RUN git clone -b amanzi-${ats_version} --recursive --depth=1 https://github.com/amanzi/amanzi amanzi-ats 23 | WORKDIR /home/${user}/ats/amanzi-ats/src/physics/ats 24 | RUN git checkout -b ats-${ats_version} 25 | RUN git pull 26 | 27 | # install amanzi_xml 28 | WORKDIR /home/${user}/ats/amanzi-ats/tools/amanzi_xml 29 | RUN ${CONDA_BIN} run -n ${env_name} python -m pip install -e . 30 | 31 | # set up environment for ats 32 | ENV AMANZI_SRC_DIR=/home/${user}/ats/amanzi-ats 33 | ENV ATS_SRC_DIR=/home/${user}/ats/amanzi-ats/src/physics/ats 34 | ENV PYTHONPATH=/home/${user}/ats/amanzi-ats/src/physics/ats/tools/utils 35 | 36 | # get ats_input_spec and install 37 | WORKDIR /home/${user}/ats 38 | RUN git clone --depth=1 https://github.com/ecoon/ats_input_spec ats_input_spec 39 | WORKDIR /home/${user}/ats/ats_input_spec 40 | RUN ${CONDA_BIN} run -n ${env_name} python -m pip install -e . 41 | 42 | # leave us in the right spot 43 | WORKDIR /home/${user}/workdir 44 | -------------------------------------------------------------------------------- /watershed_workflow/sources/filenames.py: -------------------------------------------------------------------------------- 1 | """A utility class for generating file and folder names.""" 2 | 3 | import attr 4 | import sys, os 5 | import watershed_workflow.config 6 | 7 | 8 | @attr.s 9 | class Names: 10 | """File system meta data for downloading a file.""" 11 | name = attr.ib(type=str) 12 | base_folder = attr.ib(type=str) 13 | folder_template = attr.ib(type=str) 14 | file_template = attr.ib(type=str) 15 | raw_template = attr.ib(type=str, default=None) 16 | 17 | def data_dir(self): 18 | return os.path.join(watershed_workflow.config.rcParams['DEFAULT']['data_directory'], 19 | self.base_folder) 20 | 21 | def folder_name(self, *args, **kwargs): 22 | if self.folder_template is None: 23 | return os.path.join(self.data_dir()) 24 | else: 25 | return os.path.join(self.data_dir(), self.folder_template.format(*args, **kwargs)) 26 | 27 | def raw_folder_name(self, *args, **kwargs): 28 | if self.raw_template is None: 29 | if self.folder_template is None: 30 | self.raw_template = 'raw' 31 | else: 32 | self.raw_template = os.path.join(self.folder_template, 'raw') 33 | return os.path.join(self.data_dir(), self.raw_template.format(*args, **kwargs)) 34 | 35 | def file_name_base(self, *args, **kwargs): 36 | return self.file_template.format(*args, **kwargs) 37 | 38 | def file_name(self, *args, **kwargs): 39 | return os.path.join(self.folder_name(*args, **kwargs), self.file_name_base(*args, **kwargs)) 40 | -------------------------------------------------------------------------------- /watershed_workflow/test/FAIL_test_dem_interp.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | import rasterio.transform 4 | import rasterio.crs 5 | import numpy as np 6 | 7 | import watershed_workflow 8 | import watershed_workflow.config 9 | 10 | 11 | @pytest.fixture 12 | def dem_and_points(): 13 | # create a dem 14 | dem = np.ones((2, 2)) 15 | dem[1, 0] = 2 16 | dem[1, 1] = 10 17 | 18 | # with a profile 19 | dem_profile = dict() 20 | dem_profile['crs'] = rasterio.crs.CRS.from_epsg(5070) 21 | dem_profile['transform'] = rasterio.transform.Affine(1, 0, 0, 0, 1, 0) 22 | dem_profile['height'] = 2 23 | dem_profile['width'] = 2 24 | dem_profile['offset'] = (0, 0) 25 | 26 | # create some points to sample 27 | xy = np.array([(0.000001, 0.000001), (.5, .5), (.9999999, .9999999), (1, 1), 28 | (1.0000001, 1.0000001), (1.5, 1.5), (1.9999999, 1.9999999), (.5, 1.5), (1.5, .5), 29 | ]) 30 | 31 | return dem, dem_profile, xy 32 | 33 | 34 | def test_nearest(dem_and_points): 35 | dem, dem_profile, xy = dem_and_points 36 | vals = watershed_workflow.values_from_raster( 37 | xy, watershed_workflow.crs.from_rasterio(dem_profile['crs']), dem, dem_profile, 'nearest') 38 | assert (np.allclose(np.array([1, 1, 1, 10, 10, 10, 10, 2, 1]), vals)) 39 | 40 | 41 | def test_interp(dem_and_points): 42 | dem, dem_profile, xy = dem_and_points 43 | vals = watershed_workflow.values_from_raster( 44 | xy, watershed_workflow.crs.from_rasterio(dem_profile['crs']), dem, dem_profile, 45 | 'piecewise bilinear') 46 | assert (np.allclose(np.array([1, 1, 3.5, 3.5, 3.5, 10, 10, 2, 1]), vals, 1.e-4)) 47 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | SOURCEDIR = source 8 | BUILDDIR = build 9 | 10 | .PHONY: all sphinx_help worktree html pdf deploy clean 11 | 12 | all: html 13 | 14 | help: 15 | @echo "Builds Watershed Workflow documentation" 16 | @echo "---------------------------------------" 17 | @echo "To build WW documentation from scratch (new repo setup, do this once):" 18 | @echo " > make worktree" 19 | @echo "" 20 | @echo "To build input spec/user guide:" 21 | @echo " > make" 22 | @echo "" 23 | @echo "To deploy to github:" 24 | @echo " > make deploy" 25 | 26 | sphinx_help: 27 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 28 | 29 | worktree: 30 | cd deploy && git worktree add html gh-pages 31 | 32 | html: 33 | @$(SPHINXBUILD) -M html "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 34 | 35 | pdf: 36 | @$(SPHINXBUILD) -M pdf "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 37 | 38 | deploy_master: 39 | mkdir -p deploy/html/dev 40 | cp -r build/html/* deploy/html/dev/ 41 | 42 | deploy_stable: 43 | mkdir -p deploy/html/stable 44 | cp -r build/html/* deploy/html/stable/ 45 | 46 | deploy: 47 | cd deploy/html && \ 48 | git add --all && \ 49 | git commit -m "Deployment to gh-pages" && \ 50 | git push origin gh-pages 51 | 52 | clean: 53 | rm -rf build/html 54 | 55 | allclean: 56 | rm -rf build/* 57 | 58 | 59 | # Catch-all target: route all unknown targets to Sphinx using the new 60 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 61 | #%: Makefile 62 | # @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 63 | 64 | -------------------------------------------------------------------------------- /watershed_workflow/test/crs_fixtures.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import shapely 3 | 4 | 5 | def point(): 6 | return shapely.geometry.Point(-90, 38) 7 | 8 | 9 | def point_ak(): 10 | return shapely.geometry.Point(-147, 65) 11 | 12 | 13 | def shift(p, t): 14 | return shapely.geometry.Point(p.xy[0][0] + t[0], p.xy[1][0] + t[1]) 15 | 16 | 17 | @pytest.fixture 18 | def points(): 19 | def _points(p): 20 | ps = [p, shift(p, (2, 0)), shift(p, (1, 1)), ] 21 | return ps 22 | 23 | return _points 24 | 25 | 26 | @pytest.fixture 27 | def lines(): 28 | def _lines(p): 29 | ls = [ 30 | shapely.geometry.LineString([p, shift(p, (0, 1)), shift(p, (0, 2))]), 31 | shapely.geometry.LineString([p, shift(p, (1, 0)), shift(p, (2, 0))]), 32 | shapely.geometry.LineString([p, shift(p, (1, 1)), shift(p, (2, 2))]), 33 | ] 34 | return ls 35 | 36 | return _lines 37 | 38 | 39 | @pytest.fixture 40 | def polygons(): 41 | def _polygons(p): 42 | polys = [ 43 | shapely.geometry.Polygon([[ 44 | p.x, p.y 45 | ] for p in [p, shift(p, ( 46 | -1, 47 | 0)), shift(p, (-1, 48 | -1)), shift(p, (0, -1)), p]]), 49 | shapely.geometry.Polygon([[ 50 | p.x, p.y 51 | ] for p in [p, shift(p, ( 52 | 1, 53 | 0)), shift(p, (1, 54 | -1)), shift(p, (0, -1)), p]]), 55 | shapely.geometry.Polygon([[ 56 | p.x, p.y 57 | ] for p in [p, shift(p, ( 58 | -1, 1)), shift(p, (0, 59 | 2)), shift(p, (1, 1)), p]]), 60 | ] 61 | return polys 62 | 63 | return _polygons 64 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | ---------------------------------------------- 2 | Berkeley Software Distribution (BSD) License 3 | ---------------------------------------------- 4 | 5 | Redistribution and use in source and binary forms, with or without 6 | modification, are permitted provided that the following conditions 7 | are met: 8 | 9 | * Redistributions of source code must retain the above copyright 10 | notice, this list of conditions and the following disclaimer. 11 | 12 | * Redistributions in binary form must reproduce the above copyright 13 | notice, this list of conditions and the following disclaimer in the 14 | documentation and/or other materials provided with the distribution. 15 | 16 | * Neither the names of the national laboratories listed above, 17 | nor their operating companies, nor the names of its 18 | contributors may be used to endorse or promote products derived 19 | from this software without specific prior written permission. 20 | 21 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 24 | FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 25 | NATIONAL LABORATORIES LISTED ABOVE, OR THEIR OPERATING COMPANIES, BE 26 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 29 | BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 30 | LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 31 | NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 32 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 | -------------------------------------------------------------------------------- /watershed_workflow/test/test_12_soil_properties.py: -------------------------------------------------------------------------------- 1 | """Test van Genuchten from Rosetta""" 2 | import numpy as np 3 | import watershed_workflow.soil_properties 4 | 5 | 6 | def test_vgm(): 7 | # headers: sand %, silt %, clay %, bulk dens 8 | data = np.array([70, 15, 15, 1.4]) 9 | 10 | vgm = watershed_workflow.soil_properties.computeVanGenuchtenModel_Rosetta(data) 11 | print(vgm) 12 | 13 | 14 | def test_vgm2(): 15 | # headers: sand %, silt %, clay %, bulk dens 16 | data = np.array([[70, 15, 15, 1.4], [50, 25, 25, 1.4]]).transpose() 17 | 18 | vgm = watershed_workflow.soil_properties.computeVanGenuchtenModel_Rosetta(data) 19 | ats = watershed_workflow.soil_properties.convertRosettaToATS(vgm) 20 | print(ats.keys()) 21 | assert (all(ats['residual saturation [-]'] < 1)) 22 | assert (all(ats['residual saturation [-]'] >= 0)) 23 | assert (all(ats['Rosetta porosity [-]'] < 1)) 24 | assert (all(ats['Rosetta porosity [-]'] >= 0)) 25 | assert (all(ats['Rosetta porosity [-]'] > ats['residual saturation [-]'])) 26 | assert (all(ats['van Genuchten alpha [Pa^-1]'] > 0)) 27 | assert (all(ats['van Genuchten alpha [Pa^-1]'] < 1.e-2)) 28 | assert (all(ats['van Genuchten n [-]'] > 1)) 29 | assert (all(ats['van Genuchten n [-]'] < 12)) 30 | assert (all(ats['Rosetta permeability [m^2]'] > 0)) 31 | assert (all(ats['Rosetta permeability [m^2]'] < 1.e-10)) 32 | 33 | 34 | def test_cluster(): 35 | arr_in = np.array([[1.01, 1, 1], [1, 2, 2], [2, 2.01, 2]]) 36 | arr_gd = np.array([[1, 1, 1], [1, 0, 0], [0, 0, 0]]) 37 | 38 | arr_in = np.expand_dims(arr_in, -1) 39 | codebook, arr_out, dists = watershed_workflow.soil_properties.cluster(arr_in, 2) 40 | print(arr_out) 41 | 42 | assert ((arr_out[arr_gd == 0] == arr_out[-1, -1]).all()) 43 | assert ((arr_out[arr_gd == 1] == arr_out[0, 0]).all()) 44 | -------------------------------------------------------------------------------- /watershed_workflow/sources/test/test_manager_glhymps.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | import os 4 | import shapely 5 | import numpy as np 6 | 7 | import watershed_workflow.config 8 | import watershed_workflow.utils 9 | import watershed_workflow.sources.manager_glhymps 10 | import watershed_workflow.sources.standard_names as names 11 | 12 | from fixtures import coweeta 13 | 14 | 15 | def test_glhymps_coweeta(coweeta): 16 | """Test GLHYMPS manager using Coweeta example data.""" 17 | # Use the local Coweeta GLHYMPS file 18 | glhymps_file = os.path.join(os.path.dirname(__file__), '..', '..', '..', 19 | 'examples', 'Coweeta', 'input_data', 'soil_structure', 'GLHYMPS', 'GLHYMPS.shp') 20 | glhymps_file = os.path.abspath(glhymps_file) 21 | 22 | # Skip test if file doesn't exist 23 | if not os.path.exists(glhymps_file): 24 | pytest.skip(f"GLHYMPS test file not found: {glhymps_file}") 25 | 26 | # Create manager with local file 27 | glhymps = watershed_workflow.sources.manager_glhymps.ManagerGLHYMPS(glhymps_file) 28 | 29 | # Test getShapesByGeometry with coweeta GeoDataFrame 30 | data = glhymps.getShapesByGeometry(coweeta) 31 | 32 | # Check basic properties 33 | assert len(data) > 0 34 | assert data.crs is not None 35 | assert names.ID in data.columns 36 | assert names.NAME in data.columns 37 | 38 | # Test that we get raw GLHYMPS data (OBJECTID_1 mapped to ID) 39 | assert 'OBJECTID_1' in data.columns # Original GLHYMPS field should still be present 40 | 41 | # Test getShapesByID using the IDs we just retrieved 42 | test_ids = data[names.ID].iloc[:2].astype(str).tolist() # Get first 2 IDs as strings 43 | data_by_id = glhymps.getShapesByID(test_ids) 44 | 45 | assert len(data_by_id) == len(test_ids) 46 | assert names.ID in data_by_id.columns 47 | -------------------------------------------------------------------------------- /watershed_workflow/config.py: -------------------------------------------------------------------------------- 1 | """Configuration and global defaults.""" 2 | 3 | import os 4 | import subprocess 5 | import configparser 6 | import getpass 7 | 8 | 9 | def getHome() -> str: 10 | return os.path.expanduser('~') 11 | 12 | 13 | def getDefaultConfig() -> configparser.ConfigParser: 14 | """Dictionary of all config option defaults. 15 | 16 | Returns 17 | ------- 18 | rcParams : configparser.ConfigParser 19 | A dict-like object containing parameters. 20 | """ 21 | rcParams = configparser.ConfigParser() 22 | 23 | rcParams['DEFAULT']['data_directory'] = "" 24 | rcParams['DEFAULT']['ssl_cert'] = "True" # note this can be True, 25 | # False (bad 26 | # idea/permissive) or a 27 | # path to ssl certs, 28 | # e.g. /etc/ssl/cert.perm 29 | # or similar 30 | rcParams['DEFAULT']['proj_network'] = "False" 31 | 32 | rcParams.add_section('AppEEARS') 33 | rcParams['AppEEARS']['username'] = 'NOT_PROVIDED' 34 | rcParams['AppEEARS']['password'] = 'NOT_PROVIDED' 35 | return rcParams 36 | 37 | 38 | def getConfig() -> configparser.ConfigParser: 39 | try: 40 | data_directory = os.path.join(os.environ['WATERSHED_WORKFLOW_DATA_DIR']) 41 | except KeyError: 42 | data_directory = os.path.join(os.getcwd(), 'data') 43 | rc = getDefaultConfig() 44 | rc['DEFAULT']['data_directory'] = data_directory 45 | 46 | # paths to search for rc files 47 | rc_paths = [ 48 | os.path.join(getHome(), '.watershed_workflowrc'), 49 | os.path.join(os.getcwd(), '.watershed_workflowrc'), 50 | os.path.join(os.getcwd(), 'watershed_workflowrc'), 51 | ] 52 | 53 | # this is a bit fragile -- it checks if the user is the docker user 54 | if getpass.getuser() == 'jovyan': 55 | rc_paths.append('/home/jovyan/workdir/.docker_watershed_workflowrc') 56 | 57 | # read the rc files 58 | rc.read(rc_paths) 59 | return rc 60 | 61 | 62 | def setDataDirectory(path : str) -> None: 63 | """Sets the directory in which all data is stored.""" 64 | rcParams['DEFAULT']['data_directory'] = path 65 | 66 | 67 | # global config 68 | rcParams = getConfig() 69 | -------------------------------------------------------------------------------- /bin/extrude_mesh.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """Vertically extrudes a VTK surface mesh. 3 | 4 | Note that this script is quite simple, and is not as rich as the 5 | possible use of the extrude capability here. It is quite likely that 6 | you don't really want to use this, and should instead write your own 7 | script or use a Jupyter notebook to do your own extrusion process. 8 | The command line simply isn't rich enough to express things 9 | efficiently. 10 | 11 | Mostly this script exists for testing and debugging. 12 | """ 13 | 14 | import sys,os 15 | import logging 16 | 17 | import watershed_workflow.ui 18 | import watershed_workflow.mesh 19 | 20 | def get_args(): 21 | parser = watershed_workflow.ui.get_basic_argparse(__doc__) 22 | parser.add_argument("-n", "--num-cells", default=10, type=int, 23 | help="number of cells to extrude") 24 | parser.add_argument("-d", "--depth", default=40.0, type=float, 25 | help="depth to extrude") 26 | parser.add_argument("-p", "--plot", default=False, action="store_true", 27 | help="plot the 2D mesh") 28 | parser.add_argument("input_file", type=watershed_workflow.ui.vtkfile, 29 | help="input filename of surface mesh (expects VTK)") 30 | parser.add_argument("output_file", type=str, 31 | help="output filename (expects EXO)") 32 | 33 | args = parser.parse_args() 34 | 35 | if os.path.isfile(args.output_file): 36 | print('Output file "%s" exists, cowardly not overwriting.'%args.output_file) 37 | sys.exit(1) 38 | 39 | return args 40 | 41 | 42 | if __name__ == "__main__": 43 | args = get_args() 44 | watershed_workflow.ui.setup_logging(args.verbosity, args.logfile) 45 | 46 | logging.info("Reading file: {}".format(args.input_file)) 47 | m2 = watershed_workflow.mesh.Mesh2D.read_VTK(args.input_file) 48 | if args.plot: 49 | m2.plot() 50 | 51 | logging.info("Extruding:") 52 | extrusion = ['constant',], [args.depth,], [args.num_cells,], [101,] 53 | watershed_workflow.mesh.Mesh3D.summarize_extrusion(*extrusion) 54 | m3 = watershed_workflow.mesh.Mesh3D.extruded_Mesh2D(m2, *extrusion) 55 | 56 | logging.info("Writing file: {}".format(args.output_file)) 57 | m3.write_exodus(args.output_file) 58 | -------------------------------------------------------------------------------- /docs/source/concepts.rst: -------------------------------------------------------------------------------- 1 | Workflow library Concepts 2 | ========================= 3 | 4 | Package configuration 5 | ~~~~~~~~~~~~~~~~~~~~~ 6 | 7 | Watershed Workflow is configured through a limited set of parameters 8 | specified in a file `".watershed_workflowrc`", located in the current 9 | working directory or the user's home directory. An example including 10 | all defaults is shown in the top level directory as 11 | `"watershed_workflowrc`". 12 | 13 | Coordinate Reference Systems (CRS) 14 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 15 | 16 | Coordinate Reference Systems are used to locate geographic positions. 17 | These define a specific map projection, transforming 3D positions on 18 | the Earth's surface to 2D coordinates. Different projections can be 19 | used to optimize for different things, but typically hydrologic 20 | simulations work on equal area projections. These projects maintain, 21 | at least regionally, proportional areas for polygons, which is 22 | critical for ensuring accurate water balances. 23 | 24 | CRSs are specified by a dataset, and differ across datasets; 25 | standardizing and managing these across the workflow is a necessary 26 | technical detail. That said, rarely does the user care what 27 | coordinate system is being used, as long as it is appropriate for the 28 | watershed in question. Watershed Workflow aims to make using datasets 29 | in different CRSs as streamlined as possible. Typically, a workflow 30 | will pick a CRS based upon either a default for the region or by 31 | simply using the CRS of the shapefile that specifies the watershed 32 | boundary. This CRS is the passed into each function that acquires 33 | more data, and that data's coordinates are changed to the CRS 34 | requested. 35 | 36 | Often it can be a good idea to work with a CRS that is used by a 37 | raster dataset, for instance meterological data. Interpolating from a 38 | raster to a set of points (e.g. mesh cell centroids) is done by first 39 | transforming those points into the CRS of the raster and then 40 | interpolating. While reprojecting rasters is possible (and supported 41 | by rasterio), it involves some error and is tricky. Working in a 42 | raster's native CRS allows interpolation without reprojection, which 43 | is especially useful for rasters that must be repeatedly interpolated 44 | (i.e. meterological data or other time-dependent datasets). 45 | 46 | See :ref:`CRS` for detailed documentation of working with CRSs. 47 | 48 | 49 | -------------------------------------------------------------------------------- /watershed_workflow/test/fixture_data/hucs.dbf: -------------------------------------------------------------------------------- 1 | y AQHUCCP 0601 060102 060101 06010204 06010201 06010207 06010208 06010205 06010206 06010203 06010202 0601020202 0601020205 0601020204 0601020201 0601020203 060102020103 060102020101 060102020102 060102020105 060102020104 060102020106 060101080204 06010103 06010105 06010106 06010104 06010102 06010101 06010107 06010108  -------------------------------------------------------------------------------- /watershed_workflow/test/test_08_triangulate.py: -------------------------------------------------------------------------------- 1 | """ 2 | 'tests' for triangulate 3 | 4 | These aren't actually tests -- they just exercise the capability 5 | and make sure it runs and does something. Can plot for debugging. 6 | Not sure how to test them...""" 7 | 8 | import pytest 9 | import shapely 10 | import geopandas 11 | from matplotlib import pyplot as plt 12 | 13 | import watershed_workflow.triangulation 14 | import watershed_workflow.hydrography 15 | import watershed_workflow.split_hucs 16 | import watershed_workflow.plot 17 | from watershed_workflow.test.shapes import * 18 | 19 | _plot = False 20 | _assert_plot = False 21 | def plot(hucs, rivers, points, tris, force = False): 22 | if _plot or force: 23 | fig, ax = plt.subplots(1,1) 24 | ax.triplot(points[:,0], points[:,1], tris) 25 | hucs.plot(color='r', ax=ax) 26 | for river in rivers: 27 | river.plot(color='b', ax=ax) 28 | plt.show() 29 | assert not _assert_plot 30 | 31 | 32 | def test_triangulate_nofunc(watershed_rivers1): 33 | hucs, rivers = watershed_rivers1 34 | watershed_workflow.simplify(hucs, rivers, 1) 35 | 36 | points, tris = watershed_workflow.triangulation.triangulate(hucs, tol=0.01) 37 | plot(hucs, rivers, points, tris) 38 | 39 | 40 | def test_triangulate_max_area(watershed_rivers1): 41 | hucs, rivers = watershed_rivers1 42 | watershed_workflow.simplify(hucs, rivers, 1) 43 | 44 | func = watershed_workflow.triangulation.refineByMaxArea(1.) 45 | points, tris = watershed_workflow.triangulation.triangulate(hucs, refinement_func=func, tol=0.01) 46 | plot(hucs, rivers, points, tris) 47 | 48 | 49 | def test_triangulate_distance(watershed_rivers1): 50 | hucs, rivers = watershed_rivers1 51 | watershed_workflow.simplify(hucs, rivers, 1) 52 | 53 | func = watershed_workflow.triangulation.refineByRiverDistance(1., 0.5, 4, 2, rivers) 54 | points, tris = watershed_workflow.triangulation.triangulate(hucs, refinement_func=func, tol=0.01) 55 | plot(hucs, rivers, points, tris) 56 | 57 | 58 | def test_triangulate_internal_boundary(watershed_rivers1): 59 | hucs, rivers = watershed_rivers1 60 | watershed_workflow.simplify(hucs, rivers, 1) 61 | 62 | func = watershed_workflow.triangulation.refineByRiverDistance(1., 0.5, 4, 2, rivers) 63 | points, tris = watershed_workflow.triangulation.triangulate(hucs, 64 | internal_boundaries=[r.linestring for river in rivers for r in river], 65 | refinement_func=func, tol=0.01) 66 | plot(hucs, rivers, points, tris) 67 | 68 | -------------------------------------------------------------------------------- /watershed_workflow/sources/test/test_manager_nrcs.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | import watershed_workflow.sources.manager_nrcs 4 | 5 | from fixtures import coweeta 6 | 7 | def test_nrcs2(coweeta): 8 | # get imgs 9 | nrcs = watershed_workflow.sources.manager_nrcs.ManagerNRCS(force_download=True) 10 | df = nrcs.getShapesByGeometry(coweeta.geometry[0], coweeta.crs) 11 | 12 | # check df 13 | mukeys = set(df['ID']) 14 | assert len(df) == len(mukeys) # one per unique key 15 | assert 50 > len(df) > 40 16 | assert df.crs is not None 17 | 18 | # Test that standard names are applied 19 | import watershed_workflow.sources.standard_names as names 20 | assert names.ID in df.columns 21 | assert names.NAME in df.columns 22 | 23 | # Check that all names follow NRCS-{mukey} pattern 24 | for name in df[names.NAME]: 25 | assert name.startswith('NRCS-') 26 | 27 | # Check metadata 28 | assert df.attrs['name'] == nrcs.name 29 | assert df.attrs['source'] == nrcs.source 30 | 31 | 32 | def test_nrcs_constructor(): 33 | """Test NRCS constructor properties""" 34 | nrcs = watershed_workflow.sources.manager_nrcs.ManagerNRCS() 35 | assert nrcs.name == 'National Resources Conservation Service Soil Survey (NRCS Soils)' 36 | assert nrcs.source == 'USDA NRCS SSURGO Database' 37 | assert nrcs.native_id_field == 'mukey' 38 | assert nrcs.force_download == False 39 | 40 | # Test with force_download=True 41 | nrcs_force = watershed_workflow.sources.manager_nrcs.ManagerNRCS(force_download=True) 42 | assert nrcs_force.force_download == True 43 | 44 | 45 | def test_nrcs_geodataframe_input(coweeta): 46 | """Test getShapesByGeometry with GeoDataFrame input""" 47 | import geopandas as gpd 48 | nrcs = watershed_workflow.sources.manager_nrcs.ManagerNRCS() 49 | 50 | # Create GeoDataFrame from coweeta fixture 51 | gdf = gpd.GeoDataFrame([{'test': 1}], geometry=[coweeta.geometry[0]], crs=coweeta.crs) 52 | 53 | df = nrcs.getShapesByGeometry(gdf) 54 | 55 | assert isinstance(df, gpd.GeoDataFrame) 56 | assert 50 > len(df) > 40 57 | import watershed_workflow.sources.standard_names as names 58 | assert names.ID in df.columns 59 | assert names.NAME in df.columns 60 | 61 | 62 | def test_nrcs_getShapesByID_not_supported(): 63 | """Test that getShapesByID raises NotImplementedError""" 64 | import pytest 65 | nrcs = watershed_workflow.sources.manager_nrcs.ManagerNRCS() 66 | 67 | with pytest.raises(NotImplementedError, match="ManagerNRCS doesn't support getShapesByID"): 68 | nrcs.getShapesByID(['123456']) 69 | -------------------------------------------------------------------------------- /bin/plot_hucs.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """Downloads and plots HUCs, hydrography, and DEM data.""" 3 | 4 | import logging 5 | import numpy as np 6 | from matplotlib import pyplot as plt 7 | 8 | import watershed_workflow 9 | import watershed_workflow.ui 10 | import watershed_workflow.sources 11 | import watershed_workflow.bin_utils 12 | 13 | def get_args(): 14 | # set up parser 15 | parser = watershed_workflow.ui.get_basic_argparse(__doc__+'\n\n'+watershed_workflow.sources.__doc__) 16 | watershed_workflow.ui.projection(parser) 17 | watershed_workflow.ui.huc_arg(parser) 18 | watershed_workflow.ui.huc_level_arg(parser) 19 | 20 | watershed_workflow.ui.simplify_options(parser) 21 | watershed_workflow.ui.plot_options(parser) 22 | 23 | data_ui = parser.add_argument_group('Data Sources') 24 | watershed_workflow.ui.huc_source_options(data_ui) 25 | watershed_workflow.ui.hydro_source_options(data_ui) 26 | watershed_workflow.ui.dem_source_options(data_ui) 27 | 28 | # parse args, log 29 | return parser.parse_args() 30 | 31 | def plot_hucs(args): 32 | sources = watershed_workflow.sources.get_sources(args) 33 | 34 | if args.level == 0: 35 | args.level = len(args.HUC) 36 | 37 | logging.info("") 38 | logging.info("Plotting level {} HUCs in HUC: {}".format(args.level, args.HUC)) 39 | logging.info("="*30) 40 | try: 41 | logging.info('Target projection: "{}"'.format(args.projection['init'])) 42 | except TypeError: 43 | pass 44 | 45 | # collect data 46 | crs, hucs = watershed_workflow.get_split_form_hucs(sources['HUC'], args.HUC, args.level, out_crs=args.projection) 47 | args.projection = crs 48 | 49 | # hydrography 50 | _, reaches = watershed_workflow.get_reaches(sources['hydrography'], args.HUC, None, crs, crs) 51 | 52 | # raster 53 | dem_profile, dem = watershed_workflow.get_raster_on_shape(sources['DEM'], hucs.exterior(), crs, crs, 54 | mask=True, nodata=np.nan) 55 | logging.info('dem crs: {}'.format(dem_profile['crs'])) 56 | 57 | return hucs, reaches, dem, dem_profile 58 | 59 | if __name__ == '__main__': 60 | args = get_args() 61 | watershed_workflow.ui.setup_logging(args.verbosity, args.logfile) 62 | hucs, reaches, dem, profile = plot_hucs(args) 63 | 64 | if args.title is None: 65 | args.title = 'HUC: {}'.format(args.HUC) 66 | 67 | fig, ax = watershed_workflow.bin_utils.plot_with_dem(args, hucs, reaches, dem, profile) 68 | 69 | logging.info("SUCESS") 70 | if args.output_filename is not None: 71 | fig.savefig(args.output_filename, dpi=150) 72 | plt.show() 73 | 74 | -------------------------------------------------------------------------------- /docker/CI-Env.Dockerfile: -------------------------------------------------------------------------------- 1 | # Does everything except running tests... 2 | # 3 | # Stage 1 -- setup base CI environment 4 | # 5 | FROM condaforge/miniforge3:latest AS ww_env_base_ci 6 | LABEL Description="Base env for CI of Watershed Workflow" 7 | 8 | ARG env_name=watershed_workflow_CI 9 | ENV CONDA_BIN=mamba 10 | 11 | # figure out and print out conda platform info 12 | ARG TARGETARCH 13 | ARG TARGETOS 14 | 15 | RUN echo "TARGETARCH=${TARGETARCH}" && \ 16 | echo "TARGETOS=${TARGETOS}" && \ 17 | uname -m && \ 18 | conda info | grep platform 19 | 20 | # copy over create_envs 21 | WORKDIR /ww/tmp 22 | COPY environments/create_envs.py /ww/tmp/create_envs.py 23 | RUN mkdir environments 24 | 25 | # Create the environment 26 | RUN --mount=type=cache,target=/opt/conda/pkgs \ 27 | /opt/conda/bin/python create_envs.py --OS=Linux --manager=${CONDA_BIN} \ 28 | --env-type=CI --with-tools-env=watershed_workflow_tools ${env_name} 29 | 30 | # test the environment 31 | RUN ${CONDA_BIN} run --name ${env_name} python -c "import pymetis; import geopandas" 32 | 33 | # set compilers from watershed_workflow_tools environment 34 | ENV COMPILERS=/opt/conda/envs/watershed_workflow_tools 35 | ENV PATH="${COMPILERS}/bin:${PATH}" 36 | 37 | # 38 | # Stage 2 -- add in the pip 39 | # 40 | FROM ww_env_base_ci AS ww_env_pip_ci 41 | 42 | WORKDIR /ww/tmp 43 | COPY requirements.txt /ww/tmp/requirements.txt 44 | 45 | RUN ${CONDA_BIN} run --name ${env_name} python -m pip install -r requirements.txt 46 | 47 | # test the environment 48 | RUN ${CONDA_BIN} run --name ${env_name} python -c "import meshpy" 49 | 50 | # 51 | # Stage 3 -- add in Exodus 52 | # 53 | FROM ww_env_pip_ci AS ww_env_exodus_ci 54 | 55 | ENV SEACAS_DIR="/opt/conda/envs/${env_name}" 56 | ENV CONDA_PREFIX="/opt/conda/envs/${env_name}" 57 | 58 | # get the source 59 | WORKDIR /opt/conda/envs/${env_name}/src 60 | RUN git clone -b v2025-08-28 --depth=1 https://github.com/gsjaardema/seacas/ seacas 61 | 62 | # apply the patch 63 | COPY environments/exodus_py.patch /opt/conda/envs/${env_name}/src/exodus_py.patch 64 | WORKDIR /opt/conda/envs/${env_name}/src/seacas 65 | RUN git apply ../exodus_py.patch 66 | 67 | # configure 68 | WORKDIR /ww/tmp 69 | COPY docker/configure-seacas.sh /ww/tmp/configure-seacas.sh 70 | RUN chmod +x /ww/tmp/configure-seacas.sh 71 | WORKDIR /ww/tmp/seacas-build 72 | RUN ${CONDA_BIN} run -n watershed_workflow_CI ../configure-seacas.sh 73 | RUN make -j4 install 74 | 75 | # exodus installs its wrappers in an invalid place for python... 76 | # -- get and save the python version 77 | RUN SITE_PACKAGES=$(conda run -n ${env_name} python -c "import site; print(site.getsitepackages()[0])") && \ 78 | cp /opt/conda/envs/${env_name}/lib/exodus3.py ${SITE_PACKAGES} 79 | 80 | # test the environment 81 | RUN ${CONDA_BIN} run --name ${env_name} python -c "import exodus3" 82 | -------------------------------------------------------------------------------- /watershed_workflow/test/FAIL_test_hilev.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import watershed_workflow.crs 3 | import watershed_workflow 4 | 5 | from source_fixtures import sources, sources_download 6 | 7 | 8 | # def test_river_tree_properties(sources_download): 9 | # crs = watershed_workflow.crs.default_crs 10 | # nhd = sources_download['hydrography'] 11 | # _, cc = watershed_workflow.get_split_form_hucs(nhd, '060102020103', 12, crs) 12 | # _, reaches = watershed_workflow.get_reaches(nhd, 13 | # '060102020103', 14 | # cc.exterior(), 15 | # crs, 16 | # crs, 17 | # properties=True) 18 | 19 | # rivers = watershed_workflow.construct_rivers(reaches, method='hydroseq') 20 | # assert (len(rivers) == 1) 21 | # assert (rivers[0].is_consistent()) 22 | # assert (len(rivers[0]) == 94) 23 | 24 | 25 | # def test_river_tree_properties_prune(sources_download): 26 | # crs = watershed_workflow.crs.default_crs 27 | # nhd = sources_download['hydrography'] 28 | # _, cc = watershed_workflow.get_split_form_hucs(nhd, '060102020103', 12, crs) 29 | # _, reaches = watershed_workflow.get_reaches(nhd, 30 | # '060102020103', 31 | # cc.exterior(), 32 | # crs, 33 | # crs, 34 | # properties=True) 35 | 36 | # rivers = watershed_workflow.construct_rivers(reaches, 37 | # method='hydroseq', 38 | # prune_by_area=0.03 * cc.exterior().area * 1.e-6) 39 | # assert (len(rivers) == 1) 40 | # assert (rivers[0].is_consistent()) 41 | # assert (len(rivers[0]) == 49) 42 | 43 | 44 | # def test_river_tree_geometry(sources): 45 | # crs = watershed_workflow.crs.default_crs 46 | # nhd = sources['HUC'] 47 | # _, cc = watershed_workflow.get_split_form_hucs(nhd, '060102020103', 12, crs) 48 | # _, reaches = watershed_workflow.get_reaches(nhd, 49 | # '060102020103', 50 | # cc.exterior(), 51 | # crs, 52 | # crs, 53 | # properties=False) 54 | 55 | # rivers = watershed_workflow.construct_rivers(reaches) 56 | # assert (len(rivers) == 1) 57 | # assert (rivers[0].is_consistent()) 58 | # assert (len(rivers[0]) == 98) 59 | -------------------------------------------------------------------------------- /bin/plot_shape.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """Plots watersheds and their context within a HUC. 3 | """ 4 | import logging 5 | from matplotlib import pyplot as plt 6 | import numpy as np 7 | 8 | import watershed_workflow 9 | import watershed_workflow.ui 10 | import watershed_workflow.sources 11 | import watershed_workflow.bin_utils 12 | import watershed_workflow.plot 13 | 14 | def get_args(): 15 | # set up parser 16 | parser = watershed_workflow.ui.get_basic_argparse(__doc__+'\n\n'+watershed_workflow.sources.__doc__) 17 | watershed_workflow.ui.projection(parser) 18 | watershed_workflow.ui.inshape_args(parser) 19 | watershed_workflow.ui.huc_hint_options(parser) 20 | 21 | watershed_workflow.ui.simplify_options(parser) 22 | watershed_workflow.ui.plot_options(parser) 23 | 24 | data_ui = parser.add_argument_group('Data Sources') 25 | watershed_workflow.ui.huc_source_options(data_ui) 26 | watershed_workflow.ui.hydro_source_options(data_ui) 27 | watershed_workflow.ui.dem_source_options(data_ui) 28 | 29 | # parse args, log 30 | return parser.parse_args() 31 | 32 | def plot_shape(args): 33 | sources = watershed_workflow.sources.get_sources(args) 34 | 35 | logging.info("") 36 | logging.info("Plotting shapes from file: {}".format(args.input_file)) 37 | logging.info("="*30) 38 | try: 39 | logging.info('Target projection: "{}"'.format(args.projection['init'])) 40 | except TypeError: 41 | pass 42 | 43 | # collect data 44 | # -- get the shapes 45 | crs, shapes = watershed_workflow.get_split_form_shapes(args.input_file, args.shape_index, args.projection) 46 | args.projection = crs 47 | 48 | # -- get the containing huc 49 | hucstr = watershed_workflow.find_huc(sources['HUC'], shapes.exterior(), crs, args.hint, shrink_factor=0.1) 50 | logging.info("found shapes in HUC %s"%hucstr) 51 | _, huc = watershed_workflow.get_huc(sources['HUC'], hucstr, crs) 52 | 53 | # -- get reaches of that huc 54 | _, reaches = watershed_workflow.get_reaches(sources['hydrography'], hucstr, 55 | shapes.exterior().bounds, crs, crs) 56 | 57 | # -- dem 58 | dem_profile, dem = watershed_workflow.get_raster_on_shape(sources['DEM'], shapes.exterior(), crs, crs, 59 | mask=True, nodata=np.nan) 60 | logging.info('dem crs: {}'.format(dem_profile['crs'])) 61 | 62 | return shapes, huc, reaches, dem, dem_profile 63 | 64 | 65 | if __name__ == '__main__': 66 | args = get_args() 67 | watershed_workflow.ui.setup_logging(args.verbosity, args.logfile) 68 | 69 | # get objects 70 | shapes, huc, reaches, dem, dem_profile = plot_shape(args) 71 | 72 | # plot 73 | fig, ax = watershed_workflow.bin_utils.plot_with_dem(args, shapes, reaches, dem, dem_profile, river_color='white') 74 | # watershed_workflow.plot.shply([huc,], args.projection, color='k', ax=ax) 75 | 76 | logging.info("SUCESS") 77 | if args.output_filename is not None: 78 | fig.savefig(args.output_filename, dpi=150) 79 | plt.show() 80 | 81 | 82 | 83 | 84 | 85 | -------------------------------------------------------------------------------- /watershed_workflow/test/test_14_find_huc.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import geopandas 3 | import math 4 | import numpy as np 5 | import watershed_workflow.crs 6 | 7 | #nhdplus giving service errors that are not WW's fault... 8 | from async_retriever.exceptions import ServiceError 9 | 10 | from watershed_workflow.sources.manager_wbd import ManagerWBD 11 | 12 | from watershed_workflow.test.source_fixtures import datadir 13 | 14 | 15 | 16 | def get_shapes(filename): 17 | gdf = geopandas.read_file(filename) 18 | gdf = gdf.to_crs(watershed_workflow.crs.default_crs) 19 | return gdf 20 | 21 | 22 | def test_find12(datadir): 23 | nhd = ManagerWBD() 24 | 25 | testshpfile = datadir.join('test_shapefile.shp') 26 | shp = get_shapes(testshpfile) 27 | radius = math.sqrt(float(shp.area.iloc[0]) / np.pi) 28 | shp = shp.buffer(-.001 * radius) 29 | try: 30 | found = watershed_workflow.findHUC(nhd, shp.geometry.iloc[0], shp.crs, '0601') 31 | except ServiceError: 32 | pass 33 | else: 34 | assert '060102020103' == found 35 | 36 | 37 | def test_find12_exact(datadir): 38 | nhd = ManagerWBD() 39 | 40 | testshpfile = datadir.join('test_shapefile.shp') 41 | shp = get_shapes(testshpfile) 42 | radius = np.sqrt(float(shp.area[0]) / np.pi) 43 | shp = shp.buffer(-.001 * radius) 44 | 45 | try: 46 | found = watershed_workflow.findHUC(nhd, shp.geometry[0], shp.crs, '060102020103') 47 | except ServiceError: 48 | pass 49 | else: 50 | assert '060102020103' == found 51 | 52 | 53 | def test_find12_raises(datadir): 54 | """This throws because the shape is not in this huc""" 55 | nhd = ManagerWBD() 56 | 57 | testshpfile = datadir.join('test_shapefile.shp') 58 | shp = get_shapes(testshpfile) 59 | radius = np.sqrt(float(shp.area[0]) / np.pi) 60 | shp = shp.buffer(-.001 * radius) 61 | print(shp.area) 62 | with pytest.raises(RuntimeError): 63 | watershed_workflow.findHUC(nhd, shp.geometry[0], shp.crs, '060101080204') 64 | 65 | 66 | def test_find8(datadir): 67 | nhd = ManagerWBD() 68 | 69 | testshpfile = datadir.join('test_polygon.shp') 70 | shp = get_shapes(testshpfile) 71 | 72 | try: 73 | found = watershed_workflow.findHUC(nhd, shp.geometry[0], shp.crs, '0601') 74 | except ServiceError: 75 | pass 76 | else: 77 | assert '06010202' == found 78 | 79 | 80 | def test_find8_exact(datadir): 81 | nhd = ManagerWBD() 82 | 83 | testshpfile = datadir.join('test_polygon.shp') 84 | shp = get_shapes(testshpfile) 85 | 86 | try: 87 | found = watershed_workflow.findHUC(nhd, shp.geometry[0], shp.crs, '06010202') 88 | except ServiceError: 89 | pass 90 | else: 91 | assert '06010202' == found 92 | 93 | 94 | def test_find8_raises(datadir): 95 | nhd = ManagerWBD() 96 | 97 | testshpfile = datadir.join('copper_creek.shp') 98 | shp = get_shapes(testshpfile) 99 | with pytest.raises(RuntimeError): 100 | watershed_workflow.findHUC(nhd, shp.geometry[0], shp.crs, '0601') 101 | 102 | 103 | -------------------------------------------------------------------------------- /watershed_workflow/sources/test/test_manager_soilgrids.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import os 3 | import numpy as np 4 | 5 | import watershed_workflow.config 6 | import watershed_workflow.sources.manager_soilgrids_2017 as manager_soilgrids 7 | import watershed_workflow.crs 8 | 9 | 10 | @pytest.fixture 11 | def soilgrids(): 12 | return manager_soilgrids.ManagerSoilGrids2017() 13 | 14 | 15 | @pytest.fixture 16 | def soilgrids_us(): 17 | return manager_soilgrids.ManagerSoilGrids2017('US') 18 | 19 | 20 | def test_constructor(soilgrids): 21 | """Test basic constructor and properties.""" 22 | assert soilgrids.name == 'SoilGrids2017' 23 | assert soilgrids.source == manager_soilgrids.ManagerSoilGrids2017.URL 24 | assert soilgrids.native_crs_in == watershed_workflow.crs.from_epsg(4326) 25 | assert soilgrids.native_crs_out == watershed_workflow.crs.from_epsg(4326) 26 | assert soilgrids.native_start is None 27 | assert soilgrids.native_end is None 28 | assert soilgrids.default_variables == ['BDTICM'] 29 | 30 | 31 | def test_constructor_us_variant(soilgrids_us): 32 | """Test US variant constructor.""" 33 | assert soilgrids_us.name == 'SoilGrids2017_US' 34 | assert soilgrids_us.source == manager_soilgrids.ManagerSoilGrids2017.URL 35 | assert soilgrids_us.default_variables == ['BDTICM'] 36 | 37 | 38 | def test_valid_variables(soilgrids): 39 | """Test that all expected variables are present.""" 40 | expected_vars = set(['BDTICM']) # bedrock variable 41 | 42 | # Add all layer variables 43 | for base_var in manager_soilgrids.ManagerSoilGrids2017.BASE_VARIABLES: 44 | for layer in manager_soilgrids.ManagerSoilGrids2017.LAYERS: 45 | expected_vars.add(f'{base_var}_layer_{layer}') 46 | 47 | assert set(soilgrids.valid_variables) == expected_vars 48 | assert len(soilgrids.valid_variables) == 1 + 5 * 7 # BDTICM + 5 vars * 7 layers 49 | 50 | 51 | def test_parse_variable(soilgrids): 52 | """Test variable name parsing.""" 53 | # Test bedrock variable 54 | base_var, layer = soilgrids._parseVariable('BDTICM') 55 | assert base_var == 'BDTICM' 56 | assert layer is None 57 | 58 | # Test layered variables 59 | base_var, layer = soilgrids._parseVariable('BLDFIE_layer_3') 60 | assert base_var == 'BLDFIE' 61 | assert layer == 3 62 | 63 | # Test invalid variables 64 | with pytest.raises(ValueError): 65 | soilgrids._parseVariable('INVALID_VAR') 66 | 67 | with pytest.raises(ValueError): 68 | soilgrids._parseVariable('BLDFIE_layer_8') # layer 8 doesn't exist 69 | 70 | with pytest.raises(ValueError): 71 | soilgrids._parseVariable('BLDFIE_layer_abc') # invalid layer number 72 | 73 | 74 | def test_variable_categories(soilgrids): 75 | """Test that variables are properly categorized.""" 76 | # Check base variables 77 | expected_base_vars = ['BLDFIE', 'CLYPPT', 'SLTPPT', 'SNDPPT', 'WWP'] 78 | assert soilgrids.BASE_VARIABLES == expected_base_vars 79 | 80 | # Check bedrock variable 81 | assert soilgrids.BEDROCK_VARIABLE == 'BDTICM' 82 | 83 | # Check layers 84 | assert soilgrids.LAYERS == list(range(1, 8)) 85 | 86 | 87 | -------------------------------------------------------------------------------- /watershed_workflow/sources/manager_3dep.py: -------------------------------------------------------------------------------- 1 | """Manager for downloading 3DEP data.""" 2 | 3 | from typing import Tuple, Optional, List 4 | import cftime 5 | import logging 6 | 7 | import shapely.geometry 8 | import xarray as xr 9 | import py3dep 10 | 11 | import watershed_workflow.crs 12 | from watershed_workflow.crs import CRS 13 | 14 | from . import manager_dataset 15 | 16 | class Manager3DEP(manager_dataset.ManagerDataset): 17 | """3D Elevation Program (3DEP) data manager. 18 | 19 | Provides access to USGS 3DEP elevation and derived products through 20 | the py3dep library. Supports multiple resolution options and various 21 | topographic layers including DEM, slope, aspect, and hillshade products. 22 | """ 23 | 24 | def __init__(self, resolution : int): 25 | """Downloads DEM data from the 3DEP. 26 | 27 | Parameters 28 | ---------- 29 | resolution : int 30 | Resolution in meters. Valid resolutions are: 60, 30, or 10. 31 | """ 32 | self._resolution = resolution 33 | resolution_in_degrees = 2 * resolution * 9e-6 34 | 35 | in_crs = CRS.from_epsg(4326) # lat-long 36 | out_crs = CRS.from_epsg(5070) # CONUS Albers Equal Area 37 | 38 | valid_variables = [ 39 | 'DEM', 'Hillshade Gray', 'Aspect Degrees', 'Aspect Map', 40 | 'GreyHillshade_elevationFill', 'Hillshade Multidirectional', 41 | 'Slope Map', 'Slope Degrees', 'Hillshade Elevation Tinted', 42 | 'Height Ellipsoidal', 'Contour 25', 'Contour Smoothed 25' 43 | ] 44 | default_variables = ['DEM'] 45 | 46 | # Initialize base class with native properties 47 | super().__init__(f'3DEP {resolution}m', 'py3dep', resolution_in_degrees, in_crs, out_crs, 48 | None, None, valid_variables, default_variables) 49 | 50 | def _requestDataset(self, request : manager_dataset.ManagerDataset.Request 51 | ) -> manager_dataset.ManagerDataset.Request: 52 | """Request the data -- ready upon request.""" 53 | request.is_ready = True 54 | return request 55 | 56 | 57 | def _fetchDataset(self, request : manager_dataset.ManagerDataset.Request) -> xr.Dataset: 58 | """Implementation of abstract method to get 3DEP data.""" 59 | 60 | # Base class ensures these for multi-variable, time independent class 61 | assert request.variables is not None 62 | assert request.start is None 63 | assert request.end is None 64 | 65 | # Use instance resolution and native CRS 66 | logging.info(f'Getting DEM with map of area = {request.geometry.area}') 67 | result = py3dep.get_map(request.variables, request.geometry, self._resolution, 68 | geo_crs=self.native_crs_in, crs=self.native_crs_out) 69 | 70 | # py3dep returns DataArray for single layer, Dataset for multiple layers 71 | if isinstance(result, xr.DataArray): 72 | # Convert DataArray to Dataset 73 | result = result.to_dataset(name=request.variables[0].lower().replace(' ', '_')) 74 | 75 | return result 76 | 77 | -------------------------------------------------------------------------------- /bin/mesh_hucs.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """Downloads and meshes HUCs based on hydrography data.""" 3 | 4 | from matplotlib import pyplot as plt 5 | import logging 6 | 7 | import watershed_workflow 8 | import watershed_workflow.ui 9 | import watershed_workflow.sources 10 | import watershed_workflow.bin_utils 11 | 12 | def get_args(): 13 | # set up parser 14 | parser = watershed_workflow.ui.get_basic_argparse(__doc__+'\n\n'+ 15 | watershed_workflow.sources.__doc__) 16 | watershed_workflow.ui.projection(parser) 17 | watershed_workflow.ui.huc_arg(parser) 18 | watershed_workflow.ui.outmesh_args(parser) 19 | 20 | watershed_workflow.ui.simplify_options(parser) 21 | watershed_workflow.ui.triangulate_options(parser) 22 | watershed_workflow.ui.plot_options(parser) 23 | 24 | data_ui = parser.add_argument_group('Data Sources') 25 | watershed_workflow.ui.huc_source_options(data_ui) 26 | watershed_workflow.ui.hydro_source_options(data_ui) 27 | watershed_workflow.ui.dem_source_options(data_ui) 28 | 29 | # parse args, log 30 | return parser.parse_args() 31 | 32 | def mesh_hucs(args): 33 | sources = watershed_workflow.sources.get_sources(args) 34 | 35 | logging.info("") 36 | logging.info("Meshing HUC: {}".format(args.HUC)) 37 | logging.info("="*30) 38 | try: 39 | logging.info('Target projection: "{}"'.format(args.projection['init'])) 40 | except TypeError: 41 | pass 42 | 43 | # collect data 44 | crs, hucs = watershed_workflow.get_split_form_hucs(sources['HUC'], args.HUC, out_crs=args.projection) 45 | args.projection = crs 46 | 47 | # hydrography 48 | _, rivers = watershed_workflow.get_reaches(sources['hydrography'], args.HUC, None, crs, crs) 49 | rivers = watershed_workflow.simplify_and_prune(hucs, rivers, args.simplify, args.prune_reach_size, args.cut_intersections) 50 | 51 | # make 2D mesh 52 | mesh_points2, mesh_tris, _, _ = watershed_workflow.triangulate(hucs, rivers, 53 | verbosity=args.verbosity, 54 | refine_max_area=args.refine_max_area, 55 | refine_distance=args.refine_distance, 56 | refine_max_edge_length=args.refine_max_edge_length, 57 | refine_min_angle=args.refine_min_angle, 58 | enforce_delaunay=args.enforce_delaunay) 59 | 60 | # elevate to 3D 61 | dem_profile, dem = watershed_workflow.get_raster_on_shape(sources['DEM'], hucs.exterior(), crs) 62 | mesh_points3 = watershed_workflow.elevate(mesh_points2, crs, dem, dem_profile) 63 | 64 | return hucs, rivers, (mesh_points3, mesh_tris) 65 | 66 | 67 | if __name__ == '__main__': 68 | args = get_args() 69 | watershed_workflow.ui.setup_logging(args.verbosity, args.logfile) 70 | hucs, rivers, triangulation = mesh_hucs(args) 71 | fig, ax = watershed_workflow.bin_utils.plot_with_triangulation(args, hucs, rivers, triangulation) 72 | watershed_workflow.bin_utils.save(args, triangulation) 73 | logging.info("SUCESS") 74 | plt.show() 75 | 76 | -------------------------------------------------------------------------------- /watershed_workflow/test/test_00_shapely.py: -------------------------------------------------------------------------------- 1 | """A series of tests to see how shapely works.""" 2 | 3 | import numpy as np 4 | import shapely.geometry 5 | import watershed_workflow.utils 6 | 7 | 8 | def test_intersection_intersects(): 9 | """Does the intersection of two shapes always intersect those shapes? SURPRISE""" 10 | shp = shapely.geometry.Polygon([(1.03425, 0.0013), (0.0035, 1.03523), (-1.09824, 0.0033), 11 | (0.0012, -1.04856)]) 12 | line = shapely.geometry.LineString([(0.1394, 0.0492), (3.1415, 1.1394)]) 13 | 14 | # check if a linestring intersecting a linestring intersects the line 15 | p = shp.boundary.intersection(line) 16 | 17 | # Lesson learned: 18 | # The intersection of two lines does not always intersect either line! 19 | # Nonrobustness of point geometry! 20 | assert (not shp.boundary.intersects(p)) 21 | assert (not line.intersects(p)) 22 | 23 | # same as 24 | assert (not shp.boundary.contains(p)) 25 | 26 | # is this obviously safe? NO! 27 | #assert (shp.contains(p)) 28 | 29 | 30 | def test_simplify(): 31 | """Does simplify ever move the end points of a linestring? YAY!""" 32 | coords = np.array([(-.001, -.001), (0, 0), (100, 0), (100.001, .001)]) 33 | 34 | def wiggle(coords): 35 | random = np.random.random((len(coords), 2)) 36 | random = 2 * (random-.5) * .001 37 | return coords + random 38 | 39 | good = [] 40 | for i in range(100): 41 | newc = wiggle(coords) 42 | ls = shapely.geometry.LineString(newc) 43 | ls_s = ls.simplify(.01) 44 | mygood = ((len(ls_s.coords) == 2) 45 | and watershed_workflow.utils.isClose(ls_s.coords[0], ls.coords[0], 1.e-10) 46 | and watershed_workflow.utils.isClose(ls_s.coords[-1], ls.coords[-1], 1.e-10)) 47 | good.append(mygood) 48 | 49 | print("Good % = ", sum(1 for i in good if i) / 100.0) 50 | assert (all(good)) 51 | 52 | 53 | def test_snap(): 54 | """How does snap work? IT DOESN'T merge points""" 55 | coords = np.array([(-.001, -.001), (100, 0), (100.001, .001)]) 56 | l = shapely.geometry.LineString(coords) 57 | 58 | c2 = np.array([(100, -100), (0, 0), (-100, 100)]) 59 | l2 = shapely.geometry.LineString(c2) 60 | 61 | ls = shapely.ops.snap(l, l2, 1) 62 | print(list(ls.coords)) 63 | assert (len(ls.coords) is 3) 64 | assert (watershed_workflow.utils.isClose(l2.coords[1], ls.coords[0], 1.e-8)) 65 | 66 | 67 | def test_snap2(): 68 | """How does snap work? will it snap to a midpoint of the linestring? NO""" 69 | coords = np.array([(-.001, -.001), (100, 0)]) 70 | l = shapely.geometry.LineString(coords) 71 | 72 | c2 = np.array([(100, -100), (-100, 100)]) 73 | l2 = shapely.geometry.LineString(c2) 74 | 75 | ls = shapely.ops.snap(l, l2, 1) 76 | print(list(ls.coords)) 77 | assert (len(ls.coords) is 2) 78 | assert (watershed_workflow.utils.isClose(ls.coords[0], l.coords[0], 1.e-8)) 79 | 80 | 81 | def test_kdtree(): 82 | """Does kdtree replicate duplicated points? YES""" 83 | import scipy.spatial 84 | coords = np.array([(0., 0.), (0., 0.)]) 85 | kdtree = scipy.spatial.cKDTree(coords) 86 | closest = kdtree.query_ball_point(np.array([0.0000001, 0.0000001]), 1.e-5) 87 | assert (len(closest) is 2) 88 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import os 2 | from os import path 3 | from setuptools import setup, find_packages 4 | import sys 5 | import versioneer 6 | import warnings 7 | 8 | # NOTE: This file must remain Python 2 compatible for the foreseeable future, 9 | # to ensure that we error out properly for people with outdated setuptools 10 | # and/or pip. 11 | min_version = (3, 7) 12 | if sys.version_info < min_version: 13 | error = """ 14 | watershed-workflow does not support Python {0}.{1}. 15 | Python {2}.{3} and above is required. Check your Python version like so: 16 | 17 | python3 --version 18 | 19 | This may be due to an out-of-date pip. Make sure you have pip >= 9.0.1. 20 | Upgrade pip like so: 21 | 22 | pip install --upgrade pip 23 | """.format(*(sys.version_info[:2] + min_version)) 24 | sys.exit(error) 25 | 26 | here = path.abspath(path.dirname(__file__)) 27 | 28 | with open(path.join(here, 'README.md'), encoding='utf-8') as readme_file: 29 | readme = readme_file.read() 30 | 31 | 32 | # NOTE: the requirements.txt listed here is VERY incomplete. This is 33 | # intentional -- most of the pip-based GIS packages don't correctly 34 | # deal with dependencies on GIS libraries. Instead, the majority of 35 | # packages here MUST be installed via Anaconda or done manually. 36 | # However, there are a few requirements that cannot be provided via 37 | # conda, but CAN be provided via pip, so we get those here... 38 | with open(path.join(here, 'requirements.txt')) as requirements_file: 39 | # Parse requirements.txt, ignoring any commented-out lines. 40 | requirements = [line for line in requirements_file.read().splitlines() 41 | if not line.startswith('#')] 42 | 43 | 44 | setup( 45 | name='watershed-workflow', 46 | version=versioneer.get_version(), 47 | cmdclass=versioneer.get_cmdclass(), 48 | description="Workflow tool that synthesizes datasets for use in integrated hydrologic models.", 49 | long_description=readme, 50 | author="Ethan Coon", 51 | author_email='etcoon@gmail.com', 52 | url='https://github.com/environmental-modeling-workflow/watershed-workflow', 53 | python_requires='>={}'.format('.'.join(str(n) for n in min_version)), 54 | packages=find_packages(exclude=['docs', 'tests']), 55 | entry_points={ 56 | 'console_scripts': [ 57 | # 'command = some.module:some_function', 58 | ], 59 | }, 60 | include_package_data=True, 61 | package_data={ 62 | 'watershed_workflow': [ 63 | # When adding files here, remember to update MANIFEST.in as well, 64 | # or else they will not be included in the distribution on PyPI! 65 | # 'path/to/data_file', 66 | ] 67 | }, 68 | install_requires=requirements, 69 | license="BSD (3-clause)", 70 | classifiers=[ 71 | 'Development Status :: 5 - Production/Stable', 72 | 'Natural Language :: English', 73 | 'Programming Language :: Python :: 3', 74 | ], 75 | ) 76 | 77 | # copy over the rc file as a template 78 | try: 79 | rcfile = path.join(os.environ['HOME'], '.watershed_worklowrc') 80 | if not path.exists(rcfile): 81 | os.copyfile(path.join(here, 'watershed_workflowrc'), rcfile) 82 | except: 83 | warnings.warn('Warning: cannot figure out where to put .watershed_workflowrc. Manually copy this to your home directory.') 84 | 85 | -------------------------------------------------------------------------------- /docker/CI-Env-FromFile.Dockerfile: -------------------------------------------------------------------------------- 1 | # This is the same as CI-Env but it pulls from an old 2 | # environment-CI-Linux.yml file instead of letting conda try to 3 | # re-resolve the environment. This is really useful for when that 4 | # process is broken... 5 | 6 | # Does everything through running tests... 7 | # 8 | # Stage 1 -- setup base CI environment 9 | # 10 | FROM condaforge/mambaforge:4.12.0-0 AS ww_env_base_ci 11 | LABEL Description="Base env for CI of Watershed Workflow" 12 | 13 | ARG env_name=watershed_workflow_CI 14 | ENV CONDA_BIN=mamba 15 | 16 | WORKDIR /ww/tmp 17 | COPY environments/create_envs.py /ww/tmp/create_envs.py 18 | RUN mkdir environments 19 | COPY environments/environment-CI-Linux.yml /ww/tmp/environments/environment-CI-Linux.yml 20 | RUN ${CONDA_BIN} env create -f /ww/tmp/environments/environment-CI-Linux.yml 21 | 22 | RUN --mount=type=cache,target=/opt/conda/pkgs \ 23 | /opt/conda/bin/python create_envs.py --manager=${CONDA_BIN} --without-ww-env \ 24 | --with-tools-env=watershed_workflow_tools Linux 25 | 26 | # 27 | # Stage 2 -- add in the pip 28 | # 29 | FROM ww_env_base_ci AS ww_env_pip_ci 30 | 31 | WORKDIR /ww/tmp 32 | COPY requirements.txt /ww/tmp/requirements.txt 33 | RUN ${CONDA_BIN} run -n ${env_name} python -m pip install -r requirements.txt 34 | 35 | 36 | # 37 | # Stage 3 -- add in Exodus 38 | # 39 | FROM ww_env_pip_ci AS ww_env_exodus_ci 40 | 41 | ENV PATH=/opt/conda/envs/watershed_workflow_tools/bin:${PATH} 42 | ENV SEACAS_DIR="/opt/conda/envs/${env_name}" 43 | ENV CONDA_PREFIX="/opt/conda/envs/${env_name}" 44 | 45 | # get the source 46 | WORKDIR /opt/conda/envs/${env_name}/src 47 | RUN apt-get install git 48 | RUN git clone -b v2021-10-11 --depth=1 https://github.com/gsjaardema/seacas/ seacas 49 | 50 | # configure 51 | WORKDIR /ww/tmp 52 | COPY docker/configure-seacas.sh /ww/tmp/configure-seacas.sh 53 | RUN chmod +x /ww/tmp/configure-seacas.sh 54 | WORKDIR /ww/tmp/seacas-build 55 | RUN ${CONDA_BIN} run -n watershed_workflow_tools ../configure-seacas.sh 56 | RUN make -j install 57 | 58 | # exodus installs its wrappers in an invalid place for python... 59 | # -- get and save the python version 60 | RUN cp /opt/conda/envs/${env_name}/lib/exodus3.py \ 61 | /opt/conda/envs/${env_name}/lib/python3.10/site-packages/ 62 | 63 | # 64 | # Stage 4 -- move the whole thing to make simpler containers 65 | # 66 | FROM ww_env_exodus_ci AS ww_env_ci_moved 67 | 68 | # add conda-pack to the base env 69 | RUN conda install -n base -c conda-forge --yes --freeze-installed conda-pack 70 | RUN conda-pack -n ${env_name} -o /tmp/env.tar && \ 71 | mkdir /ww_env && cd /ww_env && tar xf /tmp/env.tar && \ 72 | rm /tmp/env.tar 73 | RUN /ww_env/bin/conda-unpack 74 | 75 | 76 | # 77 | # Stage 5 -- copy over just what we need for CI 78 | # 79 | FROM ubuntu:20.04 AS ww_env_ci 80 | COPY --from=ww_env_ci_moved /ww_env /ww_env 81 | ENV PATH="/ww_env/bin:${PATH}" 82 | 83 | # # 84 | # # Stage 6 -- run tests! 85 | # # 86 | # # Note, this is in CI.Dockerfile as well 87 | # # 88 | # FROM ww_env_ci AS ww_ci 89 | 90 | # WORKDIR /ww 91 | 92 | # # copy over source code 93 | # COPY . /ww 94 | # RUN python -m pip install -e . 95 | 96 | # # create a watershed_workflowrc that will be picked up 97 | # RUN cp watershed_workflowrc .watershed_workflowrc 98 | 99 | # # run the tests 100 | # RUN python -m pytest watershed_workflow/test/ 101 | 102 | 103 | 104 | 105 | -------------------------------------------------------------------------------- /watershed_workflow/sources/manager_pelletier_dtb.py: -------------------------------------------------------------------------------- 1 | """Manager for interacting with GLHYMPS v2.0 dataset.""" 2 | import os, sys 3 | import logging 4 | import xarray as xr 5 | import shapely.geometry 6 | import numpy as np 7 | 8 | from watershed_workflow.crs import CRS 9 | 10 | from . import manager_raster 11 | from . import filenames 12 | 13 | # No API for getting GLHYMPS locally -- must download the whole thing. 14 | urls = { 15 | 'Pelletier at NASA DAAC': 16 | 'https://daac.ornl.gov/SOILS/guides/Global_Soil_Regolith_Sediment.html' 17 | } 18 | 19 | 20 | class ManagerPelletierDTB(manager_raster.ManagerRaster): 21 | """The [PelletierDTB]_ global soil regolith sediment map provides global values of 22 | depth to bedrock at a 1km spatial resolution. 23 | 24 | .. note:: Pelletier DTB is served through ORNL's DAAC, does not 25 | have an API, and is a large (~1GB) download. Download the file 26 | from the below citation DOI and unzip the file into: 27 | 28 | /soil_structure/PelletierDTB/ 29 | 30 | which should yield a set of tif files, 31 | 32 | Global_Soil_Regolith_Sediment_1304/data/\\*.tif 33 | 34 | .. [PelletierDTB] Pelletier, J.D., P.D. Broxton, P. Hazenberg, 35 | X. Zeng, P.A. Troch, G. Niu, Z.C. Williams, M.A. Brunke, and 36 | D. Gochis. 2016. Global 1-km Gridded Thickness of Soil, 37 | Regolith, and Sedimentary Deposit Layers. ORNL DAAC, Oak Ridge, 38 | Tennessee, USA. http://dx.doi.org/10.3334/ORNLDAAC/1304 39 | 40 | """ 41 | def __init__(self, filename=None): 42 | if filename is None: 43 | # Use default file location via Names system 44 | self.names = filenames.Names( 45 | 'Pelletier DTB', 46 | os.path.join('soil_structure', 'PelletierDTB', 'Global_Soil_Regolith_Sediment_1304', 47 | 'data'), '', 'average_soil_and_sedimentary-deposit_thickness.tif') 48 | filename = self.names.file_name() 49 | else: 50 | # Use provided filename directly 51 | self.names = None 52 | 53 | # Initialize ManagerRaster with the resolved filename 54 | # ManagerRaster will set name and source attributes appropriately 55 | super(ManagerPelletierDTB, self).__init__(filename, None, None, None, None) 56 | 57 | 58 | def _download(self, force : bool = False): 59 | """Validate the files exist, returning the filename.""" 60 | filename = self.names.file_name() 61 | logging.info(' from file: {}'.format(filename)) 62 | if not os.path.exists(filename): 63 | logging.error(f'PelletierDTB download file {filename} not found.') 64 | logging.error('See download instructions below\n\n') 65 | logging.error(self.__doc__) 66 | raise RuntimeError(f'PelletierDTB download file {filename} not found.') 67 | return filename 68 | 69 | 70 | def _fetchDataset(self, request : manager_raster.ManagerRaster.Request) -> xr.Dataset: 71 | """Fetch the data.""" 72 | dset = super(ManagerPelletierDTB, self)._fetchDataset(request) 73 | 74 | # DTB in pelletier is an int with -1 indicating nodata 75 | dset['band_1'] = dset['band_1'].astype("float32") 76 | dset['band_1'].encoding["_FillValue"] = np.nan 77 | dset.encoding["_FillValue"] = np.nan 78 | dset['band_1'].where(dset['band_1'] < 0) 79 | return dset 80 | -------------------------------------------------------------------------------- /watershed_workflow/test/source_fixture_helpers.py: -------------------------------------------------------------------------------- 1 | """One major problem of testing is that we need some real data, but 2 | real data comes in big files (~GB) that we don't want to download, but 3 | also don't want to save to our repo. 4 | 5 | This helper works with source_fixtures.FileManagerMockNHDPlusSave to 6 | save all needed HUCs to a couple of files, which can be put into the 7 | repo, and are a much smaller file dataset. 8 | 9 | To update the saved work files: 10 | 11 | 1. In source_fixtures.py:sources, replace FileManagerMockNHDPlus with FileManagerMockNHDPlusSave. 12 | 2. Run the tests, e.g. pytest watershed_workflow/test/ 13 | 3. Run this script, e.g. python ./source_fixture_helpers.py within the test directory! 14 | 4. Change source_fixtures.py:sources back to FileManagerMockNHDPlus. 15 | 16 | Note, you probably need to also commit the updated files to the repo! 17 | """ 18 | 19 | import fiona 20 | import pickle 21 | import watershed_workflow.io 22 | import watershed_workflow.sources 23 | import watershed_workflow.crs 24 | import watershed_workflow.utils 25 | 26 | 27 | def read_and_process_dump(pkl_dump_file): 28 | """Helper function to read, process, and save a new mock HUC file.""" 29 | 30 | # read the pkl file saved when the tests were run 31 | with open(pkl_dump_file, 'rb') as fid: 32 | d = pickle.load(fid) 33 | 34 | # collect all HUCs needed 35 | nhdp = watershed_workflow.sources.FileManagerNHDPlus() 36 | hucs = dict() 37 | hydro_reqs = list() 38 | for huc, v in d.items(): 39 | for level, _ in v.items(): 40 | if level != 'hydro': 41 | print(f'reading {huc} level {level}') 42 | profile, these = nhdp.get_hucs(huc, int(level)) 43 | name_key = f'HUC{level}' 44 | for this in these: 45 | name = this['properties'][name_key] 46 | hucs[name] = this 47 | else: 48 | hydro_reqs.append(huc) 49 | 50 | # convert to shapely 51 | for h, v in hucs.items(): 52 | hucs[h] = watershed_workflow.utils.create_shply(v) 53 | for h, v in hucs.items(): 54 | # normalize the properties 55 | v.properties = dict(HUC=h) 56 | 57 | # get the crs 58 | crs = watershed_workflow.crs.from_fiona(profile['crs']) 59 | 60 | # get the hydro, bounding as much as possible 61 | hydro = dict() 62 | for huc in hydro_reqs: 63 | bounds = hucs[huc].bounds 64 | print(f'reading {huc} hydro data') 65 | _, hydro[huc] = nhdp.get_hydro(huc, bounds, crs, include_catchments=False) 66 | 67 | # convert to shply -- and simplify, I think this should be safe! 68 | for h, v in hydro.items(): 69 | print(v[0].keys()) 70 | rivers = [watershed_workflow.utils.create_shply(r) for r in v] 71 | hydro[h] = [r.simplify(50) for r in rivers] 72 | 73 | # write the files 74 | watershed_workflow.io.write_to_shapefile('watershed_workflow/test/fixture_data/hucs.shp', 75 | list(hucs.values()), crs) 76 | for h, v in hydro.items(): 77 | watershed_workflow.io.write_to_shapefile( 78 | f'watershed_workflow/test/fixture_data/river_{h}.shp', v, crs) 79 | 80 | 81 | if __name__ == '__main__': 82 | import os 83 | if not os.path.isdir('watershed_workflow/test/fixture_data'): 84 | raise RuntimeError('Run this script from the top level directory') 85 | if not os.path.isfile('/tmp/my.pkl'): 86 | raise RuntimeError( 87 | 'Modify tests to write and run the tests -- see source_fixture_helpers.__doc__') 88 | 89 | read_and_process_dump('/tmp/my.pkl') 90 | -------------------------------------------------------------------------------- /.github/workflows/user_container.yml: -------------------------------------------------------------------------------- 1 | name: User docker image 2 | 3 | on: 4 | # Allows you to run this workflow manually from the Actions tab 5 | workflow_dispatch: 6 | 7 | jobs: 8 | build: 9 | # The type of runner that the job will run on 10 | strategy: 11 | matrix: 12 | include: 13 | - arch: amd64 14 | runner: ubuntu-latest 15 | platform: linux/amd64 16 | - arch: arm64 17 | runner: ubuntu-22.04-arm 18 | platform: linux/arm64 19 | 20 | runs-on: ${{ matrix.runner }} 21 | 22 | steps: 23 | - name: Checkout 24 | uses: actions/checkout@v4 25 | 26 | - name: Extract branch name 27 | id: extract_branch 28 | run: echo "BRANCH_NAME=$GITHUB_REF_NAME" >> $GITHUB_ENV 29 | 30 | - name: Extract short sha name 31 | id: extract_short_sha 32 | run: echo "GITHUB_SHORT_SHA=$(echo ${GITHUB_SHA::7})" >> $GITHUB_ENV 33 | 34 | - name: Extract docker tag name 35 | id: extract_tag 36 | run: echo "BRANCH_TAG=$(echo ${GITHUB_REF_NAME} | sed 's/[^[:alnum:]\.\_\-]/-/g')" >> $GITHUB_ENV 37 | 38 | - name: Set up Docker Buildx 39 | id: buildx 40 | uses: docker/setup-buildx-action@v3 41 | 42 | - name: Login to Docker Hub 43 | uses: docker/login-action@v3 44 | with: 45 | username: ${{ secrets.DOCKER_HUB_USERNAME }} 46 | password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} 47 | 48 | - name: Build and push 49 | id: docker_build 50 | uses: docker/build-push-action@v6 51 | with: 52 | context: . 53 | file: ./docker/User-Env.Dockerfile 54 | push: false 55 | load: true 56 | tags: ${{ secrets.DOCKER_HUB_USERNAME }}/watershed_workflow:${{ env.BRANCH_TAG }}-${{ matrix.arch }}-${{ env.GITHUB_SHORT_SHA }} 57 | cache-from: type=gha 58 | cache-to: type=gha,mode=max 59 | 60 | - name: Manually push to avoid creating manifest 61 | id: docker_push 62 | run: | 63 | docker push ${{ secrets.DOCKER_HUB_USERNAME }}/watershed_workflow:${{ env.BRANCH_TAG }}-${{ matrix.arch }}-${{ env.GITHUB_SHORT_SHA }} 64 | 65 | fix-manifest: 66 | runs-on: ubuntu-latest 67 | needs: build 68 | steps: 69 | - name: Check out repo 70 | uses: actions/checkout@v4 71 | 72 | - name: Extract branch name 73 | id: extract_branch 74 | run: echo "BRANCH_NAME=$GITHUB_REF_NAME" >> $GITHUB_ENV 75 | 76 | - name: Extract short sha name 77 | id: extract_short_sha 78 | run: echo "GITHUB_SHORT_SHA=$(echo ${GITHUB_SHA::7})" >> $GITHUB_ENV 79 | 80 | - name: Extract docker tag name 81 | id: extract_tag 82 | run: echo "BRANCH_TAG=$(echo ${GITHUB_REF_NAME} | sed 's/[^[:alnum:]\.\_\-]/-/g')" >> $GITHUB_ENV 83 | 84 | - name: Login to Docker Hub 85 | uses: docker/login-action@v3 86 | with: 87 | username: ${{secrets.DOCKER_HUB_USERNAME}} 88 | password: ${{secrets.DOCKER_HUB_ACCESS_TOKEN}} 89 | 90 | - name: Create and push multi-arch manifest 91 | run: | 92 | docker manifest create ${{ secrets.DOCKER_HUB_USERNAME }}/watershed_workflow:${{ env.BRANCH_TAG }} \ 93 | --amend ${{ secrets.DOCKER_HUB_USERNAME }}/watershed_workflow:${{ env.BRANCH_TAG }}-amd64-${{ env.GITHUB_SHORT_SHA }} \ 94 | --amend ${{ secrets.DOCKER_HUB_USERNAME }}/watershed_workflow:${{ env.BRANCH_TAG }}-arm64-${{ env.GITHUB_SHORT_SHA }} 95 | 96 | docker manifest push ${{ secrets.DOCKER_HUB_USERNAME }}/watershed_workflow:${{ env.BRANCH_TAG }} 97 | -------------------------------------------------------------------------------- /bin/mesh_shape.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """Downloads and meshes shapes based upon hydrography data.""" 3 | 4 | from matplotlib import pyplot as plt 5 | import logging 6 | 7 | import watershed_workflow 8 | import watershed_workflow.ui 9 | import watershed_workflow.sources 10 | import watershed_workflow.bin_utils 11 | 12 | def get_args(): 13 | # set up parser 14 | parser = watershed_workflow.ui.get_basic_argparse(__doc__+'\n\n'+ 15 | watershed_workflow.sources.__doc__) 16 | watershed_workflow.ui.projection(parser) 17 | watershed_workflow.ui.inshape_args(parser) 18 | watershed_workflow.ui.huc_hint_options(parser) 19 | watershed_workflow.ui.outmesh_args(parser) 20 | 21 | watershed_workflow.ui.simplify_options(parser) 22 | watershed_workflow.ui.triangulate_options(parser) 23 | watershed_workflow.ui.plot_options(parser) 24 | 25 | data_ui = parser.add_argument_group('Data Sources') 26 | watershed_workflow.ui.huc_source_options(data_ui) 27 | watershed_workflow.ui.hydro_source_options(data_ui) 28 | watershed_workflow.ui.dem_source_options(data_ui) 29 | 30 | # parse args, log 31 | return parser.parse_args() 32 | 33 | def mesh_shapes(args): 34 | sources = watershed_workflow.sources.get_sources(args) 35 | 36 | logging.info("") 37 | logging.info("Meshing shapes from: {}".format(args.input_file)) 38 | logging.info(" with index: {}".format(args.shape_index)) 39 | logging.info("="*30) 40 | try: 41 | logging.info('Target projection: "{}"'.format(args.projection['init'])) 42 | except TypeError: 43 | pass 44 | 45 | # collect data 46 | # -- get the shapes 47 | crs, shapes = watershed_workflow.get_split_form_shapes(args.input_file, args.shape_index, out_crs=args.projection) 48 | args.projection = crs 49 | 50 | # -- get the containing huc 51 | hucstr = watershed_workflow.find_huc(sources['HUC'], shapes.exterior(), crs, args.hint, shrink_factor=0.1) 52 | logging.info("found shapes in HUC %s"%hucstr) 53 | 54 | # -- get reaches of that huc 55 | _, reaches = watershed_workflow.get_reaches(sources['hydrography'], hucstr, shapes.exterior().bounds, crs, crs) 56 | rivers = watershed_workflow.simplify_and_prune(shapes, reaches, args.simplify, args.prune_reach_size, args.cut_intersections) 57 | 58 | # make 2D mesh 59 | mesh_points2, mesh_tris, _, _ = watershed_workflow.triangulate(shapes, rivers, 60 | verbosity=args.verbosity, 61 | refine_max_area=args.refine_max_area, 62 | refine_distance=args.refine_distance, 63 | refine_max_edge_length=args.refine_max_edge_length, 64 | refine_min_angle=args.refine_min_angle, 65 | enforce_delaunay=args.enforce_delaunay) 66 | 67 | # elevate to 3D 68 | dem_profile, dem = watershed_workflow.get_raster_on_shape(sources['DEM'], shapes.exterior(), crs) 69 | mesh_points3 = watershed_workflow.elevate(mesh_points2, crs, dem, dem_profile) 70 | 71 | return shapes, rivers, (mesh_points3, mesh_tris) 72 | 73 | 74 | if __name__ == '__main__': 75 | args = get_args() 76 | watershed_workflow.ui.setup_logging(args.verbosity, args.logfile) 77 | 78 | shapes, rivers, triangulation = mesh_shapes(args) 79 | fig, ax = watershed_workflow.bin_utils.plot_with_triangulation(args, shapes, rivers, triangulation) 80 | watershed_workflow.bin_utils.save(args, triangulation) 81 | logging.info("SUCESS") 82 | plt.show() 83 | -------------------------------------------------------------------------------- /.github/workflows/env.yml: -------------------------------------------------------------------------------- 1 | name: CI Environment to Docker Hub 2 | 3 | # Controls when the workflow will run 4 | on: 5 | # Allows you to run this workflow manually from the Actions tab 6 | workflow_dispatch: 7 | 8 | jobs: 9 | build: 10 | # The type of runner that the job will run on 11 | strategy: 12 | matrix: 13 | include: 14 | - arch: amd64 15 | runner: ubuntu-latest 16 | platform: linux/amd64 17 | - arch: arm64 18 | runner: ubuntu-22.04-arm 19 | platform: linux/arm64 20 | 21 | runs-on: ${{ matrix.runner }} 22 | 23 | steps: 24 | - name: Checkout 25 | uses: actions/checkout@v4 26 | 27 | - name: Extract branch name 28 | id: extract_branch 29 | run: echo "BRANCH_NAME=$GITHUB_REF_NAME" >> $GITHUB_ENV 30 | 31 | - name: Extract short sha name 32 | id: extract_short_sha 33 | run: echo "GITHUB_SHORT_SHA=$(echo ${GITHUB_SHA::7})" >> $GITHUB_ENV 34 | 35 | - name: Extract docker tag name 36 | id: extract_tag 37 | run: echo "BRANCH_TAG=$(echo ${GITHUB_REF_NAME} | sed 's/[^[:alnum:]\.\_\-]/-/g')" >> $GITHUB_ENV 38 | 39 | - name: Set up Docker Buildx 40 | id: buildx 41 | uses: docker/setup-buildx-action@v3 42 | 43 | - name: Login to Docker Hub 44 | uses: docker/login-action@v3 45 | with: 46 | username: ${{ secrets.DOCKER_HUB_USERNAME }} 47 | password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} 48 | 49 | - name: Build and push 50 | id: docker_build 51 | uses: docker/build-push-action@v6 52 | with: 53 | context: . 54 | file: ./docker/CI-Env.Dockerfile 55 | push: false 56 | load: true 57 | tags: ${{ secrets.DOCKER_HUB_USERNAME }}/watershed_workflow-ci_env:${{ env.BRANCH_TAG }}-${{ matrix.arch }}-${{ env.GITHUB_SHORT_SHA }} 58 | cache-from: type=gha 59 | cache-to: type=gha,mode=max 60 | 61 | - name: Manually push to avoid creating manifest 62 | id: docker_push 63 | run: | 64 | docker push ${{ secrets.DOCKER_HUB_USERNAME }}/watershed_workflow-ci_env:${{ env.BRANCH_TAG }}-${{ matrix.arch }}-${{ env.GITHUB_SHORT_SHA }} 65 | 66 | 67 | fix-manifest: 68 | runs-on: ubuntu-latest 69 | needs: build 70 | steps: 71 | - name: Check out repo 72 | uses: actions/checkout@v4 73 | 74 | - name: Extract branch name 75 | id: extract_branch 76 | run: echo "BRANCH_NAME=$GITHUB_REF_NAME" >> $GITHUB_ENV 77 | 78 | - name: Extract short sha name 79 | id: extract_short_sha 80 | run: echo "GITHUB_SHORT_SHA=$(echo ${GITHUB_SHA::7})" >> $GITHUB_ENV 81 | 82 | - name: Extract docker tag name 83 | id: extract_tag 84 | run: echo "BRANCH_TAG=$(echo ${GITHUB_REF_NAME} | sed 's/[^[:alnum:]\.\_\-]/-/g')" >> $GITHUB_ENV 85 | 86 | - name: Login to Docker Hub 87 | uses: docker/login-action@v3 88 | with: 89 | username: ${{secrets.DOCKER_HUB_USERNAME}} 90 | password: ${{secrets.DOCKER_HUB_ACCESS_TOKEN}} 91 | 92 | - name: Create and push multi-arch manifest 93 | run: | 94 | docker manifest create ${{ secrets.DOCKER_HUB_USERNAME }}/watershed_workflow-ci_env:${{ env.BRANCH_TAG }} \ 95 | --amend ${{ secrets.DOCKER_HUB_USERNAME }}/watershed_workflow-ci_env:${{ env.BRANCH_TAG }}-amd64-${{ env.GITHUB_SHORT_SHA }} \ 96 | --amend ${{ secrets.DOCKER_HUB_USERNAME }}/watershed_workflow-ci_env:${{ env.BRANCH_TAG }}-arm64-${{ env.GITHUB_SHORT_SHA }} 97 | 98 | docker manifest push ${{ secrets.DOCKER_HUB_USERNAME }}/watershed_workflow-ci_env:${{ env.BRANCH_TAG }} 99 | -------------------------------------------------------------------------------- /docs/source/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Configuration file for the Sphinx documentation builder. 4 | # 5 | # This file does only contain a selection of the most common options. For a 6 | # full list see the documentation: 7 | # http://www.sphinx-doc.org/en/master/config 8 | 9 | # -- Path setup -------------------------------------------------------------- 10 | 11 | # If extensions (or modules to document with autodoc) are in another directory, 12 | # add these directories to sys.path here. If the directory is relative to the 13 | # documentation root, use os.path.abspath to make it absolute, like shown here. 14 | # 15 | # import os 16 | # import sys 17 | # sys.path.insert(0, os.path.abspath('.')) 18 | 19 | 20 | # -- Project information ----------------------------------------------------- 21 | 22 | project = 'Watershed Workflow' 23 | copyright = '2019-202X, UT Battelle, Ethan Coon' 24 | author = 'Ethan Coon' 25 | 26 | # The short X.Y version 27 | version = 'dev' 28 | # The full version, including alpha/beta/rc tags 29 | release = 'dev' 30 | 31 | 32 | # -- General configuration --------------------------------------------------- 33 | 34 | # If your documentation needs a minimal Sphinx version, state it here. 35 | # 36 | # needs_sphinx = '1.0' 37 | 38 | # Add any Sphinx extension module names here, as strings. They can be 39 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 40 | # ones. 41 | extensions = [ 42 | 'sphinx.ext.autodoc', 43 | 'sphinx.ext.autosummary', 44 | 'sphinx.ext.coverage', 45 | 'sphinx.ext.mathjax', 46 | 'sphinx.ext.viewcode', 47 | 'sphinx.ext.githubpages', 48 | 'sphinx.ext.napoleon', 49 | 'myst_nb', 50 | 'sphinxcontrib.jquery', 51 | ] 52 | 53 | # Add any paths that contain templates here, relative to this directory. 54 | templates_path = ['_templates'] 55 | source_suffix = '.rst' 56 | master_doc = 'index' 57 | language = 'en' 58 | 59 | exclude_patterns = [] 60 | 61 | 62 | # -- Options for HTML output ------------------------------------------------- 63 | 64 | # The theme to use for HTML and HTML Help pages. See the documentation for 65 | # a list of builtin themes. 66 | # 67 | html_theme = "pydata_sphinx_theme" 68 | html_title = "Watershed Workflow" 69 | #html_favicon = "_static/images/favicon.ico" 70 | 71 | html_sidebars = { 72 | "**" : ["version", 73 | "version-switcher", 74 | "sidebar-nav-bs.html", 75 | "page-toc.html", 76 | ] 77 | } 78 | 79 | html_theme_options = { 80 | # "logo": { 81 | # "alt_text": "Watershed Workflow documentation -- Home", 82 | # "image_light": "_static/images/logo_full.png", 83 | # "image_dark": "_static/images/logo_full.png", # todo -- make a dark logo! 84 | # }, 85 | "secondary_sidebar_items": [], 86 | "switcher": { 87 | "json_url": "https://raw.githubusercontent.com/environmental-modeling-workflows/watershed-workflow/master/docs/source/_static/versions.json", 88 | "version_match": 'v2.0', 89 | }, 90 | # "navbar_start" : ["navbar-logo", ], 91 | } 92 | 93 | nb_execution_excludepatterns = ['*',] 94 | 95 | # Add any paths that contain custom static files (such as style sheets) here, 96 | # relative to this directory. They are copied after the builtin static files, 97 | # so a file named "default.css" will overwrite the builtin "default.css". 98 | html_static_path = ['_static'] 99 | 100 | 101 | html_css_files = [ 102 | 'https://cdn.datatables.net/2.0.8/css/dataTables.dataTables.css', 103 | 'styles/custom_theme.css', 104 | ] 105 | 106 | html_js_files = [ 107 | 'https://cdn.datatables.net/2.0.8/js/dataTables.js', 108 | 'main.js', 109 | ] 110 | -------------------------------------------------------------------------------- /watershed_workflow/sources/manager_wbd.py: -------------------------------------------------------------------------------- 1 | from typing import List, Optional 2 | import logging 3 | import geopandas as gpd 4 | 5 | import watershed_workflow.crs 6 | 7 | from . import standard_names as names 8 | from . import manager_hyriver 9 | 10 | class ManagerWBD(manager_hyriver.ManagerHyRiver): 11 | """Leverages pygeohydro to download WBD data.""" 12 | lowest_level = 12 13 | 14 | def __init__(self, protocol_name : str = 'WBD'): 15 | """Also valid is WaterData""" 16 | self._level : Optional[int] = None 17 | 18 | # WBD data is typically in lat/lon coordinates 19 | native_crs_in = watershed_workflow.crs.from_epsg(4269) 20 | native_resolution = 0.001 # ~100m at mid-latitudes 21 | 22 | super().__init__(protocol_name, native_crs_in, native_resolution) 23 | self.name = 'WBD' 24 | 25 | def set(self, **kwargs): 26 | if 'level' in kwargs: 27 | self.setLevel(kwargs['level']) 28 | 29 | def setLevel(self, level : int) -> None: 30 | self._level = level 31 | if self._protocol_name == 'WBD': 32 | self._layer = f'huc{level}' 33 | self._id_name = self._layer 34 | else: 35 | self._layer = f'wbd{level:02d}' 36 | self._id_name = f'huc{level}' 37 | 38 | def _getShapesByID(self, 39 | hucs : List[str]) -> gpd.GeoDataFrame: 40 | """Finds all HUs in the WBD dataset of a given level contained in a list of HUCs.""" 41 | req_levels = set(len(l) for l in hucs) 42 | if len(req_levels) != 1: 43 | raise ValueError("ManagerWBD.getShapesByID can only be called with a list of HUCs of the same level") 44 | req_level = req_levels.pop() 45 | 46 | if self._level is not None and self._level != req_level: 47 | level = self._level 48 | self.setLevel(req_level) 49 | geom_df = self._getShapesByID(hucs) 50 | self.setLevel(level) 51 | 52 | df = self.getShapesByGeometry(geom_df.union_all(), geom_df.crs, geom_df.crs) 53 | return df[df.ID.apply(lambda l : any(l.startswith(huc) for huc in hucs))] 54 | else: 55 | self.setLevel(req_level) 56 | df = super()._getShapesByID(hucs) 57 | return df 58 | 59 | def _addStandardNames(self, df: gpd.GeoDataFrame) -> gpd.GeoDataFrame: 60 | """Convert native column names to standard names. 61 | 62 | Parameters 63 | ---------- 64 | df : gpd.GeoDataFrame 65 | GeoDataFrame with native column names. 66 | 67 | Returns 68 | ------- 69 | gpd.GeoDataFrame 70 | GeoDataFrame with standard column names added. 71 | """ 72 | # Add ID column from current ID field (set by setLevel) 73 | if hasattr(self, '_id_name') and self._id_name in df.columns: 74 | df[names.ID] = df[self._id_name].astype('string') 75 | 76 | # Add WBD-specific standard name mappings 77 | if 'areasqkm' in df.columns: 78 | df[names.AREA] = df['areasqkm'] 79 | 80 | # Add HUC field if level is set 81 | if self._level is not None: 82 | huc_field = f'huc{self._level}' 83 | if huc_field in df.columns: 84 | df[names.HUC] = df[huc_field] 85 | 86 | return df 87 | 88 | def getAll(self, 89 | level : int) -> gpd.GeoDataFrame: 90 | """Download all HUCs at a given level.""" 91 | # this is a shortcut... 92 | import pygeohydro.watershed 93 | df = pygeohydro.watershed.huc_wb_full(level) 94 | df[names.HUC] = df[f'huc{level}'] 95 | df[names.AREA] = df[f'areasqkm'] 96 | return df 97 | -------------------------------------------------------------------------------- /docs/source/intro.rst: -------------------------------------------------------------------------------- 1 | ******************* 2 | Watershed Workflow 3 | ******************* 4 | 5 | .. image:: _static/images/watershed_workflow.png 6 | 7 | .. include:: ../AUTHORS.rst 8 | 9 | Watershed Workflow is a python-based, open source chain of tools for 10 | generating meshes and other data inputs for hyper-resolution 11 | hydrology, anywhere in the US. 12 | 13 | Fully distributed ydrologic models have huge data requirements, thanks 14 | to their large extent (full river basins) and often very high 15 | resolution (~1-500 meters). Furthermore, most process-rich 16 | models of integrated, distributed hydrology at this scale require 17 | meshes that understand both surface land cover and subsurface 18 | structure. Typical data needs for simulations such as these include: 19 | 20 | * Watershed delineation (what is your domain?) 21 | * Hydrography data (river network geometry, hydrographs for model evaluation) 22 | * A digital elevation model (DEM) for surface topography 23 | * Surface land use / land cover 24 | * Subsurface soil types and properties 25 | * Meterological data, 26 | 27 | and more. 28 | 29 | This package is a python library of tools and a set of jupyter 30 | notebooks for interacting with these types of data streams using free 31 | and open (both free as in freedom and free as in free beer) python and 32 | GIS libraries and data. Critically, this package aims to provide a 33 | way for **automatically and quickly** downloading, interpreting, and 34 | processing data needed to **generate a "first" simulation on any 35 | watershed** in the United States. Some, but not all, of the data 36 | products used here are global; the tools are directly applicable to 37 | global datasets as well. 38 | 39 | To do this, this package provides tools to automate querying and 40 | downloading a wide range of **open datasets** from various data 41 | portals, including data from United States governmental agencies such 42 | as USGS, USDA, DOE, NASA, and others. These datasets are then 43 | colocated on a mesh which is generated based on a watershed 44 | delineation and a river network, and that mesh is written in one of a 45 | variety of mesh formats for use in simulation tools. 46 | 47 | 48 | Workflows via Jupyter notebooks 49 | ------------------------------------ 50 | 51 | Workflows are the composition of partially automated steps to 52 | accomplish a range of tasks. Manual intervention is most commonly 53 | needed when there are problems or inconsistencies with the datasets 54 | themselves, or corner cases in data that these authors have not yet 55 | found. Combining automated and manual steps in a single workflow is 56 | reasonably supported by Jupyter notebooks. 57 | 58 | Note that the majority of code is NOT in notebooks. Notebooks have 59 | `all sorts of issues for software development, demonstration, and 60 | reproducibility 61 | `_ 62 | but they are great for providing a template for **modifiable** 63 | tutorials. 64 | 65 | 66 | Acknowledgements, citation, etc 67 | ----------------------------------- 68 | 69 | This work was supported by multiple US Department of Energy projects, 70 | including ORNL LDRD funds, the ExaSheds project, and the IDEAS 71 | project, and has been contributed to by authors at the Oak Ridge 72 | National Laboratory, Pacific Northwest National Laboratory, and Utah 73 | State University. Use of this codebase in the academic literature 74 | should cite: 75 | 76 | * Coon, Ethan T., and Pin Shuai. "Watershed Workflow: A toolset for parameterizing data-intensive, integrated hydrologic models." Environmental Modelling & Software 157 (2022): 105502. : `https://doi.org/10.1016/j.envsoft.2022.105502 `_ 77 | 78 | Collaborators and contributions are very welcome! 79 | 80 | -------------------------------------------------------------------------------- /watershed_workflow/sources/manager_glhymps.py: -------------------------------------------------------------------------------- 1 | """Manager for interacting with GLHYMPS v2.0 dataset.""" 2 | import os, sys 3 | import logging 4 | import numpy as np 5 | import pandas, geopandas 6 | import shapely 7 | 8 | from watershed_workflow.crs import CRS 9 | 10 | from . import manager_shapefile 11 | from . import filenames 12 | 13 | # No API for getting GLHYMPS locally -- must download the whole thing. 14 | urls = { 'GLHYMPS version 2.0': 'https://doi.org/10.5683/SP2/TTJNIU'} 15 | 16 | 17 | class ManagerGLHYMPS(manager_shapefile.ManagerShapefile): 18 | """The [GLHYMPS]_ global hydrogeology map provides global values of a 19 | two-layer (unconsolidated, consolidated) structure. 20 | 21 | .. note:: GLHYMPS does not have an API, and is a large (~4GB) 22 | download. Download the file from the below citation DOI and 23 | unzip the file into: 24 | 25 | /soil_structure/GLHYMPS/ 26 | 27 | which should yield GLHYMPS.shp (amongst other files). 28 | 29 | .. [GLHYMPS] Huscroft, J.; Gleeson, T.; Hartmann, J.; Börker, J., 30 | 2018, "Compiling and mapping global permeability of the 31 | unconsolidated and consolidated Earth: GLobal HYdrogeology MaPS 32 | 2.0 (GLHYMPS 2.0). [Supporting Data]", 33 | https://doi.org/10.5683/SP2/TTJNIU, Scholars Portal Dataverse, 34 | V1 35 | 36 | """ 37 | def __init__(self, filename=None): 38 | if filename is None: 39 | self.name = 'GLHYMPS version 2.0' 40 | self.names = filenames.Names( 41 | self.name, os.path.join('soil_structure', 'GLHYMPS'), '', 'GLHYMPS.shp') 42 | super(ManagerGLHYMPS, self).__init__(self.names.file_name(), id_name='OBJECTID_1') 43 | else: 44 | self.name = filename 45 | self.names = None 46 | super(ManagerGLHYMPS, self).__init__(self.name, id_name='OBJECTID_1') 47 | 48 | 49 | def _download(self, force : bool = False): 50 | """Download the files, returning downloaded filename.""" 51 | # check directory structure 52 | if self.names is None: 53 | return self.name 54 | filename = self.names.file_name() 55 | logging.info(' from file: {}'.format(filename)) 56 | if not os.path.exists(filename): 57 | logging.error(f'GLHYMPS download file {filename} not found.') 58 | logging.error('See download instructions below\n\n') 59 | logging.error(self.__doc__) 60 | raise RuntimeError(f'GLHYMPS download file {filename} not found.') 61 | return filename 62 | 63 | 64 | def _getShapesByGeometry(self, geometry_gdf: geopandas.GeoDataFrame) -> geopandas.GeoDataFrame: 65 | """Fetch shapes for the given geometry, ensuring file exists first. 66 | 67 | Parameters 68 | ---------- 69 | geometry_gdf : geopandas.GeoDataFrame 70 | GeoDataFrame with geometries in native_crs_in to search for shapes. 71 | 72 | Returns 73 | ------- 74 | geopandas.GeoDataFrame 75 | Raw GeoDataFrame with native column names and CRS properly set. 76 | """ 77 | # Ensure GLHYMPS file exists before attempting to read 78 | self._download() 79 | return super()._getShapesByGeometry(geometry_gdf) 80 | 81 | 82 | def _getShapesByID(self, ids: list[str]) -> geopandas.GeoDataFrame: 83 | """Fetch shapes by ID list, ensuring file exists first. 84 | 85 | Parameters 86 | ---------- 87 | ids : list[str] 88 | List of IDs to retrieve. 89 | 90 | Returns 91 | ------- 92 | geopandas.GeoDataFrame 93 | Raw GeoDataFrame with native column names and CRS properly set. 94 | """ 95 | # Ensure GLHYMPS file exists before attempting to read 96 | self._download() 97 | return super()._getShapesByID(ids) 98 | 99 | -------------------------------------------------------------------------------- /watershed_workflow/sources/utils.py: -------------------------------------------------------------------------------- 1 | """Utilities for working with sources.""" 2 | 3 | import sys, os 4 | import logging 5 | import requests 6 | import shutil 7 | import numpy as np 8 | import shapely 9 | import math 10 | import urllib.request 11 | import attr 12 | 13 | import watershed_workflow.utils 14 | import watershed_workflow.config 15 | 16 | 17 | def getCode(fiona_or_shply_obj, level): 18 | """Gets the huc string from a HUC shape.""" 19 | try: 20 | prop = fiona_or_shply_obj.properties 21 | except AttributeError: 22 | prop = fiona_or_shply_obj['properties'] 23 | 24 | key = 'HUC{:d}'.format(level) 25 | try: 26 | return prop[key] 27 | except KeyError: 28 | return prop[key.lower()] 29 | 30 | 31 | def getVerifyOption(): 32 | """Returns the 'verify' option for requests as provided in config files.""" 33 | verify = watershed_workflow.config.rcParams['DEFAULT']['ssl_cert'] 34 | logging.debug(' cert: "%s"' % verify) 35 | if verify == "True": 36 | verify = True 37 | elif verify == "False": 38 | verify = False 39 | return verify 40 | 41 | 42 | def download(url, location, force=False, **kwargs): 43 | """Download a file from a URL to a location. If force, clobber whatever is there. 44 | 45 | Note that kwargs are supplied to the requests call. 46 | """ 47 | if os.path.isfile(location) and force: 48 | os.remove(location) 49 | 50 | if not os.path.isfile(location): 51 | logging.info('Downloading: "%s"' % url) 52 | logging.info(' to: "%s"' % location) 53 | 54 | with requests.get(url, stream=True, verify=getVerifyOption(), **kwargs) as r: 55 | r.raise_for_status() 56 | with open(location, 'wb') as f: 57 | shutil.copyfileobj(r.raw, f) 58 | 59 | return os.path.isfile(location) 60 | 61 | 62 | def downloadWithProgressBar(url, location, force=False): 63 | """Download a file from URL to location, with a progress bar. 64 | 65 | If force, clobber whatever is there. 66 | """ 67 | from tqdm.autonotebook import tqdm 68 | 69 | if os.path.isfile(location) and force: 70 | os.remove(location) 71 | 72 | if not os.path.isfile(location): 73 | logging.info('Downloading: "%s"' % url) 74 | logging.info(' to: "%s"' % location) 75 | verify = watershed_workflow.config.rcParams['DEFAULT']['ssl_cert'] 76 | logging.info(' cert: "%s"' % verify) 77 | if verify == "True": 78 | verify = True 79 | elif verify == "False": 80 | verify = False 81 | 82 | r = requests.get(url, stream=True) 83 | total = int(r.headers.get('content-length', 0)) 84 | with open(location, 'wb') as file, tqdm(desc=os.path.split(location)[-1], 85 | total=total, 86 | unit='iB', 87 | unit_scale=True, 88 | unit_divisor=1024, 89 | ) as bar: 90 | for data in r.iter_content(chunk_size=1024): 91 | size = file.write(data) 92 | bar.update(size) 93 | return os.path.isfile(location) 94 | 95 | 96 | def unzip(filename, to_location, format=None): 97 | """Unzip the corresponding, assumed to exist, zipped DEM into the DEM directory.""" 98 | logging.info(f'Unzipping: "{filename}"') 99 | logging.info(f' to: "{to_location}"') 100 | 101 | import shutil 102 | shutil.unpack_archive(filename, to_location, format) 103 | return to_location 104 | 105 | 106 | def move(filename, to_location): 107 | """Move a file to a folder.""" 108 | logging.info('Moving: "%s"' % filename) 109 | logging.info(' to: "%s"' % to_location) 110 | shutil.move(filename, to_location) 111 | -------------------------------------------------------------------------------- /docker/User-Env.Dockerfile: -------------------------------------------------------------------------------- 1 | # 2 | # Sets up baseline watershed workflow container for a user/jupyterlab 3 | # 4 | 5 | # 6 | # Stage 1 -- setup base CI environment 7 | # 8 | FROM quay.io/jupyter/minimal-notebook AS ww_env_base_user 9 | LABEL Description="Base env for CI of Watershed Workflow" 10 | 11 | ARG env_name=watershed_workflow 12 | ENV CONDA_BIN=mamba 13 | 14 | # Fix mamba cache permissions 15 | USER root 16 | RUN mkdir -p /home/jovyan/.cache/mamba && \ 17 | chown -R jovyan:users /home/jovyan/.cache 18 | USER jovyan 19 | 20 | WORKDIR ${HOME}/tmp 21 | RUN mkdir ${HOME}/tmp/environments 22 | COPY environments/create_envs.py environments/create_envs.py 23 | 24 | # compilers 25 | USER root 26 | RUN apt-get update --yes && \ 27 | apt-get install --yes --no-install-recommends gcc gfortran g++ make cmake ca-certificates && \ 28 | apt-get clean && \ 29 | rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \ 30 | apt-get autoremove -y 31 | USER jovyan 32 | 33 | # Create the base environment 34 | RUN --mount=type=cache,target=/opt/conda/pkgs \ 35 | /opt/conda/bin/python environments/create_envs.py --OS=Linux --manager=${CONDA_BIN} \ 36 | --env-type=STANDARD --use-local ${env_name} 37 | 38 | # install the kernel on base's jupyterlab 39 | USER root 40 | RUN conda run -n ${env_name} python -m ipykernel install \ 41 | --name watershed_workflow --display-name "Python3 (watershed_workflow)" 42 | USER jovyan 43 | 44 | # 45 | # Stage 2 -- add in the pip 46 | # 47 | FROM ww_env_base_user AS ww_env_pip_user 48 | 49 | WORKDIR ${HOME}/tmp 50 | COPY requirements.txt ${HOME}/tmp/requirements.txt 51 | RUN ${CONDA_BIN} run -n ${env_name} python -m pip install -r requirements.txt 52 | 53 | RUN ${CONDA_BIN} run -n ${env_name} python -c 'import geopandas; import meshpy; meshpy.__file__' 54 | 55 | # 56 | # Stage 3 -- add in Exodus 57 | # 58 | FROM ww_env_pip_user AS ww_env_user 59 | 60 | ENV SEACAS_DIR="/opt/conda/envs/${env_name}" 61 | ENV CONDA_ENV_PREFIX="/opt/conda/envs/${env_name}" 62 | 63 | # get the source 64 | WORKDIR /opt/conda/envs/${env_name}/src 65 | COPY environments/exodus_py.patch /opt/conda/envs/${env_name}/src/exodus_py.patch 66 | RUN git clone -b v2021-10-11 --depth=1 https://github.com/gsjaardema/seacas/ seacas 67 | 68 | WORKDIR /opt/conda/envs/${env_name}/src/seacas 69 | RUN git apply ../exodus_py.patch 70 | RUN sed -i '/const int NC_SZIP_NN =/d' packages/seacas/libraries/exodus/src/ex_utils.c 71 | 72 | # configure 73 | ENV COMPILERS=/usr 74 | 75 | WORKDIR ${HOME}/tmp 76 | COPY --chown=jovyan:jovyan docker/configure-seacas.sh ${HOME}/tmp/configure-seacas.sh 77 | RUN chmod +x ${HOME}/tmp/configure-seacas.sh 78 | WORKDIR ${HOME}/tmp/seacas-build 79 | RUN ../configure-seacas.sh 80 | RUN make -j4 install 81 | 82 | # exodus installs its wrappers in an invalid place for python... 83 | # -- get and save the python version 84 | RUN SITE_PACKAGES=$(conda run -n ${env_name} python -c "import site; print(site.getsitepackages()[0])") && \ 85 | cp /opt/conda/envs/${env_name}/lib/exodus3.py ${SITE_PACKAGES} 86 | 87 | RUN ${CONDA_BIN} run -n ${env_name} python -c "import exodus3; print(exodus3.__file__)" 88 | 89 | # clean up 90 | RUN rm -rf ${HOME}/tmp 91 | 92 | # unclear where this comes from, must be in the jupyter/minimal-notebook? 93 | RUN rm -rf ${HOME}/work 94 | 95 | # 96 | # Stage 6 -- copy over source and run tests 97 | # 98 | FROM ww_env_user AS ww_user 99 | 100 | 101 | WORKDIR ${HOME}/watershed_workflow 102 | 103 | # copy over source code 104 | COPY --chown=jovyan:jovyan . ${HOME}/watershed_workflow 105 | RUN ${CONDA_BIN} run -n watershed_workflow python -m pip install -e . 106 | 107 | # change the default port to something not used by ATS container 108 | ENV NOTEBOOK_ARGS="--NotebookApp.port=9999" 109 | 110 | # Set up the workspace. 111 | # 112 | # create a watershed_workflowrc that will be picked up 113 | RUN cp watershed_workflowrc ${HOME}/.watershed_workflowrc 114 | 115 | # create a directory for data -- NOTE, the user should mount a 116 | # persistent volume at this location! 117 | RUN mkdir ${HOME}/data 118 | 119 | # create a working directory -- NOTE, the user should mount a 120 | # persistent volume at this location! 121 | RUN mkdir ${HOME}/workdir 122 | WORKDIR ${HOME}/workdir 123 | 124 | 125 | -------------------------------------------------------------------------------- /watershed_workflow/sources/test/test_manager_basin3d.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import os 3 | import numpy as np 4 | import shapely.geometry 5 | import geopandas as gpd 6 | 7 | import watershed_workflow.config 8 | import watershed_workflow.crs 9 | import watershed_workflow.sources.standard_names as names 10 | 11 | 12 | pytest.skip("Skipping all Basin-3D tests -- Basin-3D is not in the default environment.", allow_module_level=True) 13 | 14 | import watershed_workflow.sources.manager_basin3d as manager_basin3d 15 | 16 | 17 | @pytest.fixture 18 | def basin3d_mgr(): 19 | """Create Basin3D manager with USGS plugin.""" 20 | return manager_basin3d.ManagerBasin3D() 21 | 22 | 23 | @pytest.fixture 24 | def basin3d_mgr_multi(): 25 | """Create Basin3D manager with multiple feature types.""" 26 | return manager_basin3d.ManagerBasin3D(feature_types=['point', 'site']) 27 | 28 | 29 | def test_constructor(basin3d_mgr): 30 | """Test basic constructor and properties.""" 31 | assert basin3d_mgr.name == 'Basin3D-usgs' 32 | assert 'Basin3D with plugins: usgs' in basin3d_mgr.source 33 | assert basin3d_mgr.native_crs_in == watershed_workflow.crs.from_epsg(4326) 34 | assert basin3d_mgr.native_id_field == 'id' 35 | assert basin3d_mgr.feature_types == ['point'] 36 | assert basin3d_mgr.data_sources == ['usgs'] 37 | 38 | 39 | def test_constructor_multi_features(basin3d_mgr_multi): 40 | """Test constructor with multiple feature types.""" 41 | assert basin3d_mgr_multi.feature_types == ['point', 'site'] 42 | assert basin3d_mgr_multi.data_sources == ['usgs'] 43 | 44 | 45 | def test_constructor_invalid_feature_type(): 46 | """Test constructor with invalid feature type.""" 47 | with pytest.raises(ValueError, match="Invalid feature_type"): 48 | manager_basin3d.ManagerBasin3D(feature_types=['INVALID_TYPE']) 49 | 50 | 51 | def test_constructor_invalid_data_source(): 52 | """Test constructor with no valid plugins.""" 53 | with pytest.raises(ValueError, match="No valid plugins found"): 54 | manager_basin3d.ManagerBasin3D(data_sources=['invalid_source']) 55 | 56 | 57 | def test_plugin_registration(basin3d_mgr): 58 | """Test that Basin3D synthesizer was registered correctly.""" 59 | assert hasattr(basin3d_mgr, 'synthesizer') 60 | assert basin3d_mgr.synthesizer is not None 61 | assert len(basin3d_mgr.plugin_classes) == 1 62 | assert 'basin3d.plugins.usgs.USGSDataSourcePlugin' in basin3d_mgr.plugin_classes 63 | 64 | 65 | def test_getShapesByGeometry(basin3d_mgr): 66 | """Test geometry-based query returns proper GeoDataFrame.""" 67 | 68 | polygon = shapely.geometry.box(-90.6, 34.4, -90.5, 34.6) 69 | result = basin3d_mgr.getShapesByGeometry(polygon, watershed_workflow.crs.latlon_crs) 70 | 71 | # Check result structure 72 | assert isinstance(result, gpd.GeoDataFrame) 73 | assert result.crs == basin3d_mgr.native_crs_in 74 | 75 | # Check expected columns 76 | expected_cols = ['id', 'name', 'feature_type', 'description', 'data_source', 'elevation', 'geometry'] 77 | for col in expected_cols: 78 | assert col in result.columns 79 | 80 | # Check standard names were added 81 | if names.ID in result.columns: 82 | assert names.ID in result.columns 83 | if names.NAME in result.columns: 84 | assert names.NAME in result.columns 85 | 86 | assert len(result) == 2 87 | 88 | 89 | def test_getShapesByID(basin3d_mgr): 90 | """Test geometry-based query returns proper GeoDataFrame.""" 91 | 92 | result = basin3d_mgr.getShapesByID(['USGS-13010000', 'USGS-385508107021201',]) 93 | 94 | # Check result structure 95 | assert isinstance(result, gpd.GeoDataFrame) 96 | assert result.crs == basin3d_mgr.native_crs_in 97 | 98 | # Check expected columns 99 | expected_cols = ['id', 'name', 'feature_type', 'description', 'data_source', 'elevation', 'geometry'] 100 | for col in expected_cols: 101 | assert col in result.columns 102 | 103 | # Check standard names were added 104 | if names.ID in result.columns: 105 | assert names.ID in result.columns 106 | if names.NAME in result.columns: 107 | assert names.NAME in result.columns 108 | 109 | print(result) 110 | assert len(result) == 2 111 | 112 | 113 | -------------------------------------------------------------------------------- /bin/run_ww_lab.py: -------------------------------------------------------------------------------- 1 | """Starts Watershed Workflow Jupyterlab in Docker container""" 2 | 3 | epilog = """ 4 | A confusing aspect of running Watershed Workflow in a Docker container 5 | is that paths in the host are not the same as paths in the container, 6 | and files must be explicitly shared with the container. 7 | 8 | Therefore, when run in the container, `~/.watershed_workflowrc` will not 9 | be the same as that in the host system. To avoid issues like this, we 10 | always generate an rc file in the working directory using the 11 | parameters supplied in the host system files. This generated config 12 | should not be modified -- instead create/modify 13 | `~/.watershed_workflowrc` or `WORKDIR/watershed_workflowrc`. 14 | 15 | """ 16 | 17 | import os 18 | import subprocess 19 | import configparser 20 | 21 | 22 | def set_up_docker_config(workdir, data_library): 23 | # read config, not including the dockerfile's config 24 | rc = configparser.ConfigParser() 25 | rc.read([os.path.join(os.path.expanduser('~'), '.watershed_workflowrc'), 26 | os.path.join(workdir, '.watershed_workflowrc'), 27 | os.path.join(workdir, 'watershed_workflowrc')]) 28 | 29 | if data_library is None: 30 | if 'data_directory' in rc['DEFAULT']: 31 | data_library = rc['DEFAULT']['data_directory'] 32 | else: 33 | data_library = os.path.join(workdir, 'data_library') 34 | if not os.path.isdir(data_library): 35 | os.mkdir(data_library) 36 | 37 | # set the config's data library to the location, in the container, 38 | # where we will mount the data_library volume 39 | rc['DEFAULT']['data_directory'] = '/home/jovyan/data' 40 | with open(os.path.join(workdir, '.docker_watershed_workflowrc'), 'w') as fid: 41 | rc.write(fid) 42 | 43 | return data_library 44 | 45 | def start_docker(data_library, workdir, port, pull='missing', tag='master', for_ats=False): 46 | if for_ats: 47 | repo = 'watershed_workflow-ats' 48 | else: 49 | repo = 'watershed_workflow' 50 | 51 | abspath_data_library = os.path.abspath(data_library) 52 | if not os.path.isdir(abspath_data_library): 53 | raise FileNotFoundError(f'Data library directory {abspath_data_library} does not exist.') 54 | 55 | abspath_workdir = os.path.abspath(workdir) 56 | if not os.path.isdir(abspath_workdir): 57 | raise FileNotFoundError(f'Working directory {abspath_workdir} does not exist.') 58 | 59 | cmd = ['docker', 'run', '-it', '--rm', # remove the image after completion 60 | '-p', f'{port}:9999', # port here is the host port 61 | '--pull', pull, # options for whether to update 62 | '-e', 'JUPYTER_ENABLE_LAB=yes', # use jupyterlab, not notebook 63 | '-v', f'{abspath_data_library}:/home/jovyan/data:delegated', # volume for data 64 | '-v', f'{abspath_workdir}:/home/jovyan/workdir:delegated', # volume for output 65 | f'ecoon/{repo}:{tag}' 66 | ] 67 | print(f'Running: {" ".join(cmd)}') 68 | subprocess.run(cmd, check=True) 69 | 70 | 71 | if __name__ == '__main__': 72 | import argparse 73 | parser = argparse.ArgumentParser(description=__doc__, 74 | epilog=epilog) 75 | parser.add_argument('--ats', action='store_true', help='Use docker image with ATS and ats_input_spec included.') 76 | parser.add_argument('--rc', type=str, default=None, help='Configuration file, see below.') 77 | parser.add_argument('--data-library', type=str, default=None, help='Location of data library.') 78 | parser.add_argument('-p', '--port', type=int, default='9999', help='Port to open for jupyterlab.') 79 | parser.add_argument('--pull', action='store_true', help='Pull latest changes from dockerhub') 80 | parser.add_argument('-t', '--tag', type=str, default='master', 81 | help='Tag of the watershed_workflow container.') 82 | parser.add_argument('WORKDIR', type=str, help='Where to store output files.') 83 | args = parser.parse_args() 84 | 85 | if not os.path.isdir(args.WORKDIR): 86 | raise FileNotFoundError(f'Invalid working directory: {args.WORKDIR}') 87 | 88 | data_library = set_up_docker_config(args.WORKDIR, args.data_library) 89 | 90 | if args.pull: 91 | pull = 'always' 92 | else: 93 | pull = 'missing' 94 | start_docker(data_library, args.WORKDIR, args.port, pull=pull, tag=args.tag, for_ats=args.ats) 95 | -------------------------------------------------------------------------------- /watershed_workflow/sources/manager_hyriver.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import abc 4 | from typing import List, Optional, Any 5 | from shapely.geometry.base import BaseGeometry 6 | import geopandas as gpd 7 | 8 | from watershed_workflow.crs import CRS 9 | from watershed_workflow.sources.manager_shapes import ManagerShapes 10 | import watershed_workflow.sources.standard_names as names 11 | 12 | 13 | class ManagerHyRiver(ManagerShapes): 14 | """A generic base class for working with HyRiver""" 15 | 16 | def __init__(self, 17 | protocol_name: str, 18 | native_crs_in: CRS, 19 | native_resolution: float, 20 | layer: str = '', 21 | id_name: Optional[str] = None): 22 | """Initialize HyRiver manager. 23 | 24 | Parameters 25 | ---------- 26 | protocol_name : str 27 | HyRiver protocol name (NHD, NHDPlusHR, WBD, WaterData). 28 | native_crs_in : CRS 29 | Expected CRS of incoming geometry for API queries. 30 | native_resolution : float 31 | Native resolution in native_crs_in units. 32 | layer : str, optional 33 | Layer name for the protocol. 34 | id_name : str, optional 35 | Name of the ID field, defaults to layer name. 36 | """ 37 | self._layer = layer 38 | if id_name is None: 39 | id_name = layer 40 | self._id_name = id_name 41 | self._protocol_name = protocol_name 42 | self._protocol: Any = None 43 | 44 | # Set up protocol-specific API 45 | if protocol_name == 'NHD': 46 | import pynhd.pynhd 47 | self._protocol = pynhd.pynhd.NHD 48 | elif protocol_name == 'NHDPlusHR': 49 | import pynhd.pynhd 50 | self._protocol = pynhd.pynhd.NHDPlusHR 51 | elif protocol_name == 'WBD': 52 | import pygeohydro.watershed 53 | self._protocol = pygeohydro.watershed.WBD 54 | elif protocol_name == 'WaterData': 55 | import pynhd.pynhd 56 | self._protocol = pynhd.pynhd.WaterData 57 | else: 58 | raise ValueError(f'Invalid HyRiver protocol "{protocol_name}"') 59 | 60 | # Create name and source for base class 61 | name = f'HyRiver {protocol_name}: {layer}' if layer else f'HyRiver {protocol_name}' 62 | source = f'HyRiver.{protocol_name}' 63 | 64 | # Initialize base class 65 | super().__init__(name, source, native_crs_in, native_resolution, id_name) 66 | 67 | def _getShapes(self): 68 | """Fetch all shapes in a dataset. 69 | 70 | Returns 71 | ------- 72 | gpd.GeoDataFrame 73 | Raw GeoDataFrame with native column names and CRS properly set. 74 | """ 75 | raise NotImplementedError(f'Manager source {self.source} does not support getting all shapes.') 76 | 77 | 78 | def _getShapesByGeometry(self, geometry_gdf: gpd.GeoDataFrame) -> gpd.GeoDataFrame: 79 | """Fetch shapes for the given geometry using HyRiver API. 80 | 81 | Parameters 82 | ---------- 83 | geometry_gdf : gpd.GeoDataFrame 84 | GeoDataFrame with geometries in native_crs_in to search for shapes. 85 | 86 | Returns 87 | ------- 88 | gpd.GeoDataFrame 89 | Raw GeoDataFrame with native column names and CRS properly set. 90 | """ 91 | # HyRiver APIs take the union of geometries in the GeoDataFrame 92 | union_geometry = geometry_gdf.union_all() 93 | df = self._protocol(self._layer).bygeom(union_geometry, self.native_crs_in) 94 | return df 95 | 96 | def _getShapesByID(self, ids: List[str]) -> gpd.GeoDataFrame: 97 | """Fetch shapes by ID list using HyRiver API. 98 | 99 | Parameters 100 | ---------- 101 | ids : List[str] 102 | List of IDs to retrieve. 103 | 104 | Returns 105 | ------- 106 | gpd.GeoDataFrame 107 | Raw GeoDataFrame with native column names and CRS properly set. 108 | """ 109 | protocol = self._protocol(self._layer) 110 | if hasattr(protocol, 'byid'): 111 | df = protocol.byid(self._id_name, ids) 112 | else: 113 | df = protocol.byids(self._id_name, ids) 114 | return df 115 | -------------------------------------------------------------------------------- /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | name: CI to Docker Hub 2 | 3 | on: 4 | # Triggers the workflow on push or pull request events but only for the master branch 5 | push: 6 | branches: [ master ] 7 | pull_request: 8 | branches: [ master ] 9 | 10 | # Allows you to run this workflow manually from the Actions tab 11 | workflow_dispatch: 12 | 13 | jobs: 14 | build: 15 | # The type of runner that the job will run on 16 | strategy: 17 | matrix: 18 | include: 19 | - arch: amd64 20 | runner: ubuntu-latest 21 | platform: linux/amd64 22 | - arch: arm64 23 | runner: ubuntu-22.04-arm 24 | platform: linux/arm64 25 | 26 | runs-on: ${{ matrix.runner }} 27 | 28 | steps: 29 | - name: Checkout 30 | uses: actions/checkout@v4 31 | 32 | - name: Extract branch name 33 | id: extract_branch 34 | run: echo "BRANCH_NAME=$GITHUB_REF_NAME" >> $GITHUB_ENV 35 | 36 | - name: Extract short sha name 37 | id: extract_short_sha 38 | run: echo "GITHUB_SHORT_SHA=$(echo ${GITHUB_SHA::7})" >> $GITHUB_ENV 39 | 40 | - name: Extract docker tag name 41 | id: extract_tag 42 | run: echo "BRANCH_TAG=$(echo ${GITHUB_REF_NAME} | sed 's/[^[:alnum:]\.\_\-]/-/g')" >> $GITHUB_ENV 43 | 44 | - name: Set up Docker Buildx 45 | id: buildx 46 | uses: docker/setup-buildx-action@v3 47 | 48 | - name: Check to see if CI_ENV_DOCKER_TAG exists for current branch 49 | run: | 50 | if docker manifest inspect ${{ secrets.DOCKER_HUB_USERNAME }}/watershed_workflow-ci_env:${{ env.BRANCH_TAG }} > /dev/null 2>&1; then 51 | echo "CI_ENV_BRANCH=${{ env.BRANCH_TAG }}" >> $GITHUB_ENV 52 | else 53 | echo "CI_ENV_BRANCH=master" >> $GITHUB_ENV 54 | fi 55 | 56 | - name: check environment variable 57 | run: | 58 | # verify that logic above has worked 59 | echo ${{ env.CI_ENV_BRANCH }} 60 | 61 | - name: Login to Docker Hub 62 | uses: docker/login-action@v3 63 | with: 64 | username: ${{ secrets.DOCKER_HUB_USERNAME }} 65 | password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} 66 | 67 | - name: Build and push 68 | id: docker_build 69 | uses: docker/build-push-action@v6 70 | with: 71 | context: . 72 | file: ./docker/CI.Dockerfile 73 | push: false 74 | load: true 75 | tags: ${{ secrets.DOCKER_HUB_USERNAME }}/watershed_workflow-ci:${{ env.BRANCH_TAG }}-${{ matrix.arch }}-${{ env.GITHUB_SHORT_SHA }} 76 | build-args: CI_ENV_DOCKER_TAG=${{ env.CI_ENV_BRANCH }} 77 | cache-from: type=gha 78 | cache-to: type=gha,mode=max 79 | 80 | - name: Manually push to avoid creating manifest 81 | id: docker_push 82 | run: | 83 | docker push ${{ secrets.DOCKER_HUB_USERNAME }}/watershed_workflow-ci:${{ env.BRANCH_TAG }}-${{ matrix.arch }}-${{ env.GITHUB_SHORT_SHA }} 84 | 85 | fix-manifest: 86 | runs-on: ubuntu-latest 87 | needs: build 88 | steps: 89 | - name: Check out repo 90 | uses: actions/checkout@v4 91 | 92 | - name: Extract branch name 93 | id: extract_branch 94 | run: echo "BRANCH_NAME=$GITHUB_REF_NAME" >> $GITHUB_ENV 95 | 96 | - name: Extract short sha name 97 | id: extract_short_sha 98 | run: echo "GITHUB_SHORT_SHA=$(echo ${GITHUB_SHA::7})" >> $GITHUB_ENV 99 | 100 | - name: Extract docker tag name 101 | id: extract_tag 102 | run: echo "BRANCH_TAG=$(echo ${GITHUB_REF_NAME} | sed 's/[^[:alnum:]\.\_\-]/-/g')" >> $GITHUB_ENV 103 | 104 | - name: Login to Docker Hub 105 | uses: docker/login-action@v3 106 | with: 107 | username: ${{secrets.DOCKER_HUB_USERNAME}} 108 | password: ${{secrets.DOCKER_HUB_ACCESS_TOKEN}} 109 | 110 | - name: Create and push multi-arch manifest 111 | run: | 112 | docker manifest create ${{ secrets.DOCKER_HUB_USERNAME }}/watershed_workflow-ci:${{ env.BRANCH_TAG }} \ 113 | --amend ${{ secrets.DOCKER_HUB_USERNAME }}/watershed_workflow-ci:${{ env.BRANCH_TAG }}-amd64-${{ env.GITHUB_SHORT_SHA }} \ 114 | --amend ${{ secrets.DOCKER_HUB_USERNAME }}/watershed_workflow-ci:${{ env.BRANCH_TAG }}-arm64-${{ env.GITHUB_SHORT_SHA }} 115 | 116 | docker manifest push ${{ secrets.DOCKER_HUB_USERNAME }}/watershed_workflow-ci:${{ env.BRANCH_TAG }} 117 | 118 | -------------------------------------------------------------------------------- /.github/workflows/ats_user_container.yml: -------------------------------------------------------------------------------- 1 | name: ATS User docker image 2 | 3 | on: 4 | # Allows you to run this workflow manually from the Actions tab 5 | workflow_dispatch: 6 | 7 | jobs: 8 | build: 9 | # The type of runner that the job will run on 10 | strategy: 11 | matrix: 12 | include: 13 | - arch: amd64 14 | runner: ubuntu-latest 15 | platform: linux/amd64 16 | - arch: arm64 17 | runner: ubuntu-22.04-arm 18 | platform: linux/arm64 19 | 20 | runs-on: ${{ matrix.runner }} 21 | 22 | steps: 23 | - name: Checkout 24 | uses: actions/checkout@v4 25 | 26 | - name: Extract branch name 27 | id: extract_branch 28 | run: echo "BRANCH_NAME=$GITHUB_REF_NAME" >> $GITHUB_ENV 29 | 30 | - name: Extract short sha name 31 | id: extract_short_sha 32 | run: echo "GITHUB_SHORT_SHA=$(echo ${GITHUB_SHA::7})" >> $GITHUB_ENV 33 | 34 | - name: Extract docker tag name 35 | id: extract_tag 36 | run: echo "BRANCH_TAG=$(echo ${GITHUB_REF_NAME} | sed 's/[^[:alnum:]\.\_\-]/-/g')" >> $GITHUB_ENV 37 | 38 | - name: Set up Docker Buildx 39 | id: buildx 40 | uses: docker/setup-buildx-action@v3 41 | 42 | - name: Check to see if USER_ENV_DOCKER_TAG exists for current branch 43 | run: | 44 | # check to see if manifest for requested image exists, echo $? returns 0 if it exists 45 | docker manifest inspect ${{secrets.DOCKER_HUB_USERNAME}}/watershed_workflow:${{ env.BRANCH_TAG }} > /dev/null 2>&1 ; 46 | # returns 0 if image exists, 1 if it is missing (so default to master if 1) 47 | if [ $? -eq 1 ]; then 48 | echo "USER_ENV_BRANCH=master" >> $GITHUB_ENV 49 | else 50 | echo "USER_ENV_BRANCH=${{ env.BRANCH_TAG }}" >> $GITHUB_ENV 51 | fi 52 | 53 | - name: check environment variable 54 | run: | 55 | # verify that logic above has worked 56 | echo ${{ env.USER_ENV_BRANCH }} 57 | 58 | - name: Login to Docker Hub 59 | uses: docker/login-action@v3 60 | with: 61 | username: ${{ secrets.DOCKER_HUB_USERNAME }} 62 | password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} 63 | 64 | - name: Build and push 65 | id: docker_build 66 | uses: docker/build-push-action@v6 67 | with: 68 | context: . 69 | file: ./docker/ATS-User-Env.Dockerfile 70 | push: false 71 | load: true 72 | tags: ${{ secrets.DOCKER_HUB_USERNAME }}/watershed_workflow-ats:${{ env.BRANCH_TAG }}-${{ matrix.arch }}-${{ env.GITHUB_SHORT_SHA }} 73 | build-args: USER_ENV_DOCKER_TAG=${{ env.USER_ENV_BRANCH }} 74 | cache-from: type=gha 75 | cache-to: type=gha,mode=max 76 | 77 | - name: Manually push to avoid creating manifest 78 | id: docker_push 79 | run: | 80 | docker push ${{ secrets.DOCKER_HUB_USERNAME }}/watershed_workflow-ats:${{ env.BRANCH_TAG }}-${{ matrix.arch }}-${{ env.GITHUB_SHORT_SHA }} 81 | 82 | fix-manifest: 83 | runs-on: ubuntu-latest 84 | needs: build 85 | steps: 86 | - name: Check out repo 87 | uses: actions/checkout@v4 88 | 89 | - name: Extract branch name 90 | id: extract_branch 91 | run: echo "BRANCH_NAME=$GITHUB_REF_NAME" >> $GITHUB_ENV 92 | 93 | - name: Extract short sha name 94 | id: extract_short_sha 95 | run: echo "GITHUB_SHORT_SHA=$(echo ${GITHUB_SHA::7})" >> $GITHUB_ENV 96 | 97 | - name: Extract docker tag name 98 | id: extract_tag 99 | run: echo "BRANCH_TAG=$(echo ${GITHUB_REF_NAME} | sed 's/[^[:alnum:]\.\_\-]/-/g')" >> $GITHUB_ENV 100 | 101 | - name: Login to Docker Hub 102 | uses: docker/login-action@v3 103 | with: 104 | username: ${{secrets.DOCKER_HUB_USERNAME}} 105 | password: ${{secrets.DOCKER_HUB_ACCESS_TOKEN}} 106 | 107 | - name: Create and push multi-arch manifest 108 | run: | 109 | docker manifest create ${{ secrets.DOCKER_HUB_USERNAME }}/watershed_workflow-ats:${{ env.BRANCH_TAG }} \ 110 | --amend ${{ secrets.DOCKER_HUB_USERNAME }}/watershed_workflow-ats:${{ env.BRANCH_TAG }}-amd64-${{ env.GITHUB_SHORT_SHA }} \ 111 | --amend ${{ secrets.DOCKER_HUB_USERNAME }}/watershed_workflow-ats:${{ env.BRANCH_TAG }}-arm64-${{ env.GITHUB_SHORT_SHA }} 112 | 113 | docker manifest push ${{ secrets.DOCKER_HUB_USERNAME }}/watershed_workflow-ats:${{ env.BRANCH_TAG }} 114 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Watershed Workflow 2 | [![Docs](https://img.shields.io/badge/docs-link-blue?style=for-the-badge)](https://environmental-modeling-workflows.github.io/watershed-workflow/build/html/index.html) 3 | [![Release](https://img.shields.io/github/v/release/environmental-modeling-workflows/watershed-workflow?display_name=release&style=for-the-badge)](https://github.com/environmental-modeling-workflows/watershed-workflow/releases/tag/watershed-workflow-1.1.0) 4 | 5 | [![Build Status](https://img.shields.io/github/actions/workflow/status/environmental-modeling-workflows/watershed-workflow/main.yml?label=tests&style=for-the-badge)](https://github.com/environmental-modeling-workflows/watershed-workflow/actions) 6 | [![Issues](https://img.shields.io/github/issues/environmental-modeling-workflows/watershed-workflow?style=for-the-badge)](https://github.com/environmental-modeling-workflows/watershed-workflow/issues) 7 | [![Issues](https://img.shields.io/github/issues-pr/environmental-modeling-workflows/watershed-workflow?style=for-the-badge)](https://github.com/environmental-modeling-workflows/watershed-workflow/pulls) 8 | 9 | ![sample image](https://environmental-modeling-workflows.github.io/watershed-workflow/build/html/_images/watershed_workflow.png "Example output of the Coweeta Hydrologic Lab watersheds across scales.") 10 | 11 | [Please prefer to see our documentation.](https://environmental-modeling-workflows.github.io/watershed-workflow/build/html/index.html) 12 | 13 | Watershed Workflow is a python-based, open source chain of tools for generating meshes and other data inputs for hyper-resolution hydrology, anywhere in the (conterminous + Alaska?) US. 14 | 15 | Hyper-resolution hydrologic models have huge data requirements, thanks to their large extent (full river basins) and very high resolution (often ~10-100 meters). Furthermore, most process-rich models of integrated, distributed hydrology at this scale require meshes that understand both surface land cover and subsurface structure. Typical data needs for simulations such as these include: 16 | 17 | * Watershed delineation (what is your domain?) 18 | * Hydrography data (river network geometry, hydrographs for model evaluation) 19 | * A digital elevation model (DEM) for surface topography 20 | * Surface land use / land cover 21 | * Subsurface soil types and properties 22 | * Meterological data, 23 | 24 | and more. 25 | 26 | This package is a python library of tools and a set of jupyter notebooks for interacting with these types of data streams using free and open (both free as in freedom and free as in free beer) python and GIS libraries and data. Critically, this package provides a way for **automatically and quickly** downloading, interpreting, and processing data needed to **generate a "first" hyper-resolution simulation on any watershed** in the conterminous United States (and most of Alaska/Hawaii/Puerto Rico). 27 | 28 | To do this, this package provides tools to automate downloading a wide range of **open data streams,** including data from United States governmental agencies, including USGS, USDA, DOE, and others. These data streams are then colocated on a mesh which is generated based on a watershed delineation and a river network, and that mesh is written in one of a variety of mesh formats for use in hyper-resolution simulation tools. 29 | 30 | Note: Hypothetically, this package works on all of Linux, Mac, and Windows. It has been tested on the first two, but not the third. 31 | 32 | ## Installation 33 | 34 | [Visit our Installation documentation.](https://environmental-modeling-workflows.github.io/watershed-workflow/stable/install.html) 35 | 36 | ## For more... 37 | 38 | * [See the documentation](https://environmental-modeling-workflows.github.io/watershed-workflow) 39 | * [See a gallery of data product images (work in progress)](https://environmental-modeling-workflows.github.io/watershed-workflow/build/html/gallery.html) 40 | 41 | ## Funding, attribution, etc 42 | 43 | This work was supported by multiple US Department of Energy projects, and was mostly developed at the Oak Ridge National Laboratory. Use of this codebase in the academic literature should cite: 44 | 45 | [Coon, E. T., & Shuai, P. (2022). Watershed Workflow: A toolset for parameterizing data-intensive, integrated hydrologic models. Environmental Modelling & Software, 157, 105502.](https://doi.org/10.1016/j.envsoft.2022.105502) 46 | 47 | The use of stream-aligned mixed-polyhedral mesh should cite: 48 | 49 | [Rathore, S. S., Coon E. T., and Painter S. L. (2024). A stream-aligned mixed polyhedral meshing strategy for integrated surface-subsurface hydrological models. Computers & Geosciences, 188, 105617.](https://doi.org/10.1016/j.cageo.2024.105617) 50 | 51 | Collaborators and contributions are very welcome! 52 | -------------------------------------------------------------------------------- /watershed_workflow/sources/test/test_manager_wbd.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | import os 4 | import numpy as np 5 | import shapely 6 | 7 | import geopandas as gpd 8 | from matplotlib import pyplot as plt 9 | 10 | from watershed_workflow.sources.manager_wbd import ManagerWBD 11 | import watershed_workflow.crs 12 | import watershed_workflow.sources.standard_names as names 13 | 14 | bounds4_ll = np.array([-76.3955534, 36.8008194, -73.9026218, 42.4624454]) 15 | bounds8_ll = np.array([-75.5722117, 41.487746, -74.5581047, 42.4624454]) 16 | 17 | def test_wbd_get() -> None: 18 | wbd = ManagerWBD() 19 | huc = wbd.getShapesByID('02040101') 20 | bounds = huc[huc.ID=='02040101'].geometry.bounds 21 | assert (np.allclose(bounds8_ll, np.array(bounds), 1.e-6)) 22 | 23 | def test_wbd_get_many() -> None: 24 | wbd = ManagerWBD() 25 | wbd.setLevel(12) 26 | huc = wbd.getShapesByID('02040101') 27 | print(huc) 28 | assert len(huc) == len(set(huc.ID)) # unique 29 | assert all(l.startswith('02040101') for l in huc.ID) # all in the HUC8 30 | assert (len(huc) == 38) # right number 31 | 32 | def test_wbd_get_geometry() -> None: 33 | wbd = ManagerWBD() 34 | wbd.setLevel(8) 35 | shp = shapely.geometry.box(*bounds8_ll) 36 | huc = wbd.getShapesByGeometry(shp, watershed_workflow.crs.latlon_crs) 37 | huc = huc.to_crs(watershed_workflow.crs.latlon_crs) 38 | huc = huc[[shp.buffer(0.001).contains(h) for h in huc.geometry]] 39 | assert len(huc) == 1 40 | 41 | 42 | 43 | def test_wbd_waterdata_get() -> None: 44 | wbd = ManagerWBD(protocol_name='WaterData') 45 | huc = wbd.getShapesByID('02040101') 46 | bounds = huc[huc.ID=='02040101'].geometry.bounds 47 | assert (np.allclose(bounds8_ll, np.array(bounds), 1.e-6)) 48 | 49 | def test_wbd_waterdata_get_many() -> None: 50 | wbd = ManagerWBD(protocol_name='WaterData') 51 | wbd.setLevel(12) 52 | huc = wbd.getShapesByID('02040101') 53 | print(huc) 54 | assert len(huc) == len(set(huc.ID)) # unique 55 | assert all(l.startswith('02040101') for l in huc.ID) # all in the HUC8 56 | assert (len(huc) == 38) # right number 57 | 58 | def test_wbd_waterdata_get_geometry() -> None: 59 | wbd = ManagerWBD(protocol_name='WaterData') 60 | wbd.setLevel(8) 61 | shp = shapely.geometry.box(*bounds8_ll) 62 | huc = wbd.getShapesByGeometry(shp, watershed_workflow.crs.latlon_crs) 63 | huc = huc.to_crs(watershed_workflow.crs.latlon_crs) 64 | huc = huc[[shp.buffer(0.001).contains(h) for h in huc.geometry]] 65 | assert len(huc) == 1 66 | 67 | 68 | def test_constructor_properties() -> None: 69 | """Test that constructor sets properties correctly""" 70 | wbd = ManagerWBD() 71 | assert wbd.name == 'WBD' # Name should be WBD regardless of protocol 72 | assert wbd.source == 'HyRiver.WBD' 73 | assert wbd.native_crs_in == watershed_workflow.crs.latlon_crs 74 | assert wbd._protocol_name == 'WBD' # Protocol name is the string 75 | assert wbd._protocol.__name__ == 'WBD' # _protocol is the class 76 | 77 | def test_constructor_waterdata_properties() -> None: 78 | """Test that constructor with WaterData protocol sets properties correctly""" 79 | wbd = ManagerWBD(protocol_name='WaterData') 80 | assert wbd.name == 'WBD' # Name should still be WBD 81 | assert wbd.source == 'HyRiver.WaterData' 82 | assert wbd._protocol_name == 'WaterData' 83 | assert wbd._protocol.__name__ == 'WaterData' 84 | 85 | def test_standard_naming_applied() -> None: 86 | """Test that standard naming is properly applied""" 87 | wbd = ManagerWBD() 88 | wbd.setLevel(8) 89 | huc = wbd.getShapesByID('02040101') 90 | 91 | # Check required standard columns 92 | assert names.ID in huc.columns 93 | assert names.HUC in huc.columns 94 | assert names.AREA in huc.columns 95 | 96 | # Check that ID values are strings (as specified in _addStandardNames) 97 | assert huc[names.ID].dtype == 'string' 98 | 99 | # Check metadata 100 | assert huc.attrs['name'] == wbd.name 101 | assert huc.attrs['source'] == wbd.source 102 | 103 | def test_geodataframe_input() -> None: 104 | """Test getShapesByGeometry with GeoDataFrame input""" 105 | wbd = ManagerWBD() 106 | wbd.setLevel(8) 107 | 108 | # Create GeoDataFrame from bounds 109 | shp = shapely.geometry.box(*bounds8_ll) 110 | gdf = gpd.GeoDataFrame([{'test': 1}], geometry=[shp], crs=watershed_workflow.crs.latlon_crs) 111 | 112 | huc = wbd.getShapesByGeometry(gdf) 113 | 114 | assert isinstance(huc, gpd.GeoDataFrame) 115 | assert len(huc) >= 1 116 | assert names.ID in huc.columns 117 | assert names.HUC in huc.columns 118 | 119 | -------------------------------------------------------------------------------- /watershed_workflow/test/test_09_mesh.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import numpy as np 3 | import warnings 4 | 5 | import watershed_workflow.mesh 6 | 7 | from watershed_workflow.test.shapes import two_boxes 8 | 9 | 10 | def check_2D_geometry(m2): 11 | assert (2 == m2.num_cells) 12 | assert (7 == m2.num_edges) 13 | assert (6 == m2.num_vertices) 14 | assert (6 == len(m2.boundary_edges)) 15 | assert (6 == len(m2.boundary_vertices)) 16 | assert (np.allclose(np.array([5, 0, 0]), m2.computeCentroid(0), 1.e-6)) 17 | assert (np.allclose(np.array([15, 0, 0]), m2.computeCentroid(1), 1.e-6)) 18 | 19 | 20 | def test_2D(two_boxes): 21 | """Create a 2D mesh, extrude, write.""" 22 | two_boxes = list(two_boxes.geometry) 23 | coords1 = np.array(two_boxes[0].exterior.coords)[:-1] 24 | coords2 = np.array(two_boxes[1].exterior.coords)[:-1] 25 | coords_xy = np.concatenate([coords1, coords2[1:3]], axis=0) 26 | coords_z = np.zeros((len(coords_xy), 1), 'd') 27 | 28 | coords = np.concatenate([coords_xy, coords_z], axis=1) 29 | print(coords) 30 | conn = [[0, 1, 2, 3], [1, 4, 5, 2]] 31 | 32 | m2 = watershed_workflow.mesh.Mesh2D(coords, conn) 33 | check_2D_geometry(m2) 34 | 35 | 36 | def test_from_transect(): 37 | m2 = watershed_workflow.mesh.Mesh2D.from_Transect(np.array([0, 10, 20]), 38 | np.array([0, 0, 0]), 39 | width=10) 40 | check_2D_geometry(m2) 41 | 42 | 43 | def test_extrude(): 44 | m2 = watershed_workflow.mesh.Mesh2D.from_Transect(np.array([0, 10, 20]), 45 | np.array([0, 0, 0]), 46 | width=10) 47 | m3 = watershed_workflow.mesh.Mesh3D.extruded_Mesh2D(m2, ['constant', ], [5.0, ], [10, ], 48 | [1001, ]) 49 | assert (20 == m3.num_cells) 50 | assert (7*10 + 2*11 == m3.num_faces) 51 | assert (6 * 11 == m3.num_vertices) 52 | 53 | 54 | def test_reorder(): 55 | m2 = watershed_workflow.mesh.Mesh2D.from_Transect(np.array([0, 10, 20, 30, 40, 50]), 56 | np.array([0, 0, 0, 0, 0, 0]), 57 | width=10) 58 | assert m2.num_cells == 5 59 | m2.labeled_sets.append(watershed_workflow.mesh.LabeledSet('myset', 101, 'CELL', [3,])) 60 | assert np.allclose(m2.centroids[m2.labeled_sets[0].ent_ids[0], 0:2], [35.,0.], 1.e-10) 61 | 62 | # reorder 63 | new_order = [0, 4, 3, 1, 2] 64 | m2new = m2.reorder(new_order) 65 | 66 | # check the new mesh 67 | assert m2new.num_cells == 5 68 | assert np.allclose(m2new.centroids[0,0:2], [5.,0.], 1.e-10) 69 | assert np.allclose(m2new.centroids[1,0:2], [45.,0.], 1.e-10) 70 | assert np.allclose(m2new.centroids[4,0:2], [25.,0.], 1.e-10) 71 | assert np.allclose(m2new.centroids[m2new.labeled_sets[0].ent_ids[0], 0:2], [35.,0.], 1.e-10) 72 | 73 | 74 | # partition 75 | m2newpart = m2new.partition(2, True) 76 | 77 | assert m2new.num_cells == 5 78 | # -- the partitioned mesh wants to put [0,20) on rank 1, (20,50] on rank 0 79 | assert np.allclose(m2newpart.centroids[0,0:2], [ 5.,0.], 1.e-10) 80 | assert np.allclose(m2newpart.centroids[1,0:2], [15.,0.], 1.e-10) 81 | assert np.allclose(m2newpart.centroids[2,0:2], [45.,0.], 1.e-10) 82 | assert np.allclose(m2newpart.centroids[3,0:2], [35.,0.], 1.e-10) 83 | assert np.allclose(m2newpart.centroids[4,0:2], [25.,0.], 1.e-10) 84 | assert np.allclose(m2newpart.centroids[m2newpart.labeled_sets[0].ent_ids[0], 0:2], [35.,0.], 1.e-10) 85 | 86 | # check that the partition is good 87 | assert 'partition' in m2newpart.cell_data 88 | for i in range(1, len(m2newpart.cell_data['partition'])): 89 | assert m2newpart.cell_data.loc[i-1, 'partition'] <= m2newpart.cell_data.loc[i,'partition'] 90 | 91 | 92 | def test_write(): 93 | m2 = watershed_workflow.mesh.Mesh2D.from_Transect(np.array([0, 10, 20]), 94 | np.array([0, 0, 0]), 95 | width=10) 96 | m2.cell_data['partition'] = [0,1] 97 | m3 = watershed_workflow.mesh.Mesh3D.extruded_Mesh2D(m2, ['constant', ], [5.0, ], [10, ], 98 | [1001, ]) 99 | 100 | import os 101 | if os.path.isfile('./mesh.exo'): 102 | os.remove('./mesh.exo') 103 | try: 104 | m3.writeExodus('./mesh.exo') 105 | except ImportError: 106 | warnings.warn('ExodusII is not enabled with this python.') 107 | else: 108 | assert (os.path.isfile('./mesh.exo')) 109 | 110 | import xarray 111 | with xarray.open_dataset('./mesh.exo') as fid: 112 | assert 20 == fid.sizes['num_el_in_blk1'] 113 | -------------------------------------------------------------------------------- /watershed_workflow/sources/manager_raster.py: -------------------------------------------------------------------------------- 1 | """Basic manager for interacting with raster files. 2 | """ 3 | from typing import Tuple, List, Optional, Iterable 4 | 5 | import os 6 | import xarray as xr 7 | import shapely 8 | import rioxarray 9 | import cftime 10 | import logging 11 | 12 | import watershed_workflow.crs 13 | from watershed_workflow.crs import CRS 14 | 15 | from . import manager_dataset 16 | from . import utils as source_utils 17 | 18 | 19 | class ManagerRaster(manager_dataset.ManagerDataset): 20 | """A simple class for reading rasters.""" 21 | 22 | def __init__(self, 23 | filename : str, 24 | url : Optional[str] = None, 25 | native_resolution : Optional[float] = None, 26 | native_crs : Optional[CRS] = None, 27 | bands : Optional[Iterable[str] | int] = None, 28 | ): 29 | """Initialize raster manager. 30 | 31 | Parameters 32 | ---------- 33 | filename : str 34 | Path to the raster file. 35 | """ 36 | self.filename = filename 37 | self.url = url 38 | 39 | # a flag to only preprocess once 40 | self._file_preprocessed = False 41 | 42 | # Use basename of file as name 43 | name = f'raster: "{os.path.basename(filename)}"' 44 | 45 | # Use absolute path as source for complete provenance 46 | source = os.path.abspath(filename) 47 | 48 | if bands is None: 49 | valid_variables = None 50 | default_variables = None 51 | elif isinstance(bands, int): 52 | valid_variables = [f'band {i+1}' for i in range(num_bands)] 53 | default_variables = [valid_variables[0],] 54 | else: 55 | valid_variables = bands 56 | default_variables = [valid_variables[0],] 57 | 58 | # Initialize base class 59 | super().__init__( 60 | name, source, 61 | native_resolution, native_crs, native_crs, 62 | None, None, valid_variables, default_variables 63 | ) 64 | 65 | 66 | def _prerequestDataset(self) -> None: 67 | # first download -- this is done here and not in _request so 68 | # that we can set the resolution and CRS for input geometry 69 | # manipulation. 70 | self._download() 71 | 72 | if not self._file_preprocessed: 73 | # Inspect raster to get native properties 74 | with rioxarray.open_rasterio(self.filename) as temp_ds: 75 | # Get native CRS 76 | self.native_crs_in = temp_ds.rio.crs 77 | self.native_crs_out = temp_ds.rio.crs 78 | 79 | # Get native resolution (approximate from first pixel) 80 | if len(temp_ds.coords['x']) > 1 and len(temp_ds.coords['y']) > 1: 81 | x_res = abs(float(temp_ds.coords['x'][1] - temp_ds.coords['x'][0])) 82 | y_res = abs(float(temp_ds.coords['y'][1] - temp_ds.coords['y'][0])) 83 | self.native_resolution = max(x_res, y_res) 84 | else: 85 | self.native_resolution = 1.0 # fallback 86 | 87 | # Create variable names for each band 88 | if self.valid_variables is None: 89 | if hasattr(temp_ds, 'band'): 90 | # pull from bands 91 | self.valid_variables = [f'band_{i}' for i in temp_ds.band.values] 92 | 93 | # First band as default 94 | self.default_variables = [self.valid_variables[0],] 95 | elif len(d.values.shape) == 3: 96 | num_bands = d.values.shape[0] 97 | self.valid_variables = [f'band_{i}' for i in range(num_bands)] 98 | self.default_variables = [self.valid_variables[0],] 99 | 100 | # only do this work once 101 | self._file_preprocessed = True 102 | 103 | 104 | def _requestDataset(self, request : manager_dataset.ManagerDataset.Request 105 | ) -> manager_dataset.ManagerDataset.Request: 106 | """Request the data -- ready upon request.""" 107 | request.is_ready = True 108 | return request 109 | 110 | 111 | def _fetchDataset(self, request : manager_dataset.ManagerDataset.Request) -> xr.Dataset: 112 | """Fetch the data.""" 113 | bounds = request.geometry.bounds 114 | 115 | # Open raster and clip to bounds 116 | if not self.filename.lower().endswith('.tif'): 117 | dataset = rioxarray.open_rasterio(self.filename, chunk='auto') 118 | else: 119 | dataset = rioxarray.open_rasterio(self.filename, cache=False) 120 | 121 | # Clip to bounds 122 | dataset = dataset.rio.clip_box(*bounds, crs=watershed_workflow.crs.to_rasterio(self.native_crs_out)) 123 | 124 | # Convert to Dataset with band variables 125 | result_dataset = xr.Dataset() 126 | 127 | if request.variables is None: 128 | # single-variable case 129 | if len(dataset.shape) > 2: 130 | result_dataset['raster'] = dataset[0, :, :] # Take first band 131 | else: 132 | result_dataset['raster'] = dataset 133 | 134 | else: 135 | for var in request.variables: 136 | assert var.startswith('band_') 137 | band_idx = int(var.split('_')[1]) - 1 # Convert to 0-indexed 138 | if len(dataset.shape) > 2 and band_idx < dataset.shape[0]: 139 | band_data = dataset[band_idx, :, :] 140 | band_data = band_data.drop_vars('band', errors='ignore') 141 | result_dataset[var] = band_data 142 | elif len(dataset.shape) == 2: # Single band raster 143 | if band_idx == 0: 144 | result_dataset[var] = dataset 145 | else: 146 | raise ValueError(f"Band {band_idx + 1} not available in raster") 147 | 148 | return result_dataset 149 | 150 | 151 | def _download(self, force : bool = False): 152 | """A default download implementation.""" 153 | os.makedirs(os.path.dirname(self.filename), exist_ok=True) 154 | if not os.path.exists(self.filename) or force: 155 | source_utils.download(self.url, self.filename, force) 156 | 157 | 158 | -------------------------------------------------------------------------------- /watershed_workflow/io.py: -------------------------------------------------------------------------------- 1 | """I/O Utilities""" 2 | 3 | from typing import Optional, Dict, Union, Any 4 | import os 5 | import numpy as np 6 | import logging 7 | import h5py 8 | import cftime 9 | import rasterio.transform 10 | import xarray as xr 11 | import pandas as pd 12 | 13 | import watershed_workflow.crs 14 | 15 | 16 | def writeDatasetToHDF5(filename: str, 17 | dataset: xr.Dataset, 18 | attributes: Optional[Dict[str, Any]] = None, 19 | time0: Optional[Union[str, cftime.datetime]] = None, 20 | calendar: str = 'noleap') -> None: 21 | """ 22 | Write an xarray.Dataset and attributes to an HDF5 file. 23 | 24 | Parameters 25 | ---------- 26 | filename : str 27 | Name of the file to write. 28 | dataset : xarray.Dataset 29 | Dataset containing the data to write. Must have 'time', 'x', and 'y' coordinates. 30 | attributes : dict, optional 31 | Dictionary of attributes to write to the HDF5 file. Default is None. 32 | time0 : str or cftime.datetime, optional 33 | Time to use as the zero time for the time series. If not provided, the first 34 | time in the time series is used. If string, should be in 'YYYY-MM-DD' format. 35 | calendar : str, optional 36 | Calendar type to use for time conversion. Default is 'noleap'. 37 | 38 | Raises 39 | ------ 40 | KeyError 41 | If required coordinates ('time', 'x', 'y') are missing from dataset. 42 | ValueError 43 | If dataset dimensions don't match expected shapes. 44 | 45 | Notes 46 | ----- 47 | The function writes time as seconds since time0, with y coordinates in reverse order 48 | to match typical geospatial conventions. Data arrays are also flipped vertically 49 | to match the y coordinate ordering. 50 | """ 51 | try: 52 | os.remove(filename) 53 | except FileNotFoundError: 54 | pass 55 | 56 | keys = list(dataset.data_vars.keys()) 57 | times = dataset.time.values 58 | 59 | # construct the x,y arrays from the dataset coordinates 60 | x = dataset.x.values 61 | y = dataset.y.values 62 | 63 | if time0 is None: 64 | time0 = times[0] 65 | 66 | if isinstance(time0, str): 67 | time0_split = time0.split('-') 68 | time0 = cftime.datetime(int(time0_split[0]), 69 | int(time0_split[1]), 70 | int(time0_split[2]), 71 | calendar=calendar) 72 | if attributes is None: 73 | attributes = dict() 74 | attributes['origin date'] = str(time0) 75 | if dataset.attrs is not None: 76 | attributes.update(dataset.attrs) 77 | 78 | times = np.array([(t - time0).total_seconds() for t in times]) 79 | times = times.astype(np.int32) 80 | logging.info('Writing HDF5 file: {}'.format(filename)) 81 | with h5py.File(filename, 'w') as fid: 82 | fid.create_dataset('time [s]', data=times) 83 | 84 | # make y increasing order 85 | rev_y = y[::-1] 86 | fid.create_dataset('y [m]', data=rev_y) 87 | fid.create_dataset('x [m]', data=x) 88 | 89 | for key in keys: 90 | # dat has shape (ntime, ny, nx) 91 | data = dataset[key].values 92 | assert (data.shape[0] == times.shape[0]) 93 | assert (data.shape[1] == y.shape[0]) 94 | assert (data.shape[2] == x.shape[0]) 95 | 96 | grp = fid.create_group(key) 97 | for i in range(len(times)): 98 | idat = data[i, :, :] 99 | # flip rows to match the order of y 100 | rev_idat = np.flip(idat, axis=0) 101 | grp.create_dataset(str(i), data=rev_idat) 102 | 103 | if attributes is not None: 104 | for key, val in attributes.items(): 105 | fid.attrs[key] = val 106 | 107 | 108 | def writeTimeseriesToHDF5(filename: str, 109 | ts: Union[Dict[str, Any], pd.DataFrame], 110 | attributes: Optional[Dict[str, Any]] = None, 111 | time0: Optional[Union[str, cftime.datetime]] = None) -> None: 112 | """ 113 | Write a time series and attributes to an HDF5 file. 114 | 115 | Parameters 116 | ---------- 117 | filename : str 118 | Name of the file to write. 119 | ts : dict or pandas.DataFrame 120 | Dictionary or DataFrame of time series data. Must contain a 'time [datetime]' key/column 121 | with datetime values. Other keys/columns contain the time series data. 122 | attributes : dict, optional 123 | Dictionary of attributes to write to the HDF5 file. Default is None. 124 | time0 : str or cftime.datetime, optional 125 | Time to use as the zero time for the time series. If not provided, the first 126 | time in the time series is used. If string, should be in 'YYYY-MM-DD' format. 127 | 128 | Raises 129 | ------ 130 | KeyError 131 | If 'time [datetime]' key is not found in the input data. 132 | ValueError 133 | If time0 string format is invalid. 134 | 135 | Notes 136 | ----- 137 | Time values are converted to seconds since time0 and stored as 32-bit integers. 138 | The function automatically adds 'origin date', 'start date', and 'end date' 139 | attributes to the output file. 140 | """ 141 | try: 142 | os.remove(filename) 143 | except FileNotFoundError: 144 | pass 145 | 146 | keys = list(ts.keys()) 147 | keys.remove('time') 148 | times = ts['time'] 149 | 150 | if time0 is None: 151 | time0 = times.values[0] 152 | if isinstance(time0, str): 153 | time0 = cftime.datetime.strptime(time0, '%Y-%m-%d').date() 154 | if attributes is None: 155 | attributes = dict() 156 | attributes['origin date'] = str(time0) 157 | attributes['start date'] = str(times.values[0]) 158 | attributes['end date'] = str(times.values[-1]) 159 | 160 | times = np.array([(t - time0).total_seconds() for t in times]) 161 | times = times.astype(np.int32) 162 | logging.info('Writing HDF5 file: {}'.format(filename)) 163 | with h5py.File(filename, 'w') as fid: 164 | fid.create_dataset('time [s]', data=times) 165 | 166 | for key in keys: 167 | fid.create_dataset(key, data=ts[key][:]) 168 | if attributes is not None: 169 | for key, val in attributes.items(): 170 | fid.attrs[key] = val 171 | -------------------------------------------------------------------------------- /watershed_workflow/sources/__init__.py: -------------------------------------------------------------------------------- 1 | """This module provides a dictionary of sources, broken out by data type, and a 2 | dictionary of default sources. 3 | 4 | These dictionaries are provided as module-local (singleton) variables. 5 | 6 | * huc_sources : A dictionary of sources that provide USGS HUC boundaries. 7 | * hydrography_sources : A dictionary of sources that provide river reaches by HUC. 8 | * dem_sources : A dictionary of available digital elevation models. 9 | * soil_sources : A dictionary of available sources for soil properties. 10 | * land_cover_sources : A dictionary of available land cover datasets. 11 | 12 | # """ 13 | import logging 14 | from typing import Dict, Any 15 | 16 | from .manager_shapefile import ManagerShapefile 17 | from .manager_raster import ManagerRaster 18 | 19 | from .manager_wbd import ManagerWBD 20 | from .manager_nhd import ManagerNHD 21 | from .manager_3dep import Manager3DEP 22 | from .manager_nrcs import ManagerNRCS 23 | from .manager_glhymps import ManagerGLHYMPS 24 | from .manager_soilgrids_2017 import ManagerSoilGrids2017 25 | from .manager_pelletier_dtb import ManagerPelletierDTB 26 | from .manager_nlcd import ManagerNLCD 27 | 28 | # DayMet THREDDS API is disabled -- this only works for previously-downloaded files! 29 | from .manager_daymet import ManagerDaymet 30 | from .manager_aorc import ManagerAORC 31 | 32 | from .manager_modis_appeears import ManagerMODISAppEEARS 33 | 34 | 35 | # available and default water boundary datasets 36 | huc_sources = { 37 | 'WBD' : ManagerWBD('WBD'), 38 | 'WaterData WBD' : ManagerWBD('WaterData'), 39 | } 40 | default_huc_source = 'WBD' 41 | 42 | # available and default hydrography datasets 43 | hydrography_sources = { 'NHDPlus MR v2.1' : ManagerNHD('NHDPlus MR v2.1'), 44 | 'NHD MR' : ManagerNHD('NHD MR'), 45 | 'NHDPlus HR' : ManagerNHD('NHDPlus HR') 46 | } 47 | default_hydrography_source = 'NHDPlus MR v2.1' 48 | 49 | # available and default digital elevation maps 50 | dem_sources : Dict[str,Any] = { 51 | '3DEP 60m': Manager3DEP(60), 52 | '3DEP 30m': Manager3DEP(30), 53 | '3DEP 10m': Manager3DEP(10), 54 | } 55 | default_dem_source = '3DEP 60m' 56 | 57 | # available and default soil survey datasets 58 | structure_sources : Dict[str,Any] = { 59 | 'NRCS SSURGO': ManagerNRCS(), 60 | 'GLHYMPS': ManagerGLHYMPS(), 61 | # 'SoilGrids2017': ManagerSoilGrids2017(), 62 | 'Pelletier DTB': ManagerPelletierDTB(), 63 | } 64 | default_structure_source = 'NRCS SSURGO' 65 | 66 | # available and default land cover 67 | land_cover_sources : Dict[str,Any] = { 68 | 'NLCD (L48)': ManagerNLCD(location='L48'), 69 | 'NLCD (AK)': ManagerNLCD(location='AK'), 70 | 'MODIS': ManagerMODISAppEEARS() 71 | } 72 | default_land_cover = 'NLCD (L48)' 73 | 74 | lai_sources : Dict[str,Any] = { 75 | 'MODIS': ManagerMODISAppEEARS() 76 | } 77 | default_lai = 'MODIS' 78 | 79 | # available and default meteorology 80 | met_sources : Dict[str,Any] = { 81 | 'AORC': ManagerAORC(), 82 | 'DayMet': ManagerDaymet() 83 | } 84 | default_met = 'AORC' 85 | 86 | 87 | def getDefaultSources() -> Dict[str, Any]: 88 | """Provides a default set of data sources. 89 | 90 | Returns a dictionary with default sources for each type. 91 | """ 92 | sources : Dict[str,Any] = dict() 93 | sources['HUC'] = huc_sources[default_huc_source] 94 | sources['hydrography'] = hydrography_sources[default_hydrography_source] 95 | sources['DEM'] = dem_sources[default_dem_source] 96 | sources['soil structure'] = structure_sources['NRCS SSURGO'] 97 | sources['geologic structure'] = structure_sources['GLHYMPS'] 98 | sources['land cover'] = land_cover_sources[default_land_cover] 99 | sources['LAI'] = lai_sources[default_lai] 100 | sources['depth to bedrock'] = structure_sources['Pelletier DTB'] 101 | sources['meteorology'] = met_sources[default_met] 102 | return sources 103 | 104 | 105 | def getSources(args) -> Dict[str, Any]: 106 | """Parsers the command line argument struct from argparse and provides an 107 | updated set of data sources. 108 | 109 | Parameters 110 | ---------- 111 | args : struct 112 | A python struct generated from an argparse.ArgumentParser object with 113 | source options set by watershed_workflow.ui.*_source_options 114 | 115 | Returns 116 | ------- 117 | sources : dict 118 | Dictionary of defaults for each of "HUC", "hydrography", "DEM", "soil 119 | type", and "land cover". 120 | """ 121 | sources = getDefaultSources() 122 | try: 123 | source_huc = args.source_huc 124 | except AttributeError: 125 | pass 126 | else: 127 | sources['HUC'] = huc_sources[source_huc] 128 | 129 | try: 130 | source_hydrography = args.source_hydro 131 | except AttributeError: 132 | pass 133 | else: 134 | sources['hydrography'] = hydrography_sources[source_hydrography] 135 | 136 | try: 137 | source_dem = args.source_dem 138 | except AttributeError: 139 | pass 140 | else: 141 | sources['DEM'] = dem_sources[source_dem] 142 | 143 | try: 144 | source_soil = args.soil_structure 145 | except AttributeError: 146 | pass 147 | else: 148 | sources['soil structure'] = structure_sources[source_soil] 149 | 150 | try: 151 | source_geo = args.geologic_structure 152 | except AttributeError: 153 | pass 154 | else: 155 | sources['geologic structure'] = structure_sources[source_geo] 156 | 157 | try: 158 | source_dtb = args.dtb_structure 159 | except AttributeError: 160 | pass 161 | else: 162 | sources['depth to bedrock'] = structure_sources[source_dtb] 163 | 164 | try: 165 | land_cover = args.land_cover 166 | except AttributeError: 167 | pass 168 | else: 169 | sources['land cover'] = land_cover_sources[land_cover] 170 | 171 | try: 172 | met = args.meteorology 173 | except AttributeError: 174 | pass 175 | else: 176 | sources['meteorology'] = met_sources[met] 177 | 178 | return sources 179 | 180 | 181 | def logSources(sources : Dict[str, Any]) -> None: 182 | """Pretty print source dictionary to log.""" 183 | logging.info('Using sources:') 184 | logging.info('--------------') 185 | for stype, s in sources.items(): 186 | if s is not None: 187 | logging.info('{}: {}'.format(stype, s.name)) 188 | else: 189 | logging.info('{}: None'.format(stype)) 190 | -------------------------------------------------------------------------------- /watershed_workflow/bin_utils.py: -------------------------------------------------------------------------------- 1 | """Collection of common functionality across multiple scripts in bin.""" 2 | import os, sys 3 | from matplotlib import pyplot as plt 4 | import logging 5 | import rasterio.transform 6 | import shapely 7 | import numpy as np 8 | import cartopy.feature 9 | 10 | import watershed_workflow 11 | import watershed_workflow.vtk_io 12 | import watershed_workflow.plot 13 | import watershed_workflow.config 14 | import watershed_workflow.utils 15 | 16 | 17 | def plot_with_triangulation(args, 18 | hucs, 19 | rivers, 20 | triangulation, 21 | shape_color='k', 22 | river_color='white', 23 | fig=None, 24 | ax=None): 25 | logging.info('Plotting') 26 | logging.info('--------') 27 | 28 | # get a figure and axis 29 | if fig is None: 30 | fig = plt.figure(figsize=args.figsize) 31 | if ax is None: 32 | ax = watershed_workflow.plot.get_ax(args.projection, fig=fig) 33 | 34 | if triangulation is not None: 35 | mesh_points3, mesh_tris = triangulation 36 | mp = watershed_workflow.plot.triangulation(mesh_points3, 37 | mesh_tris, 38 | args.projection, 39 | color='elevation', 40 | ax=ax, 41 | linewidth=0) 42 | #fig.colorbar(mp, orientation="horizontal", pad=0.1) 43 | if rivers is not None: 44 | watershed_workflow.plot.rivers(rivers, args.projection, river_color, ax, linewidth=0.5) 45 | 46 | if hucs is not None: 47 | watershed_workflow.plot.hucs(hucs, args.projection, shape_color, ax, linewidth=.7) 48 | 49 | ax.set_aspect('equal', 'datalim') 50 | return fig, ax 51 | 52 | 53 | def plot_with_dem(args, 54 | hucs, 55 | reaches, 56 | dem, 57 | profile, 58 | shape_color='k', 59 | river_color='white', 60 | cb=True, 61 | cb_label='elevation [m]', 62 | vmin=None, 63 | vmax=None, 64 | fig=None, 65 | ax=None): 66 | 67 | logging.info('Plotting') 68 | logging.info('--------') 69 | 70 | # get a figure and axis 71 | if fig is None: 72 | fig = plt.figure(figsize=args.figsize) 73 | if ax is None: 74 | ax = watershed_workflow.plot.get_ax(args.projection, fig=fig) 75 | 76 | # get a plot extent 77 | if args.extent is None: 78 | args.extent = hucs.exterior().bounds 79 | 80 | if args.pad_fraction is not None: 81 | if len(args.pad_fraction) == 1: 82 | dxp = (args.extent[2] - args.extent[0]) * args.pad_fraction[0] 83 | dxm = dxp 84 | dym = dxp 85 | dyp = dxp 86 | elif len(args.pad_fraction) == 2: 87 | dxp = (args.extent[2] - args.extent[0]) * args.pad_fraction[0] 88 | dxm = dxp 89 | dyp = (args.extent[3] - args.extent[1]) * args.pad_fraction[1] 90 | dym = dyp 91 | elif len(args.pad_fraction) == 4: 92 | dxm = (args.extent[2] - args.extent[0]) * args.pad_fraction[0] 93 | dym = (args.extent[3] - args.extent[1]) * args.pad_fraction[1] 94 | dxp = (args.extent[2] - args.extent[0]) * args.pad_fraction[2] 95 | dyp = (args.extent[3] - args.extent[1]) * args.pad_fraction[3] 96 | else: 97 | raise ValueError('Option: --pad-fraction must be of length 1, 2, or 4') 98 | 99 | args.extent = [ 100 | args.extent[0] - dxm, args.extent[1] - dym, args.extent[2] + dxp, 101 | args.extent[3] + dyp 102 | ] 103 | 104 | logging.info('plot extent: {}'.format(args.extent)) 105 | 106 | # continents 107 | if args.basemap: 108 | watershed_workflow.plot.basemap(args.projection, 109 | ax=ax, 110 | resolution=args.basemap_resolution, 111 | land_kwargs={ 'zorder': 0 }, 112 | ocean_kwargs={ 'zorder': 2 }) 113 | 114 | # plot the raster 115 | # -- pad the raster to have the same extent 116 | if dem is not None: 117 | mappable = watershed_workflow.plot.dem(profile, dem, ax, vmin, vmax) 118 | if args.basemap: 119 | mappable.set_zorder(1) 120 | if cb: 121 | cb = fig.colorbar(mappable, orientation="horizontal", pad=0) 122 | cb.set_label(cb_label) 123 | 124 | # plot HUCs and reaches on top 125 | if reaches is not None: 126 | watershed_workflow.plot.river(reaches, 127 | args.projection, 128 | river_color, 129 | ax, 130 | linewidth=0.5, 131 | zorder=3) 132 | 133 | if hucs is not None: 134 | watershed_workflow.plot.hucs(hucs, args.projection, shape_color, ax, linewidth=.7, zorder=4) 135 | 136 | ax.set_xlim(args.extent[0], args.extent[2]) 137 | ax.set_ylim(args.extent[1], args.extent[3]) 138 | ax.set_aspect('equal', 'box') 139 | ax.set_title(args.title) 140 | return fig, ax 141 | 142 | 143 | def save(args, triangulation): 144 | mesh_points3, mesh_tris = triangulation 145 | if hasattr(args, 'HUC'): 146 | metadata_lines = [ 147 | 'Mesh of HUC: %s' % args.HUC, '', 148 | ' coordinate system = epsg:{}'.format(args.projection), 149 | ] 150 | else: 151 | metadata_lines = [ 152 | 'Mesh of shape: %s' % args.input_file, '', 153 | ' coordinate system = epsg:{}'.format(args.projection), 154 | ] 155 | 156 | metadata_lines.extend([ 157 | '', 'Mesh generated by workflow mesh_hucs.py script.', '', watershed_workflow.__version__, 158 | '', 'with calling sequence:', ' ' + ' '.join(sys.argv) 159 | ]) 160 | 161 | logging.info("") 162 | logging.info("File I/O") 163 | logging.info("-" * 30) 164 | logging.info("Saving mesh: %s" % args.output_file) 165 | watershed_workflow.vtk_io.write(args.output_file, mesh_points3, { 'triangle': mesh_tris }) 166 | 167 | logging.info("Saving README: %s" % args.output_file + '.readme') 168 | with open(args.output_file + '.readme', 'w') as fid: 169 | fid.write('\n'.join(metadata_lines)) 170 | -------------------------------------------------------------------------------- /watershed_workflow/test/shapes.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | import numpy as np 4 | import shapely.geometry 5 | import geopandas 6 | import watershed_workflow.utils 7 | 8 | _tol = 1.e-7 9 | 10 | 11 | def assert_close(s1, s2, tol=_tol): 12 | assert (watershed_workflow.utils.isClose(s1, s2, tol)) 13 | 14 | 15 | # ===== river shapes ===== 16 | 17 | def to_dataframe(shape_func): 18 | def _func(): 19 | shapes = shape_func() 20 | df = geopandas.GeoDataFrame({'index':range(len(shapes)), 21 | 'geometry':shapes}).set_index('index') 22 | return df 23 | return _func 24 | 25 | 26 | @pytest.fixture 27 | @to_dataframe 28 | def y(): 29 | points = [[(1, 0), (0, 0)], [(1, 1), (1, 0)], [(1, -1), (1, 0)]] 30 | return list(shapely.geometry.MultiLineString(points).geoms) 31 | 32 | 33 | @pytest.fixture 34 | @to_dataframe 35 | def y_with_extension(): 36 | points = [[(1, 0), (0, 0)], [(1, 1), (1, 0)], [(1, -1), (1, 0)], [(2, -1), (1, -1)]] 37 | return list(shapely.geometry.MultiLineString(points).geoms) 38 | 39 | 40 | @pytest.fixture 41 | def two_ys(): 42 | points = [[(1, 0), (0, 0)], [(1, 1), (1, 0)], [(1, -1), (1, 0)], [(12, 0), (11, 0)], 43 | [(12, 1), (12, 0)], [(12, -1), (12, 0)]] 44 | mls = list(shapely.geometry.MultiLineString(points).geoms) 45 | hydroseqs = [1, 2, 3, 4, 5, 6] 46 | dnstream = [-1, 1, 1, -1, 4, 4] 47 | 48 | 49 | df = geopandas.GeoDataFrame({'index' : hydroseqs, 50 | 'hydroseq' : hydroseqs, 51 | 'dnhydroseq' : dnstream, 52 | 'geometry' : mls}).set_index('index') 53 | return df 54 | 55 | 56 | @pytest.fixture 57 | def braided_stream(): 58 | points = [[(1, 0), (0, 0)], [(2, 1), (1, 0)], [(3, 0), (2, 1)], [(4, 0), (3, 0)], 59 | [(2, -1), (1, 0)], [(3, 0), (2, -1)]] 60 | mls = list(shapely.geometry.MultiLineString(points).geoms) 61 | hydroseqs = [1, 2, 3, 6, 4, 5] 62 | dnstream = [-1, 1, 2, 3, 1, 4] 63 | upstream = [2, 3, 6, -1, 5, 6] 64 | divergence = [0, 0, 1, 0, 0, 2] 65 | 66 | df = geopandas.GeoDataFrame({'index' : range(len(mls)), 67 | 'hydroseq' : hydroseqs, 68 | 'dnhydroseq' : dnstream, 69 | 'uphydroseq' : upstream, 70 | 'divergence' : divergence, 71 | 'geometry' : mls}).set_index('index') 72 | return df 73 | 74 | 75 | @pytest.fixture 76 | @to_dataframe 77 | def rivers(): 78 | return [shapely.geometry.LineString([(5, 0), (0, 0)]), 79 | shapely.geometry.LineString([(8, 3), (5, 0)]), 80 | shapely.geometry.LineString([(12, -3), (8, -3), (5, 0)]), 81 | shapely.geometry.LineString([(15, -3), (12, -3)]), 82 | shapely.geometry.LineString([(12, 0), (12, -3)]), 83 | ] 84 | 85 | 86 | # 87 | # Note, this is not valid input for a River object, or at least may 88 | # not return a single river! 89 | # 90 | @pytest.fixture 91 | @to_dataframe 92 | def y_with_junction(): 93 | return [shapely.geometry.LineString([(1, 0), (0, 0)]), 94 | shapely.geometry.LineString([(1, 1), (1, 0)]), 95 | shapely.geometry.LineString([(1, -1), (0.5, 0)]), 96 | ] 97 | 98 | 99 | # ===== polygons ===== 100 | @pytest.fixture 101 | @to_dataframe 102 | def two_boxes(): 103 | return [shapely.geometry.Polygon([(0, -5), (10, -5), (10, 5), (0, 5)]), 104 | shapely.geometry.Polygon([(10, -5), (20, -5), (20, 5), (10, 5)]), 105 | ] 106 | 107 | 108 | @pytest.fixture 109 | @to_dataframe 110 | def three_boxes(): 111 | return [shapely.geometry.Polygon([(0, -5), (10, -5), (10, 5), (0, 5)]), 112 | shapely.geometry.Polygon([(10, -5), (20, -5), (20, 5), (10, 5)]), 113 | shapely.geometry.Polygon([(20, -5), (30, -5), (30, 5), (20, 5)]), 114 | ] 115 | 116 | 117 | @pytest.fixture 118 | @to_dataframe 119 | def three_more_boxes(): 120 | return [ shapely.geometry.Polygon([(0, -5), (10, -5), (10, 5), (0, 5)]), 121 | shapely.geometry.Polygon([(10, -5), (20, -5), (20, 5), (10, 5)]), 122 | shapely.geometry.Polygon([(0, 5), (10, 5), (20, 5), (20, 10), (0, 10)]), 123 | ] 124 | 125 | 126 | @pytest.fixture 127 | @to_dataframe 128 | def watershed_poly1(): 129 | return [ shapely.geometry.Polygon([(0, -5), (10, -5), (10, 5), (0, 5)]), 130 | shapely.geometry.Polygon([(10, -5), (20, -5), (20, 5), (10, 5)]), 131 | shapely.geometry.Polygon([(0, 5), (10, 5), (20, 5), (20, 10), (0, 10)]), 132 | ] 133 | 134 | 135 | @pytest.fixture 136 | @to_dataframe 137 | def watershed_reaches1(): 138 | return [ 139 | shapely.geometry.LineString([(5., 0.), (10., 5), ]), 140 | shapely.geometry.LineString([(15., 0.), (10., 5), ]), 141 | shapely.geometry.LineString([(10., 5.), (10, 10)]), 142 | ] 143 | 144 | 145 | @pytest.fixture 146 | @to_dataframe 147 | def watershed_poly2(): 148 | """Create watershed polygon, mocking NHDPLus dataset""" 149 | return [shapely.geometry.Polygon( 150 | 100 * np.array([[0, 0], [1, 0], [3, 0], [4, 0], [4, 1], [4, 2], [4, 3], [4, 4], [3, 4.5], 151 | [2, 5], [1, 4.5], [0, 4], [0, 3], [0, 2], [0, 1]], 'd')),] 152 | 153 | 154 | @pytest.fixture 155 | @to_dataframe 156 | def watershed_reaches2(): 157 | """Create a list of reaches, mocking NHDPLus dataset""" 158 | reach1 = shapely.geometry.LineString([(200, 200), (200, 0)]) 159 | reach2 = shapely.geometry.LineString([(50, 300), (100, 300), (100, 200), (200, 200)]) 160 | reach3 = shapely.geometry.LineString([(350, 400), (350, 300), (300, 300), (300, 200), 161 | (200, 200)]) 162 | reach4 = shapely.geometry.LineString([(100, 400), (200, 300)]) 163 | reaches = [reach1, reach2, reach3, reach4] 164 | return reaches 165 | 166 | 167 | @pytest.fixture 168 | def watershed_rivers1(watershed_poly1, watershed_reaches1): 169 | if watershed_poly1 is not None: 170 | hucs = watershed_workflow.split_hucs.SplitHUCs(watershed_poly1) 171 | else: 172 | hucs = None 173 | if watershed_reaches1 is not None: 174 | rivers = watershed_workflow.river_tree.createRivers(watershed_reaches1) 175 | else: 176 | rivers = None 177 | return hucs, rivers 178 | 179 | 180 | 181 | 182 | @pytest.fixture 183 | def watershed_rivers2(watershed_poly2, watershed_reaches2): 184 | """The goalpost river network with two rivers.""" 185 | if watershed_poly2 is not None: 186 | hucs = watershed_workflow.split_hucs.SplitHUCs(watershed_poly2) 187 | else: 188 | hucs = None 189 | if watershed_reaches2 is not None: 190 | rivers = watershed_workflow.river_tree.createRivers(watershed_reaches2) 191 | else: 192 | rivers = None 193 | return hucs, rivers 194 | 195 | --------------------------------------------------------------------------------