├── .gitignore ├── CHANGELOG.md ├── LICENSE ├── README.md ├── doc ├── example_config.ini └── tile_schema.csv ├── get_runtime_stats.sh ├── lib ├── __init__.py ├── mosaic.py ├── ortho_functions.py ├── taskhandler.py ├── utils.py └── version.py ├── pgc_mosaic.py ├── pgc_mosaic_build_tile.py ├── pgc_mosaic_query_index.py ├── pgc_ndvi.py ├── pgc_ortho.py ├── pgc_pansharpen.py ├── qsub_mosaic.sh ├── qsub_ndvi.sh ├── qsub_ortho.sh ├── qsub_pansharpen.sh ├── slurm_mosaic.sh ├── slurm_ndvi.sh ├── slurm_ortho.sh ├── slurm_pansharpen.sh ├── tests ├── __init__.py ├── test_mosaic.py ├── test_mosaic_lib.py ├── test_ndvi.py ├── test_ortho.py ├── test_ortho_functions.py ├── test_ortho_functions_nodata.py ├── test_pansharpen.py ├── test_taskhandler.py └── test_utils.py └── utility_scripts ├── pgc_get_scene_overlaps_standalone.py └── stack_landsat.py /.gitignore: -------------------------------------------------------------------------------- 1 | **/*.pyc 2 | __pycache__/ 3 | /mosaic_bash_scripts 4 | /tests/testdata 5 | *.pyc 6 | .pytest_cache/ 7 | .idea/ 8 | .coverage 9 | htmlcov/ 10 | tests/tmp_output/ 11 | tests/tmp_pytest/ -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | History 2 | ======= 3 | 4 | 2.2.0 (2024-10-31) 5 | ------------------ 6 | 7 | * Add slurm queue arg to ortho and pansharpen scripts by @bakkerbakker in https://github.com/PolarGeospatialCenter/imagery_utils/pull/84 8 | * Fix ortho metadata file for IKONOS imagery by @dannyim in https://github.com/PolarGeospatialCenter/imagery_utils/pull/85 9 | * add stack_landsat.py script to the repo by @bakkerbakker in https://github.com/PolarGeospatialCenter/imagery_utils/pull/87 10 | * Mosaic exclude list from sandwich by @bakkerbakker in https://github.com/PolarGeospatialCenter/imagery_utils/pull/86 11 | * Pansharpened mosaic selection by @bakkerbakker in https://github.com/PolarGeospatialCenter/imagery_utils/pull/88 12 | * Remove default cmd txt behavior by @bakkerbakker in https://github.com/PolarGeospatialCenter/imagery_utils/pull/89 13 | * Update radiometric calibration factors and add CAVIS by @clairecporter in https://github.com/PolarGeospatialCenter/imagery_utils/pull/90 14 | 15 | 2.1.3 (2024-07-03) 16 | ------------------ 17 | 18 | * Minor bugfix for slurm job submission memory settings 19 | 20 | 2.1.2 (2024-07-02) 21 | ------------------ 22 | 23 | * add option for passing custom slurm job name by @bakkerbakker in https://github.com/PolarGeospatialCenter/imagery_utils/pull/80 24 | * Patch for running pansharpen in mamba env on windows by @bakkerbakker in https://github.com/PolarGeospatialCenter/imagery_utils/pull/81 25 | * Slurm script default settings by @bakkerbakker in https://github.com/PolarGeospatialCenter/imagery_utils/pull/82 26 | * auto DEM windows bug fix by @bakkerbakker in https://github.com/PolarGeospatialCenter/imagery_utils/pull/83 27 | 28 | 2.1.1 (2024-05-23) 29 | ------------------ 30 | 31 | * patch to fix processing bug with `auto` DEM flag 32 | 33 | 2.1.0 (2024-05-17) 34 | ------------------ 35 | 36 | * add option to write input command to txt file next to output dir by @bakkerbakker in https://github.com/PolarGeospatialCenter/imagery_utils/pull/52 37 | * Add functionality for accepting ESRI codes in the EPSG arg by @bakkerbakker in https://github.com/PolarGeospatialCenter/imagery_utils/pull/53 38 | * Minor bug fixes for writing out the input command by @bakkerbakker in https://github.com/PolarGeospatialCenter/imagery_utils/pull/54 39 | * Add `--queue` arg to ortho/pansh scripts by @ehusby in https://github.com/PolarGeospatialCenter/imagery_utils/pull/56 40 | * Fix `ElementTree.getiterator()` call, removed in Python 3.8 by @ehusby in https://github.com/PolarGeospatialCenter/imagery_utils/pull/57 41 | * Fix ortho of multi-band-separate-files Ikonos imagery by @ehusby in https://github.com/PolarGeospatialCenter/imagery_utils/pull/58 42 | * Apply all script arg settings to pansharpen outputs by @ehusby in https://github.com/PolarGeospatialCenter/imagery_utils/pull/62 43 | * use EARLIESTACQTIME if FIRSTLINETIME is not in the metadata file by @dannyim in https://github.com/PolarGeospatialCenter/imagery_utils/pull/61 44 | * Add `--epsg-auto-nad83` option to use NAD83 datum for auto UTM projection by @ehusby in https://github.com/PolarGeospatialCenter/imagery_utils/pull/63 45 | * Remove stacked Ikonos NTF temp file with other temp files by @ehusby in https://github.com/PolarGeospatialCenter/imagery_utils/pull/64 46 | * Set SLURM job log filenames to match PBS job log filenames by @ehusby in https://github.com/PolarGeospatialCenter/imagery_utils/pull/65 47 | * Repo readme update by @ehusby in https://github.com/PolarGeospatialCenter/imagery_utils/pull/66 48 | * Fix: Assign NoData to a value outside of the valid data range for outputs of pgc_ortho.py and pgc_pansharpen.py by @power720 in https://github.com/PolarGeospatialCenter/imagery_utils/pull/74 49 | * bug fix for inadvertent testing commit by @bakkerbakker in https://github.com/PolarGeospatialCenter/imagery_utils/pull/77 50 | * Add auto DEM functionality to pgc_ortho.py by @SAliHossaini in https://github.com/PolarGeospatialCenter/imagery_utils/pull/76 51 | * Slurm log location option by @bakkerbakker in https://github.com/PolarGeospatialCenter/imagery_utils/pull/78 52 | * Update orthoing code to handle SWIR and CAVIS by @clairecporter in https://github.com/PolarGeospatialCenter/imagery_utils/pull/75 53 | 54 | 2.1.0 (2022-06-14) 55 | ------------------ 56 | 57 | * Added "REGION_ID" field to DemInfo attributes by @ehusby in https://github.com/PolarGeospatialCenter/imagery_utils/pull/18 58 | * bugfix: add allow_invalid_geom to qsub key removal list by @stevefoga in https://github.com/PolarGeospatialCenter/imagery_utils/pull/19 59 | * Changes to to address rare bug in pgc_ortho.py --tasks-per-job feature by @ehusby in https://github.com/PolarGeospatialCenter/imagery_utils/pull/20 60 | * Threading for gdalwarp and gdal_pansharpen by @stevefoga in https://github.com/PolarGeospatialCenter/imagery_utils/pull/21 61 | * Automatically scale default memory request for ortho and pansharpen jobs by @ehusby in https://github.com/PolarGeospatialCenter/imagery_utils/pull/22 62 | * Update osgeo import syntax by @ehusby in https://github.com/PolarGeospatialCenter/imagery_utils/pull/24 63 | * Bugfix escaped quotes in command string for parallel processing by @ehusby in https://github.com/PolarGeospatialCenter/imagery_utils/pull/25 64 | * Change gdal version to 2.1.3 in qsub scripts to resolve numpy issue by @bakkerbakker in https://github.com/PolarGeospatialCenter/imagery_utils/pull/28 65 | * Show full stack trace in error messages/logs by @ehusby in https://github.com/PolarGeospatialCenter/imagery_utils/pull/30 66 | * Refactor changes to remove duplicate code by @ehusby in https://github.com/PolarGeospatialCenter/imagery_utils/pull/32 67 | * Updated pgc_ortho.py to check for existing .vrt files that are left b… by @bagl0025 in https://github.com/PolarGeospatialCenter/imagery_utils/pull/31 68 | * Enable footprinting DG ortho imagery using image GCPs by @ehusby in https://github.com/PolarGeospatialCenter/imagery_utils/pull/33 69 | * Added JPEG support to format list. by @bagl0025 in https://github.com/PolarGeospatialCenter/imagery_utils/pull/35 70 | * pgc_ortho.py new CSV argument list source type by @ehusby in https://github.com/PolarGeospatialCenter/imagery_utils/pull/34 71 | * Update regex in doesCross180() to accept lat/lon integer values, not just floats by @bakkerbakker in https://github.com/PolarGeospatialCenter/imagery_utils/pull/36 72 | * Added gdalwarp --tap argument, couple bugfixes by @ehusby in https://github.com/PolarGeospatialCenter/imagery_utils/pull/37 73 | * Automatic output ortho/pansh EPSG settings by @ehusby in https://github.com/PolarGeospatialCenter/imagery_utils/pull/38 74 | * Fix bytes to string error when extracting RPB from tarfile by @ehusby in https://github.com/PolarGeospatialCenter/imagery_utils/pull/40 75 | * Subset VRT tile mosaic DEM argument using src CSV argument list by @ehusby in https://github.com/PolarGeospatialCenter/imagery_utils/pull/39 76 | * Miscellaneous fixes by @ehusby in https://github.com/PolarGeospatialCenter/imagery_utils/pull/42 77 | * Fix bug where float32 outputs are integer values on pgc_ortho.py by @clairecporter in https://github.com/PolarGeospatialCenter/imagery_utils/pull/43 78 | * Revert "Fix bug where float32 outputs are integer values on pgc_ortho.py" by @clairecporter in https://github.com/PolarGeospatialCenter/imagery_utils/pull/44 79 | * Repair introduced bug in taskhandler.py by @clairecporter in https://github.com/PolarGeospatialCenter/imagery_utils/pull/45 80 | * Undo unintented revert by @clairecporter in https://github.com/PolarGeospatialCenter/imagery_utils/pull/46 81 | * Change standard GDAL GTiff creation option from 'BIGTIFF=IF_SAFER' to… by @ehusby in https://github.com/PolarGeospatialCenter/imagery_utils/pull/47 82 | * Add check in pgc_pansharpen.py to match pan and mul scenes that differ by 1 sec by @bakkerbakker in https://github.com/PolarGeospatialCenter/imagery_utils/pull/48 83 | * Adjust ortho image metadata for old GeoEye and Ikonos imagery so fp.py works on them by @ehusby in https://github.com/PolarGeospatialCenter/imagery_utils/pull/50 84 | * Update Versioning Scheme by @clairecporter in https://github.com/PolarGeospatialCenter/imagery_utils/pull/51 -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (C) 2025 Polar Geospatial Center 2 | 3 | This program is free software; you can redistribute it and/or 4 | modify it under the terms of the GNU General Public License 5 | as published by the Free Software Foundation; either version 2 6 | of the License, or (at your option) any later version. 7 | 8 | This program is distributed in the hope that it will be useful, 9 | but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | GNU General Public License for more details. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # PGC Imagery Utils 2 | 3 | 4 | ## Introduction 5 | PGC Imagery Utils is a collection of commercial satellite imagery manipulation tools to handle batch processing of 6 | Geoeye and DigitalGlobe/Maxar imagery. The tools can: 7 | 8 | 1) Correct for terrain and radiometry 9 | 2) Mosaic several images into one set of tiles 10 | 3) Pansharpen a multispectral image with its panchromatic partner 11 | 4) Calculate an NDVI raster from a multispectral image. 12 | 13 | These tools are build on the GDAL/OGR image processing API using Python. The code is built to run primarily on a Linux 14 | HPC cluster running PBS or Slurm for queue management. Some of the tools will work on a Windows platform. 15 | 16 | The code is tightly coupled to the systems on which it was developed. You should have no expectation of it running 17 | perfectly on another system without some patching. 18 | 19 | ## Utilites 20 | Files starting with "qsub" and "slurm" are PBS and SLURM submission scripts. See the script-specific documentation for 21 | more details on usage. 22 | 23 | ### pgc_ortho 24 | 25 | The orthorectification script can correct for terrain displacement and radiometric settings as well as alter the bit 26 | depth of the imagery. Using the --pbs or --slurm options will submit the jobs to a job scheduler. Alternatively, 27 | using the --parallel-processes option will instruct the script to run multiple tasks in parallel. Using --threads N 28 | will enable threading for gdalwarp, where N is the number of threads (or ALL_CPUS); this option will not work with 29 | --pbs/--slurm, and (threads * parallel processes) cannot exceed number of threads available on system. 30 | 31 | Example: 32 | ``` 33 | python pgc_ortho.py --epsg 3031 --dem DEM.tif --format GTiff --stretch ns --outtype UInt16 input_dir output dir 34 | ``` 35 | 36 | This example will take all the nitf or tif files in the input_dir and orthorectify them using DEM.tif. The output files 37 | will be written to output_dir and be 16 bit (like the original image) GeoTiffs with no stretch applied with a spatial 38 | reference of EPSG 3031, or Antarctic Polar Stereographic -71. 39 | 40 | #### DEM Auto-Selection Configuration (when using `--dem auto`) 41 | 42 | When using the `--dem auto` setting in `pgc_ortho.py`, the script will automatically attempt to select an appropriate 43 | DEM based on image location and geometry. For this to work, a configuration file must be specified using the `--config` 44 | option. This configuration file should contain a valid `gpkg_path` entry, which points to the GeoPackage file that holds 45 | DEM coverage information. 46 | 47 | **Configuration Requirements** 48 | 49 | The config file should point to a file path for checking image overlap with reference dems. This path should locate a 50 | geopackage file which includes geometries of a list of reference DEMs. Each feature in each layer of the geopackage 51 | should have a field named 'dempath' pointing to the corresponding reference DEM. 52 | 53 | 1. **Config File Path**: Ensure that the config file exists at the specified path provided to the `--config` argument. 54 | 2. **`gpkg_path` Setting**: The config file should have a `gpkg_path` entry under the `[default]` section. This path 55 | should point to a GeoPackage file containing a 'dempath' field to the corresponding DEM. 56 | 3. **Valid DEM File**: The path specified by `dempath` should be accessible and valid. 57 | 58 | **Example Configuration File (`config.ini`)** 59 | 60 | ```ini 61 | [default] 62 | gpkg_path = /path/to/dem_list.gpkg 63 | ``` 64 | 65 | ### pgc_mosaic 66 | 67 | The mosaicking toolset mosaics multiple input images into a set of non-overlapping output tile images. It can sort the 68 | images according to several factors including cloud cover, sun elevation angle, off-nadir angle, probability of 69 | overexposure, and proximity to a specific date. It consists of 3 scripts: 70 | 71 | 1. pgc_mosaic.py - initializes the output mosaic, creates cutlines, and run the subtile processes. 72 | 2. pgc_mosaic_query_index.py - takes mosaic parameters and a shapefile index and determines which images will contribute 73 | to the resulting mosaic. The resulting list can be used to reduce the number of images that are run through the 74 | orthorectification script to those that will be eventually used. 75 | 3. pgc_mosaic_build_tile.py - builds an individual mosaic tile. This script is invoked by pgc_mosaic. 76 | 77 | Example: 78 | ``` 79 | python pgc_mosaic.py --slurm --bands 1 --tilesize 20000 20000 --resolution 0.5 0.5 input_dir output_mosaic_name 80 | ``` 81 | 82 | This example will evaluate all the 1-band images in input_dir and sort them according to their quality score. It will 83 | submit a job to the cluster queue to build each tile of size 20,000 x 20,000 pixels at 0.5 meters resolution. The 84 | output tiles will be Geotiffs named by appending a row and column identifier to the output_mosaic_name. 85 | 86 | ### pgc_pansharpen 87 | 88 | The pansharpening utility applies the orthorectification process to both the pan and multi image in a pair and then 89 | pansharpens them using the GDAL tool gdal_pansharpen. GDAL 2.1+ is required for this tool to function. The --threads 90 | flag will apply threading to both gdalwarp and gdal_pansharpen operations. 91 | 92 | ### pgc_ndvi 93 | 94 | The NDVI utility calculates NDVI from multispectral image(s). The tool is designed to run on data that have already 95 | been run through the pgc_ortho utility. 96 | 97 | ## Miscellaneous Utility Scripts 98 | 99 | ### Building RGB Composite Landsat TIFs - stack_landsat.py 100 | 101 | `stack_landsat.py` is a command line tool to combine individual Landsat band .tif files into a stacked RGB composite 102 | .tif. To run, set the input directory to a folder with the downloaded Landsat imagery you want to combine, with each of 103 | the bands as a separate .tiff file. The script will need to be run within the same environment as the other PGC 104 | utilities in this repo; it only uses standard python and gdal functionality, so there is nothing further to install. 105 | 106 | Show tool help text: 107 | ```python C:\path\to\stack_landsat.py -h``` 108 | 109 | Example usage with long options: 110 | ```python C:\path\to\stack_landsat.py --input-dir C:\path\to\landsat\directory --output-dir C:\path\to\output\dir``` 111 | 112 | Example usage with short options: 113 | ```python C:\path\to\stack_landsat.py -i C:\path\to\landsat\directory -o C:\path\to\output\dir``` 114 | 115 | The script will: 116 | - Verify that the provided input directory exists and is, in fact, a directory 117 | - Create the output directory if it does not already exist 118 | - Find all the Landsat scenes in the input directory 119 | - Attempt to create a composite RGB TIF of the scenes it finds 120 | - Report the scenes it fails to build. For instance, an RGB TIF will not be built if all of bands 4, 3, and 2 do 121 | not exist 122 | - Write the console messages to a log file in the input directory (stack_landsat_{date}.log). There is no need to 123 | retain the logs long term if the script is operating smoothly 124 | 125 | The script will not: 126 | - Know anything about previous runs. If you rerun the script, it will process whatever inputs are present, even if 127 | they have been run previously. It will also overwrite any corresponding outputs if pointed to the same output 128 | directory. 129 | 130 | ### Identifying Overaping Images - pgc_get_scene_overlap_standalone.py 131 | `pgc_get_scene_overlap_standalone.py` is a tool to identify which images are stereo-photogrammetry. 132 | candidates. 133 | 134 | ## Installation and dependencies 135 | PGC uses the Miniforge installer to build our Python/GDAL software stack. You can find installers for your OS here: 136 | https://github.com/conda-forge/miniforge?tab=readme-ov-file#miniforge3 137 | 138 | Users should expect a recent (less than 1-2 years old) version of Python and GDAL to be compatible with tools in this 139 | repo. 140 | The following conda/mamba environment likely contains more dependencies than are needed for tools in this repo, but 141 | should suffice: 142 | ``` 143 | mamba create --name pgc -c conda-forge git python=3.11 gdal=3.6.4 globus-sdk globus-cli numpy scipy pandas geopandas 144 | rasterio shapely postgresql psycopg2 sqlalchemy configargparse lxml pathlib2 python-dateutil pytest rtree xlsxwriter 145 | tqdm alive-progress pyperclip --yes 146 | ``` 147 | 148 | ## Running Tests 149 | Tests for imagery-utils use python's pytest. They require licensed commercial data that cannot be distributed freely 150 | but is available to project contributors. 151 | 152 | On Linux systems, make a symlink to the test data location: 153 | ```sh 154 | # first time only 155 | ln -s /tests/testdata tests/ 156 | 157 | # run the tests 158 | pytest 159 | ``` 160 | 161 | On Windows, you have to use the full network path and not a mounted drive letter path: 162 | ```sh 163 | # first time only 164 | mklink /d tests\testdata <\\server.school.edu\test_data_location>\tests\testdata 165 | 166 | # run the tests 167 | pytest 168 | ``` 169 | 170 | ## Contact 171 | To report any questions or issues, please open a github issue or contact the Polar Geospatial Center: 172 | pgc-support@umn.edu 173 | -------------------------------------------------------------------------------- /doc/example_config.ini: -------------------------------------------------------------------------------- 1 | [default] 2 | # To use, copy this file, rename it to config.ini, and fill in the variables below 3 | 4 | #### Required paths 5 | # Base file for checking image overlap with reference dems 6 | # this path should locate a geopackage file which includes geometries of a list of reference DEMs 7 | # each feature in each layer of the geopackage file should have a field named 'dempath' pointing to the corresponding reference DEM 8 | gpkg_path = drive:/path/to/file.gpkg -------------------------------------------------------------------------------- /doc/tile_schema.csv: -------------------------------------------------------------------------------- 1 | row,col,name,status,xmin,xmax,ymin,ymax,epsg 2 | 17,21,17_21,1,-1000000,-900000,-1400000,-1300000,3031 3 | 17,22,17_22,1,-900000,-800000,-1400000,-1300000,3031 4 | 17,23,17_23,1,-800000,-700000,-1400000,-1300000,3031 5 | -------------------------------------------------------------------------------- /lib/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """ 4 | PGC image processing utils and classes 5 | """ 6 | 7 | from .version import __version__, VERSION 8 | 9 | 10 | -------------------------------------------------------------------------------- /lib/taskhandler.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """ 4 | task handler classes and methods 5 | """ 6 | 7 | import codecs 8 | import logging 9 | import multiprocessing as mp 10 | import os 11 | import platform 12 | import signal 13 | import subprocess 14 | 15 | #### Create Logger 16 | logger = logging.getLogger("logger") 17 | logger.setLevel(logging.DEBUG) 18 | 19 | 20 | class Task(object): 21 | 22 | def __init__(self, task_name, task_abrv, task_exe, task_cmd, task_method=None, task_method_arg_list=None): 23 | self.name = task_name 24 | self.abrv = task_abrv 25 | self.exe = task_exe 26 | self.cmd = task_cmd 27 | self.method = task_method 28 | self.method_arg_list = task_method_arg_list 29 | 30 | 31 | class PBSTaskHandler(object): 32 | 33 | def __init__(self, qsubscript, qsub_args=""): 34 | 35 | #### verify PBS is present by calling pbsnodes cmd 36 | try: 37 | cmd = "pbsnodes" 38 | p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 39 | so, se = p.communicate() 40 | except OSError: 41 | raise RuntimeError("PBS job submission is not available on this system") 42 | 43 | self.qsubscript = qsubscript 44 | if not qsubscript: 45 | raise RuntimeError("PBS job submission resuires a valid qsub script") 46 | elif not os.path.isfile(qsubscript): 47 | raise RuntimeError("Qsub script does not exist: {}".format(qsubscript)) 48 | 49 | self.qsub_args = qsub_args 50 | 51 | def run_tasks(self, tasks, dryrun=False): 52 | 53 | for task in tasks: 54 | cmd = r'qsub {} -N {} -v p1="{} {}" "{}"'.format( 55 | self.qsub_args, 56 | task.abrv, 57 | task.exe, 58 | escape_problem_jobsubmit_chars(task.cmd), 59 | self.qsubscript 60 | ) 61 | if dryrun: 62 | print(cmd) 63 | else: 64 | subprocess.call(cmd, shell=True) 65 | 66 | 67 | class SLURMTaskHandler(object): 68 | 69 | def __init__(self, qsubscript, qsub_args=""): 70 | 71 | #### verify SLURM is present by calling sinfo cmd 72 | try: 73 | cmd = "sinfo" 74 | p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 75 | so, se = p.communicate() 76 | except OSError: 77 | raise RuntimeError("SLURM job submission is not available on this system") 78 | 79 | self.qsubscript = qsubscript 80 | if not qsubscript: 81 | raise RuntimeError("SLURM job submission requires a valid qsub script") 82 | elif not os.path.isfile(qsubscript): 83 | raise RuntimeError("Qsub script does not exist: {}".format(qsubscript)) 84 | 85 | self.qsub_args = qsub_args 86 | 87 | def run_tasks(self, tasks): 88 | 89 | for task in tasks: 90 | cmd = r'sbatch {} -J {} --export=p1="{} {}" "{}"'.format( 91 | self.qsub_args, 92 | task.abrv, 93 | task.exe, 94 | escape_problem_jobsubmit_chars(task.cmd), 95 | self.qsubscript 96 | ) 97 | subprocess.call(cmd, shell=True) 98 | 99 | 100 | class ParallelTaskHandler(object): 101 | 102 | def __init__(self, num_processes=1): 103 | self.num_processes = num_processes 104 | if mp.cpu_count() < num_processes: 105 | raise RuntimeError("Specified number of processes ({0}) is higher than the system cpu count ({1})". 106 | format(num_processes, mp.cpu_count())) 107 | elif num_processes < 1: 108 | raise RuntimeError("Specified number of processes ({0}) must be greater than 0, using default". 109 | format(num_processes)) 110 | 111 | def run_tasks(self, tasks): 112 | 113 | task_queue = [[task.name, self._format_task(task)] for task in tasks] 114 | pool = mp.Pool(self.num_processes) 115 | try: 116 | pool.map(exec_cmd_mp, task_queue, 1) 117 | except KeyboardInterrupt: 118 | pool.terminate() 119 | raise RuntimeError("Processes terminated without file cleanup") 120 | 121 | def _format_task(self, task): 122 | _cmd = r'{} {}'.format( 123 | task.exe, 124 | task.cmd, 125 | ) 126 | return _cmd 127 | 128 | 129 | def exec_cmd_mp(job): 130 | job_name, cmd = job 131 | logger.info('Running job: %s', job_name) 132 | logger.debug('Cmd: %s', cmd) 133 | if platform.system() == "Windows": 134 | p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) 135 | else: 136 | p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, preexec_fn=os.setsid) 137 | try: 138 | (so, se) = p.communicate() 139 | except KeyboardInterrupt: 140 | if platform.system() == "Windows": 141 | p.terminate() 142 | else: 143 | os.killpg(p.pid, signal.SIGTERM) 144 | 145 | else: 146 | logger.debug(so) 147 | logger.debug(se) 148 | 149 | 150 | def exec_cmd(cmd): 151 | logger.debug(cmd) 152 | 153 | p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) 154 | (so, se) = p.communicate() 155 | rc = p.wait() 156 | err = 0 157 | 158 | if rc != 0: 159 | logger.error("Error found - Return Code = %s: %s", rc, cmd) 160 | err = 1 161 | else: 162 | logger.debug("Return Code = %s: %s", rc, cmd) 163 | 164 | logger.debug("STDOUT: %s", so) 165 | logger.debug("STDERR: %s", se) 166 | return err, so, se 167 | 168 | 169 | def argval2str(item): 170 | if type(item) is str: 171 | if ( (item.startswith("'") and item.endswith("'")) 172 | or (item.startswith('"') and item.endswith('"'))): 173 | item_str = item 174 | else: 175 | item_str = '"{}"'.format(item) 176 | else: 177 | item_str = '{}'.format(item) 178 | return item_str 179 | 180 | 181 | def escape_problem_jobsubmit_chars(str_item): 182 | str_item = str_item.replace("'", "\\'") 183 | str_item = str_item.replace('"', '\\"') 184 | # str_item = str_item.replace(',', '@COMMA@') 185 | # str_item = str_item.replace(' ', '@SPACE@') 186 | return str_item 187 | 188 | 189 | def convert_optional_args_to_string(args, positional_arg_keys, arg_keys_to_remove): 190 | 191 | args_dict = vars(args) 192 | arg_list = [] 193 | 194 | ## Add optional args to arg_list 195 | for k, v in args_dict.items(): 196 | if k not in positional_arg_keys and k not in arg_keys_to_remove and v is not None: 197 | k = k.replace('_', '-') 198 | if isinstance(v, (list, tuple)): 199 | if not any([item is None for item in v]): 200 | arg_list.append("--{} {}".format(k, ' '.join([argval2str(item) for item in v]))) 201 | elif isinstance(v, bool): 202 | if v is True: 203 | if len(k) == 1: 204 | arg_list.append("-{}".format(k)) 205 | else: 206 | arg_list.append("--{}".format(k)) 207 | else: 208 | arg_list.append("--{} {}".format(k, argval2str(v))) 209 | 210 | arg_str_base = " ".join(arg_list) 211 | return arg_str_base 212 | -------------------------------------------------------------------------------- /lib/version.py: -------------------------------------------------------------------------------- 1 | 2 | __version__ = (2, 2, 0) 3 | VERSION = "{}.{}.{}".format(*__version__) -------------------------------------------------------------------------------- /pgc_mosaic_build_tile.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import argparse 4 | import logging 5 | import os 6 | import shutil 7 | import sys 8 | 9 | import numpy 10 | from osgeo import gdal 11 | 12 | from lib import mosaic, taskhandler, utils 13 | from lib import VERSION 14 | 15 | logger = logging.getLogger("logger") 16 | logger.setLevel(logging.DEBUG) 17 | 18 | gdal.SetConfigOption('GDAL_PAM_ENABLED', 'NO') 19 | 20 | 21 | def main(): 22 | 23 | ######################################################### 24 | #### Handle args 25 | ######################################################### 26 | 27 | #### Set Up Arguments 28 | parser = argparse.ArgumentParser( 29 | description="Create mosaic subtile" 30 | ) 31 | 32 | parser.add_argument("tile", help="output tile name") 33 | parser.add_argument("src", help="textfile of input rasters (tif only)") 34 | 35 | parser.add_argument("-r", "--resolution", nargs=2, type=float, 36 | help="output pixel resolution -- xres yres (default is same as first input file)") 37 | parser.add_argument("-e", "--extent", nargs=4, type=float, 38 | help="extent of output mosaic -- xmin xmax ymin ymax (default is union of all inputs)") 39 | parser.add_argument("-t", "--tilesize", nargs=2, type=float, 40 | help="tile size in coordinate system units -- xsize ysize (default is 40,000 times output " 41 | "resolution)") 42 | parser.add_argument("--force-pan-to-multi", action="store_true", default=False, 43 | help="if output is multiband, force script to also use 1 band images") 44 | parser.add_argument("-b", "--bands", type=int, 45 | help="number of output bands( default is number of bands in the first image)") 46 | parser.add_argument("--median-remove", action="store_true", default=False, 47 | help="subtract the median from each input image before forming the mosaic in order to correct " 48 | "for contrast") 49 | parser.add_argument("--wd", 50 | help="scratch space (default is mosaic directory)") 51 | parser.add_argument("--gtiff-compression", choices=mosaic.GTIFF_COMPRESSIONS, default="lzw", 52 | help="GTiff compression type. Default=lzw ({})".format(','.join(mosaic.GTIFF_COMPRESSIONS))) 53 | parser.add_argument("--skip-cmd-txt", action='store_true', default=True, 54 | help='THIS OPTION IS DEPRECATED - ' 55 | 'By default this arg is True and the cmd text file will not be written. ' 56 | 'Input commands are written to the log for reference.') 57 | parser.add_argument("--version", action='version', version="imagery_utils v{}".format(VERSION)) 58 | 59 | 60 | #### Parse Arguments 61 | args = parser.parse_args() 62 | 63 | status = 0 64 | 65 | bands = args.bands 66 | inpath = args.src 67 | tile = args.tile 68 | ref_xres, ref_yres = args.resolution 69 | xmin, xmax, ymin, ymax = args.extent 70 | dims = "-tr {} {} -te {} {} {} {}".format(ref_xres, ref_yres, xmin, ymin, xmax, ymax) 71 | 72 | ##### Configure Logger 73 | logfile = os.path.splitext(tile)[0] + ".log" 74 | lfh = logging.FileHandler(logfile) 75 | lfh.setLevel(logging.DEBUG) 76 | formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s', '%m-%d-%Y %H:%M:%S') 77 | lfh.setFormatter(formatter) 78 | logger.addHandler(lfh) 79 | 80 | #### get working directory 81 | if args.wd: 82 | if os.path.isdir(args.wd): 83 | localpath = args.wd 84 | else: 85 | parser.error("scratch space directory does not exist: {0}".format(args.wd)) 86 | else: 87 | localpath = os.path.dirname(tile) 88 | 89 | #### log input command for reference 90 | command_str = ' '.join(sys.argv) 91 | logger.info("Running command: {}".format(command_str)) 92 | 93 | intersects = [] 94 | 95 | if os.path.isfile(inpath): 96 | t = open(inpath, 'r') 97 | for line in t.readlines(): 98 | line = line.strip('\n').strip('\r') 99 | 100 | if ',' in line: 101 | image, median_string = line.split(',') 102 | iinfo = mosaic.ImageInfo(image, "IMAGE") 103 | median = {} 104 | for stat in median_string.split(";"): 105 | k, v = stat.split(":") 106 | median[int(k)] = float(v) 107 | if len(median) == iinfo.bands: 108 | iinfo.set_raster_median(median) 109 | else: 110 | logger.warning("Median dct length (%i) does not match band count (%i)", len(median), iinfo.bands) 111 | 112 | else: 113 | iinfo = mosaic.ImageInfo(line, "IMAGE") 114 | 115 | intersects.append(iinfo) 116 | t.close() 117 | else: 118 | logger.error("Intersecting image file does not exist: %i", inpath) 119 | 120 | logger.info(tile) 121 | 122 | logger.info("Number of image found in source file: %i", len(intersects)) 123 | 124 | wd = os.path.join(localpath, os.path.splitext(os.path.basename(tile))[0]) 125 | if not os.path.isdir(wd): 126 | os.makedirs(wd) 127 | localtile2 = os.path.join(wd, os.path.basename(tile)) 128 | localtile1 = localtile2.replace(".tif", "_temp.tif") 129 | 130 | del_images = [] 131 | images = {} 132 | 133 | #### Get Extent geometry 134 | poly_wkt = 'POLYGON (( {} {}, {} {}, {} {}, {} {}, {} {} ))'.format(xmin, ymin, xmin, ymax, xmax, ymax, xmax, ymin, 135 | xmin, ymin) 136 | 137 | c = 0 138 | for iinfo in intersects: 139 | 140 | #### Check if bands number is correct 141 | mergefile = iinfo.srcfp 142 | 143 | if args.force_pan_to_multi and iinfo.bands > 1: 144 | if iinfo.bands == 1: 145 | mergefile = os.path.join(wd, os.path.basename(iinfo.srcfp)[:-4]) + "_merge.tif" 146 | cmd = 'gdal_merge.py -ps {} {} -separate -o "{}" "{}"'.format(ref_xres, 147 | ref_yres, 148 | mergefile, 149 | '" "'.join([iinfo.srcfp] * iinfo.bands)) 150 | taskhandler.exec_cmd(cmd) 151 | srcnodata = " ".join([str(ndv) for ndv in iinfo.nodatavalue]) 152 | 153 | if args.median_remove: 154 | dst = os.path.join(wd, os.path.basename(mergefile)[:-4]) + "_median_removed.tif" 155 | status = BandSubtractMedian(iinfo, dst) 156 | if status == 1: 157 | logger.error("BandSubtractMedian() failed on %s", mergefile) 158 | sys.exit(1) 159 | ds = gdal.Open(dst) 160 | if ds: 161 | srcnodata_val = ds.GetRasterBand(1).GetNoDataValue() 162 | srcnodata = " ".join([str(srcnodata_val)] * bands) 163 | mergefile = dst 164 | else: 165 | logger.error("BandSubtractMedian() failed at gdal.Open(%s)", dst) 166 | sys.exit(1) 167 | 168 | if c == 0: 169 | if os.path.isfile(localtile1): 170 | logger.info("localtile1 already exists") 171 | status = 1 172 | break 173 | cmd = 'gdalwarp {} -srcnodata "{}" -dstnodata "{}" "{}" "{}"'.format(dims, srcnodata, srcnodata, mergefile, 174 | localtile1) 175 | taskhandler.exec_cmd(cmd) 176 | 177 | else: 178 | cmd = 'gdalwarp -srcnodata "{}" "{}" "{}"'.format(srcnodata, mergefile, localtile1) 179 | taskhandler.exec_cmd(cmd) 180 | 181 | c += 1 182 | 183 | if not mergefile == iinfo.srcfp: 184 | del_images.append(mergefile) 185 | 186 | del_images.append(localtile1) 187 | 188 | if status == 0: 189 | #### Write to Compressed file 190 | if os.path.isfile(localtile1): 191 | if args.gtiff_compression == 'lzw': 192 | compress_option = '-co "compress=lzw"' 193 | elif args.gtiff_compression == 'jpeg95': 194 | compress_option = '-co "compress=jpeg" -co "jpeg_quality=95"' 195 | 196 | cmd = 'gdal_translate -stats -of GTiff {} -co "PHOTOMETRIC=MINISBLACK" -co "TILED=YES" -co ' \ 197 | '"BIGTIFF=YES" "{}" "{}"'.format(compress_option, localtile1, localtile2) 198 | taskhandler.exec_cmd(cmd) 199 | 200 | #### Build Pyramids 201 | if os.path.isfile(localtile2): 202 | cmd = 'gdaladdo "{}" 2 4 8 16 30'.format(localtile2) 203 | taskhandler.exec_cmd(cmd) 204 | 205 | #### Copy tile to destination 206 | if os.path.isfile(localtile2): 207 | logger.info("Copying output files to destination dir") 208 | mosaic.copyall(localtile2, os.path.dirname(tile)) 209 | 210 | del_images.append(localtile2) 211 | 212 | 213 | #### Delete temp files 214 | utils.delete_temp_files(del_images) 215 | shutil.rmtree(wd) 216 | 217 | logger.info("Done") 218 | 219 | 220 | def BandSubtractMedian(iinfo, dstfp): 221 | # Subtract the median from each band of srcfp and write the result 222 | # to dstfp. 223 | # Band types byte, uint16 and int16 will be output as int16 with nodata -32768. 224 | # Band types uint32 and int32 will be output as int32 with nodata -2147483648. 225 | 226 | if not (iinfo.datatype in [1, 2, 3, 4, 5]): 227 | logger.error("BandSubtractMedian only works on integer data types") 228 | return 1 229 | elif iinfo.datatype in [1, 2, 3]: 230 | out_datatype = 3 231 | out_nodataval = -32768 232 | out_min = -32767 233 | else: 234 | out_datatype = 5 235 | out_nodataval = -2147483648 236 | out_min = -2147483647 237 | 238 | if not os.path.isfile(dstfp): 239 | gtiff_options = ['TILED=YES', 'COMPRESS=LZW', 'BIGTIFF=YES'] 240 | driver = gdal.GetDriverByName('GTiff') 241 | out_ds = driver.Create(dstfp, iinfo.xsize, iinfo.ysize, iinfo.bands, out_datatype, gtiff_options) 242 | if not out_ds: 243 | logger.error("BandSubtractMedian(): !driver.Create(%s)", dstfp) 244 | return 1 245 | 246 | ds = gdal.Open(iinfo.srcfp) 247 | if not ds: 248 | logger.error("BandSubtractMedian(): !gdal.Open(%s)", iinfo.srcfp) 249 | return 1 250 | 251 | out_ds.SetGeoTransform(ds.GetGeoTransform()) 252 | out_ds.SetProjection(ds.GetProjectionRef()) 253 | 254 | ## check if median was passed in, calculate if not 255 | try: 256 | keys = list(iinfo.median.keys()) 257 | except KeyError: 258 | iinfo.get_raster_median() 259 | keys = list(iinfo.median.keys()) 260 | 261 | keys.sort() 262 | for band in keys: 263 | band_median = iinfo.median[band] 264 | if band_median is not None: 265 | band_data = ds.GetRasterBand(band) 266 | band_nodata = band_data.GetNoDataValue() 267 | # default nodata to zero 268 | if band_nodata is None: 269 | logger.info("Defaulting band %i nodata to zero", band) 270 | band_nodata = 0.0 271 | band_array = numpy.array(band_data.ReadAsArray()) 272 | nodata_mask = (band_array == band_nodata) 273 | 274 | if out_datatype == 3: 275 | band_corrected = numpy.full_like(band_array, fill_value=out_nodataval, dtype=numpy.int16) 276 | else: 277 | band_corrected = numpy.full_like(band_array, fill_value=out_nodataval, dtype=numpy.int32) 278 | band_valid = band_array[~nodata_mask] 279 | if band_valid.size != 0: 280 | band_min = numpy.min(band_valid) 281 | corr_min = numpy.subtract(float(band_min), float(band_median)) 282 | if corr_min < float(out_min): 283 | logger.error("BandSubtractMedian() returns min out of range for %s band %i", iinfo.srcfp, band) 284 | return 1 285 | band_corrected[~nodata_mask] = numpy.subtract(band_array[~nodata_mask], band_median) 286 | else: 287 | logger.warning("Band %i has no valid data", band) 288 | out_band = out_ds.GetRasterBand(band) 289 | out_band.WriteArray(band_corrected) 290 | out_band.SetNoDataValue(out_nodataval) 291 | 292 | else: 293 | logger.error("BandSubtractMedian(): iinfo.median[%i] is None, image %s", band, iinfo.srcfp) 294 | return 1 295 | ds = None 296 | out_ds = None 297 | 298 | # ## redo pyramids -- WHY? 299 | # cmd = 'gdaladdo "%s" 2 4 8 16' %(srcfp) 300 | # taskhandler.exec_cmd(cmd) 301 | 302 | else: 303 | logger.info("BandSubtractMedian(): %s exists", dstfp) 304 | 305 | return 0 306 | 307 | 308 | if __name__ == '__main__': 309 | main() 310 | -------------------------------------------------------------------------------- /pgc_ndvi.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import argparse 4 | import logging 5 | import math 6 | import os 7 | import shutil 8 | import sys 9 | import datetime 10 | 11 | import numpy 12 | from osgeo import gdal 13 | 14 | from lib import ortho_functions, taskhandler, utils 15 | from lib import VERSION 16 | 17 | #### Create Loggers 18 | logger = logging.getLogger("logger") 19 | logger.setLevel(logging.DEBUG) 20 | 21 | outtypes = ['Float32', 'Int16'] 22 | 23 | def main(): 24 | 25 | #### Set Up Arguments 26 | parser = argparse.ArgumentParser( 27 | description="Run/Submit batch ndvi calculation in parallel" 28 | ) 29 | 30 | parser.add_argument("src", help="source image, text file, or directory") 31 | parser.add_argument("dst", help="destination directory") 32 | pos_arg_keys = ["src", "dst"] 33 | 34 | parser.add_argument("-t", "--outtype", choices=outtypes, default='Float32', 35 | help="output data type (for Int16, output values are scaled from -1000 to 1000)") 36 | parser.add_argument("-s", "--save-temps", action="store_true", default=False, 37 | help="save temp files") 38 | parser.add_argument("--wd", 39 | help="local working directory for cluster jobs (default is dst dir)") 40 | parser.add_argument("--pbs", action='store_true', default=False, 41 | help="submit tasks to PBS") 42 | parser.add_argument("--slurm", action='store_true', default=False, 43 | help="submit tasks to SLURM") 44 | parser.add_argument("--slurm-log-dir", default=None, 45 | help="directory path for logs from slurm jobs on the cluster. " 46 | "Default is the parent directory of the output. " 47 | "To use the current working directory, use 'working_dir'") 48 | parser.add_argument("--slurm-job-name", default=None, 49 | help="assign a name to the slurm job for easier job tracking") 50 | parser.add_argument("--parallel-processes", type=int, default=1, 51 | help="number of parallel processes to spawn (default 1)") 52 | parser.add_argument("--qsubscript", 53 | help="submission script to use in PBS/SLURM submission (PBS default is qsub_ndvi.sh, SLURM " 54 | "default is slurm_ndvi.py, in script root folder)") 55 | parser.add_argument("-l", help="PBS resources requested (mimicks qsub syntax, PBS only)") 56 | parser.add_argument("--log", nargs='?', const="default", 57 | help="output log file -- top level log is not written without this arg. " 58 | "when this flag is used, log will be written to ndvi_.log next to the ) " 59 | "unless a specific file path is provided here") 60 | parser.add_argument("--skip-cmd-txt", action='store_true', default=True, 61 | help='THIS OPTION IS DEPRECATED - ' 62 | 'By default this arg is True and the cmd text file will not be written. ' 63 | 'Input commands are written to the log for reference.') 64 | parser.add_argument("--dryrun", action="store_true", default=False, 65 | help="print actions without executing") 66 | parser.add_argument("--version", action='version', version="imagery_utils v{}".format(VERSION)) 67 | 68 | 69 | #### Parse Arguments 70 | args = parser.parse_args() 71 | scriptpath = os.path.abspath(sys.argv[0]) 72 | src = os.path.abspath(args.src) 73 | dstdir = os.path.abspath(args.dst) 74 | 75 | #### Validate Required Arguments 76 | if os.path.isdir(src): 77 | srctype = 'dir' 78 | elif os.path.isfile(src) and os.path.splitext(src)[1].lower() == '.txt': 79 | srctype = 'textfile' 80 | elif os.path.isfile(src) and os.path.splitext(src)[1].lower() in ortho_functions.exts: 81 | srctype = 'image' 82 | elif os.path.isfile(src.replace('msi', 'blu')) and os.path.splitext(src)[1].lower() in ortho_functions.exts: 83 | srctype = 'image' 84 | else: 85 | parser.error("Error arg1 is not a recognized file path or file type: {}".format(src)) 86 | 87 | if not os.path.isdir(dstdir): 88 | parser.error("Error arg2 is not a valid file path: {}".format(dstdir)) 89 | 90 | ## Verify qsubscript 91 | if args.pbs or args.slurm: 92 | if args.qsubscript is None: 93 | if args.pbs: 94 | qsubpath = os.path.join(os.path.dirname(scriptpath), 'qsub_ndvi.sh') 95 | if args.slurm: 96 | qsubpath = os.path.join(os.path.dirname(scriptpath), 'slurm_ndvi.sh') 97 | else: 98 | qsubpath = os.path.abspath(args.qsubscript) 99 | if not os.path.isfile(qsubpath): 100 | parser.error("qsub script path is not valid: {}".format(qsubpath)) 101 | 102 | # Parse slurm log location 103 | if args.slurm: 104 | # by default, the parent directory of the dst dir is used for saving slurm logs 105 | if args.slurm_log_dir == None: 106 | slurm_log_dir = os.path.abspath(os.path.join(dstdir, os.pardir)) 107 | # if "working_dir" is passed in the CLI, use the default slurm behavior which saves logs in working dir 108 | elif args.slurm_log_dir == "working_dir": 109 | slurm_log_dir = None 110 | # otherwise, verify that the path for the logs is a valid path 111 | else: 112 | slurm_log_dir = os.path.abspath(args.slurm_log_dir) 113 | # Verify slurm log path 114 | if not os.path.isdir(slurm_log_dir): 115 | parser.error("Error directory for slurm logs is not a valid file path: {}".format(slurm_log_dir)) 116 | 117 | ## Verify processing options do not conflict 118 | if args.pbs and args.slurm: 119 | parser.error("Options --pbs and --slurm are mutually exclusive") 120 | if (args.pbs or args.slurm) and args.parallel_processes > 1: 121 | parser.error("HPC Options (--pbs or --slurm) and --parallel-processes > 1 are mutually exclusive") 122 | 123 | #### Set concole logging handler 124 | lso = logging.StreamHandler() 125 | lso.setLevel(logging.DEBUG) 126 | formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s', '%m-%d-%Y %H:%M:%S') 127 | lso.setFormatter(formatter) 128 | logger.addHandler(lso) 129 | 130 | #### Configure file handler if --log is passed to CLI 131 | if args.log is not None: 132 | if args.log == "default": 133 | log_fn = "ndvi_{}.log".format(datetime.datetime.now().strftime("%Y%m%d_%H%M%S")) 134 | logfile = os.path.join(os.path.abspath(os.path.join(args.dst, os.pardir)), log_fn) 135 | else: 136 | logfile = os.path.abspath(args.log) 137 | if not os.path.isdir(os.path.pardir(logfile)): 138 | parser.warning("Output location for log file does not exist: {}".format(os.path.isdir(os.path.pardir(logfile)))) 139 | 140 | lfh = logging.FileHandler(logfile) 141 | lfh.setLevel(logging.DEBUG) 142 | formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s', '%m-%d-%Y %H:%M:%S') 143 | lfh.setFormatter(formatter) 144 | logger.addHandler(lfh) 145 | 146 | # log input command for reference 147 | command_str = ' '.join(sys.argv) 148 | logger.info("Running command: {}".format(command_str)) 149 | 150 | if args.slurm: 151 | logger.info("Slurm output and error log saved here: {}".format(slurm_log_dir)) 152 | 153 | #### Get args ready to pass to task handler 154 | arg_keys_to_remove = ('l', 'qsubscript', 'pbs', 'slurm', 'parallel_processes', 'dryrun') 155 | arg_str = taskhandler.convert_optional_args_to_string(args, pos_arg_keys, arg_keys_to_remove) 156 | 157 | ## Identify source images 158 | if srctype == 'dir': 159 | image_list = utils.find_images(src, False, ortho_functions.exts) 160 | elif srctype == 'textfile': 161 | image_list = utils.find_images(src, True, ortho_functions.exts) 162 | else: 163 | image_list = [src] 164 | logger.info('Number of src images: %i', len(image_list)) 165 | 166 | ## Build task queue 167 | i = 0 168 | task_queue = [] 169 | for srcfp in image_list: 170 | srcdir, srcfn = os.path.split(srcfp) 171 | bn, ext = os.path.splitext(srcfn) 172 | dstfp = os.path.join(dstdir, bn + '_ndvi.tif') 173 | 174 | if not os.path.isfile(dstfp): 175 | i += 1 176 | 177 | # add a custom name to the job 178 | if not args.slurm_job_name: 179 | job_name = 'NDVI{:04g}'.format(i) 180 | else: 181 | job_name = str(args.slurm_job_name) 182 | 183 | task = taskhandler.Task( 184 | srcfn, 185 | job_name, 186 | 'python', 187 | '{} {} {} {}'.format(scriptpath, arg_str, srcfp, dstdir), 188 | calc_ndvi, 189 | [srcfp, dstfp, args] 190 | ) 191 | task_queue.append(task) 192 | 193 | logger.info('Number of incomplete tasks: %i', i) 194 | 195 | ## Run tasks 196 | if len(task_queue) > 0: 197 | logger.info("Submitting Tasks") 198 | if args.pbs: 199 | l = "-l {}".format(args.l) if args.l else "" 200 | try: 201 | task_handler = taskhandler.PBSTaskHandler(qsubpath, l) 202 | except RuntimeError as e: 203 | logger.error(utils.capture_error_trace()) 204 | logger.error(e) 205 | else: 206 | if not args.dryrun: 207 | task_handler.run_tasks(task_queue) 208 | 209 | elif args.slurm: 210 | qsub_args = "" 211 | if not slurm_log_dir == None: 212 | qsub_args += '-o {}/%x.o%j '.format(slurm_log_dir) 213 | qsub_args += '-e {}/%x.o%j '.format(slurm_log_dir) 214 | # adjust wallclock if submitting multiple tasks ro be run in serial for a single slurm job 215 | # default wallclock for ortho jobs is 1:00:00, refer to slurm_ndvi.sh to verify 216 | if args.tasks_per_job: 217 | qsub_args += '-t {}:00:00 '.format(args.tasks_per_job) 218 | try: 219 | task_handler = taskhandler.SLURMTaskHandler(qsubpath, qsub_args) 220 | except RuntimeError as e: 221 | logger.error(utils.capture_error_trace()) 222 | logger.error(e) 223 | else: 224 | if not args.dryrun: 225 | task_handler.run_tasks(task_queue) 226 | 227 | elif args.parallel_processes > 1: 228 | try: 229 | task_handler = taskhandler.ParallelTaskHandler(args.parallel_processes) 230 | except RuntimeError as e: 231 | logger.error(utils.capture_error_trace()) 232 | logger.error(e) 233 | else: 234 | logger.info("Number of child processes to spawn: %i", task_handler.num_processes) 235 | if not args.dryrun: 236 | task_handler.run_tasks(task_queue) 237 | 238 | else: 239 | results = {} 240 | for task in task_queue: 241 | 242 | srcfp, dstfp, task_arg_obj = task.method_arg_list 243 | 244 | #### Set up processing log handler 245 | logfile = os.path.splitext(dstfp)[0] + ".log" 246 | lfh = logging.FileHandler(logfile) 247 | lfh.setLevel(logging.DEBUG) 248 | formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s', '%m-%d-%Y %H:%M:%S') 249 | lfh.setFormatter(formatter) 250 | logger.addHandler(lfh) 251 | 252 | if not args.dryrun: 253 | results[task.name] = task.method(srcfp, dstfp, task_arg_obj) 254 | 255 | #### remove existing file handler 256 | logger.removeHandler(lfh) 257 | 258 | #### Print Images with Errors 259 | for k, v in results.items(): 260 | if v != 0: 261 | logger.warning("Failed Image: %s", k) 262 | 263 | logger.info("Done") 264 | 265 | else: 266 | logger.info("No images found to process") 267 | 268 | 269 | def calc_ndvi(srcfp, dstfp, args): 270 | 271 | # ndvi nodata value 272 | ndvi_nodata = -9999 273 | 274 | # tolerance for floating point equality 275 | tol = 0.00001 276 | 277 | # get basenames for src and dst files, get xml metadata filenames 278 | srcdir, srcfn = os.path.split(srcfp) 279 | dstdir, dstfn = os.path.split(dstfp) 280 | bn, ext = os.path.splitext(srcfn) 281 | src_xml = os.path.join(srcdir, bn + '.xml') 282 | dst_xml = os.path.join(dstdir, bn + '_ndvi.xml') 283 | 284 | #### Get working dir 285 | if args.wd is not None: 286 | wd = args.wd 287 | else: 288 | wd = dstdir 289 | if not os.path.isdir(wd): 290 | try: 291 | os.makedirs(wd) 292 | except OSError: 293 | pass 294 | logger.info("Working Dir: %s", wd) 295 | 296 | print("Image: {}".format(srcfn)) 297 | 298 | ## copy source image to working directory 299 | srcfp_local = os.path.join(wd, srcfn) 300 | if not os.path.isfile(srcfp_local): 301 | shutil.copy2(srcfp, srcfp_local) 302 | 303 | ## open image and get band numbers 304 | ds = gdal.Open(srcfp_local) 305 | if ds: 306 | bands = ds.RasterCount 307 | if bands == 8: 308 | red_band_num = 5 309 | nir_band_num = 7 310 | elif bands == 4: 311 | red_band_num = 3 312 | nir_band_num = 4 313 | else: 314 | logger.error("Cannot calculate NDVI from a %i band image: %s", bands, srcfp_local) 315 | clean_up([srcfp_local]) 316 | return 1 317 | else: 318 | logger.error("Cannot open target image: %s", srcfp_local) 319 | clean_up([srcfp_local]) 320 | return 1 321 | 322 | ## check for input data type - must be float or int 323 | datatype = ds.GetRasterBand(1).DataType 324 | if datatype not in [1, 2, 3, 4, 5, 6, 7]: 325 | logger.error("Invalid input data type %s", datatype) 326 | clean_up([srcfp_local]) 327 | return 1 328 | 329 | ## get the raster dimensions 330 | nx = ds.RasterXSize 331 | ny = ds.RasterYSize 332 | 333 | ## open output file for write and copy proj/geotransform info 334 | if not os.path.isfile(dstfp): 335 | dstfp_local = os.path.join(wd, os.path.basename(dstfp)) 336 | gtiff_options = ['TILED=YES', 'COMPRESS=LZW', 'BIGTIFF=YES'] 337 | driver = gdal.GetDriverByName('GTiff') 338 | out_ds = driver.Create(dstfp_local, nx, ny, 1, gdal.GetDataTypeByName(args.outtype), gtiff_options) 339 | if out_ds: 340 | out_ds.SetGeoTransform(ds.GetGeoTransform()) 341 | out_ds.SetProjection(ds.GetProjection()) 342 | ndvi_band = out_ds.GetRasterBand(1) 343 | ndvi_band.SetNoDataValue(float(ndvi_nodata)) 344 | else: 345 | logger.error("Couldn't open for write: %s", dstfp_local) 346 | clean_up([srcfp_local]) 347 | return 1 348 | 349 | ## for red and nir bands, get band data, nodata values, and natural block size 350 | ## if NoData is None default it to zero. 351 | red_band = ds.GetRasterBand(red_band_num) 352 | if red_band is None: 353 | logger.error("Can't load band %i from %s", red_band_num, srcfp_local) 354 | clean_up([srcfp_local]) 355 | return 1 356 | red_nodata = red_band.GetNoDataValue() 357 | if red_nodata is None: 358 | logger.info("Defaulting red band nodata to zero") 359 | red_nodata = 0.0 360 | (red_xblocksize, red_yblocksize) = red_band.GetBlockSize() 361 | 362 | nir_band = ds.GetRasterBand(nir_band_num) 363 | if nir_band is None: 364 | logger.error("Can't load band %i from %s", nir_band_num, srcfp_local) 365 | clean_up([srcfp_local]) 366 | return 1 367 | nir_nodata = nir_band.GetNoDataValue() 368 | if nir_nodata is None: 369 | logger.info("Defaulting nir band nodata to zero") 370 | nir_nodata = 0.0 371 | (nir_xblocksize, nir_yblocksize) = nir_band.GetBlockSize() 372 | 373 | ## if different block sizes choose the smaller of the two 374 | xblocksize = min([red_xblocksize, nir_xblocksize]) 375 | yblocksize = min([red_yblocksize, nir_yblocksize]) 376 | 377 | ## calculate the number of x and y blocks to read/write 378 | nxblocks = int(math.floor(nx + xblocksize - 1) / xblocksize) 379 | nyblocks = int(math.floor(ny + yblocksize - 1) / yblocksize) 380 | 381 | ## blocks loop 382 | yblockrange = range(nyblocks) 383 | xblockrange = range(nxblocks) 384 | for yblock in yblockrange: 385 | ## y offset for ReadAsArray 386 | yoff = yblock * yblocksize 387 | 388 | ## get block actual y size in case of partial block at edge 389 | if yblock < nyblocks - 1: 390 | block_ny = yblocksize 391 | else: 392 | block_ny = ny - (yblock * yblocksize) 393 | 394 | for xblock in xblockrange: 395 | ## x offset for ReadAsArray 396 | xoff = xblock * xblocksize 397 | 398 | ## get block actual x size in case of partial block at edge 399 | if xblock < (nxblocks - 1): 400 | block_nx = xblocksize 401 | else: 402 | block_nx = nx - (xblock * xblocksize) 403 | 404 | ## read a block from each band 405 | red_array = red_band.ReadAsArray(xoff, yoff, block_nx, block_ny) 406 | nir_array = nir_band.ReadAsArray(xoff, yoff, block_nx, block_ny) 407 | 408 | ## generate mask for red nodata, nir nodata, and 409 | ## (red+nir) less than tol away from zero 410 | red_mask = (red_array == red_nodata) 411 | if red_array[red_mask].size > 0: 412 | nir_mask = (nir_array == nir_nodata) 413 | if nir_array[nir_mask].size > 0: 414 | divzero_mask = abs(nir_array + red_array) < tol 415 | if red_array[divzero_mask].size > 0: 416 | ndvi_mask = red_mask | nir_mask | divzero_mask 417 | else: 418 | ndvi_mask = red_mask | nir_mask 419 | else: 420 | divzero_mask = abs(nir_array + red_array) < tol 421 | if red_array[divzero_mask].size > 0: 422 | ndvi_mask = red_mask | divzero_mask 423 | else: 424 | ndvi_mask = red_mask 425 | else: 426 | nir_mask = (nir_array == nir_nodata) 427 | if nir_array[nir_mask].size > 0: 428 | divzero_mask = abs(nir_array + red_array) < tol 429 | if red_array[divzero_mask].size > 0: 430 | ndvi_mask = nir_mask | divzero_mask 431 | else: 432 | ndvi_mask = nir_mask 433 | else: 434 | divzero_mask = abs(nir_array + red_array) < tol 435 | if red_array[divzero_mask].size > 0: 436 | ndvi_mask = divzero_mask 437 | else: 438 | ndvi_mask = numpy.full_like(red_array, fill_value=0, dtype=bool) 439 | 440 | ## declare ndvi array, init to nodata value 441 | ndvi_array = numpy.full_like(red_array, fill_value=ndvi_nodata, dtype=numpy.float32) 442 | ## cast bands to float for calc 443 | red_asfloat = numpy.array(red_array, dtype=numpy.float32) 444 | red_array = None 445 | nir_asfloat = numpy.array(nir_array, dtype=numpy.float32) 446 | nir_array = None 447 | 448 | ## calculate ndvi 449 | if ndvi_array[~ndvi_mask].size > 0: 450 | ndvi_array[~ndvi_mask] = numpy.divide(numpy.subtract(nir_asfloat[~ndvi_mask], 451 | red_asfloat[~ndvi_mask]), 452 | numpy.add(nir_asfloat[~ndvi_mask], 453 | red_asfloat[~ndvi_mask])) 454 | red_asfloat = None 455 | nir_asfloat = None 456 | 457 | ## scale and cast to int if outtype integer 458 | if args.outtype == 'Int16': 459 | ndvi_scaled = numpy.full_like(ndvi_array, fill_value=ndvi_nodata, dtype=numpy.int16) 460 | if ndvi_scaled[~ndvi_mask].size > 0: 461 | ndvi_scaled[~ndvi_mask] = numpy.array(ndvi_array[~ndvi_mask]*1000.0, dtype=numpy.int16) 462 | ndvi_array = ndvi_scaled 463 | ndvi_scaled = None 464 | 465 | ndvi_mask = None 466 | 467 | ## write valid portion of ndvi array to output file 468 | ndvi_band.WriteArray(ndvi_array, xoff, yoff) 469 | ndvi_array = None 470 | 471 | out_ds = None 472 | ds = None 473 | 474 | if os.path.isfile(dstfp_local): 475 | ## add pyramids 476 | cmd = 'gdaladdo "{}" 2 4 8 16'.format(dstfp_local) 477 | taskhandler.exec_cmd(cmd) 478 | 479 | ## copy to dst 480 | if wd != dstdir: 481 | shutil.copy2(dstfp_local, dstfp) 482 | 483 | ## copy xml to dst 484 | if os.path.isfile(src_xml): 485 | shutil.copy2(src_xml, dst_xml) 486 | else: 487 | logger.warning("xml %s not found", src_xml) 488 | 489 | ## Delete Temp Files 490 | temp_files = [srcfp_local] 491 | wd_files = [dstfp_local] 492 | if not args.save_temps: 493 | clean_up(temp_files) 494 | if wd != dstdir: 495 | clean_up(wd_files) 496 | else: 497 | logger.error("pgc_ndvi.py: %s was not created", dstfp_local) 498 | return 1 499 | 500 | else: 501 | logger.info("pgc_ndvi.py: file %s already exists", dstfp) 502 | 503 | ## copy xml to dst if missing 504 | if not os.path.isfile(dst_xml): 505 | shutil.copy2(src_xml, dst_xml) 506 | 507 | return 0 508 | 509 | def clean_up(filelist): 510 | for f in filelist: 511 | try: 512 | os.remove(f) 513 | except Exception as e: 514 | logger.error(utils.capture_error_trace()) 515 | logger.warning('Could not remove %s: %s', os.path.basename(f), e) 516 | 517 | 518 | if __name__ == '__main__': 519 | main() 520 | -------------------------------------------------------------------------------- /pgc_ortho.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from __future__ import division 4 | 5 | import argparse 6 | import logging 7 | import math 8 | import os 9 | import sys 10 | import xml.etree.ElementTree as ET 11 | import datetime 12 | 13 | import numpy as np 14 | 15 | from lib import ortho_functions, taskhandler, utils 16 | from lib.taskhandler import argval2str 17 | 18 | #### Create Loggers 19 | logger = logging.getLogger("logger") 20 | logger.setLevel(logging.DEBUG) 21 | 22 | ARGDEF_SCRATCH = os.path.join(os.path.expanduser('~'), 'scratch', 'task_bundles') 23 | 24 | 25 | def main(): 26 | ret_code = 0 27 | 28 | #### Set Up Arguments 29 | parent_parser, pos_arg_keys = ortho_functions.build_parent_argument_parser() 30 | parser = argparse.ArgumentParser( 31 | parents=[parent_parser], 32 | description="Run/submit batch image ortho and conversion tasks" 33 | ) 34 | 35 | parser.add_argument("--pbs", action='store_true', default=False, 36 | help="submit tasks to PBS") 37 | parser.add_argument("--slurm", action='store_true', default=False, 38 | help="submit tasks to SLURM") 39 | parser.add_argument("--slurm-log-dir", default=None, 40 | help="directory path for logs from slurm jobs on the cluster. " 41 | "Default is the parent directory of the output. " 42 | "To use the current working directory, use 'working_dir'") 43 | parser.add_argument("--slurm-job-name", default=None, 44 | help="assign a name to the slurm job for easier job tracking") 45 | parser.add_argument("--tasks-per-job", type=int, 46 | help="Number of tasks to bundle into a single job. (requires --pbs or --slurm option) (Warning:" 47 | " a higher number of tasks per job may require modification of default wallclock limit.)") 48 | parser.add_argument('--scratch', default=ARGDEF_SCRATCH, 49 | help="Scratch space to build task bundle text files. (default={})".format(ARGDEF_SCRATCH)) 50 | parser.add_argument("--parallel-processes", type=int, default=1, 51 | help="number of parallel processes to spawn (default 1)") 52 | parser.add_argument("--qsubscript", 53 | help="submission script to use in PBS/SLURM submission (PBS default is qsub_ortho.sh, SLURM " 54 | "default is slurm_ortho.py, in script root folder)") 55 | parser.add_argument("-l", 56 | help="PBS resources requested (mimicks qsub syntax, PBS only)") 57 | parser.add_argument("--queue", 58 | help="Cluster queue/partition to submit jobs to. Accepted slurm queues: batch (default " 59 | "partition, no need to specify it in this arg), big_mem (for large memory jobs), " 60 | "and low_priority (for background processes)") 61 | parser.add_argument("--log", nargs='?', const="default", 62 | help="output log file -- top level log is not written without this arg. " 63 | "when this flag is used, log will be written to ortho_.log next to the ) " 64 | "unless a specific file path is provided here") 65 | parser.add_argument("--dryrun", action='store_true', default=False, 66 | help='print actions without executing') 67 | parser.add_argument("-v", "--verbose", action='store_true', default=False, 68 | help='log debug messages') 69 | 70 | #### Parse Arguments 71 | args = parser.parse_args() 72 | scriptpath = os.path.abspath(sys.argv[0]) 73 | src = os.path.abspath(args.src) 74 | dstdir = os.path.abspath(args.dst) 75 | args.scratch = os.path.abspath(args.scratch) 76 | args.dst = dstdir 77 | 78 | #### Validate Required Arguments 79 | if os.path.isdir(src): 80 | srctype = 'dir' 81 | elif os.path.isfile(src) and os.path.splitext(src)[1].lower() == '.txt': 82 | srctype = 'textfile' 83 | elif os.path.isfile(src) and os.path.splitext(src)[1].lower() == '.csv': 84 | srctype = 'csvfile' 85 | elif os.path.isfile(src) and os.path.splitext(src)[1].lower() in ortho_functions.exts: 86 | srctype = 'image' 87 | elif os.path.isfile(src.replace('msi', 'blu')) and os.path.splitext(src)[1].lower() in ortho_functions.exts: 88 | srctype = 'image' 89 | else: 90 | parser.error("Error arg1 is not a recognized file path or file type: {}".format(src)) 91 | 92 | if not os.path.isdir(dstdir): 93 | parser.error("Error arg2 is not a valid file path: {}".format(dstdir)) 94 | 95 | 96 | ## Verify qsubscript 97 | if args.pbs or args.slurm: 98 | if args.qsubscript is None: 99 | if args.pbs: 100 | qsubpath = os.path.join(os.path.dirname(scriptpath), 'qsub_ortho.sh') 101 | if args.slurm: 102 | qsubpath = os.path.join(os.path.dirname(scriptpath), 'slurm_ortho.sh') 103 | else: 104 | qsubpath = os.path.abspath(args.qsubscript) 105 | if not os.path.isfile(qsubpath): 106 | parser.error("qsub script path is not valid: {}".format(qsubpath)) 107 | 108 | # Parse slurm log location 109 | if args.slurm: 110 | # by default, the parent directory of the dst dir is used for saving slurm logs 111 | if args.slurm_log_dir == None: 112 | slurm_log_dir = os.path.abspath(os.path.join(dstdir, os.pardir)) 113 | # if "working_dir" is passed in the CLI, use the default slurm behavior which saves logs in working dir 114 | elif args.slurm_log_dir == "working_dir": 115 | slurm_log_dir = None 116 | # otherwise, verify that the path for the logs is a valid path 117 | else: 118 | slurm_log_dir = os.path.abspath(args.slurm_log_dir) 119 | # check that partition names are valid 120 | if args.queue and not args.queue in ortho_functions.slurm_partitions: 121 | parser.error("--queue argument '{}' is not a valid slurm partition. " 122 | "Valid partitions: {}".format(args.queue, 123 | ortho_functions.slurm_partitions)) 124 | # Verify slurm log path 125 | if not os.path.isdir(slurm_log_dir): 126 | parser.error("Error directory for slurm logs is not a valid file path: {}".format(slurm_log_dir)) 127 | 128 | ## Verify processing options do not conflict 129 | requested_threads = ortho_functions.ARGDEF_CPUS_AVAIL if args.threads == "ALL_CPUS" else args.threads 130 | if args.pbs and args.slurm: 131 | parser.error("Options --pbs and --slurm are mutually exclusive") 132 | if (args.pbs or args.slurm) and args.parallel_processes > 1: 133 | parser.error("HPC Options (--pbs or --slurm) and --parallel-processes > 1 are mutually exclusive") 134 | if (args.pbs or args.slurm) and requested_threads > 1: 135 | parser.error("HPC Options (--pbs or --slurm) and --threads > 1 are mutually exclusive") 136 | if requested_threads < 1: 137 | parser.error("--threads count must be positive, nonzero integer or ALL_CPUS") 138 | if args.parallel_processes > 1: 139 | total_proc_count = requested_threads * args.parallel_processes 140 | if total_proc_count > ortho_functions.ARGDEF_CPUS_AVAIL: 141 | parser.error("the (threads * number of processes requested) ({0}) exceeds number of available threads " 142 | "({1}); reduce --threads and/or --parallel-processes count" 143 | .format(total_proc_count, ortho_functions.ARGDEF_CPUS_AVAIL)) 144 | 145 | if args.tasks_per_job: 146 | if not (args.pbs or args.slurm): 147 | parser.error("--tasks-per-job option requires the (--pbs or --slurm) option") 148 | if not os.path.isdir(args.scratch): 149 | print("Creating --scratch directory: {}".format(args.scratch)) 150 | os.makedirs(args.scratch) 151 | 152 | #### Verify EPSG 153 | spatial_ref = None 154 | if srctype == 'csvfile' and args.epsg is None: 155 | # Check for valid EPSG argument in CSV argument list file 156 | pass 157 | elif args.epsg is None: 158 | parser.error("--epsg argument is required") 159 | elif args.epsg in ('utm', 'auto'): 160 | # EPSG code is automatically determined in ortho_functions.get_image_stats function 161 | pass 162 | else: 163 | try: 164 | args.epsg = int(args.epsg) 165 | except ValueError: 166 | parser.error("--epsg must be 'utm', 'auto', or an integer EPSG code") 167 | try: 168 | spatial_ref = utils.SpatialRef(args.epsg) 169 | except RuntimeError as e: 170 | parser.error(e) 171 | 172 | #### Verify that dem and ortho_height are not both specified 173 | if args.dem is not None and args.ortho_height is not None: 174 | parser.error("--dem and --ortho_height options are mutually exclusive. Please choose only one.") 175 | 176 | #### Test if DEM exists 177 | if not args.dem == "auto": 178 | if args.dem is not None and not os.path.isfile(args.dem): 179 | parser.error("DEM does not exist: {}".format(args.dem)) 180 | 181 | ## Check the correct number of values are supplied for --resolution 182 | if args.resolution and len(args.resolution) > 2: 183 | parser.error("--resolution option requires one or two values") 184 | 185 | #### Set up console logging handler 186 | if args.verbose: 187 | lso_log_level = logging.DEBUG 188 | else: 189 | lso_log_level = logging.INFO 190 | lso = logging.StreamHandler() 191 | lso.setLevel(lso_log_level) 192 | formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s', '%m-%d-%Y %H:%M:%S') 193 | lso.setFormatter(formatter) 194 | logger.addHandler(lso) 195 | 196 | #### Configure file handler if --log is passed to CLI 197 | if args.log is not None: 198 | if args.log == "default": 199 | log_fn = "ortho_{}.log".format(datetime.datetime.now().strftime("%Y%m%d_%H%M%S")) 200 | logfile = os.path.join(os.path.abspath(os.path.join(args.dst, os.pardir)), log_fn) 201 | else: 202 | logfile = os.path.abspath(args.log) 203 | if not os.path.isdir(os.path.pardir(logfile)): 204 | parser.warning("Output location for log file does not exist: {}".format(os.path.isdir(os.path.pardir(logfile)))) 205 | 206 | lfh = logging.FileHandler(logfile) 207 | lfh.setLevel(logging.DEBUG) 208 | formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s', '%m-%d-%Y %H:%M:%S') 209 | lfh.setFormatter(formatter) 210 | logger.addHandler(lfh) 211 | 212 | # log input command for reference 213 | command_str = ' '.join(sys.argv) 214 | logger.info("Running command: {}".format(command_str)) 215 | 216 | if args.slurm: 217 | logger.info("Slurm output and error log saved here: {}".format(slurm_log_dir)) 218 | 219 | #### Handle thread count that exceeds system limits 220 | if requested_threads > ortho_functions.ARGDEF_CPUS_AVAIL: 221 | logger.info("threads requested ({0}) exceeds number available on system ({1}), setting thread count to " 222 | "'ALL_CPUS'".format(requested_threads, ortho_functions.ARGDEF_CPUS_AVAIL)) 223 | args.threads = 'ALL_CPUS' 224 | 225 | #### Get args ready to pass to task handler 226 | arg_keys_to_remove = ('l', 'queue', 'qsubscript', 'dryrun', 'pbs', 'slurm', 'parallel_processes', 'tasks_per_job') 227 | 228 | ## Identify source images 229 | csv_arg_data = None 230 | csv_header_argname_list = None 231 | csv_src_array = None 232 | if srctype == 'dir': 233 | image_list1 = utils.find_images(src, False, ortho_functions.exts) 234 | elif srctype == 'textfile': 235 | image_list1 = utils.find_images(src, True, ortho_functions.exts) 236 | elif srctype == 'csvfile': 237 | # Load CSV data 238 | csv_arg_data = np.char.strip(np.loadtxt(src, dtype=str, delimiter=',', encoding="utf-8-sig"), '\'"') 239 | csv_header = csv_arg_data[0, :] 240 | 241 | # Adjust CSV header argument names 242 | # Support ArcGIS export of typical fp.py output 243 | if 'OID_' in csv_header: 244 | csv_arg_data = np.delete(csv_arg_data, np.where(csv_header == 'OID_')[0], axis=1) 245 | csv_header = csv_arg_data[0, :] 246 | if 'O_FILEPATH' in csv_header: 247 | csv_header[csv_header == 'O_FILEPATH'] = 'src' 248 | if 'o_filepath' in csv_header: 249 | csv_header[csv_header == 'o_filepath'] = 'src' 250 | 251 | # Convert CSV header argument names to argparse namespace variable format 252 | csv_header_argname_list = [argname.lstrip('-').replace('-', '_').lower() for argname in csv_header] 253 | 254 | # Remove header row 255 | csv_arg_data = np.delete(csv_arg_data, 0, axis=0) 256 | 257 | # Verify CSV arguments and values 258 | if len(csv_header_argname_list) >= 1 and 'src' in csv_header_argname_list: 259 | pass 260 | else: 261 | parser.error("'src' should be the header of the first column of source CSV argument list file") 262 | if 'epsg' in csv_header_argname_list: 263 | csv_epsg_array = csv_arg_data[:, csv_header_argname_list.index('epsg')].astype(int) 264 | invalid_epsg_code = False 265 | for epsg_code in np.unique(csv_epsg_array): 266 | try: 267 | utils.SpatialRef(epsg_code) 268 | except Exception: 269 | logger.error(utils.capture_error_trace()) 270 | invalid_epsg_code = True 271 | if invalid_epsg_code: 272 | parser.error("Source CSV argument list file contains invalid EPSG code(s)") 273 | elif args.epsg is None: 274 | parser.error("A valid EPSG argument must be specified") 275 | 276 | # Create subsets of VRT DEM and trim CSV data if applicable 277 | if args.dem is not None and args.dem.endswith('.vrt') and 'dem' in csv_header_argname_list: 278 | csv_arg_data = utils.subset_vrt_dem(csv_arg_data, csv_header_argname_list, args) 279 | 280 | # Extract src image paths and send to utils.find_images 281 | csv_src_array = csv_arg_data[:, csv_header_argname_list.index('src')] 282 | csv_src_list = csv_src_array.tolist() 283 | if len(csv_src_list) != len(set(csv_src_list)): 284 | parser.error("Detected duplicate 'src' items in CSV argument list file." 285 | " If meaning to use the subset-VRT-DEM functionality, you must" 286 | " provide the path to the full tileset VRT DEM file (.vrt extension)" 287 | " with the -d argument.") 288 | image_list1 = utils.find_images(csv_src_array.tolist(), True, ortho_functions.exts) 289 | 290 | # Trim CSV data to intersection with found image paths 291 | _, _, csv_rows_src_found = np.intersect1d(np.asarray(image_list1), csv_src_array, return_indices=True) 292 | csv_arg_data = csv_arg_data[csv_rows_src_found, :] 293 | csv_src_array = csv_arg_data[:, csv_header_argname_list.index('src')] 294 | assert set(csv_src_array) == set(image_list1) 295 | else: 296 | image_list1 = [src] 297 | 298 | ## Group Ikonos 299 | image_list2 = [] 300 | for i, srcfp in enumerate(image_list1): 301 | srcdir, srcfn = os.path.split(srcfp) 302 | if "IK01" in srcfn and sum([b in srcfn for b in ortho_functions.ikMsiBands]) > 0: 303 | for b in ortho_functions.ikMsiBands: 304 | if b in srcfn: 305 | newname = os.path.join(srcdir, srcfn.replace(b, "msi")) 306 | break 307 | image_list2.append(newname) 308 | if srctype == 'csvfile': 309 | # The csv_src_array is a slice/window into the larger CSV data array; 310 | # modifications are carried through to the larger CSV data array. 311 | csv_src_array[i] = newname 312 | 313 | else: 314 | image_list2.append(srcfp) 315 | 316 | image_list = list(set(image_list2)) 317 | logger.info('Number of src images: %i', len(image_list)) 318 | if len(image_list) == 0: 319 | logger.info("No images found to process") 320 | sys.exit(0) 321 | 322 | if srctype == 'csvfile': 323 | # Trim CSV data to intersection with updated image path names 324 | # (the number of source images should not have changed, so this 325 | # is mainly a check that changes to any image names were also 326 | # properly applied to the CSV data array). 327 | _, _, csv_rows_to_keep = np.intersect1d(np.asarray(image_list), csv_src_array, return_indices=True) 328 | csv_arg_data = csv_arg_data[csv_rows_to_keep, :] 329 | csv_src_array = csv_arg_data[:, csv_header_argname_list.index('src')] 330 | assert set(csv_src_array) == set(image_list) 331 | # Use the CSV argument array in place of the standard image list 332 | image_list = csv_arg_data 333 | 334 | ## Build task queue 335 | images_to_process = [] 336 | image_info_dict = {} 337 | for task_args in utils.yield_task_args(image_list, args, 338 | argname_1D='src', 339 | argname_2D_list=csv_header_argname_list): 340 | srcfp = task_args.src 341 | dstdir = task_args.dst 342 | lso.setLevel(logging.WARNING) # temporarily reduce logging level to limit excess terminal text 343 | try: 344 | info = ortho_functions.ImageInfo(srcfp, dstdir, args.wd, args) 345 | except Exception as e: 346 | logger.error(e) 347 | else: 348 | lso.setLevel(lso_log_level) 349 | dstfp = info.dstfp 350 | vrtfile1 = os.path.splitext(dstfp)[0] + "_raw.vrt" 351 | vrtfile2 = os.path.splitext(dstfp)[0] + "_vrt.vrt" 352 | 353 | # Check to see if raw.vrt or vrt.vrt are present 354 | vrt_exists = os.path.isfile(vrtfile1) or os.path.isfile(vrtfile2) 355 | tif_done = os.path.isfile(dstfp) 356 | # If no tif file present, need to make one 357 | # If tif file is present but one of the vrt files is present, need to rebuild 358 | if (not tif_done) or vrt_exists: 359 | images_to_process.append(srcfp) 360 | image_info_dict[srcfp] = info 361 | 362 | logger.info("Number of incomplete tasks: %i", len(images_to_process)) 363 | if len(images_to_process) == 0: 364 | logger.info("No incomplete tasks to process") 365 | sys.exit(0) 366 | 367 | task_queue = [] 368 | 369 | if srctype == 'csvfile': 370 | # Trim CSV data to intersection with images yet to process 371 | _, _, csv_rows_to_process = np.intersect1d(np.asarray(images_to_process), csv_src_array, return_indices=True) 372 | csv_arg_data = csv_arg_data[csv_rows_to_process, :] 373 | csv_src_array = csv_arg_data[:, csv_header_argname_list.index('src')] 374 | assert set(csv_src_array) == set(images_to_process) 375 | # Use the CSV argument array in place of the standard image list 376 | images_to_process = csv_arg_data 377 | 378 | ## Bundle tasks into sets by the number of tasks-per-job 379 | if args.tasks_per_job and args.tasks_per_job > 1: 380 | task_srcfp_list = utils.write_task_bundles( 381 | images_to_process, args.tasks_per_job, args.scratch, 'Or_src', 382 | header_list=csv_header_argname_list, bundle_ext=('csv' if srctype == 'csvfile' else 'txt') 383 | ) 384 | else: 385 | task_srcfp_list = images_to_process 386 | 387 | ## Build task objects 388 | for job_count, task_args in enumerate( 389 | utils.yield_task_args(task_srcfp_list, args, 390 | argname_1D='src', 391 | argname_2D_list=csv_header_argname_list), 392 | 1): 393 | arg_str_base = taskhandler.convert_optional_args_to_string(task_args, pos_arg_keys, arg_keys_to_remove) 394 | srcfp = task_args.src 395 | dstdir = task_args.dst 396 | srcdir, srcfn = os.path.split(srcfp) 397 | 398 | ## If task_srcfp_list = images_to_process, then the image_info_dict is also populated 399 | if task_srcfp_list is images_to_process: 400 | info = image_info_dict[srcfp] 401 | dstfp = info.dstfp 402 | else: # this case occurs when there is a textfile or csv to resubmit so dstfp is not needed 403 | dstfp = None 404 | 405 | # add a custom name to the job 406 | if not args.slurm_job_name: 407 | job_name = 'Or{:04g}'.format(job_count) 408 | else: 409 | job_name = str(args.slurm_job_name) 410 | 411 | task = taskhandler.Task( 412 | srcfn, 413 | job_name, 414 | 'python', 415 | '{} {} {} {}'.format( 416 | argval2str(scriptpath), 417 | arg_str_base, 418 | argval2str(srcfp), 419 | argval2str(dstdir) 420 | ), 421 | ortho_functions.process_image, 422 | [srcfp, dstfp, task_args] 423 | ) 424 | task_queue.append(task) 425 | 426 | ## Run tasks 427 | if len(task_queue) > 0: 428 | logger.info("Submitting Tasks") 429 | if args.pbs: 430 | qsub_args = "" 431 | if args.l: 432 | qsub_args += " -l {}".format(args.l) 433 | if args.queue: 434 | qsub_args += " -q {}".format(args.queue) 435 | try: 436 | task_handler = taskhandler.PBSTaskHandler(qsubpath, qsub_args) 437 | except RuntimeError as e: 438 | logger.error(utils.capture_error_trace()) 439 | logger.error(e) 440 | else: 441 | if not args.dryrun: 442 | task_handler.run_tasks(task_queue, dryrun=args.dryrun) 443 | 444 | elif args.slurm: 445 | qsub_args = "" 446 | if not slurm_log_dir == None: 447 | qsub_args += '-o {}/%x.o%j '.format(slurm_log_dir) 448 | qsub_args += '-e {}/%x.o%j '.format(slurm_log_dir) 449 | # adjust wallclock if submitting multiple tasks ro be run in serial for a single slurm job 450 | # default wallclock for ortho jobs is 1:00:00, refer to slurm_ortho.sh to verify 451 | if args.tasks_per_job: 452 | qsub_args += '-t {}:00:00 '.format(args.tasks_per_job) 453 | if args.queue: 454 | qsub_args += "-p {} ".format(args.queue) 455 | try: 456 | task_handler = taskhandler.SLURMTaskHandler(qsubpath, qsub_args) 457 | except RuntimeError as e: 458 | logger.error(utils.capture_error_trace()) 459 | logger.error(e) 460 | else: 461 | if not args.dryrun: 462 | task_handler.run_tasks(task_queue) 463 | 464 | elif args.parallel_processes > 1: 465 | try: 466 | task_handler = taskhandler.ParallelTaskHandler(args.parallel_processes) 467 | except RuntimeError as e: 468 | logger.error(utils.capture_error_trace()) 469 | logger.error(e) 470 | else: 471 | logger.info("Number of child processes to spawn: %i", task_handler.num_processes) 472 | if not args.dryrun: 473 | task_handler.run_tasks(task_queue) 474 | 475 | else: 476 | 477 | results = {} 478 | for task in task_queue: 479 | 480 | src, dstfp, task_arg_obj = task.method_arg_list 481 | 482 | #### Set up processing log handler 483 | logfile = os.path.splitext(dstfp)[0] + ".log" 484 | lfh = logging.FileHandler(logfile) 485 | lfh.setLevel(logging.DEBUG) 486 | formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s', '%m-%d-%Y %H:%M:%S') 487 | lfh.setFormatter(formatter) 488 | logger.addHandler(lfh) 489 | 490 | if not args.dryrun: 491 | results[task.name] = task.method(src, dstfp, task_arg_obj) 492 | 493 | #### remove existing file handler 494 | logger.removeHandler(lfh) 495 | 496 | #### Print Images with Errors 497 | for k, v in results.items(): 498 | if v != 0: 499 | logger.warning("Failed Image: %s", k) 500 | ret_code = 1 501 | 502 | logger.info("Done") 503 | 504 | else: 505 | logger.info("No images found to process") 506 | 507 | sys.exit(ret_code) 508 | 509 | 510 | if __name__ == "__main__": 511 | main() 512 | -------------------------------------------------------------------------------- /qsub_mosaic.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | #PBS -l nodes=1:ppn=16,mem=126gb 4 | #PBS -l walltime=48:00:00 5 | #PBS -m n 6 | #PBS -k oe 7 | #PBS -j oe 8 | #PBS -q batch 9 | 10 | echo ________________________________________________________ 11 | echo 12 | echo PBS Job Log 13 | echo Start time: $(date) 14 | echo 15 | echo Job name: $PBS_JOBNAME 16 | echo Job ID: $PBS_JOBID 17 | echo Submitted by user: $USER 18 | echo User effective group ID: $(id -ng) 19 | echo 20 | echo Hostname of submission: $PBS_O_HOST 21 | echo Submitted to cluster: $PBS_SERVER 22 | echo Submitted to queue: $PBS_QUEUE 23 | echo Requested nodes per job: $PBS_NUM_NODES 24 | echo Requested cores per node: $PBS_NUM_PPN 25 | echo Requested cores per job: $PBS_NP 26 | echo Node list file: $PBS_NODEFILE 27 | echo Nodes assigned to job: $(cat $PBS_NODEFILE) 28 | echo Running node index: $PBS_O_NODENUM 29 | echo 30 | echo Running on hostname: $HOSTNAME 31 | echo Parent PID: $PPID 32 | echo Process PID: $$ 33 | echo 34 | echo Working directory: $PBS_O_WORKDIR 35 | echo ________________________________________________________ 36 | echo 37 | 38 | module load gdal/2.1.3 39 | 40 | echo $p1 41 | time eval $p1 42 | -------------------------------------------------------------------------------- /qsub_ndvi.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | #PBS -l walltime=24:00:00,nodes=1:ppn=2,mem=8gb 4 | #PBS -m n 5 | #PBS -k oe 6 | #PBS -j oe 7 | #PBS -q batch 8 | 9 | echo ________________________________________________________ 10 | echo 11 | echo PBS Job Log 12 | echo Start time: $(date) 13 | echo 14 | echo Job name: $PBS_JOBNAME 15 | echo Job ID: $PBS_JOBID 16 | echo Submitted by user: $USER 17 | echo User effective group ID: $(id -ng) 18 | echo 19 | echo Hostname of submission: $PBS_O_HOST 20 | echo Submitted to cluster: $PBS_SERVER 21 | echo Submitted to queue: $PBS_QUEUE 22 | echo Requested nodes per job: $PBS_NUM_NODES 23 | echo Requested cores per node: $PBS_NUM_PPN 24 | echo Requested cores per job: $PBS_NP 25 | echo Node list file: $PBS_NODEFILE 26 | echo Nodes assigned to job: $(cat $PBS_NODEFILE) 27 | echo Running node index: $PBS_O_NODENUM 28 | echo 29 | echo Running on hostname: $HOSTNAME 30 | echo Parent PID: $PPID 31 | echo Process PID: $$ 32 | echo 33 | echo Working directory: $PBS_O_WORKDIR 34 | echo ________________________________________________________ 35 | echo 36 | 37 | cd $PBS_O_WORKDIR 38 | 39 | source ~/.bashrc; conda activate pgc 40 | 41 | echo $p1 42 | time eval $p1 43 | -------------------------------------------------------------------------------- /qsub_ortho.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | #PBS -l walltime=20:00:00,nodes=1:ppn=2,mem=8gb 4 | #PBS -m n 5 | #PBS -k oe 6 | #PBS -j oe 7 | #PBS -q batch 8 | 9 | echo ________________________________________________________ 10 | echo 11 | echo PBS Job Log 12 | echo Start time: $(date) 13 | echo 14 | echo Job name: $PBS_JOBNAME 15 | echo Job ID: $PBS_JOBID 16 | echo Submitted by user: $USER 17 | echo User effective group ID: $(id -ng) 18 | echo 19 | echo Hostname of submission: $PBS_O_HOST 20 | echo Submitted to cluster: $PBS_SERVER 21 | echo Submitted to queue: $PBS_QUEUE 22 | echo Requested nodes per job: $PBS_NUM_NODES 23 | echo Requested cores per node: $PBS_NUM_PPN 24 | echo Requested cores per job: $PBS_NP 25 | echo Node list file: $PBS_NODEFILE 26 | echo Nodes assigned to job: $(cat $PBS_NODEFILE) 27 | echo Running node index: $PBS_O_NODENUM 28 | echo 29 | echo Running on hostname: $HOSTNAME 30 | echo Parent PID: $PPID 31 | echo Process PID: $$ 32 | echo 33 | echo Working directory: $PBS_O_WORKDIR 34 | echo ________________________________________________________ 35 | echo 36 | 37 | cd $PBS_O_WORKDIR 38 | 39 | source ~/.bashrc; conda activate pgc 40 | 41 | echo $p1 42 | time eval $p1 43 | -------------------------------------------------------------------------------- /qsub_pansharpen.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | #PBS -l walltime=24:00:00,nodes=1:ppn=2,mem=8gb 4 | #PBS -m n 5 | #PBS -k oe 6 | #PBS -j oe 7 | #PBS -q batch 8 | 9 | echo ________________________________________________________ 10 | echo 11 | echo PBS Job Log 12 | echo Start time: $(date) 13 | echo 14 | echo Job name: $PBS_JOBNAME 15 | echo Job ID: $PBS_JOBID 16 | echo Submitted by user: $USER 17 | echo User effective group ID: $(id -ng) 18 | echo 19 | echo Hostname of submission: $PBS_O_HOST 20 | echo Submitted to cluster: $PBS_SERVER 21 | echo Submitted to queue: $PBS_QUEUE 22 | echo Requested nodes per job: $PBS_NUM_NODES 23 | echo Requested cores per node: $PBS_NUM_PPN 24 | echo Requested cores per job: $PBS_NP 25 | echo Node list file: $PBS_NODEFILE 26 | echo Nodes assigned to job: $(cat $PBS_NODEFILE) 27 | echo Running node index: $PBS_O_NODENUM 28 | echo 29 | echo Running on hostname: $HOSTNAME 30 | echo Parent PID: $PPID 31 | echo Process PID: $$ 32 | echo 33 | echo Working directory: $PBS_O_WORKDIR 34 | echo ________________________________________________________ 35 | echo 36 | 37 | cd $PBS_O_WORKDIR 38 | 39 | source ~/.bashrc; conda activate pgc 40 | 41 | echo $p1 42 | time eval $p1 43 | -------------------------------------------------------------------------------- /slurm_mosaic.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # number of nodes 4 | #SBATCH -N 1 5 | 6 | # number of cpus per task 7 | #SBATCH -c 24 8 | 9 | # memory 10 | #SBATCH --mem 48G 11 | 12 | # wallclock 13 | #SBATCH -t 8:00:00 14 | 15 | # job log path 16 | #SBATCH -o %x.o%j 17 | #SBATCH -e %x.o%j 18 | 19 | #SBATCH --licenses=vida:500 20 | 21 | echo ________________________________________ 22 | echo 23 | echo SLURM Job Log 24 | echo Start time: $(date) 25 | echo 26 | echo Job name: $SLURM_JOB_NAME 27 | echo Job ID: $SLURM_JOBID 28 | echo Submitted by user: $USER 29 | echo User effective group ID: $(id -ng) 30 | echo 31 | echo SLURM account used: $SLURM_ACCOUNT 32 | echo Hostname of submission: $SLURM_SUBMIT_HOST 33 | echo Submitted to cluster: $SLURM_CLUSTER_NAME 34 | echo Submitted to node: $SLURMD_NODENAME 35 | echo Cores on node: $SLURM_CPUS_ON_NODE 36 | echo Requested cores per task: $SLURM_CPUS_PER_TASK 37 | echo Requested cores per job: $SLURM_NTASKS 38 | echo Requested walltime: $SBATCH_TIMELIMIT 39 | echo Nodes assigned to job: $SLURM_JOB_NODELIST 40 | echo Running node index: $SLURM_NODEID 41 | echo 42 | echo Running on hostname: $HOSTNAME 43 | echo Parent PID: $PPID 44 | echo Process PID: $$ 45 | echo 46 | echo Working directory: $SLURM_SUBMIT_DIR 47 | echo ________________________________________________________ 48 | echo 49 | 50 | # init gdal tools 51 | source ~/.bashrc; conda activate pgc 52 | 53 | echo $p1 54 | time eval $p1 55 | -------------------------------------------------------------------------------- /slurm_ndvi.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # number of nodes 4 | #SBATCH -N 1 5 | 6 | # number of cpus per task 7 | #SBATCH -c 1 8 | 9 | # memory 10 | #SBATCH --mem 12G 11 | 12 | # wallclock 13 | #SBATCH -t 1:00:00 14 | 15 | # job log path 16 | #SBATCH -o %x.o%j 17 | #SBATCH -e %x.o%j 18 | 19 | echo ________________________________________ 20 | echo 21 | echo SLURM Job Log 22 | echo Start time: $(date) 23 | echo 24 | echo Job name: $SLURM_JOB_NAME 25 | echo Job ID: $SLURM_JOBID 26 | echo Submitted by user: $USER 27 | echo User effective group ID: $(id -ng) 28 | echo 29 | echo SLURM account used: $SLURM_ACCOUNT 30 | echo Hostname of submission: $SLURM_SUBMIT_HOST 31 | echo Submitted to cluster: $SLURM_CLUSTER_NAME 32 | echo Submitted to node: $SLURMD_NODENAME 33 | echo Cores on node: $SLURM_CPUS_ON_NODE 34 | echo Requested cores per task: $SLURM_CPUS_PER_TASK 35 | echo Requested cores per job: $SLURM_NTASKS 36 | echo Requested walltime: $SBATCH_TIMELIMIT 37 | echo Nodes assigned to job: $SLURM_JOB_NODELIST 38 | echo Running node index: $SLURM_NODEID 39 | echo 40 | echo Running on hostname: $HOSTNAME 41 | echo Parent PID: $PPID 42 | echo Process PID: $$ 43 | echo 44 | echo Working directory: $SLURM_SUBMIT_DIR 45 | echo ________________________________________________________ 46 | echo 47 | 48 | cd $SLURM_SUBMIT_DIR 49 | 50 | # init gdal tools 51 | source ~/.bashrc; conda activate pgc 52 | 53 | echo $p1 54 | time eval $p1 55 | -------------------------------------------------------------------------------- /slurm_ortho.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # number of nodes 4 | #SBATCH -N 1 5 | 6 | # number of cpus per task 7 | #SBATCH -c 1 8 | 9 | # memory 10 | #SBATCH --mem 12G 11 | 12 | # wallclock 13 | #SBATCH -t 1:00:00 14 | 15 | # job log path 16 | #SBATCH -o %x.o%j 17 | #SBATCH -e %x.o%j 18 | 19 | echo ________________________________________ 20 | echo 21 | echo SLURM Job Log 22 | echo Start time: $(date) 23 | echo 24 | echo Job name: $SLURM_JOB_NAME 25 | echo Job ID: $SLURM_JOBID 26 | echo Submitted by user: $USER 27 | echo User effective group ID: $(id -ng) 28 | echo 29 | echo SLURM account used: $SLURM_ACCOUNT 30 | echo Hostname of submission: $SLURM_SUBMIT_HOST 31 | echo Submitted to cluster: $SLURM_CLUSTER_NAME 32 | echo Submitted to node: $SLURMD_NODENAME 33 | echo Cores on node: $SLURM_CPUS_ON_NODE 34 | echo Requested cores per task: $SLURM_CPUS_PER_TASK 35 | echo Requested cores per job: $SLURM_NTASKS 36 | echo Requested walltime: $SBATCH_TIMELIMIT 37 | echo Nodes assigned to job: $SLURM_JOB_NODELIST 38 | echo Running node index: $SLURM_NODEID 39 | echo 40 | echo Running on hostname: $HOSTNAME 41 | echo Parent PID: $PPID 42 | echo Process PID: $$ 43 | echo 44 | echo Working directory: $SLURM_SUBMIT_DIR 45 | echo ________________________________________________________ 46 | echo 47 | 48 | cd $SLURM_SUBMIT_DIR 49 | 50 | # init gdal tools 51 | source ~/.bashrc; conda activate pgc 52 | 53 | echo $p1 54 | time eval $p1 55 | -------------------------------------------------------------------------------- /slurm_pansharpen.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # number of nodes 4 | #SBATCH -N 1 5 | 6 | # number of cpus per task 7 | #SBATCH -c 1 8 | 9 | # memory 10 | #SBATCH --mem 36G 11 | 12 | # wallclock 13 | #SBATCH -t 1:00:00 14 | 15 | # job log path 16 | #SBATCH -o %x.o%j 17 | #SBATCH -e %x.o%j 18 | 19 | #SBATCH --licenses=vida:50 20 | 21 | echo ________________________________________ 22 | echo 23 | echo SLURM Job Log 24 | echo Start time: $(date) 25 | echo 26 | echo Job name: $SLURM_JOB_NAME 27 | echo Job ID: $SLURM_JOBID 28 | echo Submitted by user: $USER 29 | echo User effective group ID: $(id -ng) 30 | echo 31 | echo SLURM account used: $SLURM_ACCOUNT 32 | echo Hostname of submission: $SLURM_SUBMIT_HOST 33 | echo Submitted to cluster: $SLURM_CLUSTER_NAME 34 | echo Submitted to node: $SLURMD_NODENAME 35 | echo Cores on node: $SLURM_CPUS_ON_NODE 36 | echo Requested cores per task: $SLURM_CPUS_PER_TASK 37 | echo Requested cores per job: $SLURM_NTASKS 38 | echo Requested walltime: $SBATCH_TIMELIMIT 39 | echo Nodes assigned to job: $SLURM_JOB_NODELIST 40 | echo Running node index: $SLURM_NODEID 41 | echo 42 | echo Running on hostname: $HOSTNAME 43 | echo Parent PID: $PPID 44 | echo Process PID: $$ 45 | echo 46 | echo Working directory: $SLURM_SUBMIT_DIR 47 | echo ________________________________________________________ 48 | echo 49 | 50 | cd $SLURM_SUBMIT_DIR 51 | 52 | # init gdal tools 53 | source ~/.bashrc; conda activate pgc 54 | 55 | echo $p1 56 | time eval $p1 57 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PolarGeospatialCenter/imagery_utils/d78651651738ff195c17ea91271e59042a76c8fd/tests/__init__.py -------------------------------------------------------------------------------- /tests/test_mosaic.py: -------------------------------------------------------------------------------- 1 | import shutil 2 | import unittest, os, subprocess 3 | 4 | __test_dir__ = os.path.dirname(os.path.abspath(__file__)) 5 | __app_dir__ = os.path.dirname(__test_dir__) 6 | testdata_dir = os.path.join(__test_dir__, 'testdata') 7 | 8 | 9 | class TestMosaicFunc(unittest.TestCase): 10 | 11 | def setUp(self): 12 | self.srcdir = os.path.join(os.path.join(testdata_dir, 'mosaic', 'ortho')) 13 | self.scriptpath = os.path.join(__app_dir__, "pgc_mosaic.py") 14 | self.dstdir = os.path.join(__test_dir__, 'tmp_output') 15 | if not os.path.isdir(self.dstdir): 16 | os.makedirs(self.dstdir) 17 | 18 | def test_pan_mosaic(self): 19 | # extent = -820000.0, -800000.0, -2420000.0, -2400000.0 20 | # tilesize = 10000, 10000 21 | # bands = 1 22 | mosaicname = os.path.join(self.dstdir, 'testmosaic1') 23 | args = '--skip-cmd-txt --component-shp -e -820000.0 -800000.0 -2420000.0 -2400000.0 -t 10000 10000 -b 1' 24 | cmd = 'python {} {} {} {}'.format( 25 | self.scriptpath, 26 | self.srcdir, 27 | mosaicname, 28 | args 29 | ) 30 | p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) 31 | se, so = p.communicate() 32 | # print(so) 33 | # print(se) 34 | 35 | self.assertTrue(os.path.isfile(mosaicname + '_1_1.tif')) 36 | self.assertTrue(os.path.isfile(mosaicname + '_1_2.tif')) 37 | self.assertTrue(os.path.isfile(mosaicname + '_2_1.tif')) 38 | self.assertTrue(os.path.isfile(mosaicname + '_2_2.tif')) 39 | self.assertTrue(os.path.isfile(mosaicname + '_cutlines.shp')) 40 | self.assertTrue(os.path.isfile(mosaicname + '_components.shp')) 41 | self.assertTrue(os.path.isfile(mosaicname + '_tiles.shp')) 42 | 43 | ## test if intersects files have correct number of files 44 | intersects_files = { 45 | mosaicname + '_1_1_intersects.txt': 2, 46 | mosaicname + '_2_1_intersects.txt': 3, 47 | mosaicname + '_1_2_intersects.txt': 2, 48 | mosaicname + '_2_2_intersects.txt': 2, 49 | } 50 | 51 | for f, cnt in intersects_files.items(): 52 | fh = open(f) 53 | lines = fh.readlines() 54 | self.assertEqual(len(lines), cnt) 55 | 56 | def test_bgrn_mosaic_with_stats(self): 57 | # extent = -3260000, -3240000, 520000, 540000 58 | # tilesize = 10000, 10000 59 | # bands = 4 60 | mosaicname = os.path.join(self.dstdir, 'testmosaic2') 61 | args = '--skip-cmd-txt --component-shp -e -3260000 -3240000 520000 540000 -t 10000 10000 -b 4 --calc-stats --median-remove' 62 | cmd = 'python {} {} {} {}'.format( 63 | self.scriptpath, 64 | self.srcdir, 65 | mosaicname, 66 | args 67 | ) 68 | p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) 69 | se, so = p.communicate() 70 | # print(so) 71 | # print(se) 72 | 73 | self.assertTrue(os.path.isfile(mosaicname + '_1_1.tif')) 74 | self.assertTrue(os.path.isfile(mosaicname + '_1_2.tif')) 75 | self.assertTrue(os.path.isfile(mosaicname + '_2_1.tif')) 76 | self.assertTrue(os.path.isfile(mosaicname + '_2_2.tif')) 77 | self.assertTrue(os.path.isfile(mosaicname + '_cutlines.shp')) 78 | self.assertTrue(os.path.isfile(mosaicname + '_components.shp')) 79 | self.assertTrue(os.path.isfile(mosaicname + '_tiles.shp')) 80 | 81 | def test_ndvi_pansh_mosaic(self): 82 | # extent = -3260000, -3240000, 520000, 540000 83 | # tilesize = 10000, 10000 84 | # bands = 1 85 | srcdir = os.path.join(os.path.join(testdata_dir, 'mosaic', 'pansh_ndvi')) 86 | mosaicname = os.path.join(self.dstdir, 'testmosaic3') 87 | args = '--skip-cmd-txt --component-shp -e -3260000 -3240000 520000 540000 -t 10000 10000 -b 1' 88 | cmd = 'python {} {} {} {}'.format( 89 | self.scriptpath, 90 | srcdir, 91 | mosaicname, 92 | args 93 | ) 94 | p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) 95 | se, so = p.communicate() 96 | # print(so) 97 | # print(se) 98 | 99 | self.assertTrue(os.path.isfile(mosaicname + '_1_1.tif')) 100 | self.assertTrue(os.path.isfile(mosaicname + '_1_2.tif')) 101 | self.assertTrue(os.path.isfile(mosaicname + '_2_1.tif')) 102 | self.assertTrue(os.path.isfile(mosaicname + '_2_2.tif')) 103 | self.assertTrue(os.path.isfile(mosaicname + '_cutlines.shp')) 104 | self.assertTrue(os.path.isfile(mosaicname + '_components.shp')) 105 | self.assertTrue(os.path.isfile(mosaicname + '_tiles.shp')) 106 | 107 | def test_ndvi_pansh_mosaic_with_stats(self): 108 | # extent = -3260000, -3240000, 520000, 540000 109 | # tilesize = 10000, 10000 110 | # bands = 1 111 | srcdir = os.path.join(os.path.join(testdata_dir, 'mosaic', 'pansh_ndvi')) 112 | mosaicname = os.path.join(self.dstdir, 'testmosaic4') 113 | args = '--skip-cmd-txt --component-shp -e -3260000 -3240000 520000 540000 -t 10000 10000 -b 1 --calc-stats --median-remove' 114 | cmd = 'python {} {} {} {}'.format( 115 | self.scriptpath, 116 | srcdir, 117 | mosaicname, 118 | args 119 | ) 120 | p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) 121 | se, so = p.communicate() 122 | # print(so) 123 | # print(se) 124 | 125 | self.assertTrue(os.path.isfile(mosaicname + '_1_1.tif')) 126 | self.assertTrue(os.path.isfile(mosaicname + '_1_2.tif')) 127 | self.assertTrue(os.path.isfile(mosaicname + '_2_1.tif')) 128 | self.assertTrue(os.path.isfile(mosaicname + '_2_2.tif')) 129 | self.assertTrue(os.path.isfile(mosaicname + '_cutlines.shp')) 130 | self.assertTrue(os.path.isfile(mosaicname + '_components.shp')) 131 | self.assertTrue(os.path.isfile(mosaicname + '_tiles.shp')) 132 | 133 | def tearDown(self): 134 | shutil.rmtree(self.dstdir) 135 | 136 | 137 | if __name__ == '__main__': 138 | 139 | test_cases = [ 140 | TestMosaicFunc 141 | ] 142 | 143 | suites = [] 144 | for test_case in test_cases: 145 | suite = unittest.TestLoader().loadTestsFromTestCase(test_case) 146 | suites.append(suite) 147 | 148 | alltests = unittest.TestSuite(suites) 149 | unittest.TextTestRunner(verbosity=2).run(alltests) 150 | -------------------------------------------------------------------------------- /tests/test_mosaic_lib.py: -------------------------------------------------------------------------------- 1 | import shutil 2 | import unittest, os, sys, glob 3 | from osgeo import gdal, ogr 4 | import numpy as np 5 | 6 | __test_dir__ = os.path.dirname(os.path.abspath(__file__)) 7 | sys.path.append(os.path.dirname(__test_dir__)) 8 | testdata_dir = os.path.join(__test_dir__, 'testdata') 9 | 10 | from lib import mosaic 11 | 12 | 13 | class TestMosaicImageInfo(unittest.TestCase): 14 | 15 | def setUp(self): 16 | self.srcdir = os.path.join(os.path.join(testdata_dir, 'mosaic', 'ortho')) 17 | 18 | def test_image_info_ge01(self): 19 | image = 'GE01_20090707163115_297600_5V090707P0002976004A222012202432M_001529596_u08mr3413.tif' 20 | image_info = mosaic.ImageInfo(os.path.join(self.srcdir, image), 'IMAGE') 21 | 22 | self.assertEqual(image_info.xres, 16.0) 23 | self.assertEqual(image_info.yres, 16.0) 24 | # self.assertEqual(image_info.proj, 'PROJCS["unnamed",GEOGCS["WGS 84",DATUM["WGS_1984",SPHEROID["WGS 84",6378137,298.257223563,AUTHORITY["EPSG","7030"]],AUTHORITY["EPSG","6326"]],PRIMEM["Greenwich",0],UNIT["degree",0.0174532925199433],AUTHORITY["EPSG","4326"]],PROJECTION["Polar_Stereographic"],PARAMETER["latitude_of_origin",70],PARAMETER["central_meridian",-45],PARAMETER["scale_factor",1],PARAMETER["false_easting",0],PARAMETER["false_northing",0],UNIT["metre",1,AUTHORITY["EPSG","9001"]]]') 25 | self.assertEqual(image_info.bands, 1) 26 | self.assertEqual(image_info.datatype, 1) 27 | 28 | mosaic_args = MosaicArgs() 29 | mosaic_params = mosaic.getMosaicParameters(image_info, mosaic_args) 30 | image_info.getScore(mosaic_params) 31 | 32 | self.assertEqual(image_info.sensor, 'GE01') 33 | self.assertEqual(image_info.sunel, 45.98) 34 | self.assertEqual(image_info.ona, 26.86) 35 | self.assertEqual(image_info.cloudcover, 0.0) 36 | self.assertEqual(image_info.tdi, 8.0) 37 | self.assertEqual(image_info.panfactor, 1) 38 | #self.assertEqual(image_info.exposure_factor, 0) 39 | self.assertEqual(image_info.date_diff, -9999) 40 | self.assertEqual(image_info.year_diff, -9999) 41 | self.assertAlmostEqual(image_info.score, 79.1422222) 42 | 43 | image_info.get_raster_stats() 44 | stat_dct = {1: [57.0, 255.0, 171.47750552856309, 42.22407526523467]} 45 | datapixelcount_dct = {1: 4435601} 46 | for i in range(len(image_info.stat_dct[1])): 47 | self.assertAlmostEqual(image_info.stat_dct[1][i], stat_dct[1][i]) 48 | self.assertEqual(image_info.datapixelcount_dct, datapixelcount_dct) 49 | 50 | def test_image_info_wv01(self): 51 | image = 'WV01_20080807153945_1020010003A5AC00_08AUG07153945-P1BS-052060421010_01_P011_u08mr3413.tif' 52 | image_info = mosaic.ImageInfo(os.path.join(self.srcdir, image), 'IMAGE') 53 | 54 | self.assertEqual(image_info.xres, 16.0) 55 | self.assertEqual(image_info.yres, 16.0) 56 | # self.assertEqual(image_info.proj, 'PROJCS["unnamed",GEOGCS["WGS 84",DATUM["WGS_1984",SPHEROID["WGS 84",6378137,298.257223563,AUTHORITY["EPSG","7030"]],AUTHORITY["EPSG","6326"]],PRIMEM["Greenwich",0],UNIT["degree",0.0174532925199433],AUTHORITY["EPSG","4326"]],PROJECTION["Polar_Stereographic"],PARAMETER["latitude_of_origin",70],PARAMETER["central_meridian",-45],PARAMETER["scale_factor",1],PARAMETER["false_easting",0],PARAMETER["false_northing",0],UNIT["metre",1,AUTHORITY["EPSG","9001"]]]') 57 | self.assertEqual(image_info.bands, 1) 58 | self.assertEqual(image_info.datatype, 1) 59 | 60 | mosaic_args = MosaicArgs() 61 | mosaic_params = mosaic.getMosaicParameters(image_info, mosaic_args) 62 | image_info.getScore(mosaic_params) 63 | 64 | self.assertEqual(image_info.sensor, 'WV01') 65 | self.assertEqual(image_info.sunel, 39.0) 66 | self.assertEqual(image_info.ona, 18.5) 67 | self.assertEqual(image_info.cloudcover, 0.0) 68 | self.assertEqual(image_info.tdi, 16.0) 69 | self.assertEqual(image_info.panfactor, 1) 70 | #self.assertEqual(image_info.exposure_factor, 0) 71 | self.assertEqual(image_info.date_diff, -9999) 72 | self.assertEqual(image_info.year_diff, -9999) 73 | self.assertAlmostEqual(image_info.score, 79.2) 74 | 75 | image_info.get_raster_stats() 76 | stat_dct = {1: [6.0, 234.0, 73.77702002, 22.52309144]} 77 | datapixelcount_dct = {1: 1405893} 78 | for i in range(len(image_info.stat_dct[1])): 79 | self.assertAlmostEqual(image_info.stat_dct[1][i], stat_dct[1][i]) 80 | self.assertEqual(image_info.datapixelcount_dct, datapixelcount_dct) 81 | 82 | def test_image_info_wv02_ndvi(self): 83 | srcdir = os.path.join(os.path.join(testdata_dir, 'mosaic', 'ndvi')) 84 | image = 'WV02_20110901210434_103001000B41DC00_11SEP01210434-M1BS-052730735130_01_P007_u16rf3413_ndvi.tif' 85 | image_info = mosaic.ImageInfo(os.path.join(srcdir, image), 'IMAGE') 86 | 87 | self.assertEqual(image_info.xres, 16.0) 88 | self.assertEqual(image_info.yres, 16.0) 89 | # self.assertEqual(image_info.proj, 'PROJCS["unnamed",GEOGCS["WGS 84",DATUM["WGS_1984",SPHEROID["WGS 84",6378137,298.257223563,AUTHORITY["EPSG","7030"]],AUTHORITY["EPSG","6326"]],PRIMEM["Greenwich",0],UNIT["degree",0.0174532925199433],AUTHORITY["EPSG","4326"]],PROJECTION["Polar_Stereographic"],PARAMETER["latitude_of_origin",70],PARAMETER["central_meridian",-45],PARAMETER["scale_factor",1],PARAMETER["false_easting",0],PARAMETER["false_northing",0],UNIT["metre",1,AUTHORITY["EPSG","9001"]]]') 90 | self.assertEqual(image_info.bands, 1) 91 | self.assertEqual(image_info.datatype, 6) 92 | 93 | mosaic_args = MosaicArgs() 94 | mosaic_params = mosaic.getMosaicParameters(image_info, mosaic_args) 95 | image_info.getScore(mosaic_params) 96 | 97 | self.assertEqual(image_info.sensor, 'WV02') 98 | self.assertEqual(image_info.sunel, 37.7) 99 | self.assertEqual(image_info.ona, 19.4) 100 | self.assertEqual(image_info.cloudcover, 0.0) 101 | self.assertEqual(image_info.tdi, 24.0) 102 | self.assertEqual(image_info.panfactor, 1) 103 | #self.assertEqual(image_info.exposure_factor, 0) 104 | self.assertEqual(image_info.date_diff, -9999) 105 | self.assertEqual(image_info.year_diff, -9999) 106 | self.assertAlmostEqual(image_info.score, 78.555555555) 107 | 108 | image_info.get_raster_stats() 109 | stat_dct = {1: [-1.0, 1.0, 0.5187682, 0.35876602]} 110 | datapixelcount_dct = {1: 1208656} 111 | for i in range(len(image_info.stat_dct[1])): 112 | self.assertAlmostEqual(image_info.stat_dct[1][i], stat_dct[1][i]) 113 | self.assertEqual(image_info.datapixelcount_dct, datapixelcount_dct) 114 | 115 | def test_image_info_wv02_ndvi_int16(self): 116 | srcdir = os.path.join(os.path.join(testdata_dir, 'mosaic', 'pansh_ndvi')) 117 | image = 'WV02_20110901210434_103001000B41DC00_11SEP01210434-M1BS-052730735130_01_P007_u16rf3413_pansh_ndvi.tif' 118 | image_info = mosaic.ImageInfo(os.path.join(srcdir, image), 'IMAGE') 119 | 120 | self.assertEqual(image_info.xres, 16.0) 121 | self.assertEqual(image_info.yres, 16.0) 122 | # self.assertEqual(image_info.proj, 'PROJCS["unnamed",GEOGCS["WGS 84",DATUM["WGS_1984",SPHEROID["WGS 84",6378137,298.257223563,AUTHORITY["EPSG","7030"]],AUTHORITY["EPSG","6326"]],PRIMEM["Greenwich",0],UNIT["degree",0.0174532925199433],AUTHORITY["EPSG","4326"]],PROJECTION["Polar_Stereographic"],PARAMETER["latitude_of_origin",70],PARAMETER["central_meridian",-45],PARAMETER["scale_factor",1],PARAMETER["false_easting",0],PARAMETER["false_northing",0],UNIT["metre",1,AUTHORITY["EPSG","9001"]]]') 123 | self.assertEqual(image_info.bands, 1) 124 | self.assertEqual(image_info.datatype, 3) 125 | 126 | mosaic_args = MosaicArgs() 127 | mosaic_params = mosaic.getMosaicParameters(image_info, mosaic_args) 128 | image_info.getScore(mosaic_params) 129 | 130 | self.assertEqual(image_info.sensor, 'WV02') 131 | self.assertEqual(image_info.sunel, 37.7) 132 | self.assertEqual(image_info.ona, 19.4) 133 | self.assertEqual(image_info.cloudcover, 0.0) 134 | self.assertEqual(image_info.tdi, 24.0) 135 | self.assertEqual(image_info.panfactor, 1) 136 | #self.assertEqual(image_info.exposure_factor, 0) 137 | self.assertEqual(image_info.date_diff, -9999) 138 | self.assertEqual(image_info.year_diff, -9999) 139 | self.assertAlmostEqual(image_info.score, 78.555555555) 140 | 141 | image_info.get_raster_stats() 142 | stat_dct = {1: [-1000.0, 1000.0, 549.7191938, 308.80771976]} 143 | datapixelcount_dct = {1: 1206259} 144 | for i in range(len(image_info.stat_dct[1])): 145 | self.assertAlmostEqual(image_info.stat_dct[1][i], stat_dct[1][i]) 146 | self.assertEqual(image_info.datapixelcount_dct, datapixelcount_dct) 147 | 148 | def test_image_info_multispectral_dg_ge01(self): 149 | image = 'GE01_20130728161916_1050410002608900_13JUL28161916-M1BS-054448357040_01_P002_u08mr3413.tif' 150 | image_info = mosaic.ImageInfo(os.path.join(self.srcdir, image), 'IMAGE') 151 | self.maxDiff = None 152 | 153 | self.assertEqual(image_info.xres, 16.0) 154 | self.assertEqual(image_info.yres, 16.0) 155 | # self.assertEqual(image_info.proj, 'PROJCS["unnamed",GEOGCS["WGS 84",DATUM["WGS_1984",SPHEROID["WGS 84",6378137,298.257223563,AUTHORITY["EPSG","7030"]],AUTHORITY["EPSG","6326"]],PRIMEM["Greenwich",0],UNIT["degree",0.0174532925199433],AUTHORITY["EPSG","4326"]],PROJECTION["Polar_Stereographic"],PARAMETER["latitude_of_origin",70],PARAMETER["central_meridian",-45],PARAMETER["scale_factor",1],PARAMETER["false_easting",0],PARAMETER["false_northing",0],UNIT["metre",1,AUTHORITY["EPSG","9001"]]]') 156 | self.assertEqual(image_info.bands, 4) 157 | self.assertEqual(image_info.datatype, 1) 158 | 159 | mosaic_args = MosaicArgs() 160 | mosaic_params = mosaic.getMosaicParameters(image_info, mosaic_args) 161 | image_info.getScore(mosaic_params) 162 | 163 | self.assertEqual(image_info.sensor, 'GE01') 164 | self.assertEqual(image_info.sunel, 42.2) 165 | self.assertEqual(image_info.ona, 25.0) 166 | self.assertEqual(image_info.cloudcover, 0.0) 167 | self.assertEqual(image_info.tdi, 6.0) 168 | self.assertEqual(image_info.panfactor, 1) 169 | #self.assertEqual(image_info.exposure_factor, 0) 170 | self.assertEqual(image_info.date_diff, -9999) 171 | self.assertEqual(image_info.year_diff, -9999) 172 | self.assertAlmostEqual(image_info.score, 78.462222222) 173 | 174 | image_info.get_raster_stats() 175 | stat_dct = { 176 | 1: [1.0, 245.0, 89.28827106, 18.75882356], 177 | 2: [2.0, 245.0, 72.48547016, 21.73902804], 178 | 3: [1.0, 251.0, 58.33183442, 21.82633595], 179 | 4: [1.0, 235.0, 61.06978454, 26.274778] 180 | } 181 | datapixelcount_dct = {1: 1152457, 2: 1152456, 3: 1152394, 4: 1146271} 182 | for i in range(len(image_info.stat_dct[1])): 183 | self.assertAlmostEqual(image_info.stat_dct[1][i], stat_dct[1][i]) 184 | self.assertEqual(image_info.datapixelcount_dct, datapixelcount_dct) 185 | 186 | def test_image_info_wv01_with_tday_and_exposure(self): 187 | image = 'WV01_20080807153945_1020010003A5AC00_08AUG07153945-P1BS-052060421010_01_P011_u08mr3413.tif' 188 | image_info = mosaic.ImageInfo(os.path.join(self.srcdir, image), 'IMAGE') 189 | 190 | self.assertEqual(image_info.xres, 16.0) 191 | self.assertEqual(image_info.yres, 16.0) 192 | self.assertEqual(image_info.bands, 1) 193 | self.assertEqual(image_info.datatype, 1) 194 | 195 | mosaic_args = MosaicArgs() 196 | mosaic_args.tday = '09-01' 197 | mosaic_args.use_exposure = True 198 | mosaic_params = mosaic.getMosaicParameters(image_info, mosaic_args) 199 | 200 | self.assertEqual(mosaic_params.m, 9) 201 | self.assertEqual(mosaic_params.d, 1) 202 | 203 | image_info.getScore(mosaic_params) 204 | self.assertEqual(image_info.sensor, 'WV01') 205 | self.assertEqual(image_info.sunel, 39.0) 206 | self.assertEqual(image_info.ona, 18.5) 207 | self.assertEqual(image_info.cloudcover, 0.0) 208 | self.assertEqual(image_info.tdi, 16.0) 209 | self.assertEqual(image_info.panfactor, 1) 210 | #self.assertEqual(image_info.exposure_factor, 0) 211 | self.assertEqual(image_info.date_diff, 24) 212 | self.assertEqual(image_info.year_diff, -9999) 213 | self.assertAlmostEqual(image_info.score, 86.0924408) 214 | 215 | def test_image_info_wv01_with_tyear(self): 216 | image = 'WV01_20080807153945_1020010003A5AC00_08AUG07153945-P1BS-052060421010_01_P011_u08mr3413.tif' 217 | image_info = mosaic.ImageInfo(os.path.join(self.srcdir, image), 'IMAGE') 218 | 219 | mosaic_args = MosaicArgs() 220 | mosaic_args.tyear = 2008 221 | mosaic_params = mosaic.getMosaicParameters(image_info, mosaic_args) 222 | 223 | image_info.getScore(mosaic_params) 224 | self.assertEqual(image_info.date_diff, -9999) 225 | self.assertEqual(image_info.year_diff, 0) 226 | self.assertAlmostEqual(image_info.score, 134.2) 227 | 228 | def test_image_info_wv01_with_tyear_and_tday(self): 229 | image = 'WV01_20080807153945_1020010003A5AC00_08AUG07153945-P1BS-052060421010_01_P011_u08mr3413.tif' 230 | image_info = mosaic.ImageInfo(os.path.join(self.srcdir, image), 'IMAGE') 231 | 232 | mosaic_args = MosaicArgs() 233 | mosaic_args.tyear = 2008 234 | mosaic_args.tday = '09-01' 235 | mosaic_params = mosaic.getMosaicParameters(image_info, mosaic_args) 236 | 237 | image_info.getScore(mosaic_params) 238 | self.assertEqual(image_info.date_diff, 24) 239 | self.assertEqual(image_info.year_diff, 0) 240 | self.assertAlmostEqual(image_info.score, 90.6334244) 241 | 242 | def test_image_info_wv02_with_cc_max(self): 243 | image = 'WV02_20110504155551_103001000BA45E00_11MAY04155551-P1BS-500085264180_01_P002_u08mr3413.tif' 244 | image_info = mosaic.ImageInfo(os.path.join(self.srcdir, image), 'IMAGE') 245 | 246 | self.assertEqual(image_info.xres, 16.0) 247 | self.assertEqual(image_info.yres, 16.0) 248 | self.assertEqual(image_info.bands, 1) 249 | self.assertEqual(image_info.datatype, 1) 250 | 251 | mosaic_args = MosaicArgs() 252 | mosaic_args.max_cc = 0.20 253 | mosaic_params = mosaic.getMosaicParameters(image_info, mosaic_args) 254 | image_info.getScore(mosaic_params) 255 | 256 | self.assertEqual(image_info.sensor, 'WV02') 257 | self.assertEqual(image_info.sunel, 39.2) 258 | self.assertEqual(image_info.ona, 19.0) 259 | self.assertEqual(image_info.cloudcover, 0.29) 260 | self.assertEqual(image_info.tdi, 48.0) 261 | self.assertEqual(image_info.panfactor, 1) 262 | #self.assertEqual(image_info.exposure_factor, 0) 263 | self.assertEqual(image_info.date_diff, -9999) 264 | self.assertEqual(image_info.year_diff, -9999) 265 | self.assertAlmostEqual(image_info.score, -1) 266 | 267 | image_info.get_raster_stats() 268 | stat_dct = {1: [1.0, 239.0, 232.17920063, 11.26401958]} 269 | datapixelcount_dct = {1: 1155208} 270 | for i in range(len(image_info.stat_dct[1])): 271 | self.assertAlmostEqual(image_info.stat_dct[1][i], stat_dct[1][i]) 272 | self.assertEqual(image_info.datapixelcount_dct, datapixelcount_dct) 273 | 274 | def test_filter_images(self): 275 | image_list = glob.glob(os.path.join(self.srcdir, '*.tif')) 276 | imginfo_list = [mosaic.ImageInfo(image, "IMAGE") for image in image_list] 277 | filter_list = [iinfo.srcfn for iinfo in imginfo_list] 278 | self.assertIn('WV02_20110901210434_103001000B41DC00_11SEP01210434-M1BS-052730735130_01_P007_u08mr3413.tif', 279 | filter_list) 280 | # self.assertIn('GE01_20130728161916_1050410002608900_13JUL28161916-P1BS-054448357040_01_P002_u16rf3413.tif', 281 | # filter_list) 282 | self.assertIn('WV02_20131123162834_10300100293C3400_13NOV23162834-P1BS-500408660030_01_P005_u08mr3413.tif', 283 | filter_list) 284 | 285 | mosaic_args = MosaicArgs() 286 | mosaic_args.extent = [-820000.0, -800000.0, -2420000.0, -2400000.0] 287 | mosaic_args.tilesize = [20000, 20000] 288 | mosaic_args.bands = 1 289 | mosaic_params = mosaic.getMosaicParameters(imginfo_list[0], mosaic_args) 290 | imginfo_list2 = mosaic.filterMatchingImages(imginfo_list, mosaic_params) 291 | filter_list = [iinfo.srcfn for iinfo in imginfo_list2] 292 | 293 | self.assertEqual(len(imginfo_list2), 8) 294 | self.assertNotIn('WV02_20110901210434_103001000B41DC00_11SEP01210434-M1BS-052730735130_01_P007_u08mr3413.tif', 295 | filter_list) 296 | self.assertNotIn('GE01_20130728161916_1050410002608900_13JUL28161916-P1BS-054448357040_01_P002_u16rf3413.tif', 297 | filter_list) 298 | 299 | imginfo_list3 = mosaic.filter_images_by_geometry(imginfo_list2, mosaic_params) 300 | filter_list = [iinfo.srcfn for iinfo in imginfo_list3] 301 | self.assertEqual(len(imginfo_list3), 7) 302 | self.assertNotIn('WV02_20131123162834_10300100293C3400_13NOV23162834-P1BS-500408660030_01_P005_u08mr3413.tif', 303 | filter_list) 304 | 305 | 306 | class TestMiscFunctions(unittest.TestCase): 307 | def setUp(self): 308 | self.srcdir = os.path.join(testdata_dir, 'metadata_files') 309 | self.srcfn = 'QB02_20021009211710_101001000153C800_02OCT09211710-M2AS_R1C1-052075481010_01_P001.xml' 310 | self.srcfile = os.path.join(self.srcdir, self.srcfn) 311 | #print(self.srcfile) 312 | self.dstdir = os.path.join(__test_dir__, 'tmp_output') 313 | self.dstfile_xml = os.path.join(self.dstdir, self.srcfn) 314 | self.dem = os.path.join(testdata_dir, 'dem', 'ramp_lowres.tif') 315 | 316 | self.resolution = None 317 | self.xres = 0.5 318 | self.yres = 0.5 319 | self.bands = 1 320 | self.proj = 'PROJCS["WGS_1984_Stereographic_South_Pole",GEOGCS["WGS 84",DATUM["WGS_1984",SPHEROID["WGS 84",6378137,298.257223563,AUTHORITY["EPSG","7030"]],AUTHORITY["EPSG","6326"]],PRIMEM["Greenwich",0],UNIT["degree",0.0174532925199433],AUTHORITY["EPSG","4326"]],PROJECTION["Polar_Stereographic"],PARAMETER["latitude_of_origin",-71],PARAMETER["central_meridian",0],PARAMETER["scale_factor",1],PARAMETER["false_easting",0],PARAMETER["false_northing",0],UNIT["metre",1,AUTHORITY["EPSG","9001"]]]' 321 | self.datatype = 3 322 | self.use_exposure = True 323 | self.tday = None 324 | self.tyear = None 325 | self.extent = [150000, -1400000, 420000, -1200000] 326 | self.tilesize = [20000, 20000] 327 | self.max_cc = 0.6 328 | self.force_pan_to_multi = False 329 | self.include_all_ms = False 330 | self.median_remove = False 331 | self.min_contribution_area = 20000000 332 | 333 | self.imginfo_list = mosaic.ImageInfo(self.dem, 'IMAGE') 334 | 335 | poly_wkt = 'POLYGON (( {} {}, {} {}, {} {}, {} {}, {} {} ))'.format(self.extent[0], self.extent[1], 336 | self.extent[0], self.extent[3], 337 | self.extent[2], self.extent[3], 338 | self.extent[2], self.extent[1], 339 | self.extent[0], self.extent[1]) 340 | self.extent_geom = ogr.CreateGeometryFromWkt(poly_wkt) 341 | if not os.path.isdir(self.dstdir): 342 | os.makedirs(self.dstdir) 343 | 344 | def test_get_exact_trimmed_geom(self): 345 | xs_expected = [2502000.0, 2868000.0, 2868000.0, -2868000.0, -2868000.0, -2501000.0] 346 | ys_expected = [2457900.0, 457900.0, -1542100.0, -1542100.0, 457900.0, 2457900.0] 347 | geom, xs, ys = mosaic.GetExactTrimmedGeom(self.dem, step=10000) 348 | self.assertEqual(xs, xs_expected) 349 | self.assertEqual(ys, ys_expected) 350 | 351 | ''' 352 | NOTE: findVertices() is not used in the codebase, and will not be tested here 353 | ''' 354 | 355 | def test_pl2xy(self): 356 | # test using random, but plausible values 357 | gtf = [0, 50, 10, 1000, 5, 50] 358 | p_var = 10 359 | l_var = 10 360 | x, y = mosaic.pl2xy(gtf, None, p_var, l_var) 361 | self.assertEqual(x, 500) 362 | self.assertEqual(y, 1525.0) 363 | 364 | # same test as above, but negative x coordinate (gtf[0]) 365 | gtf = [-50, 50, 10, 1000, 5, 50] 366 | p_var = 10 367 | l_var = 10 368 | x, y = mosaic.pl2xy(gtf, None, p_var, l_var) 369 | self.assertEqual(x, 450) 370 | self.assertEqual(y, 1525.0) 371 | 372 | def test_drange(self): 373 | self.assertEqual(list(mosaic.drange(0, 5, 1)), [0, 1, 2, 3, 4]) 374 | self.assertEqual(list(mosaic.drange(5, 0, 1)), []) 375 | 376 | def test_buffernum(self): 377 | # note: buffernum() gives strange value if 'num' is negative (buffernum(-5, 3) returns '0-5') 378 | self.assertEqual(mosaic.buffernum(10, 5), '00010') 379 | self.assertEqual(mosaic.buffernum(5, 2), '05') 380 | 381 | def test_copyall(self): 382 | # make sure basic file copying works 383 | mosaic.copyall(self.srcfile, self.dstdir) 384 | self.assertTrue(os.path.isfile(self.dstfile_xml)) 385 | 386 | # should return AttributeError 387 | with self.assertRaises(TypeError) as cm: 388 | mosaic.copyall(None, None) 389 | 390 | def tearDown(self): 391 | shutil.rmtree(self.dstdir, ignore_errors=True) 392 | 393 | 394 | class MosaicArgs(object): 395 | def __init__(self): 396 | self.resolution = None 397 | self.bands = None 398 | self.use_exposure = False 399 | self.tday = None 400 | self.tyear = None 401 | self.extent = None 402 | self.tilesize = None 403 | self.max_cc = 0.5 404 | self.force_pan_to_multi = False 405 | self.include_all_ms = False 406 | self.median_remove = False 407 | 408 | 409 | if __name__ == '__main__': 410 | 411 | test_cases = [ 412 | TestMosaicImageInfo, 413 | TestMiscFunctions 414 | ] 415 | 416 | suites = [] 417 | for test_case in test_cases: 418 | suite = unittest.TestLoader().loadTestsFromTestCase(test_case) 419 | suites.append(suite) 420 | 421 | alltests = unittest.TestSuite(suites) 422 | unittest.TextTestRunner(verbosity=2).run(alltests) 423 | -------------------------------------------------------------------------------- /tests/test_ndvi.py: -------------------------------------------------------------------------------- 1 | import shutil 2 | import unittest, os, subprocess 3 | from osgeo import gdal 4 | 5 | __test_dir__ = os.path.dirname(os.path.abspath(__file__)) 6 | __app_dir__ = os.path.dirname(__test_dir__) 7 | testdata_dir = os.path.join(__test_dir__, 'testdata') 8 | 9 | 10 | class TestNdviFunc(unittest.TestCase): 11 | 12 | def setUp(self): 13 | 14 | self.scriptpath = os.path.join(__app_dir__, "pgc_ndvi.py") 15 | self.dstdir = os.path.join(__test_dir__, 'tmp_output') 16 | if not os.path.isdir(self.dstdir): 17 | os.makedirs(self.dstdir) 18 | 19 | # @unittest.skip("skipping") 20 | def test_ndvi(self): 21 | 22 | srcdir = os.path.join(os.path.join(testdata_dir, 'ndvi', 'ortho')) 23 | 24 | cmd = 'python {} {} {} --skip-cmd-txt '.format( 25 | self.scriptpath, 26 | srcdir, 27 | self.dstdir, 28 | ) 29 | p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) 30 | se, so = p.communicate() 31 | # print(so) 32 | # print(se) 33 | 34 | for f in os.listdir(srcdir): 35 | if f.endswith('.tif'): 36 | dstfp = os.path.join(self.dstdir, f[:-4] + '_ndvi.tif') 37 | dstfp_xml = os.path.join(self.dstdir, f[:-4] + '_ndvi.xml') 38 | self.assertTrue(os.path.isfile(dstfp)) 39 | self.assertTrue(os.path.isfile(dstfp_xml)) 40 | ds = gdal.Open(dstfp) 41 | dt = ds.GetRasterBand(1).DataType 42 | self.assertEqual(dt, 6) 43 | ds = None 44 | 45 | # @unittest.skip("skipping") 46 | def test_ndvi_int16(self): 47 | 48 | srcdir = os.path.join(os.path.join(testdata_dir, 'ndvi', 'ortho')) 49 | 50 | cmd = 'python {} {} {} --skip-cmd-txt -t Int16'.format( 51 | self.scriptpath, 52 | srcdir, 53 | self.dstdir, 54 | ) 55 | p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) 56 | se, so = p.communicate() 57 | # print(so) 58 | # print(se) 59 | 60 | for f in os.listdir(srcdir): 61 | if f.endswith('.tif'): 62 | dstfp = os.path.join(self.dstdir, f[:-4] + '_ndvi.tif') 63 | dstfp_xml = os.path.join(self.dstdir, f[:-4] + '_ndvi.xml') 64 | self.assertTrue(os.path.isfile(dstfp)) 65 | self.assertTrue(os.path.isfile(dstfp_xml)) 66 | ds = gdal.Open(dstfp) 67 | dt = ds.GetRasterBand(1).DataType 68 | self.assertEqual(dt, 3) 69 | ds = None 70 | 71 | # @unittest.skip("skipping") 72 | def test_ndvi_from_pansharp(self): 73 | 74 | srcdir = os.path.join(os.path.join(testdata_dir, 'ndvi', 'pansh')) 75 | 76 | cmd = 'python {} {} {} -t Int16 --skip-cmd-txt'.format( 77 | self.scriptpath, 78 | srcdir, 79 | self.dstdir, 80 | ) 81 | p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) 82 | se, so = p.communicate() 83 | # print(so) 84 | # print(se) 85 | 86 | for f in os.listdir(srcdir): 87 | if f.endswith('.tif'): 88 | dstfp = os.path.join(self.dstdir, f[:-4] + '_ndvi.tif') 89 | dstfp_xml = os.path.join(self.dstdir, f[:-4] + '_ndvi.xml') 90 | self.assertTrue(os.path.isfile(dstfp)) 91 | self.assertTrue(os.path.isfile(dstfp_xml)) 92 | ds = gdal.Open(dstfp) 93 | dt = ds.GetRasterBand(1).DataType 94 | self.assertEqual(dt, 3) 95 | 96 | def tearDown(self): 97 | shutil.rmtree(self.dstdir) 98 | 99 | 100 | if __name__ == '__main__': 101 | 102 | test_cases = [ 103 | TestNdviFunc 104 | ] 105 | 106 | suites = [] 107 | for test_case in test_cases: 108 | suite = unittest.TestLoader().loadTestsFromTestCase(test_case) 109 | suites.append(suite) 110 | 111 | alltests = unittest.TestSuite(suites) 112 | unittest.TextTestRunner(verbosity=2).run(alltests) 113 | -------------------------------------------------------------------------------- /tests/test_ortho.py: -------------------------------------------------------------------------------- 1 | """Runs pgc_ortho with a variety of images and input parameters to achieve test coverage.""" 2 | import shutil 3 | import unittest, os, subprocess 4 | import platform 5 | 6 | from setuptools import glob 7 | 8 | __test_dir__ = os.path.dirname(os.path.abspath(__file__)) 9 | __app_dir__ = os.path.dirname(__test_dir__) 10 | testdata_dir = os.path.join(__test_dir__, 'testdata') 11 | 12 | 13 | class TestOrthoFunc(unittest.TestCase): 14 | 15 | def setUp(self): 16 | self.srcdir = os.path.join(os.path.join(testdata_dir, 'ortho')) 17 | self.scriptpath = os.path.join(__app_dir__, "pgc_ortho.py") 18 | self.dstdir = os.path.join(__test_dir__, 'tmp_output') 19 | 20 | if platform.system() == 'Windows': 21 | self.gimpdem = r'\\ad.umn.edu\geo\pgc\data\elev\dem\gimp\GIMPv1\gimpdem_v1_30m.tif' 22 | self.rampdem = r'\\ad.umn.edu\geo\pgc\data\elev\dem\ramp\RAMPv2_wgs84_200m.tif' 23 | else: 24 | self.gimpdem = '/mnt/pgc/data/elev/dem/gimp/GIMPv1/gimpdem_v1_30m.tif' 25 | self.rampdem = '/mnt/pgc/data/elev/dem/ramp/RAMPv2_wgs84_200m.tif' 26 | 27 | # if os.path.isdir(self.dstdir): 28 | # shutil.rmtree(self.dstdir) 29 | if not os.path.isdir(self.dstdir): 30 | os.makedirs(self.dstdir) 31 | 32 | # @unittest.skip("skipping") 33 | def test_image_types(self): 34 | 35 | test_images = [ 36 | # (image_path, egsg, result) 37 | ('GE01_11OCT122053047-P1BS-10504100009FD100.ntf', 3031, True), #### GE01 image wth abscalfact in W/m2/um 38 | ('GE01_14APR022119147-M1BS-1050410010473600.ntf', 3413, True), #### GE01 image wth abscalfact in W/cm2/nm 39 | ('GE01_20110108171314_1016023_5V110108M0010160234A222000100252M_000500940.ntf', 26914, True), 40 | ('GE01_20140402211914_1050410010473600_14APR02211914-M1BS-053720734020_01_P003.ntf', 3413, True), 41 | ('IK01_19991222080400_1999122208040550000011606084_po_82037_pan_0000000.tif', 32636, False), # Corrupt 42 | ('IK01_20050319201700_2005031920171340000011627450_po_333838_blu_0000000.ntf', 3413, False), # Corrupt 43 | ('QB02_20021009211710_101001000153C800_02OCT09211710-M2AS_R1C1-052075481010_01_P001.tif', 3413, False), 44 | ('QB02_20070918204906_10100100072E5100_07SEP18204906-M3AS_R1C1-005656156020_01_P001.ntf', 3413, False), 45 | ('QB02_20120827132242_10100100101AD000_12AUG27132242-M1BS-500122876080_01_P006.ntf', 3413, True), 46 | ('WV01_20091004222215_1020010009B33500_09OCT04222215-P1BS-052532098020_01_P019.ntf', 3031, True), 47 | ('WV01_20120326222942_102001001B02FA00_12MAR26222942-P1BS-052596100010_03_P007.ntf', 3413, True), 48 | ('WV02_20100423190859_1030010005C7AF00_10APR23190859-M2AS_R1C1-052462689010_01_P001.ntf', 26910, True), 49 | ('WV02_20100804230742_1030010006A15800_10AUG04230742-M3DM_R1C3-052672098020_01_P001.tif', 3413, False), 50 | ('WV02_20120719233558_103001001B998D00_12JUL19233558-M1BS-052754253040_01_P001.tif', 3413, True), 51 | ('WV02_20131005052802_10300100278D8500_13OCT05052802-P1BS-500099283010_01_P004.ntf', 3031, True), 52 | ('WV03_20140919212947_104001000227BF00_14SEP19212947-M1BS-500191821040_01_P002.ntf', 3413, True), 53 | ('WV03_20190114103353_104C0100462B2500_19JAN14103353-C1BA-502817502010_01_P001.ntf', 3031, True) 54 | ] 55 | 56 | for test_image, epsg, result in test_images: 57 | 58 | srcfp = os.path.join(self.srcdir, test_image) 59 | dstfp = os.path.join(self.dstdir, '{}_u08rf{}.tif'.format( 60 | os.path.splitext(test_image)[0], epsg)) 61 | print(srcfp) 62 | cmd = r"""python "{}" -r 10 -p {} "{}" "{}" """.format( 63 | self.scriptpath, epsg, srcfp, self.dstdir) 64 | p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) 65 | se, so = p.communicate() 66 | # print(so) 67 | # print(se) 68 | self.assertTrue(os.path.isfile(dstfp) == result) 69 | 70 | def test_input_parameters(self): 71 | 72 | # Build configs 73 | no_dempath_config = os.path.join(self.dstdir, "no_dempath_config.ini") 74 | no_dempath_gpkg = os.path.join(os.path.join(testdata_dir, 'auto_dem', 'no_dempath.gpkg')) 75 | with open(no_dempath_config, "w") as f: 76 | f.write(f"[default]\ngpkg_path = {no_dempath_gpkg}") 77 | 78 | good_config = os.path.join(self.dstdir, "good_config.ini") 79 | good_config_gpkg = os.path.join(os.path.join(testdata_dir, 'auto_dem', 'dems_list.gpkg')) 80 | with open(good_config, "w") as f: 81 | f.write(f"[default]\ngpkg_path = {good_config_gpkg}") 82 | 83 | cmds = [ 84 | # (file name, arg string, should succeed, output extension) 85 | # epsg: 3413 86 | # stretch: mr 87 | # resample: cubic 88 | # outtype: Byte 89 | # gtiff compression: jpeg95 90 | # dem: gimpdem_v2_30m.tif 91 | ('QB02_20120827132242_10100100101AD000_12AUG27132242-M1BS-500122876080_01_P006.ntf', 92 | f'-r 10 --epsg 3413 --stretch mr --resample cubic --format GTiff --outtype Byte --gtiff-compression jpeg95 --dem {self.gimpdem}', 93 | True, 94 | '.tif'), 95 | 96 | # --rgb with Geoeye image 97 | # epsg: 3413 98 | # stretch: rf 99 | # outtype: Byte 100 | # gtiff compression: jpeg95 101 | ('GE01_20110108171314_1016023_5V110108M0010160234A222000100252M_000500940.ntf', 102 | '-r 10 --epsg 3413 --stretch mr --rgb --format GTiff --outtype Byte --gtiff-compression jpeg95', 103 | True, 104 | '.tif'), 105 | 106 | # ns, rgb, and Byte with CAVIS image 107 | # epsg: auto 108 | # stretch: ns 109 | # outtype: Byte 110 | # gtiff compression: jpeg95 111 | ('WV03_20190114103353_104C0100462B2500_19JAN14103353-C1BA-502817502010_01_P001.ntf', 112 | '-r 10 --epsg auto --stretch ns --rgb --format GTiff --outtype Byte --gtiff-compression jpeg95', 113 | True, 114 | '.tif'), 115 | 116 | # --rgb with SWIR image 117 | # epsg: 3413 118 | # stretch: auto 119 | ('WV03_20150712212305_104A01000E7C1F00_15JUL12212305-A1BS-500802261010_01_P001.ntf', 120 | '-r 10 --epsg 3413 --stretch au --rgb --format GTiff --outtype Byte --gtiff-compression jpeg95', 121 | True, 122 | '.tif'), 123 | 124 | # epsg: 3413 125 | # stretch: rf 126 | # resample: near 127 | # format: ENVI 128 | # outtype: Byte 129 | # gtiff compression: lzw 130 | # dem: gimpdem_v2_30m.tif 131 | ('QB02_20120827132242_10100100101AD000_12AUG27132242-M1BS-500122876080_01_P006.ntf', 132 | f'-r 10 --epsg 3413 --stretch rf --resample near --format ENVI --outtype Byte --gtiff-compression lzw --dem {self.gimpdem}', 133 | True, 134 | '.envi'), 135 | 136 | # epsg: 3413 137 | # stretch: rf 138 | # resample: near 139 | # format: .img 140 | # outtype: Float32 141 | # gtiff compression: lzw 142 | # dem: Y:/private/elevation/dem/GIMP/GIMPv2/gimpdem_v2_30m.tif 143 | ('QB02_20120827132242_10100100101AD000_12AUG27132242-M1BS-500122876080_01_P006.ntf', 144 | f'-r 10 --epsg 3413 --stretch rf --resample near --format HFA --outtype Float32 --gtiff-compression lzw --dem {self.gimpdem}', 145 | True, 146 | '.img'), 147 | 148 | # epsg: 3413 149 | # stretch: rd 150 | # outtype: UInt16 151 | # format: .jp2 152 | ('QB02_20120827132242_10100100101AD000_12AUG27132242-M1BS-500122876080_01_P006.ntf', 153 | '-r 10 --epsg 3413 --stretch rd --resample near --format JP2OpenJPEG --outtype UInt16 --gtiff-compression lzw', 154 | True, 155 | '.jp2'), 156 | 157 | # dem: Y:/private/elevation/dem/RAMP/RAMPv2/RAMPv2_wgs84_200m.tif 158 | # should fail: the image is not contained within the DEM 159 | ('QB02_20120827132242_10100100101AD000_12AUG27132242-M1BS-500122876080_01_P006.ntf', 160 | f'-r 10 --epsg 3413 --dem {self.rampdem}', 161 | False, 162 | '.tif'), 163 | 164 | # stretch, dem, and epsg: auto 165 | ('WV02_20120719233558_103001001B998D00_12JUL19233558-M1BS-052754253040_01_P001.tif', 166 | f'-r 10 --skip-cmd-txt --epsg auto --stretch au --dem auto ' 167 | f'--config {good_config}', 168 | True, 169 | '.tif'), 170 | 171 | # stretch, dem, and epsg: auto with image over the ocean 172 | ('WV03_20210811190908_104001006CD51400_21AUG11190908-M1BS-505623932030_01_P001.ntf', 173 | f'-r 10 --skip-cmd-txt --epsg auto --stretch au --dem auto ' 174 | f'--config {good_config}', 175 | True, 176 | '.tif'), 177 | 178 | # bad auto dem config 179 | (f'QB02_20120827132242_10100100101AD000_12AUG27132242-M1BS-500122876080_01_P006.ntf', 180 | f'-r 10 --skip-cmd-txt --epsg auto --stretch au --dem auto ' 181 | f'--config {no_dempath_config}', 182 | False, 183 | '.tif'), 184 | 185 | # non-existing auto dem config 186 | (f'QB02_20120827132242_10100100101AD000_12AUG27132242-M1BS-500122876080_01_P006.ntf', 187 | f'-r 10 --skip-cmd-txt --epsg auto --stretch au --dem auto ' 188 | f'--config {os.path.join(self.dstdir, "does_not_exist_config.ini")}', 189 | False, 190 | '.tif'), 191 | ] 192 | 193 | i = 0 194 | for fn, test_args, succeeded, ext in cmds: 195 | i += 1 196 | _dstdir = os.path.join(self.dstdir, str(i)) 197 | os.mkdir(_dstdir) 198 | print(fn) 199 | cmd = f'python "{self.scriptpath}" {test_args} {self.srcdir}/{fn} {_dstdir}' 200 | print(cmd) 201 | p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) 202 | se, so = p.communicate() 203 | print(so) 204 | print(se) 205 | output_files = glob.glob(os.path.join(_dstdir, f'{os.path.splitext(fn)[0]}*{ext}')) 206 | self.assertEqual(len(output_files) > 0, succeeded) 207 | 208 | def tearDown(self): 209 | shutil.rmtree(self.dstdir) 210 | 211 | 212 | if __name__ == '__main__': 213 | 214 | test_cases = [ 215 | TestOrthoFunc 216 | ] 217 | 218 | suites = [] 219 | for test_case in test_cases: 220 | suite = unittest.TestLoader().loadTestsFromTestCase(test_case) 221 | suites.append(suite) 222 | 223 | alltests = unittest.TestSuite(suites) 224 | unittest.TextTestRunner(verbosity=2).run(alltests) 225 | -------------------------------------------------------------------------------- /tests/test_ortho_functions.py: -------------------------------------------------------------------------------- 1 | import platform 2 | import unittest, os, sys 3 | from osgeo import gdal, ogr 4 | from collections import namedtuple 5 | 6 | __test_dir__ = os.path.dirname(os.path.abspath(__file__)) 7 | sys.path.append(os.path.dirname(__test_dir__)) 8 | testdata_dir = os.path.join(__test_dir__, 'testdata') 9 | 10 | from lib import ortho_functions, utils 11 | from lib import VERSION 12 | 13 | Band_data_range = namedtuple('Band_data_range', ['factor_min', 'factor_max', 'offset_min', 'offset_max']) 14 | 15 | 16 | class TestReadMetadata(unittest.TestCase): 17 | 18 | def setUp(self): 19 | self.stretch = 'rf' 20 | self.rd_stretch = 'rd' 21 | self.srcdir = os.path.join(testdata_dir, 'metadata_files') 22 | 23 | # @unittest.skip("skipping") 24 | def test_parse_DG_md_files(self): 25 | 26 | dg_files = ( 27 | ##(file name, is readable, is usable) 28 | ('10APR23190859-M2AS-052462689010_01_P001.xml', True, True), ## 2A unrenamed 29 | ('12JUL19233558-M1BS-052754253040_01_P001.xml', True, True), ## 1B unrenamed 30 | ('12AUG27132242-M1BS-500122876080_01_P006.xml', False, False), ## 1B unrenamed truncated xml 31 | ('GE01_11OCT122053047-P1BS-10504100009FD100.xml', True, True), #### GE01 image wth abscalfact in W/m2/um 32 | ('GE01_14APR022119147-M1BS-1050410010473600.xml', True, True), #### GE01 image wth abscalfact in W/cm2/nm 33 | ('GE01_20140402211914_1050410010473600_14APR02211914-M1BS-053720734020_01_P003.xml', True, True), ##GE01 pgctools3 name 34 | ('QB02_02OCT092117107-M2AS_R1C1-101001000153C800.xml', True, True), #2A tiled pgctools2 renamed 35 | ('QB02_12AUG271322429-M1BS-10100100101AD000.xml', True, True), #1B pgctools2 renamed 36 | ('QB02_20021009211710_101001000153C800_02OCT09211710-M2AS_R1C1-052075481010_01_P001.xml', True, True), 37 | ('QB02_20120827132242_10100100101AD000_12AUG27132242-M1BS-500122876080_01_P006.xml', True, True), 38 | ('WV01_09OCT042222158-P1BS-1020010009B33500.xml', True, True), 39 | ('WV01_12MAR262229422-P1BS-102001001B02FA00.xml', True, True), 40 | ('WV01_20091004222215_1020010009B33500_09OCT04222215-P1BS-052532098020_01_P019.xml', True, True), 41 | ('WV01_20120326222942_102001001B02FA00_12MAR26222942-P1BS-052596100010_03_P007.xml', True, True), 42 | ('WV02_10APR231908590-M2AS_R1C1-1030010005C7AF00.xml', True, True), 43 | ('WV02_10APR231908590-M2AS_R2C3-1030010005C7AF00.xml', True, True), 44 | ('WV02_12JUL192335585-M1BS-103001001B998D00.xml', True, True), 45 | ('WV02_13OCT050528024-P1BS-10300100278D8500.xml', True, True), 46 | ('WV02_20131005052802_10300100278D8500_13OCT05052802-P1BS-500099283010_01_P004.xml', True, True), 47 | ('WV03_14SEP192129471-M1BS-104001000227BF00.xml', True, True), 48 | ('WV03_20140919212947_104001000227BF00_14SEP19212947-M1BS-500191821040_01_P002.xml', True, True), 49 | ('QB02_20050623212833_1010010004535800_05JUN23212833-P2AS-005511498020_01_P001.xml', True, False), # uses EARLIESTACQTIME instead of FIRSTLINETIME, but has no EFFECTIVEBANDWIDTH 50 | ('WV03_20150411220541_104A01000A704D00_15APR11220541-A1BS-500802194040_01_P001.xml', True, True), # SWIR 51 | ('WV03_20150526221639_104A01000C51A100_15MAY26221639-A1BS-500802200030_01_P001.xml', True, True), # SWIR 52 | ('WV03_20190114103353_104C0100462B2500_19JAN14103353-C1BA-502817502010_01_P001.xml', True, True), # CAVIS A 53 | ('WV03_20190114103355_104C0100462B2500_19JAN14103355-C1BB-502817502010_01_P001.xml', True, True), # CAVIS B 54 | ) 55 | 56 | dg_valid_data_range = { 57 | 'rf': { 58 | 'BAND_P': (Band_data_range(0.0005, 0.0015, -0.036, 0)), # pan 59 | 'BAND_B': (Band_data_range(0.000447, 0.0012, -0.1307, 0.0012)), # blue 60 | 'BAND_S1': (Band_data_range(0.00005, 0.0015, 0, 0)), # 1st SWIR band 61 | }, 62 | 'rd': { 63 | 'BAND_P': (Band_data_range(0.08, 0.15, -5.55, 0)), 64 | 'BAND_B': (Band_data_range(0.17, 0.33, -9.84, 0)), 65 | 'BAND_S1': (Band_data_range(0.0045, 0.15, 0, 0)), 66 | } 67 | } 68 | 69 | dg_test_bands = ['BAND_P', 'BAND_B', 'BAND_S1', 'BAND_DC', 'BAND_A1'] 70 | 71 | for stretch in [self.stretch, self.rd_stretch]: 72 | stretch_params_method(self, dg_files, stretch, dg_valid_data_range, dg_test_bands, 73 | utils.get_dg_metadata_as_xml, ortho_functions.get_dg_calib_dict, ) 74 | 75 | def test_parse_GE_md_files(self): 76 | ge_files = ( 77 | ('GE01_110108M0010160234A222000100252M_000500940.txt', True, True), 78 | ) 79 | 80 | ge_valid_data_range = { 81 | 'rf': { 82 | 5: (Band_data_range(0.0002, 0.0008, 0, 0)), # pan 83 | 1: (Band_data_range(0.0002, 0.0008, 0, 0)), # blue 84 | }, 85 | 'rd': { 86 | 5: (Band_data_range(0.08, 0.18, 0, 0)), 87 | 1: (Band_data_range(0.14, 0.33, 0, 0)), 88 | } 89 | } 90 | 91 | ge_test_bands = None 92 | 93 | for stretch in [self.stretch, self.rd_stretch]: 94 | stretch_params_method(self, ge_files, stretch, ge_valid_data_range, ge_test_bands, 95 | utils.get_ge_metadata_as_xml, ortho_functions.get_ge_calib_dict) 96 | 97 | def test_parse_IK_md_files(self): 98 | 99 | ik_files = ( 100 | ('IK01_20010602215300_2001060221531300000010031227_po_387877_metadata.txt', True, True), ## test IK metadata file with multiple source IDs 101 | ('IK01_19991222080400_1999122208040550000011606084_po_82037_metadata.txt', True, True), ## test pgctools3 name 102 | ('IK01_20050319201700_2005031920171340000011627450_po_333838_metadata.txt', True, True), ## test pgctools3 name 103 | ('IK01_1999122208040550000011606084_pan_1569N.txt', True, True), ## test pgctools2 name 104 | ('IK01_2005031920171340000011627450_rgb_5817N.txt', True, True), ## test pgctools2 name 105 | ) 106 | 107 | ik_valid_data_range = { 108 | 'rf': { 109 | 4: (Band_data_range(0.0004, 0.0007, 0, 0)), # pan 110 | 0: (Band_data_range(0.0003, 0.0007, 0, 0)), # blue 111 | }, 112 | 'rd': { 113 | 4: (Band_data_range(0.1, 0.16, 0, 0)), 114 | 0: (Band_data_range(0.15, 0.25, 0, 0)), 115 | } 116 | } 117 | 118 | ik_test_bands = None 119 | 120 | for stretch in [self.stretch, self.rd_stretch]: 121 | stretch_params_method(self, ik_files, stretch, ik_valid_data_range, ik_test_bands, 122 | utils.get_ik_metadata_as_xml, ortho_functions.get_ik_calib_dict) 123 | 124 | 125 | def stretch_params_method(test_obj, file_list, stretch, valid_data_range, test_bands, meta_function, calib_function): 126 | # Test stretch factor and offset 127 | metadata_files = [(os.path.join(test_obj.srcdir, m), r1, r2) for m, r1, r2 in file_list] 128 | for mdf, is_readable, is_usable in metadata_files: 129 | # print(f'{mdf}: {is_readable} {is_usable}') 130 | metad = None 131 | calib_dict = {} 132 | img_name = os.path.basename(mdf).replace('metadata.txt', 'blu_0000000.ntf') 133 | _, _, _, _, _, regex = utils.get_sensor(img_name) 134 | try: 135 | metad = meta_function(mdf) 136 | except utils.InvalidMetadataError as e: 137 | pass 138 | if metad: 139 | try: 140 | if calib_function == ortho_functions.get_ik_calib_dict: 141 | calib_dict = calib_function(metad, mdf, regex, stretch) 142 | else: 143 | calib_dict = calib_function(metad, stretch) 144 | except utils.InvalidMetadataError as e: 145 | pass 146 | test_obj.assertEqual(bool(metad), is_readable) 147 | test_obj.assertEqual(bool(calib_dict), is_usable) 148 | if calib_dict: 149 | if test_bands: 150 | # For DG, exactly one of the listed bands should be present 151 | t = [b for b in test_bands if b in calib_dict] 152 | test_obj.assertEqual(len(t), 1) 153 | # Check band values are not equal 154 | all_band_factors = [f for f, b in calib_dict.values()] 155 | test_obj.assertEqual(len(all_band_factors), len(set(all_band_factors))) 156 | # Check stretch values are within reasonable limits 157 | bdr_dict = valid_data_range[stretch] 158 | for band, bdr in bdr_dict.items(): 159 | if band in calib_dict: 160 | test_obj.assertGreaterEqual(calib_dict[band][0], bdr.factor_min) 161 | test_obj.assertLessEqual(calib_dict[band][0], bdr.factor_max) 162 | test_obj.assertGreaterEqual(calib_dict[band][1], bdr.offset_min) 163 | test_obj.assertLessEqual(calib_dict[band][1], bdr.offset_max) 164 | 165 | 166 | class TestWriteMetadata(unittest.TestCase): 167 | 168 | def setUp(self): 169 | self.epsg = '4326' 170 | self.stretch = 'rf' 171 | self.srcfn = 'WV03_20140919212947_104001000227BF00_14SEP19212947-M1BS-500191821040_01_P002.ntf' 172 | self.srcfp = os.path.join(testdata_dir, 'ortho', self.srcfn) 173 | self.dstdir = os.path.join(__test_dir__, 'tmp_output') 174 | self.dstfp = os.path.join(self.dstdir, '{}_u08{}{}.tif'.format( 175 | os.path.splitext(self.srcfn)[0], 176 | self.stretch, 177 | self.epsg)) 178 | self.mf = f'{os.path.splitext(self.dstfp)[0]}.xml' 179 | 180 | self.test_lines = [ 181 | f'Byte', 182 | f'lzw', 183 | f'{self.epsg}', 184 | f'GTiff', 185 | f'{self.stretch}', 186 | f'imagery_utils v{VERSION}', 187 | f'near' 188 | ] 189 | if not os.path.isdir(self.dstdir): 190 | os.makedirs(self.dstdir) 191 | 192 | def test_write_DG_md_file(self): 193 | test_args = ProcessArgs(self.epsg, self.stretch) 194 | info = ortho_functions.ImageInfo(self.srcfp, self.dstdir, self.dstdir, test_args) 195 | rc = ortho_functions.write_output_metadata(test_args, info) 196 | ## read meta and check content 197 | f = open(self.mf) 198 | contents = f.read() 199 | f.close() 200 | for test_line in self.test_lines: 201 | self.assertTrue(test_line in contents) 202 | 203 | def tearDown(self): 204 | if os.path.isfile(self.mf): 205 | os.remove(self.mf) 206 | 207 | 208 | class TestCollectFiles(unittest.TestCase): 209 | 210 | def test_gather_metadata_file(self): 211 | 212 | rm_files = [ 213 | '01JAN08QB020800008JAN01102125-P1BS-005590467020_01_P001_________AAE_0AAAAABAABA0.xml' 214 | ] 215 | skip_list = [ 216 | '01JAN08QB020800008JAN01102125-P1BS-005590467020_01_P001_________AAE_0AAAAABAABA0.ntf' # tar has an issue 217 | ] 218 | 219 | for root, dirs, files in os.walk(os.path.join(testdata_dir, 'ortho')): 220 | for f in files: 221 | if (f.lower().endswith(".ntf") or f.lower().endswith(".tif")) and f not in skip_list: 222 | #### Find metadata file 223 | stretch = 'rf' 224 | epsg = '4326' 225 | srcfp = os.path.join(root, f) 226 | dstdir = os.path.join(__test_dir__, 'tmp_output') 227 | dstfp = os.path.join(dstdir, '{}_u08{}{}.tif'.format( 228 | os.path.splitext(f)[0], 229 | stretch, 230 | epsg)) 231 | test_args = ProcessArgs(epsg, stretch) 232 | info = ortho_functions.ImageInfo(srcfp, dstfp, dstdir, test_args) 233 | self.assertIsNotNone(info.metapath) 234 | 235 | if info.metapath and os.path.basename(info.metapath) in rm_files: 236 | os.remove(info.metapath) 237 | 238 | 239 | class TestDEMOverlap(unittest.TestCase): 240 | 241 | def setUp(self): 242 | self.dem = os.path.join(os.path.join(testdata_dir, 'dem', 'grimp_200m.tif')) # dem for greenland 243 | self.srs = utils.SpatialRef(4326) 244 | 245 | def test_dem_overlap(self): 246 | image_geom_wkts = [ 247 | ('POLYGON ((-52.23 -80.843333, -51.735 -80.844444, -51.736667 -80.760556, -52.23 -80.759722, -52.23 -80.843333))', 248 | False), # False 249 | ('POLYGON ((-52.23 70.843333, -51.735 70.844444, -51.736667 70.760556, -52.23 70.759722, -52.23 70.843333))', 250 | True), # True 251 | ('POLYGON ((-52.23 -50.843333, -51.735 -50.844444, -51.736667 -50.760556, -52.23 -50.759722, -52.23 -50.843333))', 252 | False) # False 253 | ] 254 | 255 | for wkt, result in image_geom_wkts: 256 | test_result = ortho_functions.overlap_check(wkt, self.srs, self.dem) 257 | self.assertEqual(test_result, result) 258 | 259 | 260 | class TestAutoDEMOverlap(unittest.TestCase): 261 | 262 | def setUp(self): 263 | self.gpkg = os.path.join(os.path.join(testdata_dir, 'auto_dem', 'dems_list.gpkg')) 264 | self.srs = utils.SpatialRef(4326) 265 | 266 | def test_auto_dem_overlap(self): 267 | image_geom_wkts = [ 268 | # polygon, linux path 269 | ('POLYGON ((-52.23 70.843333, -51.735 70.844444, -51.736667 70.760556, -52.23 70.759722, -52.23 70.843333))', 270 | '/mnt/pgc/data/elev/dem/gimp/GrIMPv2/data/grimp_v02.0_30m_dem.tif'), #greenland 271 | ('POLYGON ((-52.23 -80.843333, -51.735 -80.844444, -51.736667 -80.760556, -52.23 -80.759722, -52.23 -80.843333))', 272 | '/mnt/pgc/data/elev/dem/tandem-x/90m/mosaic/TanDEM-X_Antarctica_90m/TanDEMX_PolarDEM_90m.tif'), #antarctic 273 | ('POLYGON ((-52.23 -50.843333, -51.735 -50.844444, -51.736667 -50.760556, -52.23 -50.759722, -52.23 -50.843333))', 274 | None), # ocean 275 | ('POLYGON ((-52.3475 84.515555,-50.882 84.53222,-50.89833 84.40166,-52.330833 84.3844,-52.3475 84.5155))', 276 | None), # ocean 277 | ('POLYGON ((-49.23 61.910556, -47.735 58.844444, -47.735 61.910556, -49.23 58.844444,-49.23 61.910556))', 278 | '/mnt/pgc/data/elev/dem/gimp/GrIMPv2/data/grimp_v02.0_30m_dem.tif'), # greenland centroid 279 | ('POLYGON ((11 -68.5, 11 -70, 12 -70, 12 -68.5, 11 -68.5))', 280 | '/mnt/pgc/data/elev/dem/tandem-x/90m/mosaic/TanDEM-X_Antarctica_90m/TanDEMX_PolarDEM_90m.tif'), # antarctic centroid 281 | ('POLYGON ((-49.23 59.7, -48.23 59.7,-47.23 58.7, -49.23 58.7,-49.23 59.7))', 282 | '/mnt/pgc/data/elev/dem/gimp/GrIMPv2/data/grimp_v02.0_30m_dem.tif'), # greenland, but not contained 283 | ('POLYGON ((-56 -61, -55 -61, -55 -60,-56 -60, -56 -61))', 284 | '/mnt/pgc/data/elev/dem/tandem-x/90m/mosaic/TanDEM-X_Antarctica_90m/TanDEMX_PolarDEM_90m.tif'), # antarctic, but not contained 285 | ('POLYGON ((-89.43 81.53, -88.94 81.53, -88.94 81.33, -89.43 81.33, -89.43 81.53))', 286 | '/mnt/pgc/data/elev/dem/copernicus-dem-30m/mosaic/global/cop30_tiles_global_wgs84-height_nunatak.vrt'), #Centroid in copernicus layer which overlaps greenland layer 287 | ] 288 | 289 | windows_paths = { 290 | # linux path: windows path 291 | '/mnt/pgc/data/elev/dem/gimp/GrIMPv2/data/grimp_v02.0_30m_dem.tif': 292 | r'\\ad.umn.edu\geo\pgc\data\elev\dem\gimp\GrIMPv2\data\grimp_v02.0_30m_dem.tif', 293 | '/mnt/pgc/data/elev/dem/tandem-x/90m/mosaic/TanDEM-X_Antarctica_90m/TanDEMX_PolarDEM_90m.tif': 294 | r'\\ad.umn.edu\geo\pgc\data\elev\dem\tandem-x\90m\mosaic\TanDEM-X_Antarctica_90m\TanDEMX_PolarDEM_90m.tif', 295 | '/mnt/pgc/data/elev/dem/copernicus-dem-30m/mosaic/global/cop30_tiles_global_wgs84-height_nunatak.vrt': 296 | r'\\ad.umn.edu\geo\pgc\data\elev\dem\copernicus-dem-30m\mosaic\global\cop30_tiles_global_wgs84-height_windows.vrt' 297 | } 298 | 299 | for wkt, result in image_geom_wkts: 300 | test_result = ortho_functions.check_image_auto_dem(wkt, self.srs, self.gpkg) 301 | if result is not None and platform.system() == 'Windows': 302 | result = windows_paths[result] 303 | self.assertEqual(test_result, result) 304 | 305 | def test_auto_dem_invalid_gpkgs(self): 306 | image_geom_wkt = 'POLYGON ((-52.23 70.843333, -51.735 70.844444, -51.736667 70.760556, -52.23 70.759722, -52.23 70.843333))' 307 | 308 | gpkgs = [ 309 | os.path.join(testdata_dir, 'auto_dem', 'invalid.gpkg'), 310 | os.path.join(testdata_dir, 'auto_dem', 'no_dempath.gpkg') 311 | ] 312 | 313 | for gpkg in gpkgs: 314 | assert(os.path.isfile(gpkg)) 315 | with self.assertRaises(RuntimeError): 316 | ortho_functions.check_image_auto_dem(image_geom_wkt, self.srs, gpkg) 317 | 318 | 319 | class TestTargetExtent(unittest.TestCase): 320 | 321 | def test_target_extent(self): 322 | epsg = '32629' 323 | stretch = 'rf' 324 | wkt = 'POLYGON ((810287 2505832,811661 2487415,807201 2487233,805772 2505802,810287 2505832))' 325 | fn = 'GE01_20110307105821_1050410001518E00_11MAR07105821-M1BS-500657359080_01_P008.ntf' 326 | srcfp = os.path.join(testdata_dir, 'ortho', fn) 327 | dstdir = os.path.join(__test_dir__, 'tmp_output') 328 | target_extent_geom = ogr.CreateGeometryFromWkt(wkt) 329 | test_args = ProcessArgs(epsg, stretch) 330 | info = ortho_functions.ImageInfo(srcfp, dstdir, dstdir, test_args) 331 | rc = info.get_image_stats(test_args) 332 | rc = info.set_extent_geom(target_extent_geom) 333 | self.assertEqual(info.extent, 334 | '-te 805772.000000000000 2487233.000000000000 811661.000000000000 2505832.000000000000 ') 335 | 336 | 337 | class TestAutoStretchAndEpsg(unittest.TestCase): 338 | 339 | def test_auto_stretch_and_epsg(self): 340 | test_files = ( 341 | # file name, expected stretch, expected epsg 342 | ('WV03_20190114103353_104C0100462B2500_19JAN14103353-C1BA-502817502010_01_P001.ntf', 'rf', 3031), # CAVIS 343 | ('WV03_20190114103355_104C0100462B2500_19JAN14103355-C1BB-502817502010_01_P001.ntf', 'rf', 3031), # CAVIS 344 | ('WV03_20150526221639_104A01000C51A100_15MAY26221639-A1BS-500802200030_01_P001.ntf', 'rf', 3413), # SWIR 345 | ('GE01_20110307105821_1050410001518E00_11MAR07105821-M1BS-500657359080_01_P008.ntf', 'mr', 32630), # Nonpolar 346 | ('WV03_20140919212947_104001000227BF00_14SEP19212947-M1BS-500191821040_01_P002.ntf', 'mr', 3413), # Arctic 347 | ('WV03_20181226170822_10400100468EFA00_18DEC26170822-M1BS-502826003080_01_P009.ntf', 'rf', 3031), # Antarctic 348 | ) 349 | 350 | for fn, out_stretch, out_epsg in test_files: 351 | in_epsg = 'auto' 352 | in_stretch = 'au' 353 | srcfp = os.path.join(testdata_dir, 'ortho', fn) 354 | dstdir = os.path.join(__test_dir__, 'tmp_output') 355 | test_args = ProcessArgs(in_epsg, in_stretch) 356 | info = ortho_functions.ImageInfo(srcfp, dstdir, dstdir, test_args) 357 | self.assertEqual(info.stretch, out_stretch) 358 | self.assertEqual(info.epsg, out_epsg) 359 | 360 | 361 | class TestCalcEarthSunDist(unittest.TestCase): 362 | def setUp(self): 363 | self.year = 2010 364 | self.month = 10 365 | self.day = 20 366 | self.hour = 10 367 | self.minute = 20 368 | self.second = 10 369 | 370 | def test_calc_esd(self): 371 | esd = ortho_functions.calc_earth_sun_dist(self) 372 | self.assertEqual(esd, 0.9957508611980816) 373 | 374 | 375 | class TestRPCHeight(unittest.TestCase): 376 | def setUp(self): 377 | self.epsg = 4326 # also test epsg as an integer 378 | self.stretch = 'rf' 379 | self.srcdir = os.path.join(testdata_dir, 'ortho') 380 | self.dstdir = os.path.join(__test_dir__, 'tmp_output') 381 | self.test_args = ProcessArgs(self.epsg, self.stretch) 382 | self.srcfns = [ 383 | ('WV01_20091004222215_1020010009B33500_09OCT04222215-P1BS-052532098020_01_P019.ntf', 2568.0), 384 | ('WV02_20120719233558_103001001B998D00_12JUL19233558-M1BS-052754253040_01_P001.tif', 75.0), 385 | ('WV03_20140919212947_104001000227BF00_14SEP19212947-M1BS-500191821040_01_P002.ntf', 149.0), 386 | ('GE01_20110307105821_1050410001518E00_11MAR07105821-P1BS-500657359080_01_P008.ntf', 334.0), 387 | ('IK01_20050319201700_2005031920171340000011627450_po_333838_blu_0000000.ntf', 520.0), 388 | ('QB02_20120827132242_10100100101AD000_12AUG27132242-M1BS-500122876080_01_P006.ntf', 45.0) 389 | ] 390 | 391 | def test_rpc_height(self): 392 | for srcfn, test_h in self.srcfns: 393 | srcfp = os.path.join(self.srcdir, srcfn) 394 | info = ortho_functions.ImageInfo(srcfp, self.dstdir, self.dstdir, self.test_args) 395 | h = ortho_functions.get_rpc_height(info) 396 | self.assertEqual(h, test_h) 397 | 398 | 399 | class ProcessArgs(object): 400 | def __init__(self, epsg='4326', stretch='rf'): 401 | self.epsg = epsg 402 | self.resolution = None 403 | self.rgb = False 404 | self.bgrn = False 405 | self.skip_warp = False 406 | self.dem = None 407 | self.ortho_height = None 408 | self.resample = 'near' 409 | self.outtype = 'Byte' 410 | self.format = "GTiff" 411 | self.gtiff_compression = "lzw" 412 | self.stretch = stretch 413 | self.tap = False 414 | self.wd = None 415 | self.epsg_utm_nad83 = False 416 | 417 | 418 | if __name__ == '__main__': 419 | 420 | test_cases = [ 421 | TestReadMetadata, 422 | TestWriteMetadata, 423 | TestCollectFiles, 424 | TestDEMOverlap, 425 | TestAutoDEMOverlap, 426 | TestTargetExtent, 427 | TestAutoStretchAndEpsg, 428 | TestRPCHeight, 429 | TestCalcEarthSunDist, 430 | ] 431 | 432 | suites = [] 433 | for test_case in test_cases: 434 | suite = unittest.TestLoader().loadTestsFromTestCase(test_case) 435 | suites.append(suite) 436 | 437 | alltests = unittest.TestSuite(suites) 438 | unittest.TextTestRunner(verbosity=2).run(alltests) 439 | -------------------------------------------------------------------------------- /tests/test_ortho_functions_nodata.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import pytest 4 | 5 | # Modify the path so that modules in the project root can be imported without installing 6 | test_dir = os.path.dirname(__file__) 7 | root_dir = os.path.dirname(test_dir) 8 | sys.path.append(root_dir) 9 | 10 | from lib.ortho_functions import get_destination_nodata, OutputType 11 | 12 | def test_get_destination_nodata_succeeds(): 13 | assert get_destination_nodata(output_type=OutputType.BYTE) == 0 14 | assert get_destination_nodata(output_type="Byte") == 0 15 | 16 | assert get_destination_nodata(output_type=OutputType.UINT16) == 65535 17 | assert get_destination_nodata(output_type="UInt16") == 65535 18 | 19 | assert get_destination_nodata(output_type=OutputType.FLOAT32) == -9999.0 20 | assert get_destination_nodata(output_type="Float32") == -9999.0 21 | 22 | def test_get_destination_nodata_raises(): 23 | with pytest.raises(ValueError): 24 | get_destination_nodata(output_type="not a valid output type") -------------------------------------------------------------------------------- /tests/test_pansharpen.py: -------------------------------------------------------------------------------- 1 | import shutil 2 | import unittest, os, subprocess 3 | import sys 4 | from osgeo import gdal 5 | 6 | __test_dir__ = os.path.dirname(os.path.abspath(__file__)) 7 | __app_dir__ = os.path.dirname(__test_dir__) 8 | sys.path.append(__app_dir__) 9 | testdata_dir = os.path.join(__test_dir__, 'testdata') 10 | 11 | from lib import mosaic 12 | 13 | 14 | class TestPanshFunc(unittest.TestCase): 15 | 16 | def setUp(self): 17 | 18 | self.scriptpath = os.path.join(__app_dir__, "pgc_pansharpen.py") 19 | self.srcdir = os.path.join(testdata_dir, 'pansharpen', 'src') 20 | self.dstdir = os.path.join(__test_dir__, 'tmp_output') 21 | if not os.path.isdir(self.dstdir): 22 | os.makedirs(self.dstdir) 23 | 24 | def test_pansharpen(self): 25 | 26 | src = self.srcdir 27 | cmd = 'python {} {} {} --skip-cmd-txt -p 3413'.format( 28 | self.scriptpath, 29 | src, 30 | self.dstdir, 31 | ) 32 | 33 | p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) 34 | se, so = p.communicate() 35 | print(so) 36 | print(se) 37 | 38 | # make sure output data exist 39 | dstfp = os.path.join(self.dstdir, 'WV02_20110901210502_103001000D52C800_11SEP01210502-M1BS-052560788010_01_P008_u08rf3413_pansh.tif') 40 | dstfp_xml = os.path.join(self.dstdir, 'WV02_20110901210502_103001000D52C800_11SEP01210502-M1BS-052560788010_01_P008_u08rf3413_pansh.xml') 41 | 42 | self.assertTrue(os.path.isfile(dstfp)) 43 | self.assertTrue(os.path.isfile(dstfp_xml)) 44 | 45 | # check second image from proccessing 46 | dstfp_2 = os.path.join(self.dstdir, 'WV02_20110901210434_103001000B41DC00_11SEP01210434-M1BS-052730735130_01_P007_u08rf3413_pansh.tif') 47 | dstfp_xml_2 = os.path.join(self.dstdir, 'WV02_20110901210434_103001000B41DC00_11SEP01210434-M1BS-052730735130_01_P007_u08rf3413_pansh.xml') 48 | 49 | self.assertTrue(os.path.isfile(dstfp_2)) 50 | self.assertTrue(os.path.isfile(dstfp_xml_2)) 51 | 52 | # verify data type 53 | ds = gdal.Open(dstfp, gdal.GA_ReadOnly) 54 | dt = ds.GetRasterBand(1).DataType 55 | self.assertEqual(dt, 1) 56 | ds = None 57 | 58 | image_info = mosaic.ImageInfo(dstfp, 'IMAGE') 59 | self.assertAlmostEqual(image_info.xres, 0.564193804791, 11) 60 | self.assertAlmostEqual(image_info.yres, 0.560335413717, 11) 61 | self.assertEqual(image_info.bands, 4) 62 | self.assertEqual(image_info.datatype, 1) 63 | 64 | mosaic_args = MosaicArgs() 65 | mosaic_params = mosaic.getMosaicParameters(image_info, mosaic_args) 66 | image_info.getScore(mosaic_params) 67 | 68 | self.assertEqual(image_info.sensor, 'WV02') 69 | self.assertEqual(image_info.sunel, 37.8) 70 | self.assertEqual(image_info.ona, 23.5) 71 | self.assertEqual(image_info.cloudcover, 0.003) 72 | self.assertEqual(image_info.tdi, 18.0) 73 | self.assertEqual(image_info.panfactor, 1) 74 | self.assertEqual(image_info.date_diff, -9999) 75 | self.assertEqual(image_info.year_diff, -9999) 76 | self.assertAlmostEqual(image_info.score, 77.34933333333333) 77 | 78 | image_info.get_raster_stats() 79 | stat_dct = {1: [2.0, 153.0, 21.934843, 7.315011], 80 | 2: [1.0, 141.0, 17.149106, 6.760020], 81 | 3: [1.0, 145.0, 11.088902, 7.401054], 82 | 4: [1.0, 172.0, 37.812614, 27.618598]} 83 | datapixelcount_dct = {1: 857617457, 2: 857617457, 3: 857617457, 4: 857617457} 84 | datapixelcount_threshold = 500 85 | for i in range(1,len(image_info.stat_dct)+1):# check stats are similar 86 | for j in range(4): 87 | self.assertAlmostEqual(image_info.stat_dct[i][j], stat_dct[i][j], 4) 88 | print(f'found:{image_info.datapixelcount_dct[i]}, expected {datapixelcount_dct[i]} +-{datapixelcount_threshold}') 89 | self.assertTrue(abs(image_info.datapixelcount_dct[i] - datapixelcount_dct[i]) < datapixelcount_threshold) 90 | 91 | def tearDown(self): 92 | shutil.rmtree(self.dstdir, ignore_errors=True) 93 | 94 | # Used to test pansharpen output 95 | class MosaicArgs(object): 96 | def __init__(self): 97 | self.resolution = None 98 | self.bands = None 99 | self.use_exposure = False 100 | self.tday = None 101 | self.tyear = None 102 | self.extent = None 103 | self.tilesize = None 104 | self.max_cc = 0.5 105 | self.force_pan_to_multi = False 106 | self.include_all_ms = False 107 | self.median_remove = False 108 | 109 | if __name__ == '__main__': 110 | 111 | test_cases = [ 112 | TestPanshFunc 113 | ] 114 | 115 | suites = [] 116 | for test_case in test_cases: 117 | suite = unittest.TestLoader().loadTestsFromTestCase(test_case) 118 | suites.append(suite) 119 | 120 | alltests = unittest.TestSuite(suites) 121 | unittest.TextTestRunner(verbosity=2).run(alltests) 122 | -------------------------------------------------------------------------------- /tests/test_taskhandler.py: -------------------------------------------------------------------------------- 1 | import unittest, os, sys 2 | 3 | __test_dir__ = os.path.dirname(os.path.abspath(__file__)) 4 | sys.path.append(os.path.dirname(__test_dir__)) 5 | testdata_dir = os.path.join(__test_dir__, 'testdata') 6 | 7 | from lib import taskhandler 8 | 9 | 10 | class TestConvertArgs(unittest.TestCase): 11 | 12 | def setUp(self): 13 | self.args = Args() 14 | 15 | def test_args(self): 16 | positional_arg_keys = ['positional'] 17 | arg_keys_to_remove = ['toremove', 'to_remove'] 18 | arg_str = taskhandler.convert_optional_args_to_string(self.args, positional_arg_keys, arg_keys_to_remove) 19 | self.assertIn('--tuple "item1" "item2"', arg_str) 20 | self.assertIn('--list "item1" "item2"', arg_str) 21 | self.assertIn('--boolean', arg_str) 22 | self.assertIn('--multi-word-key "multi-word-key"', arg_str) 23 | self.assertNotIn('positional', arg_str) 24 | self.assertNotIn('toremove', arg_str) 25 | self.assertNotIn('to-remove', arg_str) 26 | self.assertNotIn('to_remove', arg_str) 27 | 28 | 29 | class Args(object): 30 | def __init__(self): 31 | self.boolean = True 32 | self.multi_word_key = "multi-word-key" 33 | self.list = ['item1', 'item2'] 34 | self.tuple = ('item1', 'item2') 35 | self.positional = 'positional' 36 | self.toremove = 'removed' 37 | self.to_remove = 'removed' 38 | 39 | 40 | if __name__ == '__main__': 41 | 42 | test_cases = [ 43 | TestConvertArgs, 44 | ] 45 | 46 | suites = [] 47 | for test_case in test_cases: 48 | suite = unittest.TestLoader().loadTestsFromTestCase(test_case) 49 | suites.append(suite) 50 | 51 | alltests = unittest.TestSuite(suites) 52 | unittest.TextTestRunner(verbosity=2).run(alltests) 53 | -------------------------------------------------------------------------------- /tests/test_utils.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import os 3 | import sys 4 | import shutil 5 | import osgeo # necessary for data type check 6 | from osgeo import ogr 7 | import platform 8 | 9 | __test_dir__ = os.path.dirname(os.path.abspath(__file__)) 10 | sys.path.append(os.path.dirname(__test_dir__)) 11 | testdata_dir = os.path.join(__test_dir__, 'testdata') 12 | 13 | from lib import utils 14 | 15 | class TestUtils(unittest.TestCase): 16 | 17 | def setUp(self): 18 | self.output = os.path.join(__test_dir__, 'tmp_output') 19 | if not os.path.isdir(self.output): 20 | os.makedirs(self.output) 21 | 22 | # test SpatialRef class 23 | self.epsg_npole = 3413 24 | self.epsg_spole = 3031 25 | self.epsg_bad_1 = 'bad' 26 | self.epsg_bad_2 = 1010 27 | 28 | # test scenes to get sensor information 29 | self.basedir = os.path.join(testdata_dir, 'ortho') 30 | self.srcdir_ge = os.path.join(self.basedir, 'GE01_110108M0010160234A222000100252M_000500940.ntf') 31 | self.srcdir_ik = os.path.join(self.basedir, 32 | 'IK01_20050319201700_2005031920171340000011627450_po_333838_blu_0000000.ntf') 33 | self.srcdir_dg = os.path.join(self.basedir, 34 | 'WV01_20120326222942_102001001B02FA00_12MAR26222942-P1BS-052596100010_03_P007.NTF' 35 | ) 36 | 37 | # find images from srcdir_ndvi, use self.imglist as verification list 38 | self.srcdir_ndvi = os.path.join(testdata_dir, 'ndvi', 'ortho') 39 | im_names = ['WV02_20110901210434_103001000B41DC00_11SEP01210434-M1BS-052730735130_01_P007_u16rf3413.tif', 40 | 'WV02_20110901210435_103001000B41DC00_11SEP01210435-M1BS-052730735130_01_P008_u16rf3413.tif', 41 | 'WV02_20110901210500_103001000D52C800_11SEP01210500-M1BS-052560788010_01_P006_u16rf3413.tif', 42 | 'WV02_20110901210501_103001000D52C800_11SEP01210501-M1BS-052560788010_01_P007_u16rf3413.tif', 43 | 'WV02_20110901210502_103001000D52C800_11SEP01210502-M1BS-052560788010_01_P008_u16rf3413.tif'] 44 | self.imglist = [os.path.join(self.srcdir_ndvi, f) for f in im_names] 45 | self.txtfile = os.path.join(self.output, 'img_list.txt') 46 | with open(self.txtfile, 'w') as f: 47 | f.write("\n".join(self.imglist)) 48 | 49 | # file to be excluded 50 | self.excllist = ['WV02_20110901210501_103001000D52C800_11SEP01210501-M1BS-052560788010_01_P007_u16rf3413.tif'] 51 | self.exclfile = os.path.join(self.output, 'excl_list.txt') 52 | with open(self.exclfile, 'w') as f: 53 | f.write("\n".join(self.excllist)) 54 | 55 | # create dir and empty files to be deleted 56 | self.dummydir = os.path.join(testdata_dir, 'dummy_dir') 57 | if not os.path.isdir(self.dummydir): 58 | os.makedirs(self.dummydir) 59 | self.dummyfns = [os.path.join(self.dummydir, 'stuff1.txt'), os.path.join(self.dummydir, 'stuff1.tif'), 60 | os.path.join(self.dummydir, 'stuff1.xml')] 61 | [open(x, 'a').close() for x in self.dummyfns] 62 | 63 | # well-known text, to be turned into geometries to test 180th parallel crossing 64 | self.poly_no180 = 'POLYGON (( {} {}, {} {}, {} {}, {} {}, {} {} ))'.format(-183.1, -75.2, 65 | -183.1, -74, 66 | -177.5, -74, 67 | -177.5, -75.2, 68 | -183.1, -75.2) 69 | self.poly_yes180 = 'POLYGON (( {} {}, {} {}, {} {}, {} {}, {} {} ))'.format(-179.1, -75.2, 70 | -179.1, -74, 71 | 179.5, -74, 72 | 179.5, -75.2, 73 | -179.1, -75.2) 74 | 75 | def test_spatial_ref(self): 76 | sref_np = utils.SpatialRef(self.epsg_npole) 77 | sref_sp = utils.SpatialRef(self.epsg_spole) 78 | with self.assertRaises(RuntimeError) as cm: 79 | utils.SpatialRef(self.epsg_bad_1) # breaks for not being an integer 80 | with self.assertRaises(RuntimeError) as cm: 81 | utils.SpatialRef(self.epsg_bad_2) # break for invalid EPSG code 82 | 83 | self.assertTrue(isinstance(sref_np.srs, osgeo.osr.SpatialReference)) 84 | self.assertTrue(isinstance(sref_sp.srs, osgeo.osr.SpatialReference)) 85 | 86 | self.assertTrue(sref_np.proj4, '+proj=stere +lat_0=90 +lat_ts=70 +lon_0=-45 +k=1 +x_0=0 +y_0=0 +datum=WGS84 +units=m +no_defs ') 87 | self.assertTrue(sref_sp.proj4, '+proj=stere +lat_0=-90 +lat_ts=-71 +lon_0=0 +k=1 +x_0=0 +y_0=0 +datum=WGS84 +units=m +no_defs ') 88 | 89 | self.assertTrue(sref_np.epsg, 3413) 90 | self.assertTrue(sref_sp.epsg, 3031) 91 | 92 | def test_get_bit_depth(self): 93 | self.assertEqual(utils.get_bit_depth("Byte"), "u08") 94 | self.assertEqual(utils.get_bit_depth("UInt16"), "u16") 95 | self.assertEqual(utils.get_bit_depth("Float32"), "f32") 96 | self.assertEqual(utils.get_bit_depth("Uint16"), None) # function logs error, and returns None 97 | 98 | def test_get_sensor(self): 99 | vendor, sat, _, _, _, _ = utils.get_sensor(self.srcdir_ge) 100 | self.assertEqual(vendor.value, 'GeoEye') 101 | self.assertEqual(sat, 'GE01') 102 | 103 | vendor, sat, _, _, _, _ = utils.get_sensor(self.srcdir_ik) 104 | self.assertEqual(vendor.value, 'GeoEye') 105 | self.assertEqual(sat, 'IK01') 106 | 107 | vendor, sat, _, _, _, _ = utils.get_sensor(self.srcdir_dg) 108 | self.assertEqual(vendor.value, 'DigitalGlobe') 109 | self.assertEqual(sat, 'WV01') 110 | 111 | def test_find_images(self): 112 | # without text file 113 | image_list = utils.find_images(self.srcdir_ndvi, False, '.tif') 114 | # if windows, convert slashes so lists are comparable 115 | if platform.system() == "Windows": 116 | image_list = [il.replace("/", "\\") for il in image_list] 117 | self.assertEqual(sorted(image_list), sorted(self.imglist)) 118 | 119 | # with text file 120 | image_list = utils.find_images(self.txtfile, True, '.tif') 121 | # if windows, convert slashes so lists are comparable 122 | if platform.system() == "Windows": 123 | image_list = [il.replace("/", "\\") for il in image_list] 124 | self.assertEqual(sorted(image_list), sorted(self.imglist)) 125 | 126 | def test_find_images_with_exclude_list(self): 127 | # without text files 128 | image_list = utils.find_images_with_exclude_list(self.srcdir_ndvi, False, '.tif', self.excllist) 129 | # if windows, convert slashes so lists are comparable 130 | if platform.system() == "Windows": 131 | image_list = [il.replace("/", "\\") for il in image_list] 132 | self.assertEqual(sorted(image_list), sorted([x for x in self.imglist if x != self.excllist[0]])) 133 | 134 | # with text files 135 | image_list = utils.find_images_with_exclude_list(self.txtfile, True, '.tif', self.exclfile) 136 | # if windows, convert slashes so lists are comparable 137 | if platform.system() == "Windows": 138 | image_list = [il.replace("/", "\\") for il in image_list] 139 | self.assertEqual(sorted(image_list), sorted([x for x in self.imglist if x != self.excllist[0]])) 140 | 141 | def test_delete_temp_files(self): 142 | utils.delete_temp_files(self.dummyfns) 143 | self.assertFalse(os.path.isfile(self.dummyfns[0])) 144 | self.assertFalse(os.path.isfile(self.dummyfns[1])) 145 | self.assertFalse(os.path.isfile(self.dummyfns[2])) 146 | 147 | ''' 148 | NOTE: not testing get_source_names(); should add test gdb later 149 | ''' 150 | 151 | def test_does_cross_180(self): 152 | self.assertFalse(utils.doesCross180(ogr.CreateGeometryFromWkt(self.poly_no180))) 153 | self.assertTrue(utils.doesCross180(ogr.CreateGeometryFromWkt(self.poly_yes180))) 154 | 155 | def test_get_wrapped_geometry(self): 156 | self.assertTrue(isinstance(utils.getWrappedGeometry(ogr.CreateGeometryFromWkt(self.poly_yes180)), 157 | osgeo.ogr.Geometry)) 158 | 159 | ''' 160 | Cannot test for calc_y_intersection_with_180 ZeroDivisionError; code before it prevents this from happening 161 | ''' 162 | 163 | def tearDown(self): 164 | if os.path.isdir(self.dummydir): 165 | shutil.rmtree(self.dummydir) 166 | if os.path.isfile(self.txtfile): 167 | os.remove(self.txtfile) 168 | if os.path.isfile(self.exclfile): 169 | os.remove(self.exclfile) 170 | 171 | 172 | if __name__ == '__main__': 173 | 174 | test_cases = [ 175 | TestUtils 176 | ] 177 | 178 | suites = [] 179 | for test_case in test_cases: 180 | suite = unittest.TestLoader().loadTestsFromTestCase(test_case) 181 | suites.append(suite) 182 | 183 | alltests = unittest.TestSuite(suites) 184 | unittest.TextTestRunner(verbosity=2).run(alltests) 185 | -------------------------------------------------------------------------------- /utility_scripts/pgc_get_scene_overlaps_standalone.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import math 3 | import os 4 | import re 5 | from collections import namedtuple 6 | 7 | import gdal 8 | import gdalconst 9 | import ogr 10 | import osr 11 | 12 | # common name id, attribute field name, storage type, field width, field precision 13 | CoordinateMap = namedtuple("CoordinateMap", ('ul', 'ur', 'lr', 'll')) 14 | 15 | wgs84 = 4326 16 | 17 | 18 | def main(): 19 | ############################# 20 | ###### Parse Arguments 21 | ############################# 22 | 23 | #### Set Up Arguments 24 | 25 | parser = argparse.ArgumentParser(description="Build of text file of stereopair scene overlaps") 26 | 27 | #### Positional Arguments 28 | parser.add_argument('src', help="source directory") 29 | parser.add_argument('dst', help="destination text file") 30 | 31 | #### Parse Arguments 32 | args = parser.parse_args() 33 | 34 | src = os.path.abspath(args.src) 35 | if not os.path.isdir(src): 36 | parser.error("src must be a valid directory") 37 | 38 | dst = os.path.abspath(args.dst) 39 | if not os.path.isdir(os.path.dirname(dst)): 40 | parser.error("dst must be in an existing directory") 41 | if os.path.isdir(dst): 42 | parser.error("dst must be a file") 43 | 44 | ########################################### 45 | ##### Sort Images into Overlaps 46 | ########################################### 47 | 48 | epsg = 4326 49 | srs = osr.SpatialReference() 50 | srs.ImportFromEPSG(epsg) 51 | 52 | print("Source: %s" % src) 53 | stereo_images = [] 54 | 55 | #### Look for images (case insensitive), parent dir is pairname 56 | scenes = {} 57 | for root, dirs, files in os.walk(src): 58 | for f in files: 59 | if f.endswith((".NTF", ".ntf", ".TIF", ".tif")) and 'browse' not in f.lower(): 60 | # "P1BS" in f and 61 | # print f 62 | 63 | ## check for duplicate scene IDs (both a nitf and a tiff file of same base name) and prefer ntfs 64 | sceneid, ext = os.path.splitext(f) 65 | if sceneid in scenes: 66 | if ext.lower() == '.ntf': # if extension is ntf, replace whatever is already there 67 | scenes[sceneid] = os.path.abspath(os.path.join(root, f)) 68 | else: 69 | scenes[sceneid] = os.path.abspath(os.path.join(root, f)) 70 | 71 | for sceneid in scenes: 72 | scenepath = scenes[sceneid] 73 | pairname = os.path.basename(os.path.dirname(scenepath)) 74 | ## Build RasterInfo object 75 | 76 | ## Build StereoImage object. Catid is not known 77 | stereo_image = StereoImage(scenepath, pairname, srs) 78 | 79 | if stereo_image.isvalid is True: 80 | if stereo_image.source.bands == 1: # return only Pan images 81 | stereo_images.append(stereo_image) 82 | else: 83 | print("Stereo image rejected due to band constraint: %s has %d bands" % (stereo_image.path, 84 | stereo_image.source.bands)) 85 | else: 86 | print("Cannot gather sufficient metadata for stereo image: %s" % (stereo_image.path)) 87 | 88 | pn_img_dict = {} # dictionary of pairname with StereoImages list 89 | stereopairs = [] 90 | # print len(stereo_images) 91 | if len(stereo_images) > 0: 92 | # sort into pairname dict 93 | for img in stereo_images: 94 | if img.pairname not in pn_img_dict: 95 | pn_img_dict[img.pairname] = [] 96 | pn_img_dict[img.pairname].append(img) 97 | 98 | # print pn_img_dict 99 | for pairname in pn_img_dict: 100 | stereopair = StereoPair(pairname, pn_img_dict[pairname], "scene") 101 | if stereopair.isvalid is True: 102 | stereopairs.append(stereopair) 103 | 104 | print("Number of stereo pairs: %i" % len(stereopairs)) 105 | txt = open(dst, 'w') 106 | 107 | for stereopair in stereopairs: 108 | i = 0 109 | print("Stereo pair: %s" % stereopair) 110 | 111 | print("Number of overlaps: %i" % len(stereopair.overlaps)) 112 | overlaps = sorted(stereopair.overlaps, key=lambda olap: olap.name) 113 | for overlap in overlaps: 114 | overlap_geom = overlap.geom.Clone() 115 | 116 | ### overlap area in meters 117 | if srs.GetLinearUnitsName != 'Meter': 118 | overlap_centroid = overlap_geom.Centroid() 119 | srs_wgs84 = osr.SpatialReference() 120 | srs_wgs84.ImportFromEPSG(wgs84) 121 | if not srs.IsSame(srs_wgs84): 122 | to_wgs84_transform = osr.CoordinateTransformation(srs, wgs84) 123 | overlap_centroid.Transform(to_wgs84_transform) 124 | lon = overlap_centroid.GetX() 125 | lat = overlap_centroid.GetY() 126 | utm = int((lon + 180) // 6 + 1) 127 | if lat >= 0: 128 | epsg_utm = 32600 + utm 129 | else: 130 | epsg_utm = 32700 + utm 131 | 132 | srs_utm = osr.SpatialReference() 133 | srs_utm.ImportFromEPSG(epsg_utm) 134 | 135 | to_utm_transform = osr.CoordinateTransformation(srs, srs_utm) 136 | overlap_geom.Transform(to_utm_transform) 137 | 138 | area_in_meters = overlap_geom.Area() 139 | 140 | i += 1 141 | 142 | txt.write("{} {} {} {} {}\n".format( 143 | i, 144 | overlap.name, 145 | overlap.images1[0].path, 146 | overlap.images2[0].path, 147 | area_in_meters / 1000000.0 148 | )) 149 | 150 | txt.close() 151 | 152 | 153 | class RasterInfo(object): 154 | """ 155 | docstring 156 | """ 157 | 158 | def __str__(self): 159 | return self.name 160 | 161 | def __init__(self, name, geom, srs, bands, elev=None, pixelcount=None, linecount=None, xres=None, yres=None, 162 | coords=None): 163 | 164 | self.name = name 165 | self.geom = geom 166 | self.srs = srs 167 | self.bands = bands 168 | self.elev = elev 169 | self.pixelcount = pixelcount 170 | self.linecount = linecount 171 | self.xres = xres 172 | self.yres = yres 173 | self.coords = coords 174 | 175 | def transform(self, target_srs): 176 | 177 | target_geom = self.geom.Clone() 178 | ul = self.coords.ul.Clone() 179 | ur = self.coords.ur.Clone() 180 | lr = self.coords.lr.Clone() 181 | ll = self.coords.ll.Clone() 182 | 183 | if not self.srs.IsSame(target_srs): 184 | #### Create srs object 185 | src_tgt_coordtf = osr.CoordinateTransformation(self.srs, target_srs) 186 | 187 | #### Transform geometry to target srs 188 | 189 | try: 190 | target_geom.Transform(src_tgt_coordtf) 191 | ul.Transform(src_tgt_coordtf) 192 | ur.Transform(src_tgt_coordtf) 193 | lr.Transform(src_tgt_coordtf) 194 | ll.Transform(src_tgt_coordtf) 195 | 196 | except TypeError as e: 197 | print("%s Cannot Transform Geometry for image %s: %s" % (e, self.name, self.geom)) 198 | return None 199 | 200 | #### Add bit to calculate coords from geom 201 | 202 | #### calc tgt res 203 | if self.pixelcount and self.linecount and self.coords: 204 | 205 | target_coords = CoordinateMap(ul, ur, lr, ll) 206 | 207 | target_xres = abs(math.sqrt((ul.GetX() - ur.GetX()) ** 2 + (ul.GetY() - ur.GetY()) ** 2) / self.pixelcount) 208 | target_yres = abs(math.sqrt((ul.GetX() - ll.GetX()) ** 2 + (ul.GetY() - ll.GetY()) ** 2) / self.linecount) 209 | 210 | else: 211 | target_coords = None 212 | target_xres = None 213 | target_yres = None 214 | 215 | # print "Rasterinfo: target xres = %s, target yres = %s" %(target_xres, target_yres) 216 | 217 | target = RasterInfo( 218 | self.name, 219 | target_geom, 220 | target_srs, 221 | self.bands, 222 | self.elev, 223 | self.pixelcount, 224 | self.linecount, 225 | target_xres, 226 | target_yres, 227 | target_coords 228 | ) 229 | 230 | return target 231 | 232 | 233 | class StereoImage(object): 234 | """ 235 | docstring 236 | """ 237 | 238 | def __str__(self): 239 | return self.basename 240 | 241 | def __init__(self, image_path, pairname, tgt_srs, rasterinfo=None): 242 | 243 | self.path = image_path 244 | self.pairname = pairname 245 | self.dir, self.name = os.path.split(image_path) 246 | self.basename, self.file_ext = os.path.splitext(self.name) 247 | self.tgt_srs = tgt_srs 248 | self.isvalid = True 249 | # print self.path 250 | 251 | #### If rasterinfo is provided, use it. Otherwise 252 | #### get the geometry and attributes from the raster 253 | if rasterinfo: 254 | self.source = rasterinfo 255 | else: 256 | if os.path.isfile(image_path): 257 | self.source = self._getRasterInfo() 258 | else: 259 | raise RuntimeError("Cannot read source image path: %s" % image_path) 260 | 261 | if self.source: 262 | self.target = self.source.transform(tgt_srs) 263 | 264 | ## Find metadata file (xml/pvl/txt) 265 | if os.path.isfile(os.path.join(self.dir, self.basename + ".XML")): 266 | self.metapath = os.path.join(self.dir, self.basename + ".XML") 267 | ## if files are not on an available filesystem, xml path 268 | # is assumed to be the same as the file path with a lower case extension 269 | else: 270 | self.metapath = os.path.join(self.dir, self.basename + ".xml") 271 | 272 | #### If Catid is provided, use it. Otherwise get the catid from 273 | #### the raster metadata file (xml/pvl,txt) 274 | self.catid, self.order_id, self.tile = getCatidFromName(self.name) 275 | 276 | #### Check if stereo image object has enough valid values 277 | if self.metapath is None or self.target is None or self.catid is None: 278 | self.isvalid = False 279 | print("StereoImage is not valid. One of the following is None:") 280 | print("metapath:", self.metapath) 281 | print("target geom:", self.target.geom) 282 | print("catid:", self.catid) 283 | else: 284 | self.isvalid = False 285 | 286 | def _getRasterInfo(self): 287 | 288 | geom = None 289 | srs = None 290 | 291 | try: 292 | ds = gdal.Open(self.path, gdalconst.GA_ReadOnly) 293 | if ds is not None: 294 | #### Get extent from GCPs 295 | num_gcps = ds.GetGCPCount() 296 | bands = ds.RasterCount 297 | xsize = ds.RasterXSize 298 | ysize = ds.RasterYSize 299 | proj = ds.GetProjectionRef() 300 | m = ds.GetMetadata("RPC") 301 | if "HEIGHT_OFF" in m: 302 | elev = m["HEIGHT_OFF"] 303 | elev = float(''.join([c for c in elev if c in '1234567890.+-'])) 304 | else: 305 | elev = 0.0 306 | 307 | if num_gcps == 4: 308 | gcps = ds.GetGCPs() 309 | proj = ds.GetGCPProjection() 310 | 311 | gcp_dict = {} 312 | 313 | id_dict = {"UpperLeft": 1, 314 | "1": 1, 315 | "UpperRight": 2, 316 | "2": 2, 317 | "LowerLeft": 4, 318 | "4": 4, 319 | "LowerRight": 3, 320 | "3": 3} 321 | 322 | for gcp in gcps: 323 | gcp_dict[id_dict[gcp.Id]] = [float(gcp.GCPPixel), float(gcp.GCPLine), float(gcp.GCPX), 324 | float(gcp.GCPY), float(gcp.GCPZ)] 325 | 326 | ulx = gcp_dict[1][2] 327 | uly = gcp_dict[1][3] 328 | urx = gcp_dict[2][2] 329 | ury = gcp_dict[2][3] 330 | llx = gcp_dict[4][2] 331 | lly = gcp_dict[4][3] 332 | lrx = gcp_dict[3][2] 333 | lry = gcp_dict[3][3] 334 | 335 | else: 336 | gtf = ds.GetGeoTransform() 337 | 338 | ulx = gtf[0] + 0 * gtf[1] + 0 * gtf[2] 339 | uly = gtf[3] + 0 * gtf[4] + 0 * gtf[5] 340 | urx = gtf[0] + xsize * gtf[1] + 0 * gtf[2] 341 | ury = gtf[3] + xsize * gtf[4] + 0 * gtf[5] 342 | llx = gtf[0] + 0 * gtf[1] + ysize * gtf[2] 343 | lly = gtf[3] + 0 * gtf[4] + ysize * gtf[5] 344 | lrx = gtf[0] + xsize * gtf[1] + ysize * gtf[2] 345 | lry = gtf[3] + xsize * gtf[4] + ysize * gtf[5] 346 | 347 | ul = ogr.Geometry(ogr.wkbPoint) 348 | ul.AddPoint(ulx, uly) 349 | ur = ogr.Geometry(ogr.wkbPoint) 350 | ur.AddPoint(urx, ury) 351 | lr = ogr.Geometry(ogr.wkbPoint) 352 | lr.AddPoint(lrx, lry) 353 | ll = ogr.Geometry(ogr.wkbPoint) 354 | ll.AddPoint(llx, lly) 355 | 356 | coords = CoordinateMap(ul, ur, lr, ll) 357 | 358 | xres = abs(math.sqrt((ulx - urx) ** 2 + (uly - ury) ** 2) / xsize) 359 | yres = abs(math.sqrt((ulx - llx) ** 2 + (uly - lly) ** 2) / ysize) 360 | 361 | #### Create geometry object 362 | ring = ogr.Geometry(ogr.wkbLinearRing) 363 | ring.AddPoint(ulx, uly) 364 | ring.AddPoint(urx, ury) 365 | ring.AddPoint(lrx, lry) 366 | ring.AddPoint(llx, lly) 367 | ring.AddPoint(ulx, uly) 368 | geom = ogr.Geometry(ogr.wkbPolygon) 369 | geom.AddGeometry(ring) 370 | # print proj 371 | 372 | #### Create srs objects 373 | srs = osr.SpatialReference(proj) 374 | 375 | ##### build rasterinfo class and return 376 | rasterinfo = RasterInfo(self.path, geom, srs, bands, elev, xsize, ysize, xres, yres, coords) 377 | return rasterinfo 378 | 379 | except Exception as e: 380 | print("Exception in _getRasterInfo: {}".format(e)) 381 | return None 382 | 383 | 384 | class StereoPair(object): 385 | """ 386 | docstring 387 | """ 388 | 389 | def __init__(self, pairname, stereo_images, sp_type): 390 | 391 | # print pairname 392 | self.pairname = pairname 393 | self.stereo_images = stereo_images 394 | self.sp_type = sp_type 395 | self.isvalid = True 396 | self.overlaps = [] 397 | 398 | self.catids = list(set([img.catid for img in stereo_images])) 399 | self.catids.sort() 400 | # print self.catids 401 | self.images = [img.path for img in stereo_images] 402 | 403 | if len(self.catids) == 2: 404 | if self.sp_type == "mosaic": 405 | self.overlaps = self._buildOverlaps_mosaic() 406 | elif self.sp_type == "scene": 407 | self.overlaps = self._buildOverlaps_scene() 408 | else: 409 | print("Incorrect value for stereopair type (%s)" % self.sp_type) 410 | self.isvalid = False 411 | else: 412 | print("Incorrect number of component catids (%i)" % len(self.catids)) 413 | self.isvalid = False 414 | 415 | if len(self.overlaps) == 0: 416 | print("no overlaps found in pair: %s" % self.pairname) 417 | self.isvalid = False 418 | 419 | rasters_band_counts = set([img.source.bands for img in self.stereo_images]) 420 | if len(rasters_band_counts) != 1: 421 | print("Images in this stereopair have varying numbers of bands. This will break things." 422 | " Try running the script with the band_constraint option.\n\t{pairname} - {bandset}".format( 423 | pairname=self.pairname, 424 | bandset=str(rasters_band_counts) 425 | )) 426 | self.isvalid = False 427 | 428 | def writeIndex(self, dstdir, srs): 429 | OGR_DRIVER = "ESRI Shapefile" 430 | ogrDriver = ogr.GetDriverByName(OGR_DRIVER) 431 | 432 | #### Make pairname destination folder 433 | if not os.path.isdir(dstdir): 434 | os.makedirs(dstdir) 435 | 436 | shp = os.path.join(dstdir, self.pairname) 437 | if os.path.isfile(shp + '.shp'): 438 | ogrDriver.DeleteDataSource(shp + '.shp') 439 | 440 | shapefile = ogrDriver.CreateDataSource(shp + '.shp') 441 | 442 | if shapefile is not None: 443 | shpn = os.path.basename(shp) 444 | layer = shapefile.CreateLayer(shpn, srs, ogr.wkbPolygon) 445 | 446 | field = ogr.FieldDefn("overlap", ogr.OFTString) 447 | field.SetWidth(250) 448 | layer.CreateField(field) 449 | 450 | field = ogr.FieldDefn("perc_ol", ogr.OFTReal) 451 | layer.CreateField(field) 452 | 453 | for overlap in self.overlaps: 454 | feature = ogr.Feature(layer.GetLayerDefn()) 455 | feature.SetField("overlap", overlap.name) 456 | feature.SetField("perc_ol", overlap.overlap_percent) 457 | feature.SetGeometry(overlap.geom) 458 | 459 | layer.CreateFeature(feature) 460 | 461 | else: 462 | print("Cannot create shapefile: %s" % shp) 463 | 464 | def getOverlapsByName(self, overlap_name): 465 | 466 | named_overlaps = [] 467 | for overlap in self.overlaps: 468 | if overlap.name == overlap_name: 469 | named_overlaps.append(overlap) 470 | 471 | if len(named_overlaps) == 0: 472 | print("Error: Cannot locate matching overlaps: %s" % overlap_name) 473 | elif len(named_overlaps) == 1: 474 | return named_overlaps[0] 475 | elif len(named_overlaps) > 1: 476 | return named_overlaps 477 | 478 | def _buildOverlaps_mosaic(self): 479 | 480 | catid_geoms = {} # dict of catid to union of all geoms with that catid 481 | catid_images = {} # dict of catid to a list of StereoImage objects 482 | 483 | for img in self.stereo_images: 484 | if img.catid in catid_images: 485 | imglist = catid_images[img.catid] 486 | imglist.append(img) 487 | catid_images[img.catid] = imglist 488 | 489 | catid_geoms[img.catid] = catid_geoms[img.catid].Union(img.target.geom) 490 | else: 491 | catid_images[img.catid] = [img] 492 | catid_geoms[img.catid] = img.target.geom 493 | 494 | overlap_images = (catid_images[self.catids[0]], catid_images[self.catids[1]]) 495 | # print overlap_images[0] 496 | # print overlap_images[1] 497 | geom1 = catid_geoms[self.catids[0]] 498 | geom2 = catid_geoms[self.catids[1]] 499 | overlap_geom = geom1.Intersection(geom2) 500 | union_geom = geom1.Union(geom2) 501 | overlap_percent = overlap_geom.GetArea() / union_geom.GetArea() 502 | 503 | overlap = Overlap(overlap_images, overlap_geom, overlap_percent, overlap_geom.Area(), self.pairname, 504 | self.sp_type, self.pairname) 505 | 506 | return [overlap] 507 | 508 | def _buildOverlaps_scene(self): 509 | 510 | overlaps = [] 511 | 512 | ### Add code to get min xres and yres from images 513 | 514 | for img1 in self.stereo_images: 515 | if img1.catid == self.catids[0]: 516 | for img2 in self.stereo_images: 517 | if img2.catid != img1.catid: 518 | #### compare geoms 519 | if img1.target.geom.Intersects(img2.target.geom): 520 | # print img1.name,img2.name 521 | 522 | overlap_images = ([img1], [img2]) 523 | overlap_geom = img1.target.geom.Intersection(img2.target.geom) 524 | union_geom = img1.target.geom.Union(img2.target.geom) 525 | overlap_percent = overlap_geom.GetArea() / union_geom.GetArea() 526 | # overlap_name = "%s_%s" %(img1.basename, img2.basename) 527 | # Overlap name: 528 | # WV01_20120603_102001001B4BFB00_102001001BE7F800_R1C1-052903570060_01_P001_052903564030_01_P001 529 | img1_identifier = "{1}-{0}".format(img1.order_id, img1.tile) if img1.tile else img1.order_id 530 | img2_identifier = "{1}-{0}".format(img2.order_id, img2.tile) if img2.tile else img2.order_id 531 | overlap_name = "{}_{}_{}".format(self.pairname, img1_identifier, img2_identifier) 532 | 533 | if overlap_percent >= 0.10: 534 | overlap = Overlap(overlap_images, overlap_geom, overlap_percent, overlap_geom.Area(), 535 | overlap_name, self.sp_type, self.pairname) 536 | 537 | if overlap is not None: 538 | overlaps.append(overlap) 539 | 540 | return overlaps 541 | 542 | def __str__(self): 543 | return self.pairname 544 | 545 | 546 | class Overlap(object): 547 | """ 548 | docstring 549 | """ 550 | 551 | def __str__(self): 552 | return self.name 553 | 554 | def __init__(self, images_list, geom, overlap_percent, overlap_area, overlap_name, overlap_type, pairname): 555 | 556 | self.geom = geom 557 | self.bbox = geom.GetEnvelope() # Get Envelope returns a tuple (minX, maxX, minY, maxY) 558 | self.overlap_type = overlap_type 559 | self.images1 = images_list[0] 560 | self.images2 = images_list[1] 561 | self.name = overlap_name 562 | self.pairname = pairname 563 | self.overlap_percent = overlap_percent 564 | self.overlap_area = overlap_area 565 | self.overlap_filename = "%s.overlap" % overlap_name 566 | self.overlap_geojson = "%s.geojson" % overlap_name 567 | self.dem_name = "%s-DEM.tif" % overlap_name 568 | self.pc_name = "%s-PC.tif" % overlap_name 569 | 570 | #### Get minimum res from Image objects: If image res values are None, then xres and yres will also be None 571 | all_images = self.images1 + self.images2 572 | self.xres = min([si.target.xres for si in all_images]) 573 | self.yres = min([si.target.yres for si in all_images]) 574 | 575 | try: 576 | self.elev = sum([si.target.elev for si in all_images]) / float(len(all_images)) 577 | except TypeError: 578 | self.elev = 0.0 579 | 580 | ##### Write overlap index files 581 | def writeOverlapFile(self, dstdir): 582 | 583 | #### Make pairname destination folder 584 | pairname_dstdir = dstdir 585 | if not os.path.isdir(pairname_dstdir): 586 | os.makedirs(pairname_dstdir) 587 | 588 | #### Write .overlap file 589 | f = open(os.path.join(pairname_dstdir, self.overlap_filename), 'w') 590 | f.write("IMAGE;CATALOGID;PAIRNAME;OVERLAP_TYPE\n") 591 | 592 | images = self.images1 + self.images2 593 | 594 | for image in images: 595 | vals = { 596 | "pairname": self.pairname, 597 | "overlap_type": self.overlap_type, 598 | "image": image.path, 599 | "catid": image.catid, 600 | } 601 | f.write("{image};{catid};{pairname};{overlap_type}\n".format(**vals)) 602 | 603 | f.close() 604 | 605 | 606 | def getCatidFromName(filename): 607 | PGC_DG_FILE = re.compile(r""" 608 | (?P # PGC prefix 609 | (?P[a-z]{2}\d{2})_ # Sensor code 610 | (?P\d{14})_ # Acquisition time (yyyymmddHHMMSS) 611 | (?P[a-f0-9]{16}) # Catalog ID 612 | )_ 613 | (?P # Original DG name 614 | (?P\d{2}[a-z]{3}\d{8})- # Acquisition time (yymmmddHHMMSS) 615 | (?P[a-z0-9]{4})_? # DG product code 616 | (?PR\d+C\d+)?- # Tile code (mosaics, optional) 617 | (?P # DG Order ID 618 | (?P\d{12}_\d{2})_ # DG Order number 619 | (?PP\d{3}) # Part number 620 | ) 621 | (?P[a-z0-9_-]+(?=\.))? # Descriptor (optional) 622 | ) 623 | (?P\.[a-z0-9][a-z0-9.]*) # File name extension 624 | """, re.I | re.X) 625 | 626 | # WV01_20120603220928_102001001B4BFB00_12JUN03220928-P1BS_R1C1-052903570060_01_P001.tif 627 | # WV01_20120603221007_102001001BE7F800_12JUN03221007-P1BS-052903564030_01_P001.tif 628 | 629 | match = PGC_DG_FILE.search(filename) 630 | if match is not None: 631 | grp_dct = match.groupdict() 632 | catid = grp_dct['catid'] 633 | order_id = grp_dct['oid'] 634 | if 'tile' in grp_dct: 635 | tile = grp_dct['tile'] 636 | else: 637 | tile = None 638 | 639 | return catid, order_id, tile 640 | 641 | else: 642 | return None, None, None 643 | 644 | 645 | if __name__ == '__main__': 646 | main() 647 | -------------------------------------------------------------------------------- /utility_scripts/stack_landsat.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import logging 3 | import re 4 | import shlex 5 | import subprocess 6 | from dataclasses import dataclass 7 | from datetime import date 8 | from pathlib import Path 9 | 10 | 11 | def get_logger(logfile: Path, logger_name: str) -> logging.Logger: 12 | logger = logging.getLogger(logger_name) 13 | logger.setLevel(logging.INFO) 14 | formatter = logging.Formatter( 15 | fmt="%(asctime)s %(levelname)s- %(message)s", datefmt="%m-%d-%Y %H:%M:%S" 16 | ) 17 | 18 | file_handler = logging.FileHandler(filename=logfile, mode="a") 19 | file_handler.setFormatter(formatter) 20 | file_handler.setLevel(logging.INFO) 21 | logger.addHandler(file_handler) 22 | 23 | stream_handler = logging.StreamHandler() 24 | stream_handler.setFormatter(formatter) 25 | stream_handler.setLevel(logging.INFO) 26 | logger.addHandler(stream_handler) 27 | return logger 28 | 29 | 30 | @dataclass 31 | class CLIArgs: 32 | input_dir: Path 33 | output_dir: Path 34 | 35 | 36 | def validate_cli_args(args: CLIArgs) -> CLIArgs: 37 | """Validate the CLI arguments. 38 | 39 | Requirements: 40 | - The input directory must exist and must be a directory""" 41 | if not args.input_dir.exists(): 42 | raise FileNotFoundError( 43 | f"The provided input-dir does not exist: {args.input_dir}" 44 | ) 45 | if not args.input_dir.is_dir(): 46 | raise NotADirectoryError( 47 | f"The provided input-dir is not a directory: {args.input_dir}" 48 | ) 49 | return CLIArgs 50 | 51 | 52 | def get_cli_args() -> CLIArgs: 53 | """Parse and validate commandline arguments, returning a CLIArgs instance if valid""" 54 | parser = argparse.ArgumentParser( 55 | prog="stack_landsat", 56 | description="Utility for combining Landsat bands 4, 3, 2 into a multi-band RGB TIF", 57 | ) 58 | parser.add_argument( 59 | "--input-dir", 60 | "-i", 61 | type=Path, 62 | required=True, 63 | help="Directory containing Landsat TIFs with one band per TIF [REQUIRED]", 64 | ) 65 | parser.add_argument( 66 | "--output-dir", 67 | "-o", 68 | type=Path, 69 | required=True, 70 | help="Directory to write multi-band TIFs. The directory will be created if it does not exist. [REQUIRED]", 71 | ) 72 | try: 73 | return validate_cli_args(parser.parse_args(namespace=CLIArgs)) 74 | except (FileNotFoundError, NotADirectoryError) as e: 75 | print(f"ERROR: {e.args[0]}") 76 | exit() 77 | 78 | 79 | def strip_band_from_scene_name(scene_name: str) -> str: 80 | """Removes the band designation (e.g. '_B2') from the Landsat scene name""" 81 | return re.sub("_B[0-9]", "", scene_name) 82 | 83 | 84 | def create_rgb_tif(input_dir: Path, output_dir: Path, scene_name: str) -> None: 85 | red_band = input_dir / f"{scene_name}_B4.TIF" 86 | green_band = input_dir / f"{scene_name}_B3.TIF" 87 | blue_band = input_dir / f"{scene_name}_B2.TIF" 88 | temp_vrt = output_dir / "temp.vrt" 89 | rgb_tif = output_dir / f"{scene_name}_RGB.TIF" 90 | 91 | missing_bands = [] 92 | if not red_band.exists(): 93 | missing_bands.append(str(red_band)) 94 | if not green_band.exists(): 95 | missing_bands.append(str(green_band)) 96 | if not blue_band.exists(): 97 | missing_bands.append(str(blue_band)) 98 | if len(missing_bands) > 0: 99 | raise FileNotFoundError( 100 | f"The following band TIFs were not found: {missing_bands}" 101 | ) 102 | 103 | gdalbuildvrt_cmd = f"gdalbuildvrt -separate -overwrite {temp_vrt} {red_band} {green_band} {blue_band}" 104 | gdal_translate_cmd = f"gdal_translate -f COG {temp_vrt} {rgb_tif}" 105 | 106 | try: 107 | subprocess.run(shlex.split(gdalbuildvrt_cmd, posix=False), check=True) 108 | subprocess.run(shlex.split(gdal_translate_cmd, posix=False), check=True) 109 | finally: 110 | temp_vrt.unlink() 111 | 112 | 113 | def cli() -> None: 114 | args = get_cli_args() 115 | 116 | logfile = args.input_dir / f"stack_landsat_{date.today()}.log" 117 | logger = get_logger( 118 | logfile=logfile, 119 | logger_name=__file__, 120 | ) 121 | 122 | logger.info("-" * 80) 123 | logger.info(f"Input directory: {args.input_dir}") 124 | logger.info(f"Output directory: {args.output_dir}") 125 | logger.info(f"Logfile: {logfile}") 126 | logger.info("-" * 80) 127 | 128 | if not args.output_dir.exists(): 129 | logger.info(f"Output directory does not exist. Creating output directory.") 130 | args.output_dir.mkdir(parents=True) 131 | 132 | logger.info(f"Scanning input directory for Landsat scenes") 133 | tifs = list(args.input_dir.glob("*_B[0-9].TIF")) 134 | scene_names = {strip_band_from_scene_name(tif.stem) for tif in tifs} 135 | if len(scene_names) == 0: 136 | logging.error("No scenes found in input directory") 137 | logging.error("Exiting process...") 138 | return 139 | 140 | logger.info(f"Number of scenes found to process: {len(scene_names)}") 141 | succeeded_scenes = [] 142 | failed_scenes = [] 143 | for scene_name in scene_names: 144 | logger.info(f"Processing scene: {scene_name}") 145 | try: 146 | create_rgb_tif( 147 | input_dir=args.input_dir, 148 | output_dir=args.output_dir, 149 | scene_name=scene_name, 150 | ) 151 | succeeded_scenes.append(scene_name) 152 | except FileNotFoundError as e: 153 | logger.error(e.args[0]) 154 | logger.error(f"Skipping scene: {scene_name}") 155 | failed_scenes.append(scene_name) 156 | continue 157 | 158 | logger.info(f"Number of scenes successfully processed: {len(succeeded_scenes)}") 159 | logger.info(f"Number of scenes that failed to process: {len(failed_scenes)}") 160 | if len(failed_scenes) > 0: 161 | logger.warning("Scenes that failed to process:") 162 | for scene_name in failed_scenes: 163 | logger.warning(f"\t{scene_name}") 164 | 165 | 166 | if __name__ == "__main__": 167 | cli() 168 | --------------------------------------------------------------------------------