├── .gitignore ├── CMakeLists.txt ├── LICENSE ├── README.md ├── cli ├── CMakeLists.txt └── src │ ├── CliProcessor.cpp │ ├── CliProcessor.h │ └── main.cpp ├── common ├── CMakeLists.txt ├── include │ ├── OpenSpaceNet.h │ └── OpenSpaceNetArgs.h ├── src │ └── OpenSpaceNet.cpp └── version.h.in └── doc ├── BUILDING.md ├── REFERENCE.md ├── modelsize.png └── modelsize.svg /.gitignore: -------------------------------------------------------------------------------- 1 | # OS generated files 2 | .DS_Store 3 | .DS_Store? 4 | ._* 5 | .Spotlight-V100 6 | .Trashes 7 | ehthumbs.db 8 | Thumbs.db 9 | 10 | #ignore .idea files 11 | .idea 12 | 13 | #ignore CMake and CLion build directories 14 | build/ 15 | cmake-build-debug/ 16 | cmake-build-release/ 17 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_policy(SET CMP0015 NEW) 2 | cmake_minimum_required(VERSION 3.5) 3 | 4 | project(OpenSpaceNet C CXX) 5 | 6 | # Options for overriding the installation directories 7 | set(INSTALL_LIB_DIR lib CACHE PATH "Installation directory for libraries") 8 | set(INSTALL_BIN_DIR bin CACHE PATH "Installation directory for executables") 9 | set(INSTALL_INCLUDE_DIR include CACHE PATH "Installation directory for header files") 10 | 11 | # 12 | # Set up OpenSpaceNet versioning 13 | # 14 | set(OpenSpaceNet_VERSION_MAJOR 1) 15 | set(OpenSpaceNet_VERSION_MINOR 2) 16 | set(OpenSpaceNet_VERSION_PATCH 0) 17 | 18 | # Set the build number 19 | if(DEFINED ENV{BUILD_NUMBER} AND NOT DEEPCORE_RELEASE) 20 | set(OpenSpaceNet_VERSION_BUILD $ENV{BUILD_NUMBER}) 21 | else() 22 | set(OpenSpaceNet_VERSION_BUILD "SNAPSHOT") 23 | endif() 24 | 25 | # 26 | # Common dependencies 27 | # 28 | set(CMAKE_CXX_STANDARD 11) 29 | 30 | if(DEEPCORE_STATIC) 31 | set(CMAKE_PREFIX_PATH ${DEEPCORE_DEP_DIR}) 32 | set(CMAKE_EXE_LINKER_FLAGS=-static) 33 | add_definitions(-DCUDA_HOME=${DEEPCORE_DEP_DIR}) 34 | add_definitions(-DCUDA_TOOLKIT_ROOT_DIR=${DEEPCORE_DEP_DIR}) 35 | #setup DeepCore dependency dir. This will mostly be used in static mode. 36 | if (NOT DEEPCORE_DEP_DIR) 37 | if(DEFINED ENV{DEEPCORE_DEP_DIR}) 38 | set(DEEPCORE_DEP_DIR $ENV{DEEPCORE_DEP_DIR}) 39 | else() 40 | set(DEEPCORE_DEP_DIR /opt/DeepCore) 41 | endif() 42 | endif() 43 | set(CMAKE_CXX_FLAGS "-I ${DEEPCORE_DEP_DIR}/include -L${DEEPCORE_DEP_DIR}/lib -L${DEEPCORE_DEP_DIR}/lib64 ${CMAKE_CXX_FLAGS}") 44 | endif() 45 | 46 | # From the CaffeConfig.cmake file 47 | # Caffe and this config file depends on opencv, 48 | # so put `find_package(OpenCV)` before searching Caffe 49 | # via `find_package(Caffe)`. All other lib/includes 50 | # dependencies are hard coded in the file 51 | find_package(OpenCV) 52 | find_package(DeepCore REQUIRED) 53 | if (DeepCore_FOUND) 54 | include_directories(${DEEPCORE_INCLUDE_DIRS}) 55 | list(APPEND CMAKE_MODULE_PATH ${DEEPCORE_CMAKE_DIR}) 56 | list(APPEND OSN_LINK_LIBRARIES ${DEEPCORE_LIBRARIES}) 57 | endif() 58 | 59 | find_package(Caffe REQUIRED) 60 | if (Caffe_FOUND) 61 | if (DEEPCORE_STATIC) 62 | if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") 63 | set(Caffe_LINK -Wl,-force_load caffe) 64 | elseif("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") 65 | set(Caffe_LINK -Wl,--whole-archive caffe -Wl,--no-whole-archive) 66 | endif() 67 | else() 68 | set(Caffe_LINK caffe) 69 | endif() 70 | include_directories(${Caffe_INCLUDE_DIRS}) 71 | list(APPEND OSN_LINK_LIBRARIES ${Caffe_LINK}) 72 | endif() 73 | 74 | find_package(CUDA REQUIRED) 75 | if (CUDA_FOUND) 76 | include_directories(${CUDA_INCLUDE_DIRS}) 77 | list(APPEND OSN_LINK_LIBRARIES ${CUDA_LIBRARIES}) 78 | endif() 79 | 80 | find_package(Jsoncpp REQUIRED) 81 | if(JSONCPP_FOUND) 82 | include_directories(${JSONCPP_INCLUDE_DIRS}) 83 | list(APPEND OSN_LINK_LIBRARIES ${JSONCPP_LIBRARIES}) 84 | endif() 85 | 86 | set(Boost_USE_STATIC_LIBS ON) 87 | set(Boost_USE_MULTITHREADED ON) 88 | find_package(Boost COMPONENTS program_options REQUIRED) 89 | if(Boost_FOUND) 90 | include_directories(${Boost_INCLUDE_DIRS}) 91 | list(APPEND OSN_LINK_LIBRARIES ${Boost_PROGRAM_OPTIONS_LIBRARY}) 92 | endif() 93 | 94 | find_package(Threads) 95 | list(APPEND OSN_LINK_LIBRARIES ${CMAKE_THREAD_LIBS_INIT} ${CMAKE_DL_LIBS}) 96 | 97 | # 98 | # Add sub-projects 99 | # 100 | add_subdirectory(common) 101 | 102 | include_directories(${CMAKE_CURRENT_BINARY_DIR}/include) 103 | include_directories(${CMAKE_CURRENT_BINARY_DIR}/common) 104 | 105 | add_subdirectory(cli) 106 | 107 | # 108 | # Package 109 | # 110 | if (UNIX AND NOT APPLE) 111 | set(CPACK_GENERATOR "TGZ") 112 | set(CPACK_PACKAGE_NAME "OpenSpaceNet") 113 | set(CPACK_PACKAGE_VENDOR "DigitalGlobe") 114 | set(CPACK_DEBIAN_PACKAGE_MAINTAINER "DeepCore Support ") 115 | set(CPACK_DEBIAN_PACKAGE_VERSION "${OpenSpaceNet_VERSION_MAJOR}.${OpenSpaceNet_VERSION_MINOR}.${OpenSpaceNet_VERSION_PATCH}+${OpenSpaceNet_VERSION_BUILD}") 116 | set(CPACK_PACKAGE_VERSION "${OpenSpaceNet_VERSION_MAJOR}.${OpenSpaceNet_VERSION_MINOR}.${OpenSpaceNet_VERSION_PATCH}+${OpenSpaceNet_VERSION_BUILD}") 117 | set(CPACK_PACKAGE_VERSION_MAJOR "${OpenSpaceNet_VERSION_MAJOR}") 118 | set(CPACK_PACKAGE_VERSION_MINOR "${OpenSpaceNet_VERSION_MINOR}") 119 | set(CPACK_PACKAGE_VERSION_PATCH "${OpenSpaceNet_VERSION_PATCH}") 120 | set(CPACK_RESOURCE_FILE_README "${PROJECT_SOURCE_DIR}/README.md") 121 | set(CPACK_DEBIAN_PACKAGE_SHLIBDEPS ON) 122 | endif() 123 | 124 | include(CPack) 125 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | 3 | Version 2.0, January 2004 4 | 5 | http://www.apache.org/licenses/ 6 | 7 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 8 | 9 | 1. Definitions. 10 | 11 | "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. 16 | 17 | "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. 18 | 19 | "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. 20 | 21 | "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. 22 | 23 | "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). 24 | 25 | "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. 26 | 27 | "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." 28 | 29 | "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 30 | 31 | 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 32 | 33 | 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 34 | 35 | 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: 36 | 37 | You must give any other recipients of the Work or Derivative Works a copy of this License; and 38 | You must cause any modified files to carry prominent notices stating that You changed the files; and 39 | You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and 40 | If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. 41 | 42 | You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 43 | 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 44 | 45 | 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 46 | 47 | 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 48 | 49 | 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 50 | 51 | 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. 52 | 53 | END OF TERMS AND CONDITIONS -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # OpenSpaceNet 2 | 3 | [![Build Status](http://52.1.7.235/buildStatus/icon?job=OpenSpaceNet_CUDA8&style=plastic)](http://52.1.7.235/job/OpenSpaceNet_CUDA8) Jenkins Build Status 4 | 5 | OpenSpaceNet is an open-source application to perform object or terrain detection against orthorectified imagery using the DeepCore libraries. This application includes and is based on CUDA 8.0 and requires NVIDIA driver version 375 or higher to run using the GPU. 6 | 7 | OpenSpaceNet takes a pre-trained neural network and applies it to imagery. It can use local files from your computer, or connect to DigitalGlobe servers to download maps. OpenSpaceNet uses your DigitalGlobe account to download the maps, so it can only use maps that you have already purchased. This guide will explain the basic usage of OpenSpaceNet. 8 | 9 | ## Train a model 10 | Train a model with Caffe or TensorFlow. OpenSpaceNet supports Caffe sliding window, DetectNet, and segmentation models, as well as limited TensorFlow support, currently limited to sliding window models. Once you have a trained model, you will need to package it as a gbdxm file, which is an archive format used by OpenSpaceNet. 11 | 12 | ## Convert model to GBDXM 13 | The GBDXM format stores a model in a compressed file which OSN uses. The `gbdxm` module acts as a link between Caffe and OSN. The executable is available on the [DeepCore webpage](https://digitalglobe.github.io/DeepCore/). You will need to make it executable before you can run it `chmod u+x gbdxm`. Then you can convert your Caffe model to gbdxm format: 14 | 15 | ```bash 16 | $ ./gbdxm pack -f out.gbdxm --caffe-model deploy.prototxt --caffe-trained model.caffemodel \ 17 | --caffe-mean mean.binaryproto -t caffe -n "Airliner" -v 0 -d "Model Description" -l labels.txt \ 18 | --image-type jpg -b -84.06163 37.22197 -84.038803 37.240162 19 | ``` 20 | 21 | For detailed documentation of the command line arguments, run `./gbdxm help` or just `./gbdxm` at the terminal. 22 | 23 | ## Run with OpenSpaceNet 24 | 1. Make sure CUDA 8.0 is installed ([instructions](http://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html)). We are working on OSN for CUDA 9, but it will only run with 8.0 for now. 25 | 26 | 2. Download OpenSpaceNet executable [by clicking the link in the right column](https://digitalglobe.github.io/DeepCore/). You do not need to install DeepCore if you use the pre-compiled executable. 27 | 28 | 3. Use OpenSpaceNet to apply models to map data 29 | 30 | For descriptions of the command line arguments, consult the [documentation](https://github.com/DigitalGlobe/OpenSpaceNet/blob/master/doc/REFERENCE.md) for OpenSpaceNet. For basic usage, you can follow these examples. 31 | 32 | If you have the images that you want to search in an image file, then you can use the `--image` option. This does not require any access keys. 33 | 34 | ### Using a local file 35 | Linux: 36 | ``` 37 | $ ./OpenSpaceNet detect --model MODEL.gbdxm --type POLYGON --format shp --output OUTPUT_FILENAME.shp \ 38 | --nms --window-step 30 --image INPUT_IMAGE_FILENAME 39 | ``` 40 | 41 | ### Using the Maps API 42 | If you access DigitalGlobe Maps API, then you can use your token to download maps. DeepCore will handle the maps for you, so all you need to provide is the API token, and the bounding box that you want to search in. More information on the Maps API is available on [the DigitalGlobe platform page](https://platform.digitalglobe.com/) 43 | 44 | Linux: 45 | ``` 46 | $ ./OpenSpaceNet detect --bbox -84.44579 33.63404 -84.40601 33.64583 --model MODEL.gbdxm \ 47 | --type POLYGON --format shp --output OUTPUT_FILENAME.shp --confidence 99.9 --nms \ 48 | --window-step 25 --num-downloads 200 --token API_TOKEN --service maps-api 49 | ``` 50 | 51 | ### Using DigitalGlobe Cloud Services (DGCS) 52 | If you have access to DigitalGlobe Cloud Services, then OpenSpaceNet can access maps using your account. You need to provide your username, password, and the appropriate token for the type of maps that you want to use. You can find information about the kinds of maps offered by DigitalGlobe of is available on [our webpage](https://www.digitalglobe.com/products/basemap-suite). 53 | 54 | Linux: 55 | ``` 56 | $ ./OpenSpaceNet detect --bbox -84.44579 33.63404 -84.40601 33.64583 --model MODEL.gbdxm \ 57 | --type POLYGON --format shp --output OUTPUT_FILENAME.shp --confidence 99.9 --nms \ 58 | --window-step 15 --num-downloads 200 --token MAP_TOKEN --credentials USER:PASS --service dgcs 59 | ``` 60 | 61 | ## Visualizing results 62 | You can visualize your results using GIS software like ArcGIS or QGIS. You can configure QGIS to use DigitalGlobe maps by following [these instructions](https://developer.digitalglobe.com/using-maps-api-with-qgis/). Then you can import the shapefile as a vector layer to see the model results. 63 | 64 | ## Links 65 | 66 | ### [User Reference Guide](doc/REFERENCE.md) 67 | ### [Building OpenSpaceNet](doc/BUILDING.md) 68 | -------------------------------------------------------------------------------- /cli/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | include_directories(src) 2 | 3 | set(SOURCES 4 | src/main.cpp 5 | src/CliProcessor.cpp 6 | src/CliProcessor.h 7 | ) 8 | 9 | add_executable(OpenSpaceNet.cli ${SOURCES}) 10 | target_link_libraries(OpenSpaceNet.cli OpenSpaceNet.common ${OSN_LINK_LIBRARIES}) 11 | set_target_properties(OpenSpaceNet.cli PROPERTIES OUTPUT_NAME OpenSpaceNet) 12 | 13 | INSTALL(TARGETS OpenSpaceNet.cli 14 | RUNTIME DESTINATION bin 15 | LIBRARY DESTINATION lib 16 | ARCHIVE DESTINATION lib 17 | ) 18 | 19 | -------------------------------------------------------------------------------- /cli/src/CliProcessor.cpp: -------------------------------------------------------------------------------- 1 | /******************************************************************************** 2 | * Copyright 2017 DigitalGlobe, Inc. 3 | * Author: Aleksey Vitebskiy 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | ********************************************************************************/ 17 | #include "CliProcessor.h" 18 | 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | #include 31 | #include 32 | #include 33 | #include 34 | #include 35 | #include 36 | #include 37 | 38 | namespace dg { namespace osn { 39 | 40 | namespace po = boost::program_options; 41 | 42 | using namespace deepcore; 43 | 44 | using boost::program_options::variables_map; 45 | using boost::program_options::name_with_default; 46 | using boost::adaptors::reverse; 47 | using boost::bad_lexical_cast; 48 | using boost::filesystem::exists; 49 | using boost::filesystem::path; 50 | using boost::iequals; 51 | using boost::join; 52 | using boost::lexical_cast; 53 | using boost::make_unique; 54 | using boost::tokenizer; 55 | using boost::escaped_list_separator; 56 | using boost::to_lower; 57 | using boost::to_lower_copy; 58 | using dg::deepcore::level_t; 59 | using std::cout; 60 | using std::find; 61 | using std::function; 62 | using std::end; 63 | using std::endl; 64 | using std::move; 65 | using std::ofstream; 66 | using std::string; 67 | using std::unique_ptr; 68 | using geometry::GeometryType; 69 | using dg::deepcore::ConsoleProgressDisplay; 70 | using dg::deepcore::ProgressCategory; 71 | using dg::deepcore::classification::GbdxModelReader; 72 | using dg::deepcore::imagery::RasterToPolygonDP; 73 | using dg::deepcore::vector::FileFeatureSet; 74 | 75 | static const string OSN_USAGE = 76 | "Usage:\n" 77 | " OpenSpaceNet \n" 78 | " OpenSpaceNet --config [other options]\n\n"; 79 | 80 | CliProcessor::CliProcessor() : 81 | localOptions_("Local Image Input Options"), 82 | webOptions_("Web Service Input Options"), 83 | outputOptions_("Output Options"), 84 | processingOptions_("Processing Options"), 85 | detectOptions_("Feature Detection Options"), 86 | segmentationOptions_("Segmentation Options"), 87 | filterOptions_("Filtering Options"), 88 | loggingOptions_("Logging Options"), 89 | generalOptions_("General Options"), 90 | supportedFormats_(FileFeatureSet::supportedFormats()) 91 | { 92 | classification::init(); 93 | 94 | detectOptions_.add_options() 95 | ("confidence", po::value()->value_name(name_with_default("PERCENT", osnArgs.confidence)), 96 | "Minimum percent score for results to be included in the output.") 97 | ("nms", po::bounded_value>()->min_tokens(0)->max_tokens(1)->value_name(name_with_default("PERCENT", osnArgs.overlap)), 98 | "Perform non-maximum suppression on the output. You can optionally specify the overlap threshold percentage " 99 | "for non-maximum suppression calculation.") 100 | ; 101 | 102 | localOptions_.add_options() 103 | ("image", po::value()->value_name("PATH"), 104 | "If this is specified, the input will be taken from a local image.") 105 | ; 106 | 107 | webOptions_.add_options() 108 | ("service", po::value()->value_name("SERVICE"), 109 | "Web service that will be the source of input. Valid values are: dgcs, evwhs, maps-api, and tile-json.") 110 | ("token", po::value()->value_name("TOKEN"), 111 | "API token used for licensing. This is the connectId for WMTS services or the API key for the Web Maps API.") 112 | ("credentials", po::value()->value_name("USERNAME[:PASSWORD]"), 113 | "Credentials for the map service. Not required for Web Maps API, optional for TileJSON. If password is not specified, you will be " 114 | "prompted to enter it. The credentials can also be set by setting the OSN_CREDENTIALS environment variable.") 115 | ("url", po::value()->value_name("URL"), 116 | "TileJSON server URL. This is only required for the tile-json service.") 117 | ("use-tiles", 118 | "If set, the \"tiles\" field in TileJSON metadata will be used as the tile service address. The default behavior" 119 | "is to derive the service address from the provided URL.") 120 | ("zoom", po::value()->value_name(name_with_default("ZOOM", osnArgs.zoom)), "Zoom level.") 121 | ("map-id", po::value()->value_name(name_with_default("MAPID", osnArgs.mapId)), "MapsAPI map id to use.") 122 | ("max-connections", po::value()->value_name(name_with_default("NUM", osnArgs.maxConnections)), 123 | "Used to speed up downloads by allowing multiple concurrent downloads to happen at once.") 124 | ; 125 | 126 | string outputDescription = "Output file format for the results. Valid values are: "; 127 | outputDescription += join(supportedFormats_, ", ") + "."; 128 | 129 | auto formatNotifier = function([this](const string& format) { 130 | DG_CHECK(find(supportedFormats_.begin(), supportedFormats_.end(), to_lower_copy(format)) != end(supportedFormats_), 131 | "Invalid output format: %s.", format.c_str()); 132 | }); 133 | 134 | outputOptions_.add_options() 135 | ("format", po::value()->value_name(name_with_default("FORMAT", osnArgs.outputFormat))->notifier(formatNotifier), 136 | outputDescription.c_str()) 137 | ("output", po::value()->value_name("PATH"), 138 | "Output location with file name and path or URL.") 139 | ("output-layer", po::value()->value_name(name_with_default("NAME", "osndetects")), 140 | "The output layer name, index name, or table name.") 141 | ("type", po::value()->value_name(name_with_default("TYPE", "polygon")), 142 | "Output geometry type. Currently only point and polygon are valid.") 143 | ("producer-info", "Add user name, application name, and application version to the output feature set.") 144 | ("dgcs-catalog-id", "Add catalog_id property to detected features, by finding the most intersected legacyId from DGCS WFS data source DigitalGlobe:FinishedFeature") 145 | ("evwhs-catalog-id", "Add catalog_id property to detected features, by finding the most intersected legacyId from EVWHS WFS data source DigitalGlobe:FinishedFeature") 146 | ("wfs-credentials", po::value()->value_name("USERNAME[:PASSWORD]"), 147 | "Credentials for the WFS service, if appending legacyId. If not specified, credentials from the credentials option will be used.") 148 | ("append", "Append to an existing vector set. If the output does not exist, it will be created.") 149 | ("extra-fields",po::value >()->multitoken()->value_name("KEY VALUE [KEY VALUE...]"), "A set of key-value string pairs that will be added to the output feature set.") 150 | ; 151 | 152 | processingOptions_.add_options() 153 | ("cpu", "Use the CPU for processing, the default is to use the GPU.") 154 | ("max-utilization", po::value()->value_name(name_with_default("PERCENT", osnArgs.maxUtilization)), 155 | "Maximum GPU utilization %. Minimum is 5, and maximum is 100. Not used if processing on CPU") 156 | ("model", po::value()->value_name("PATH"), "Path to the the trained model.") 157 | ("window-size", po::value>()->multitoken()->value_name("SIZE [SIZE...]"), 158 | "Sliding window detection box sizes. The source image is chipped with boxes of the given sizes. " 159 | "If resampled-size is not specified, all windows must fit within the model." 160 | "Default is the model size.") 161 | ("window-step", po::value>()->multitoken()->value_name("STEP [STEP...]"), 162 | "Sliding window step. Either a single step or a step for each window size may be given. Default " 163 | "is 20% of the model size.") 164 | ("resampled-size", po::value()->value_name("SIZE"), 165 | "Resample window chips to a fixed size. This must fit within the model.") 166 | ("max-cache-size", po::value()->value_name("SIZE"), 167 | "Maximum raster cache size. This can be specified as a memory amount, " 168 | "e.g. 16G, or as a percentage, e.g. 50%. Specifying 0 turns off raster " 169 | "cache size limiting. The default is 25% of the total physical RAM.") 170 | ; 171 | 172 | segmentationOptions_.add_options() 173 | ("r2p-method", po::value()->value_name(name_with_default("METHOD", "simple")), 174 | "Raster-to-polygon approximation method. Valid values are: none, simple, tc89-l1, tc89-kcos.") 175 | ("r2p-accuracy", po::value()->value_name(name_with_default("EPSILON", 3.0)), 176 | "Approximation accuracy for the raster-to-polygon operation.") 177 | ("r2p-min-area", po::value()->value_name(name_with_default("AREA", 0.0)), 178 | "Minimum polygon area (in pixels).") 179 | ; 180 | 181 | filterOptions_.add_options() 182 | ("include-labels", po::value>()->multitoken()->value_name("LABEL [LABEL...]"), 183 | "Filter results to only include specified labels.") 184 | ("exclude-labels", po::value>()->multitoken()->value_name("LABEL [LABEL...]"), 185 | "Filter results to exclude specified labels.") 186 | ("include-region", po::value()->value_name("PATH [PATH...]"), "Path to a file prescribing regions to include when filtering.") 187 | ("exclude-region", po::value()->value_name("PATH [PATH...]"), "Path to a file prescribing regions to exclude when filtering.") 188 | ("region", po::value>()->multitoken()->value_name("(include/exclude) PATH [PATH...] [(include/exclude) PATH [PATH...]...]"), 189 | "Paths to files including and excluding regions.") 190 | ; 191 | 192 | loggingOptions_.add_options() 193 | ("log", po::bounded_value>()->min_tokens(1)->max_tokens(2)->value_name("[LEVEL (=debug)] PATH"), 194 | "Log to a file, a file name preceded by an optional log level must be specified. Permitted values for log " 195 | "level are: trace, debug, info, warning, error, fatal.") 196 | ("quiet", "If set, no output will be sent to console, only a log file, if specified.") 197 | ; 198 | 199 | generalOptions_.add_options() 200 | ("config", po::value>()->value_name("PATH")->multitoken(), 201 | "Use options from a configuration file.") 202 | ("help", "Show this help message") 203 | ; 204 | 205 | // Build the options used for processing 206 | optionsDescription_.add(detectOptions_); 207 | optionsDescription_.add(localOptions_); 208 | optionsDescription_.add(webOptions_); 209 | optionsDescription_.add(outputOptions_); 210 | optionsDescription_.add(processingOptions_); 211 | optionsDescription_.add(segmentationOptions_); 212 | optionsDescription_.add(filterOptions_); 213 | optionsDescription_.add(loggingOptions_); 214 | optionsDescription_.add(generalOptions_); 215 | 216 | optionsDescription_.add_options() 217 | ("action", po::value()->value_name("ACTION"), "Action to perform.") 218 | ("debug", "Switch console output to \"debug\" log level.") 219 | ("trace", "Switch console output to \"trace\" log level.") 220 | ("log-format", po::value()->value_name("FORMAT"), "Log format. Valid values are: short, long, debug. Default value: short") 221 | // This bbox argument works for both local and web options, but we have duplicated bbox argument 222 | // description in the usage display 223 | ("bbox", po::cvRect2d_value()) 224 | ; 225 | 226 | // Add the bbox argumet to both local and web options (duplication is not allowed when parsing the arguments) 227 | localOptions_.add_options() 228 | ("bbox", po::cvRect2d_value()->value_name("WEST SOUTH EAST NORTH"), 229 | "Optional bounding box for image subset, optional for local images. Coordinates are specified in the " 230 | "following order: west longitude, south latitude, east longitude, and north latitude."); 231 | 232 | webOptions_.add_options() 233 | ("bbox", po::cvRect2d_value()->value_name("WEST SOUTH EAST NORTH"), 234 | "Bounding box for determining tiles specified in WGS84 Lat/Lon coordinate system. Coordinates are " 235 | "specified in the following order: west longitude, south latitude, east longitude, and north latitude."); 236 | 237 | visibleOptions_.add(detectOptions_); 238 | visibleOptions_.add(localOptions_); 239 | visibleOptions_.add(webOptions_); 240 | visibleOptions_.add(outputOptions_); 241 | visibleOptions_.add(processingOptions_); 242 | visibleOptions_.add(segmentationOptions_); 243 | visibleOptions_.add(filterOptions_); 244 | visibleOptions_.add(loggingOptions_); 245 | visibleOptions_.add(generalOptions_); 246 | } 247 | 248 | void CliProcessor::setupArgParsing(int argc, const char* const* argv) 249 | { 250 | setupInitialLogging(); 251 | 252 | try { 253 | parseArgs(argc, argv); 254 | validateArgs(); 255 | } catch (...) { 256 | if (displayHelp) { 257 | printUsage(); 258 | } 259 | throw; 260 | } 261 | 262 | if (displayHelp) { 263 | printUsage(); 264 | return; 265 | } 266 | 267 | setupLogging(); 268 | } 269 | 270 | void CliProcessor::startOSNProcessing() 271 | { 272 | OpenSpaceNet osn(std::move(osnArgs)); 273 | 274 | auto pd = boost::make_shared(); 275 | osn.setProgressDisplay(pd); 276 | pd->enableTiming("Detecting"); 277 | 278 | osn.process(); 279 | } 280 | 281 | void CliProcessor::setupInitialLogging() 282 | { 283 | log::init(); 284 | 285 | cerrSink_ = log::addCerrSink(dg::deepcore::level_t::warning, dg::deepcore::level_t::fatal, 286 | dg::deepcore::log::dg_log_format::dg_short_log); 287 | 288 | coutSink_ = log::addCoutSink(dg::deepcore::level_t::info, dg::deepcore::level_t::info, 289 | dg::deepcore::log::dg_log_format::dg_short_log); 290 | } 291 | 292 | void CliProcessor::setupLogging() { 293 | osnArgs.quiet = consoleLogLevel > level_t::info; 294 | 295 | // If no file is specified, assert that warning and above goes to the console 296 | if (fileLogPath.empty() && consoleLogLevel > level_t::warning) { 297 | consoleLogLevel = level_t::warning; 298 | } 299 | 300 | // If we have all the arguments parsed correctly, now is the time to turn off console logging 301 | if(consoleLogLevel > level_t::info || logFormat > log::dg_short_log) { 302 | log::removeSink(coutSink_); 303 | log::removeSink(cerrSink_); 304 | cerrSink_ = log::addCerrSink(consoleLogLevel, level_t::fatal, logFormat); 305 | } else if(consoleLogLevel < level_t::info) { 306 | log::removeSink(coutSink_); 307 | coutSink_ = log::addCoutSink(consoleLogLevel, level_t::info, logFormat); 308 | } 309 | 310 | // Setup a file logger 311 | if (!fileLogPath.empty()) { 312 | auto ofs = boost::make_shared(fileLogPath); 313 | DG_CHECK(!ofs->fail(), "Error opening log file %s for writing", fileLogPath.c_str()); 314 | log::addStreamSink(ofs, fileLogLevel, level_t::fatal, 315 | logFormat > log::dg_log_format::dg_long_log ? logFormat : log::dg_log_format::dg_long_log); 316 | } 317 | } 318 | 319 | template 320 | static bool readVariable(const char* param, const variables_map& vm, T& ret) 321 | { 322 | auto it = vm.find(param); 323 | if(it != end(vm)) { 324 | ret = it->second.as(); 325 | return true; 326 | } 327 | 328 | return false; 329 | } 330 | 331 | template 332 | static unique_ptr readVariable(const char* param, const variables_map& vm) 333 | { 334 | auto it = vm.find(param); 335 | if(it != end(vm)) { 336 | return make_unique(it->second.as()); 337 | } else { 338 | return unique_ptr(); 339 | } 340 | } 341 | 342 | template 343 | static bool readVariable(const char* param, variables_map& vm, std::vector& ret, bool splitArgs) 344 | { 345 | auto it = vm.find(param); 346 | if(it != end(vm)) { 347 | if(splitArgs) { 348 | auto args = it->second.as>(); 349 | ret.clear(); 350 | for(const auto& arg : args) { 351 | using so_tokenizer = tokenizer >; 352 | so_tokenizer tok(arg, escaped_list_separator('\\', ' ', '\"')); 353 | for(const auto& a : tok) { 354 | try { 355 | ret.push_back(lexical_cast(a)); 356 | } catch(bad_lexical_cast&) { 357 | DG_ERROR_THROW("Invalid --%s parameter: %s", param, arg.c_str()); 358 | } 359 | } 360 | } 361 | } else { 362 | ret = it->second.as>(); 363 | } 364 | 365 | return true; 366 | } 367 | 368 | return false; 369 | } 370 | 371 | static Action parseAction(string str) 372 | { 373 | to_lower(str); 374 | if(str == "help") { 375 | return Action::HELP; 376 | } else if(str == "detect") { 377 | return Action::DETECT; 378 | } 379 | 380 | return Action::UNKNOWN; 381 | } 382 | 383 | // Order of precedence: config files from env, env, config files from cli, cli. 384 | void CliProcessor::parseArgs(int argc, const char* const* argv) 385 | { 386 | if(argc > 1) { 387 | osnArgs.action = parseAction(argv[1]); 388 | if(osnArgs.action == Action::HELP) { 389 | displayHelp = true; 390 | return; 391 | } else if(osnArgs.action == Action::DETECT) { 392 | --argc; 393 | ++argv; 394 | } else { 395 | osnArgs.action = Action::DETECT; 396 | } 397 | 398 | for (int i = 1; i < argc; ++i) { 399 | if(iequals(argv[i], "--help")) { 400 | displayHelp = true; 401 | if(argc == 2) { 402 | return; 403 | } 404 | } 405 | } 406 | } 407 | 408 | // parse environment variable options 409 | variables_map environment_vm; 410 | po::store(po::parse_environment(optionsDescription_, "OSN_"), environment_vm); 411 | po::notify(environment_vm); 412 | readArgs(environment_vm, true); 413 | 414 | // parse regular options 415 | variables_map command_line_vm; 416 | po::store(po::command_line_parser(argc, argv) 417 | .extra_style_parser(po::combine_style_parsers( 418 | { &po::ignore_numbers, po::postfix_argument("region") })) 419 | .options(optionsDescription_) 420 | .run(), command_line_vm); 421 | po::notify(command_line_vm); 422 | readArgs(command_line_vm); 423 | } 424 | 425 | static Source parseService(string service) 426 | { 427 | to_lower(service); 428 | if(service == "dgcs") { 429 | return Source::DGCS; 430 | } else if(service == "evwhs") { 431 | return Source::EVWHS; 432 | } else if(service == "maps-api") { 433 | return Source::MAPS_API; 434 | } else if(service == "tile-json") { 435 | return Source::TILE_JSON; 436 | } 437 | 438 | DG_ERROR_THROW("Invalid --service parameter: %s", service.c_str()); 439 | } 440 | 441 | void CliProcessor::printUsage() const 442 | { 443 | cout << OSN_USAGE << visibleOptions_ << endl; 444 | } 445 | 446 | enum ArgUse { 447 | IGNORED = 1, 448 | MAY_USE_ONE = 2, 449 | OPTIONAL = 3, 450 | REQUIRED = 4 451 | }; 452 | 453 | inline void checkArgument(const char* argumentName, ArgUse expectedUse, bool set, const char* cause = "processing") 454 | { 455 | if (expectedUse == REQUIRED && !set) { 456 | DG_ERROR_THROW("Argument --%s is required when %s", argumentName, cause); 457 | } else if (expectedUse == IGNORED && set) { 458 | OSN_LOG(warning) << "Argument --" << argumentName << " is ignored when " << cause; 459 | } 460 | } 461 | 462 | inline void checkArgument(const char* argumentName, ArgUse expectedUse, const std::string& set, const char* cause = "processing") 463 | { 464 | checkArgument(argumentName, expectedUse, !set.empty(), cause); 465 | } 466 | 467 | template 468 | inline void checkArgument(const char* argumentName, ArgUse expectedUse, const std::vector& set, const char* cause = "processing") 469 | { 470 | if (expectedUse == REQUIRED && set.empty()) { 471 | DG_ERROR_THROW("Argument --%s is required for %s", argumentName, cause); 472 | } else if (expectedUse == MAY_USE_ONE && set.size() > 1) { 473 | OSN_LOG(warning) << "Argument --" << argumentName << " has ignored additional parameters when " << cause; 474 | } else if (expectedUse == IGNORED && !set.empty()) { 475 | OSN_LOG(warning) << "Argument --" << argumentName << " is ignored when " << cause; 476 | } 477 | } 478 | 479 | void CliProcessor::readModelPackage() 480 | { 481 | GbdxModelReader modelReader(osnArgs.modelPath); 482 | 483 | OSN_LOG(info) << "Reading model package..." ; 484 | 485 | osnArgs.modelPackage = modelReader.readModel(); 486 | DG_CHECK(osnArgs.modelPackage, "Unable to open the model package"); 487 | } 488 | 489 | void CliProcessor::validateArgs() 490 | { 491 | if (osnArgs.action == Action::HELP || displayHelp) { 492 | return; 493 | } 494 | 495 | DG_CHECK(osnArgs.action == Action::DETECT, "Try 'OpenSpaceNet --help' for more information."); 496 | 497 | // 498 | // Validate action args. 499 | // 500 | ArgUse windowStepUse(OPTIONAL); 501 | ArgUse windowSizeUse(OPTIONAL); 502 | ArgUse nmsUse(OPTIONAL); 503 | ArgUse confidenceUse(OPTIONAL); 504 | 505 | checkArgument("window-step", windowStepUse, osnArgs.windowStep); 506 | checkArgument("window-size", windowSizeUse, osnArgs.windowSize); 507 | checkArgument("nms", nmsUse, osnArgs.nms); 508 | checkArgument("confidence", confidenceUse, confidenceSet); 509 | 510 | // 511 | // Validate source args. 512 | // 513 | ArgUse tokenUse(IGNORED); 514 | ArgUse credentialsUse(IGNORED); 515 | ArgUse mapIdUse(IGNORED); 516 | ArgUse zoomUse(IGNORED); 517 | ArgUse bboxUse(IGNORED); 518 | ArgUse maxConnectionsUse(IGNORED); 519 | ArgUse urlUse(IGNORED); 520 | ArgUse useTilesUse(IGNORED); 521 | const char * sourceDescription; 522 | 523 | switch (osnArgs.source) { 524 | case Source::LOCAL: 525 | bboxUse = OPTIONAL; 526 | sourceDescription = "using a local image"; 527 | break; 528 | 529 | case Source::MAPS_API: 530 | tokenUse = REQUIRED; 531 | mapIdUse = OPTIONAL; 532 | zoomUse = OPTIONAL; 533 | bboxUse = REQUIRED; 534 | maxConnectionsUse = OPTIONAL; 535 | sourceDescription = "using maps-api"; 536 | break; 537 | 538 | case Source::DGCS: 539 | case Source::EVWHS: 540 | tokenUse = REQUIRED; 541 | credentialsUse = REQUIRED; 542 | mapIdUse = OPTIONAL; 543 | zoomUse = OPTIONAL; 544 | bboxUse = REQUIRED; 545 | maxConnectionsUse = OPTIONAL; 546 | sourceDescription = "using dgcs or evwhs"; 547 | break; 548 | 549 | case Source::TILE_JSON: 550 | credentialsUse = OPTIONAL; 551 | zoomUse = OPTIONAL; 552 | bboxUse = REQUIRED; 553 | maxConnectionsUse = OPTIONAL; 554 | urlUse = REQUIRED; 555 | useTilesUse = OPTIONAL; 556 | sourceDescription = "using tile-json"; 557 | break; 558 | 559 | default: 560 | DG_ERROR_THROW("Source is unknown or unspecified"); 561 | } 562 | 563 | if (osnArgs.dgcsCatalogID || osnArgs.evwhsCatalogID) { 564 | tokenUse = REQUIRED; 565 | } 566 | 567 | checkArgument("token", tokenUse, osnArgs.token, sourceDescription); 568 | checkArgument("credentials", credentialsUse, osnArgs.credentials, sourceDescription); 569 | checkArgument("map-id", mapIdUse, mapIdSet, sourceDescription); 570 | checkArgument("zoom", zoomUse, zoomSet, sourceDescription); 571 | checkArgument("bbox", bboxUse, (bool) osnArgs.bbox, sourceDescription); 572 | checkArgument("max-connections", maxConnectionsUse, maxConnectionsSet, sourceDescription); 573 | checkArgument("url", urlUse, osnArgs.url, sourceDescription); 574 | checkArgument("use-tiles", useTilesUse, osnArgs.useTiles, sourceDescription); 575 | 576 | // 577 | // Validate model and detection 578 | // 579 | checkArgument("model", REQUIRED, osnArgs.modelPath); 580 | 581 | DG_CHECK(osnArgs.includeLabels.empty() || osnArgs.excludeLabels.empty(), 582 | "Arguments --include-labels and --exclude-labels may not be specified at the same time"); 583 | 584 | if(windowSizeUse > MAY_USE_ONE || windowStepUse > MAY_USE_ONE) { 585 | DG_CHECK(osnArgs.windowSize.size() < 2 || osnArgs.windowStep.size() < 2 || 586 | osnArgs.windowSize.size() == osnArgs.windowStep.size(), 587 | "Arguments --window-size and --window-step must match in length"); 588 | } 589 | 590 | // 591 | // Validate output 592 | // 593 | checkArgument("output", REQUIRED, osnArgs.outputPath); 594 | if(osnArgs.outputFormat == "shp") { 595 | checkArgument("output-layer", IGNORED, osnArgs.layerName, "the output format is a shapefile"); 596 | osnArgs.layerName = path(osnArgs.outputPath).stem().filename().string(); 597 | } else if(osnArgs.layerName.empty()) { 598 | osnArgs.layerName = "osndetects"; 599 | } 600 | 601 | // 602 | // Validate filtering 603 | // 604 | if (osnArgs.filterDefinition.size()) { 605 | for (const auto& action : osnArgs.filterDefinition) { 606 | for (const auto& file : action.second) { 607 | path filePath = path(file); 608 | if (!exists(filePath)) { 609 | DG_ERROR_THROW("Argument to %s region using file \"%s\" invalid, file does not exist", 610 | action.first.c_str(), file.c_str()); 611 | } else { 612 | string fileExtension = filePath.extension().string(); 613 | fileExtension.erase(0,1); 614 | if (find(supportedFormats_.begin(), supportedFormats_.end(), fileExtension) == supportedFormats_.end()) { 615 | DG_ERROR_THROW("Argument to %s region using file \"%s\" invalid, format \"%s\" is unsupported", 616 | action.first.c_str(), file.c_str(), fileExtension.c_str()); 617 | } 618 | } 619 | } 620 | } 621 | } 622 | 623 | // Ask for password, if not specified 624 | if (credentialsUse > IGNORED && !displayHelp && !osnArgs.credentials.empty() && osnArgs.credentials.find(':') == string::npos) { 625 | promptForPassword(); 626 | } 627 | } 628 | 629 | void CliProcessor::promptForPassword() 630 | { 631 | cout << "Enter your web service password: "; 632 | auto password = readMaskedInputFromConsole(); 633 | osnArgs.credentials += ":"; 634 | osnArgs.credentials += password; 635 | } 636 | 637 | 638 | void CliProcessor::readArgs(variables_map vm, bool splitArgs) { 639 | // See if we have --config option(s), parse it if we do 640 | std::vector configFiles; 641 | if (readVariable("config", vm, configFiles, splitArgs)) { 642 | // Parse config files in reverse order because parse_config_file will not override existing options. This way 643 | // options apply as "last one wins". The arguments specified on the command line always win. 644 | for (const auto &configFile : reverse(configFiles)) { 645 | variables_map config_vm; 646 | po::store(po::parse_config_file(configFile.c_str(), optionsDescription_), config_vm); 647 | po::notify(config_vm); 648 | readArgs(config_vm, true); 649 | } 650 | } 651 | 652 | osnArgs.bbox = readVariable("bbox", vm); 653 | 654 | string service; 655 | bool serviceSet = readVariable("service", vm, service); 656 | if (serviceSet) { 657 | osnArgs.source = parseService(service); 658 | readWebServiceArgs(vm, splitArgs); 659 | } 660 | 661 | bool imageSet = readVariable("image", vm, osnArgs.image); 662 | if (imageSet) { 663 | osnArgs.source = Source::LOCAL; 664 | } 665 | 666 | DG_CHECK(!imageSet || !serviceSet, "Arguments --image and --service may not be specified at the same time"); 667 | 668 | readWebServiceArgs(vm, splitArgs); 669 | readProcessingArgs(vm, splitArgs); 670 | readOutputArgs(vm, splitArgs); 671 | readFeatureDetectionArgs(vm, splitArgs); 672 | readLoggingArgs(vm, splitArgs); 673 | 674 | if(!osnArgs.modelPath.empty()) { 675 | readModelPackage(); 676 | } 677 | 678 | readSegmentationArgs(vm, splitArgs); 679 | } 680 | 681 | void CliProcessor::readWebServiceArgs(variables_map vm, bool splitArgs) 682 | { 683 | mapIdSet |= readVariable("map-id", vm, osnArgs.mapId); 684 | readVariable("token", vm, osnArgs.token); 685 | readVariable("credentials", vm, osnArgs.credentials); 686 | readVariable("url", vm, osnArgs.url); 687 | osnArgs.useTiles = vm.find("use-tiles") != vm.end(); 688 | zoomSet |= readVariable("zoom", vm, osnArgs.zoom); 689 | maxConnectionsSet |= readVariable("max-connections", vm, osnArgs.maxConnections); 690 | } 691 | 692 | 693 | void CliProcessor::readOutputArgs(variables_map vm, bool splitArgs) 694 | { 695 | readVariable("format", vm, osnArgs.outputFormat); 696 | to_lower(osnArgs.outputFormat); 697 | 698 | readVariable("output", vm, osnArgs.outputPath); 699 | readVariable("output-layer", vm, osnArgs.layerName); 700 | 701 | string typeStr = "polygon"; 702 | readVariable("type", vm, typeStr); 703 | to_lower(typeStr); 704 | if(typeStr == "polygon") { 705 | osnArgs.geometryType = GeometryType::POLYGON; 706 | } else if(typeStr == "point") { 707 | osnArgs.geometryType = GeometryType::POINT; 708 | } else { 709 | DG_ERROR_THROW("Invalid geometry type: %s", typeStr.c_str()); 710 | } 711 | osnArgs.append = vm.find("append") != end(vm); 712 | osnArgs.producerInfo = vm.find("producer-info") != end(vm); 713 | 714 | osnArgs.dgcsCatalogID = vm.find("dgcs-catalog-id") != end(vm); 715 | osnArgs.evwhsCatalogID = vm.find("evwhs-catalog-id") != end(vm); 716 | readVariable("wfs-credentials", vm, osnArgs.wfsCredentials); 717 | readVariable("extra-fields", vm, osnArgs.extraFields, true); 718 | if (!osnArgs.extraFields.empty() && osnArgs.extraFields.size() % 2 != 0){ 719 | DG_ERROR_THROW("Invalid number of fields: Fields must be supplied pairs of strings for key and value."); 720 | } 721 | } 722 | 723 | void CliProcessor::readProcessingArgs(variables_map vm, bool splitArgs) 724 | { 725 | osnArgs.useCpu = vm.find("cpu") != end(vm); 726 | readVariable("max-utilization", vm, osnArgs.maxUtilization); 727 | readVariable("model", vm, osnArgs.modelPath); 728 | 729 | readVariable("window-size", vm, osnArgs.windowSize, splitArgs); 730 | readVariable("window-step", vm, osnArgs.windowStep, splitArgs); 731 | osnArgs.resampledSize = readVariable("resampled-size", vm); 732 | 733 | readVariable("include-labels", vm, osnArgs.includeLabels, splitArgs); 734 | readVariable("exclude-labels", vm, osnArgs.excludeLabels, splitArgs); 735 | if (vm.find("region") != end(vm)) { 736 | parseFilterArgs(vm["region"].as>()); 737 | } 738 | 739 | string sizeString("25%"); 740 | readVariable("max-cache-size", vm, sizeString); 741 | try { 742 | osnArgs.maxCacheSize = memory::stringToRam(sizeString); 743 | } catch(...) { 744 | DG_ERROR_THROW("Argument --max-cache-size is invalid"); 745 | } 746 | } 747 | 748 | void CliProcessor::readFeatureDetectionArgs(variables_map vm, bool /* splitArgs */) 749 | { 750 | confidenceSet |= readVariable("confidence", vm, osnArgs.confidence); 751 | 752 | if(vm.find("nms") != end(vm)) { 753 | osnArgs.nms = true; 754 | std::vector args; 755 | readVariable("nms", vm, args); 756 | if(args.size()) { 757 | osnArgs.overlap = args[0]; 758 | } 759 | } 760 | } 761 | 762 | void CliProcessor::readSegmentationArgs(boost::program_options::variables_map vm, bool /* splitArgs */) 763 | { 764 | bool isSegmentation = (osnArgs.modelPackage && osnArgs.modelPackage->metadata().category() == "segmentation"); 765 | static const char* CAUSE = "input model is not a segmentation model."; 766 | 767 | string method; 768 | if(readVariable("r2p-method", vm, method)) { 769 | if(iequals(method, "none")) { 770 | osnArgs.method = RasterToPolygonDP::NONE; 771 | } else if(iequals(method, "simple")) { 772 | osnArgs.method = RasterToPolygonDP::SIMPLE; 773 | } else if(iequals(method, "tc89-l1")) { 774 | osnArgs.method = RasterToPolygonDP::TC89_L1; 775 | } else if(iequals(method, "tc89-kcos")) { 776 | osnArgs.method = RasterToPolygonDP::TC89_KCOS; 777 | } else { 778 | DG_ERROR_THROW("Invalid --r2p-method parameter: '%s'", method.c_str()); 779 | } 780 | 781 | if(!isSegmentation) { 782 | checkArgument("r2p-method", IGNORED, true, CAUSE); 783 | } 784 | } 785 | 786 | if(readVariable("r2p-accuracy", vm, osnArgs.epsilon) && !isSegmentation) { 787 | checkArgument("r2p-accuracy", IGNORED, true, CAUSE); 788 | } 789 | 790 | if(readVariable("r2p-min-area", vm, osnArgs.minArea) && !isSegmentation) { 791 | checkArgument("r2p-min-area", IGNORED, true, CAUSE); 792 | } 793 | } 794 | 795 | void CliProcessor::readLoggingArgs(variables_map vm, bool splitArgs) 796 | { 797 | if(vm.find("quiet") != end(vm)) { 798 | consoleLogLevel = level_t::fatal; 799 | } if(vm.find("trace") != end(vm)) { 800 | consoleLogLevel = level_t::trace; 801 | } else if(vm.find("debug") != end(vm)) { 802 | consoleLogLevel = level_t::debug; 803 | } 804 | 805 | string strLogFormat; 806 | if(readVariable("log-format", vm, strLogFormat)) { 807 | to_lower(strLogFormat); 808 | if(strLogFormat == "short") { 809 | logFormat = log::dg_log_format::dg_short_log; 810 | } else if(strLogFormat == "long") { 811 | logFormat = log::dg_log_format::dg_long_log; 812 | } else if(strLogFormat == "debug") { 813 | logFormat = log::dg_log_format::dg_debug_log; 814 | } else { 815 | DG_ERROR_THROW("Invalid --log-format parameter: %s", strLogFormat.c_str()); 816 | } 817 | } 818 | 819 | std::vector logArgs; 820 | if(readVariable("log", vm, logArgs, splitArgs)) { 821 | DG_CHECK(!logArgs.empty(), "Log path must be specified"); 822 | fileLogLevel = level_t::debug; 823 | 824 | if(logArgs.size() == 1) { 825 | fileLogPath = logArgs[0]; 826 | } else { 827 | fileLogLevel = log::stringToLevel(logArgs[0]); 828 | fileLogPath = logArgs[1]; 829 | } 830 | } 831 | } 832 | 833 | void CliProcessor::parseFilterArgs(const std::vector& filterList) 834 | { 835 | string filterAction = ""; 836 | string finalEntry = ""; 837 | std::vector filterActionFileSet; 838 | for (auto entry : filterList) { 839 | if (entry == filterAction) { 840 | finalEntry = entry; 841 | continue; 842 | } 843 | else if (entry == "include" || 844 | entry == "exclude") { 845 | if (filterAction != "") { 846 | if (filterActionFileSet.empty()) { 847 | DG_ERROR_THROW("Argument to %s region without file input", filterAction.c_str()); 848 | } 849 | osnArgs.filterDefinition.push_back(std::make_pair(filterAction, move(filterActionFileSet))); 850 | } 851 | 852 | filterActionFileSet.clear(); 853 | filterAction = entry.c_str(); 854 | } else { 855 | filterActionFileSet.push_back(entry); 856 | } 857 | 858 | finalEntry = entry; 859 | } 860 | if (finalEntry == "include" || finalEntry == "exclude") { 861 | DG_ERROR_THROW("Argument to %s region without file input", finalEntry.c_str()); 862 | } 863 | if (!filterActionFileSet.empty()) { 864 | osnArgs.filterDefinition.push_back(std::make_pair(filterAction, move(filterActionFileSet))); 865 | } 866 | } 867 | 868 | bool CliProcessor::showHelp() 869 | { 870 | return displayHelp; 871 | } 872 | 873 | } } // namespace dg { namespace osn { 874 | -------------------------------------------------------------------------------- /cli/src/CliProcessor.h: -------------------------------------------------------------------------------- 1 | /******************************************************************************** 2 | * Copyright 2017 DigitalGlobe, Inc. 3 | * Author: Joe White 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | ********************************************************************************/ 17 | 18 | #ifndef OPENSPACENET_CLIPROCESSOR_H 19 | #define OPENSPACENET_CLIPROCESSOR_H 20 | 21 | #include 22 | #include 23 | #include 24 | 25 | namespace dg { namespace osn { 26 | 27 | class CliProcessor 28 | { 29 | public: 30 | dg::deepcore::level_t consoleLogLevel = dg::deepcore::level_t::info; 31 | std::string fileLogPath; 32 | dg::deepcore::level_t fileLogLevel = dg::deepcore::level_t::debug; 33 | dg::deepcore::log::dg_log_format logFormat = dg::deepcore::log::dg_log_format::dg_short_log; 34 | 35 | CliProcessor(); 36 | void setupArgParsing(int argc, const char* const* argv); 37 | void startOSNProcessing(); 38 | bool showHelp(); 39 | OpenSpaceNetArgs osnArgs; 40 | 41 | private: 42 | bool confidenceSet = false; 43 | bool mapIdSet = false; 44 | bool displayHelp = false; 45 | bool maxConnectionsSet = false; 46 | bool zoomSet = false; 47 | 48 | void setupInitialLogging(); 49 | void setupLogging(); 50 | void parseArgs(int argc, const char* const* argv); 51 | void printUsage() const; 52 | void readArgs(boost::program_options::variables_map vm, bool splitArgs=false); 53 | void readWebServiceArgs(boost::program_options::variables_map vm, bool splitArgs=false); 54 | void promptForPassword(); 55 | void readOutputArgs(boost::program_options::variables_map vm, bool splitArgs=false); 56 | void readProcessingArgs(boost::program_options::variables_map vm, bool splitArgs=false); 57 | void readFeatureDetectionArgs(boost::program_options::variables_map vm, bool splitArgs=false); 58 | void readSegmentationArgs(boost::program_options::variables_map vm, bool splitArgs=false); 59 | void readLoggingArgs(boost::program_options::variables_map vm, bool splitArgs=false); 60 | void parseFilterArgs(const std::vector& filterList); 61 | 62 | void readModelPackage(); 63 | void validateArgs(); 64 | 65 | boost::program_options::options_description localOptions_; 66 | boost::program_options::options_description webOptions_; 67 | boost::program_options::options_description outputOptions_; 68 | boost::program_options::options_description processingOptions_; 69 | boost::program_options::options_description detectOptions_; 70 | boost::program_options::options_description segmentationOptions_; 71 | boost::program_options::options_description filterOptions_; 72 | boost::program_options::options_description loggingOptions_; 73 | boost::program_options::options_description generalOptions_; 74 | 75 | boost::program_options::options_description allOptions_; 76 | boost::program_options::options_description visibleOptions_; 77 | boost::program_options::options_description optionsDescription_; 78 | 79 | std::vector supportedFormats_; 80 | 81 | boost::shared_ptr cerrSink_; 82 | boost::shared_ptr coutSink_; 83 | }; 84 | 85 | } } // namespace dg { namespace osn { 86 | #endif //OPENSPACENET_CLIPROCESSOR_H 87 | -------------------------------------------------------------------------------- /cli/src/main.cpp: -------------------------------------------------------------------------------- 1 | /******************************************************************************** 2 | * Copyright 2017 DigitalGlobe, Inc. 3 | * Author: Aleksey Vitebskiy 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | ********************************************************************************/ 17 | 18 | #include 19 | #include "CliProcessor.h" 20 | #include 21 | #include 22 | 23 | using namespace dg::osn; 24 | using namespace dg::deepcore; 25 | 26 | using std::cout; 27 | using std::exception; 28 | using std::string; 29 | 30 | static const string OSN_LOGO = 31 | "Radiant Solutions\n" 32 | " ____ _____ _ _ _ \n" 33 | " / __ \\ / ____| | \\ | | | | \n" 34 | " | | | |_ __ ___ _ __ | (___ _ __ __ _ ___ ___| \\| | ___| |_ \n" 35 | " | | | | '_ \\ / _ \\ '_ \\ \\___ \\| '_ \\ / _` |/ __/ _ \\ . ` |/ _ \\ __| \n" 36 | " | |__| | |_) | __/ | | |____) | |_) | (_| | (_| __/ |\\ | __/ |_ _ _ _ \n" 37 | " \\____/| .__/ \\___|_| |_|_____/| .__/ \\__,_|\\___\\___|_| \\_|\\___|\\__(_|_|_) \n" 38 | " | | | | \n" 39 | " |_| |_| \n\n" 40 | "Version: " OPENSPACENET_VERSION_STRING "\n" 41 | "DeepCore Version: " DEEPCORE_VERSION_STRING "\n"; 42 | 43 | int main (int argc, const char* const* argv) 44 | { 45 | cout << OSN_LOGO; 46 | cout << "GBDXM Metadata Version: " << classification::gbdxm::METADATA_VERSION << "\n\n"; 47 | 48 | try { 49 | CliProcessor osnCliProcessor; 50 | osnCliProcessor.setupArgParsing(argc, argv); 51 | if(!osnCliProcessor.showHelp()) { 52 | osnCliProcessor.startOSNProcessing(); 53 | } 54 | } catch (const Error& e) { 55 | DG_ERROR_LOG(OpenSpaceNet, e); 56 | return 1; 57 | } catch (const exception &e) { 58 | OSN_LOG(error) << e.what(); 59 | return 1; 60 | } catch (...) { 61 | OSN_LOG(error) << "Unknown error."; 62 | return 1; 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /common/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | include(InstallHeaders) 2 | 3 | set(CMAKE_INCLUDE_CURRENT_DIR ON) 4 | include_directories(include) 5 | 6 | set(HEADERS 7 | include/OpenSpaceNet.h 8 | include/OpenSpaceNetArgs.h 9 | ) 10 | 11 | set(SOURCES 12 | src/OpenSpaceNet.cpp 13 | ) 14 | 15 | add_library(OpenSpaceNet.common ${SOURCES} ${HEADERS}) 16 | target_link_libraries(OpenSpaceNet.common ${OSN_LINK_LIBRARIES}) 17 | set_target_properties(OpenSpaceNet.common PROPERTIES OUTPUT_NAME openspacenet) 18 | 19 | configure_file(version.h.in ${CMAKE_CURRENT_BINARY_DIR}/OpenSpaceNetVersion.h) 20 | install_public_headers(${HEADERS}) 21 | 22 | INSTALL(TARGETS OpenSpaceNet.common 23 | RUNTIME DESTINATION bin 24 | LIBRARY DESTINATION lib 25 | ARCHIVE DESTINATION lib 26 | ) 27 | -------------------------------------------------------------------------------- /common/include/OpenSpaceNet.h: -------------------------------------------------------------------------------- 1 | /******************************************************************************** 2 | * Copyright 2017 DigitalGlobe, Inc. 3 | * Author: Joe White 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | ********************************************************************************/ 17 | 18 | 19 | #ifndef OPENSPACENET_OPENSPACENET_H 20 | #define OPENSPACENET_OPENSPACENET_H 21 | 22 | #include "OpenSpaceNetArgs.h" 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | #include 31 | #include 32 | #include 33 | #include 34 | #include 35 | #include 36 | #include 37 | 38 | namespace dg { namespace osn { 39 | 40 | class OpenSpaceNet 41 | { 42 | public: 43 | OpenSpaceNet(OpenSpaceNetArgs&& args); 44 | void process(); 45 | void setProgressDisplay(boost::shared_ptr display); 46 | 47 | private: 48 | deepcore::imagery::node::GeoBlockSource::Ptr initLocalImage(); 49 | deepcore::imagery::node::GeoBlockSource::Ptr initMapServiceImage(); 50 | deepcore::geometry::node::SubsetRegionFilter::Ptr initSubsetRegionFilter(); 51 | deepcore::classification::node::Detector::Ptr initDetector(); 52 | void initSegmentation(deepcore::classification::Model::Ptr model); 53 | deepcore::imagery::node::SlidingWindow::Ptr initSlidingWindow(); 54 | deepcore::geometry::node::LabelFilter::Ptr initLabelFilter(bool isSegmentation); 55 | deepcore::vector::node::PredictionToFeature::Ptr initPredictionToFeature(); 56 | deepcore::vector::node::WfsFeatureFieldExtractor::Ptr initWfs(); 57 | deepcore::vector::node::FileFeatureSink::Ptr initFeatureSink(); 58 | 59 | void printModel(); 60 | void skipLine() const; 61 | deepcore::imagery::SizeSteps calcWindows() const; 62 | 63 | OpenSpaceNetArgs args_; 64 | std::shared_ptr cleanup_; 65 | boost::shared_ptr pd_; 66 | 67 | cv::Size imageSize_; 68 | cv::Rect bbox_; 69 | deepcore::geometry::SpatialReference imageSr_; 70 | deepcore::geometry::SpatialReference sr_; 71 | std::unique_ptr pixelToProj_; 72 | std::unique_ptr pixelToLL_; 73 | 74 | std::unique_ptr metadata_; 75 | cv::Size primaryWindowSize_; 76 | cv::Point primaryWindowStep_; 77 | float modelAspectRatio_; 78 | bool haveAlpha_ = false; 79 | }; 80 | 81 | } } // namespace dg { namespace osn { 82 | 83 | #endif //OPENSPACENET_LIBOPENSPACENET_H 84 | -------------------------------------------------------------------------------- /common/include/OpenSpaceNetArgs.h: -------------------------------------------------------------------------------- 1 | /******************************************************************************** 2 | * Copyright 2017 DigitalGlobe, Inc. 3 | * Author: Joe White 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | ********************************************************************************/ 17 | 18 | #ifndef OPENSPACENET_OPENSPACENETARGS_H 19 | #define OPENSPACENET_OPENSPACENETARGS_H 20 | 21 | #include 22 | #include 23 | #include 24 | 25 | #define OSN_LOG(sev) DG_LOG(OpenSpaceNet, sev) 26 | #define MAPSAPI_MAPID "digitalglobe.nal0g75k" 27 | #define WFS_TYPENAME "DigitalGlobe:FinishedFeature" 28 | 29 | 30 | namespace dg { namespace osn { 31 | 32 | enum class Source 33 | { 34 | UNKNOWN, 35 | LOCAL, 36 | DGCS, 37 | EVWHS, 38 | MAPS_API, 39 | TILE_JSON 40 | }; 41 | 42 | enum class Action 43 | { 44 | UNKNOWN, 45 | HELP, 46 | DETECT 47 | }; 48 | 49 | struct OpenSpaceNetArgs 50 | { 51 | // Input options 52 | Action action = Action::UNKNOWN; 53 | Source source = Source::UNKNOWN; 54 | 55 | std::string image; 56 | std::unique_ptr bbox; 57 | 58 | // Web service input options 59 | std::string token; 60 | std::string credentials; 61 | int zoom = 18; 62 | int maxConnections = 10; 63 | std::string mapId = MAPSAPI_MAPID; 64 | std::string url; 65 | bool useTiles=false; 66 | 67 | 68 | // Output options 69 | deepcore::geometry::GeometryType geometryType = deepcore::geometry::GeometryType::POLYGON; 70 | std::string outputFormat = "shp"; 71 | std::string outputPath; 72 | std::string layerName; 73 | bool producerInfo = false; 74 | bool dgcsCatalogID = false; 75 | bool evwhsCatalogID = false; 76 | std::string wfsCredentials; 77 | bool append = false; 78 | std::vector extraFields; 79 | 80 | // Processing options 81 | std::string modelPath; 82 | std::unique_ptr modelPackage; 83 | bool useCpu = false; 84 | float maxUtilization = 95; 85 | std::vector windowSize; 86 | std::vector windowStep; 87 | std::unique_ptr resampledSize; 88 | size_t maxCacheSize = 0ULL; 89 | 90 | // Feature detection options 91 | float confidence = 95; 92 | bool nms = false; 93 | float overlap = 30; 94 | std::vector includeLabels; 95 | std::vector excludeLabels; 96 | std::vector>> filterDefinition; 97 | 98 | // Segmentation options 99 | deepcore::imagery::RasterToPolygonDP::Method method = deepcore::imagery::RasterToPolygonDP::SIMPLE; 100 | double epsilon = 3.0; 101 | double minArea = 0.0; 102 | 103 | // Logging options 104 | bool quiet = false; 105 | }; 106 | 107 | } } // namespace dg { namespace osn { 108 | 109 | #endif //OPENSPACENET_OPENSPACENETARGS_H 110 | -------------------------------------------------------------------------------- /common/src/OpenSpaceNet.cpp: -------------------------------------------------------------------------------- 1 | /******************************************************************************** 2 | * Copyright 2017 DigitalGlobe, Inc. 3 | * Author: Joe White 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | ********************************************************************************/ 17 | 18 | #include "OpenSpaceNet.h" 19 | #include 20 | 21 | #include 22 | 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | #include 31 | #include 32 | #include 33 | #include 34 | #include 35 | #include 36 | #include 37 | #include 38 | #include 39 | #include 40 | #include 41 | #include 42 | #include 43 | #include 44 | #include 45 | #include 46 | #include 47 | #include 48 | #include 49 | #include 50 | 51 | namespace dg { namespace osn { 52 | 53 | using namespace dg::deepcore::classification; 54 | using namespace dg::deepcore::classification::node; 55 | using namespace dg::deepcore::geometry; 56 | using namespace dg::deepcore::geometry::node; 57 | using namespace dg::deepcore::imagery; 58 | using namespace dg::deepcore::imagery::node; 59 | using namespace dg::deepcore::network; 60 | using namespace dg::deepcore::vector; 61 | using namespace dg::deepcore::vector::node; 62 | 63 | using boost::format; 64 | using boost::is_any_of; 65 | using boost::join; 66 | using boost::lexical_cast; 67 | using boost::make_unique; 68 | using boost::posix_time::from_time_t; 69 | using boost::posix_time::to_simple_string; 70 | using boost::split; 71 | using std::async; 72 | using std::chrono::duration; 73 | using std::chrono::high_resolution_clock; 74 | using std::map; 75 | using std::move; 76 | using std::string; 77 | using std::thread; 78 | using std::vector; 79 | using std::unique_ptr; 80 | using dg::deepcore::loginUser; 81 | using dg::deepcore::memory::prettyBytes; 82 | using dg::deepcore::Metric; 83 | using dg::deepcore::NodeState; 84 | using dg::deepcore::ProgressDisplayHelper; 85 | using dg::deepcore::Value; 86 | 87 | OpenSpaceNet::OpenSpaceNet(OpenSpaceNetArgs&& args) : 88 | args_(move(args)) 89 | { 90 | if(args_.source > Source::LOCAL) { 91 | cleanup_ = HttpCleanup::get(); 92 | } 93 | } 94 | 95 | void OpenSpaceNet::process() 96 | { 97 | deepcore::classification::init(); 98 | deepcore::vector::init(); 99 | 100 | GeoBlockSource::Ptr blockSource; 101 | if(args_.source > Source::LOCAL) { 102 | OSN_LOG(info) << "Opening map service image..." ; 103 | blockSource = initMapServiceImage(); 104 | } else if(args_.source == Source::LOCAL) { 105 | OSN_LOG(info) << "Opening local image..." ; 106 | blockSource = initLocalImage(); 107 | } else { 108 | DG_ERROR_THROW("Input source not specified"); 109 | } 110 | 111 | auto blockCache = BlockCache::create("blockCache"); 112 | blockCache->connectAttrs(*blockSource); 113 | blockCache->attr("bufferSize") = args_.maxCacheSize / 2; 114 | 115 | if(args_.maxCacheSize > 0) { 116 | OSN_LOG(info) << "Maximum raster cache size is set to " << prettyBytes(args_.maxCacheSize); 117 | } else { 118 | OSN_LOG(info) << "Maximum raster cache size is not limited"; 119 | } 120 | 121 | //Note: Model must be initialized before sliding window 122 | //and subset filter for model size and stepping 123 | OSN_LOG(info) << "Reading model..." ; 124 | auto model = initDetector(); 125 | 126 | printModel(); 127 | 128 | auto subsetWithBorder = SubsetWithBorder::create("border"); 129 | if(args_.resampledSize) { 130 | subsetWithBorder->attr("paddedSize") = metadata_->modelSize(); 131 | } 132 | subsetWithBorder->connectAttrs(*blockSource); 133 | 134 | auto subsetFilter = initSubsetRegionFilter(); 135 | auto slidingWindow = initSlidingWindow(); 136 | slidingWindow->connectAttrs(*blockSource); 137 | 138 | RemoveBandByColorInterp::Ptr removeAlpha; 139 | if(haveAlpha_) { 140 | removeAlpha = RemoveBandByColorInterp::create("removeAlpha"); 141 | removeAlpha->attr("bandToRemove") = ColorInterpretation::ALPHA_BAND; 142 | removeAlpha->connectAttrs(*blockSource); 143 | 144 | blockCache->connectAttrs(*removeAlpha); 145 | subsetWithBorder->connectAttrs(*removeAlpha); 146 | slidingWindow->connectAttrs(*removeAlpha); 147 | } 148 | 149 | bool isSegmentation = (metadata_->category() == "segmentation"); 150 | 151 | auto labelFilter = initLabelFilter(isSegmentation); 152 | NonMaxSuppression::Ptr nmsNode; 153 | if(args_.nms) { 154 | if (isSegmentation) { 155 | nmsNode = PolyNonMaxSuppression::create("nms"); 156 | } else { 157 | nmsNode = BoxNonMaxSuppression::create("nms"); 158 | } 159 | 160 | nmsNode->attr("overlapThreshold") = args_.overlap / 100; 161 | } 162 | 163 | auto predictionToFeature = initPredictionToFeature(); 164 | auto wfsExtractor = initWfs(); 165 | auto featureSink = initFeatureSink(); 166 | 167 | if(removeAlpha) { 168 | removeAlpha->input("blocks") = blockSource->output("blocks"); 169 | blockCache->input("blocks") = removeAlpha->output("blocks"); 170 | } else { 171 | blockCache->input("blocks") = blockSource->output("blocks"); 172 | } 173 | 174 | subsetWithBorder->input("subsets") = blockCache->output("subsets"); 175 | if (subsetFilter) { 176 | subsetFilter->input("subsets") = subsetWithBorder->output("subsets"); 177 | slidingWindow->input("subsets") = subsetFilter->output("subsets"); 178 | } else { 179 | slidingWindow->input("subsets") = subsetWithBorder->output("subsets"); 180 | } 181 | 182 | model->input("subsets") = slidingWindow->output("subsets"); 183 | if (labelFilter) { 184 | labelFilter->input("predictions") = model->output("predictions"); 185 | if (nmsNode) { 186 | nmsNode->input("predictions") = labelFilter->output("predictions"); 187 | } 188 | } else if (nmsNode) { 189 | nmsNode->input("predictions") = model->output("predictions"); 190 | } 191 | 192 | PredictionBoxToPoly::Ptr toPoly; 193 | if (!isSegmentation) { 194 | toPoly = PredictionBoxToPoly::create("predictionToPoly"); 195 | predictionToFeature->input("predictions") = toPoly->output("predictions"); 196 | } 197 | 198 | if (nmsNode) { 199 | if (isSegmentation) { 200 | predictionToFeature->input("predictions") = nmsNode->output("predictions"); 201 | } else { 202 | toPoly->input("predictions") = nmsNode->output("predictions"); 203 | } 204 | } else if (labelFilter) { 205 | if (isSegmentation) { 206 | predictionToFeature->input("predictions") = labelFilter->output("predictions"); 207 | } else { 208 | toPoly->input("predictions") = labelFilter->output("predictions"); 209 | } 210 | } else { 211 | if (isSegmentation) { 212 | predictionToFeature->input("predictions") = model->output("predictions"); 213 | } else { 214 | toPoly->input("predictions") = model->output("predictions"); 215 | } 216 | } 217 | 218 | if (wfsExtractor) { 219 | wfsExtractor->input("features") = predictionToFeature->output("features"); 220 | featureSink->input("features") = wfsExtractor->output("features"); 221 | } else { 222 | featureSink->input("features") = predictionToFeature->output("features"); 223 | } 224 | 225 | auto startTime = high_resolution_clock::now(); 226 | 227 | if (!args_.quiet && pd_) { 228 | pd_->start(); 229 | 230 | ProgressDisplayHelper pdHelper(*pd_); 231 | 232 | auto subsetsRequested = slidingWindow->metric("total").changed().connect( 233 | [&pdHelper, &featureSink, this] (const std::weak_ptr&, Value value) { 234 | if(!pd_->isRunning()) { 235 | featureSink->cancel(); 236 | } else { 237 | pdHelper.updateMaximum("Reading", value.convert()); 238 | pdHelper.updateMaximum("Detecting", value.convert()); 239 | } 240 | }); 241 | 242 | auto subsetsRead = slidingWindow->metric("forwarded").changed().connect( 243 | [&pdHelper, &featureSink, this] (const std::weak_ptr&, Value value) { 244 | if(!pd_->isRunning()) { 245 | featureSink->cancel(); 246 | } else { 247 | pdHelper.updateCurrent("Reading", value.convert()); 248 | } 249 | }); 250 | 251 | auto subsetsProcessed = model->metric("processed").changed().connect( 252 | [&pdHelper, &featureSink, this] (const std::weak_ptr&, Value value) { 253 | if(!pd_->isRunning()) { 254 | featureSink->cancel(); 255 | } else { 256 | pdHelper.updateCurrent("Detecting", value.convert()); 257 | } 258 | }); 259 | 260 | featureSink->run(); 261 | featureSink->wait(true); 262 | pd_->stop(); 263 | } else { 264 | featureSink->run(); 265 | featureSink->wait(); 266 | } 267 | 268 | if (!args_.quiet) { 269 | skipLine(); 270 | duration duration = high_resolution_clock::now() - startTime; 271 | OSN_LOG(info) << featureSink->metric("processed").convert() << " features detected."; 272 | OSN_LOG(info) << "Processing time " << duration.count() << " s"; 273 | } 274 | } 275 | 276 | void OpenSpaceNet::setProgressDisplay(boost::shared_ptr display) 277 | { 278 | pd_ = std::move(display); 279 | pd_->setCategories({ 280 | {"Reading", "Reading the image" }, 281 | {"Detecting", "Detecting the object(s)" } 282 | }); 283 | } 284 | 285 | GeoBlockSource::Ptr OpenSpaceNet::initLocalImage() 286 | { 287 | auto image = make_unique(args_.image); 288 | imageSize_ = image->size(); 289 | pixelToProj_ = image->pixelToProj().clone(); 290 | imageSr_ = image->spatialReference(); 291 | 292 | bbox_ = cv::Rect{ { 0, 0 }, imageSize_ }; 293 | bool ignoreArgsBbox = false; 294 | 295 | TransformationChain llToPixel; 296 | if (!imageSr_.isLocal()) { 297 | llToPixel = { 298 | imageSr_.fromLatLon(), 299 | pixelToProj_->inverse() 300 | }; 301 | sr_ = SpatialReference::WGS84; 302 | } else { 303 | OSN_LOG(warning) << "Image has geometric metadata which cannot be converted to WGS84. " 304 | "Output will be in native space, and some output formats will fail."; 305 | 306 | if (args_.bbox) { 307 | OSN_LOG(warning) << "Supplying the --bbox option implicitly requests a conversion from " 308 | "WGS84 to pixel space however there is no conversion from WGS84 to " 309 | "pixel space."; 310 | OSN_LOG(warning) << "Ignoring user-supplied bounding box"; 311 | 312 | ignoreArgsBbox = true; 313 | } 314 | 315 | llToPixel = { pixelToProj_->inverse() }; 316 | } 317 | 318 | pixelToLL_ = llToPixel.inverse(); 319 | 320 | if(args_.bbox && !ignoreArgsBbox) { 321 | auto bbox = llToPixel.transformToInt(*args_.bbox); 322 | 323 | auto intersect = bbox_ & (cv::Rect)bbox; 324 | DG_CHECK(intersect.width && intersect.height, "Input image and the provided bounding box do not intersect"); 325 | 326 | if(bbox != intersect) { 327 | auto llIntersect = pixelToLL_->transform(intersect); 328 | OSN_LOG(info) << "Bounding box adjusted to " << llIntersect.tl() << " : " << llIntersect.br(); 329 | } 330 | 331 | bbox_ = intersect; 332 | } 333 | 334 | haveAlpha_ = RasterBand::haveAlpha(image->rasterBands()); 335 | 336 | GeoBlockSource::Ptr blockSource = GdalBlockSource::create("blockSource"); 337 | blockSource->attr("path") = args_.image; 338 | return blockSource; 339 | } 340 | 341 | GeoBlockSource::Ptr OpenSpaceNet::initMapServiceImage() 342 | { 343 | DG_CHECK(args_.bbox, "Bounding box must be specified"); 344 | 345 | std::unique_ptr client; 346 | bool wmts = true; 347 | string url; 348 | switch(args_.source) { 349 | case Source::MAPS_API: 350 | OSN_LOG(info) << "Connecting to MapsAPI..." ; 351 | client = make_unique(args_.mapId, args_.token); 352 | wmts = false; 353 | break; 354 | 355 | case Source ::EVWHS: 356 | OSN_LOG(info) << "Connecting to EVWHS..." ; 357 | client = make_unique(args_.token, args_.credentials); 358 | break; 359 | 360 | case Source::TILE_JSON: 361 | OSN_LOG(info) << "Connecting to TileJSON..."; 362 | client = make_unique(args_.url, args_.credentials, args_.useTiles); 363 | wmts = false; 364 | break; 365 | 366 | default: 367 | OSN_LOG(info) << "Connecting to DGCS..." ; 368 | client = make_unique(args_.token, args_.credentials); 369 | break; 370 | } 371 | 372 | client->connect(); 373 | 374 | if(wmts) { 375 | client->setImageFormat("image/jpeg"); 376 | client->setLayer("DigitalGlobe:ImageryTileService"); 377 | client->setTileMatrixSet("EPSG:3857"); 378 | client->setTileMatrixId((format("EPSG:3857:%1d") % args_.zoom).str()); 379 | } else { 380 | client->setTileMatrixId(lexical_cast(args_.zoom)); 381 | } 382 | 383 | auto llToProj = client->spatialReference().fromLatLon(); 384 | auto projBbox = llToProj->transform(*args_.bbox); 385 | auto image = client->imageFromArea(projBbox); 386 | imageSize_ = image->size(); 387 | pixelToProj_ = image->pixelToProj().clone(); 388 | imageSr_ = image->spatialReference(); 389 | 390 | unique_ptr projToPixel(image->pixelToProj().inverse()); 391 | bbox_ = projToPixel->transformToInt(projBbox); 392 | pixelToLL_ = TransformationChain { move(llToProj), move(projToPixel) }.inverse(); 393 | sr_ = SpatialReference::WGS84; 394 | 395 | haveAlpha_ = RasterBand::haveAlpha(client->rasterBands()); 396 | 397 | auto blockSource = MapServiceBlockSource::create("blockSource"); 398 | blockSource->attr("config") = client->configFromArea(projBbox); 399 | blockSource->attr("maxConnections") = args_.maxConnections; 400 | return blockSource; 401 | } 402 | 403 | SubsetRegionFilter::Ptr OpenSpaceNet::initSubsetRegionFilter() 404 | { 405 | if (!args_.filterDefinition.empty()) { 406 | OSN_LOG(info) << "Initializing the subset filter..." ; 407 | 408 | RegionFilter::Ptr regionFilter = MaskedRegionFilter::create(cv::Rect(0, 0, bbox_.width, bbox_.height), 409 | primaryWindowStep_, 410 | MaskedRegionFilter::FilterMethod::ANY); 411 | bool firstAction = true; 412 | for (const auto& filterAction : args_.filterDefinition) { 413 | string action = filterAction.first; 414 | std::vector filterPolys; 415 | for (const auto& filterFile : filterAction.second) { 416 | FileFeatureSet filter(filterFile); 417 | for (auto& layer : filter) { 418 | auto pixelToProj = dynamic_cast(*pixelToLL_); 419 | 420 | if(layer.spatialReference().isLocal() != sr_.isLocal()) { 421 | DG_CHECK(layer.spatialReference().isLocal(), "Error applying region filter: %d doesn't have a spatial reference, but the input image does", filterFile.c_str()); 422 | DG_CHECK(sr_.isLocal(), "Error applying region filter: Input image doesn't have a spatial reference, but the %d does", filterFile.c_str()); 423 | } else if(!sr_.isLocal()) { 424 | pixelToProj.append(*layer.spatialReference().from(SpatialReference::WGS84)); 425 | } 426 | 427 | auto transform = pixelToProj.inverse(); 428 | transform->compact(); 429 | 430 | for (const auto& feature: layer) { 431 | if (feature.type() != GeometryType::POLYGON) { 432 | DG_ERROR_THROW("Filter from file \"%s\" contains a geometry that is not a POLYGON", filterFile.c_str()); 433 | } 434 | auto poly = dynamic_cast(feature.geometry->transform(*transform).release()); 435 | filterPolys.emplace_back(move(*poly)); 436 | } 437 | } 438 | } 439 | if (action == "include") { 440 | regionFilter->add(filterPolys); 441 | firstAction = false; 442 | } else if (action == "exclude") { 443 | if (firstAction) { 444 | OSN_LOG(info) << "User excluded regions first...automatically including the bounding box..."; 445 | regionFilter->add(Polygon(LinearRing(cv::Rect(0, 0, bbox_.width, bbox_.height)))); 446 | } 447 | regionFilter->subtract(filterPolys); 448 | firstAction = false; 449 | } else { 450 | DG_ERROR_THROW("Unknown filtering action \"%s\"", action.c_str()); 451 | } 452 | } 453 | 454 | 455 | auto subsetFilter = SubsetRegionFilter::create("regionFilter"); 456 | subsetFilter->attr("regionFilter") = regionFilter; 457 | 458 | return subsetFilter; 459 | } 460 | 461 | return nullptr; 462 | } 463 | 464 | Detector::Ptr OpenSpaceNet::initDetector() 465 | { 466 | auto model = Model::create(*args_.modelPackage, !args_.useCpu, args_.maxUtilization / 100); 467 | args_.modelPackage.reset(); 468 | 469 | metadata_ = model->metadata().clone(); 470 | modelAspectRatio_ = (float) metadata_->modelSize().height / metadata_->modelSize().width; 471 | float confidence = args_.confidence / 100; 472 | 473 | if(!args_.windowSize.empty()) { 474 | primaryWindowSize_ = { args_.windowSize[0], (int) roundf(modelAspectRatio_ * args_.windowSize[0]) }; 475 | } else if (args_.resampledSize) { 476 | primaryWindowSize_ = { *args_.resampledSize, (int) roundf(modelAspectRatio_ * (*args_.resampledSize)) }; 477 | } else { 478 | primaryWindowSize_ = metadata_->modelSize(); 479 | } 480 | 481 | if(!args_.windowStep.empty()) { 482 | primaryWindowStep_ = {args_.windowStep[0], (int) roundf(modelAspectRatio_ * args_.windowStep[0])}; 483 | } else { 484 | primaryWindowStep_ = model->defaultStep(primaryWindowSize_); 485 | } 486 | 487 | DG_CHECK(!args_.resampledSize || *args_.resampledSize <= metadata_->modelSize().width, 488 | "Argument --resample-size (size: %d) does not fit within the model (width: %d).", 489 | *args_.resampledSize, metadata_->modelSize().width) 490 | 491 | if (!args_.resampledSize) { 492 | for (auto c : args_.windowSize) { 493 | DG_CHECK(c <= metadata_->modelSize().width, 494 | "Argument --window-size contains a size that does not fit within the model (width: %d).", 495 | metadata_->modelSize().width) 496 | } 497 | } 498 | 499 | Detector::Ptr detectorNode; 500 | if(metadata_->category() == "segmentation") { 501 | initSegmentation(model); 502 | detectorNode = deepcore::classification::node::PolyDetector::create("detector"); 503 | } else { 504 | detectorNode = deepcore::classification::node::BoxDetector::create("detector"); 505 | } 506 | 507 | detectorNode->attr("model") = model; 508 | detectorNode->attr("confidence") = confidence; 509 | return detectorNode; 510 | } 511 | 512 | void OpenSpaceNet::initSegmentation(Model::Ptr model) 513 | { 514 | auto segmentation = std::dynamic_pointer_cast(model); 515 | DG_CHECK(segmentation, "Unsupported model type"); 516 | 517 | segmentation->setRasterToPolygon(make_unique(args_.method, args_.epsilon, args_.minArea)); 518 | } 519 | 520 | dg::deepcore::imagery::node::SlidingWindow::Ptr OpenSpaceNet::initSlidingWindow() 521 | { 522 | auto slidingWindow = dg::deepcore::imagery::node::SlidingWindow::create("slidingWindow"); 523 | auto resampledSize = args_.resampledSize ? 524 | cv::Size {*args_.resampledSize, (int) roundf(modelAspectRatio_ * (*args_.resampledSize))} : 525 | metadata_->modelSize(); 526 | auto windowSizes = calcWindows(); 527 | slidingWindow->attr("windowSizes") = windowSizes; 528 | slidingWindow->attr("resampledSize") = resampledSize; 529 | slidingWindow->attr("aoi") = bbox_; 530 | slidingWindow->attr("bufferSize") = args_.maxCacheSize / 2; 531 | 532 | return slidingWindow; 533 | } 534 | 535 | LabelFilter::Ptr OpenSpaceNet::initLabelFilter(bool isSegmentation) 536 | { 537 | LabelFilter::Ptr labelFilter; 538 | if (isSegmentation) { 539 | labelFilter = PolyLabelFilter::create("labelFilter"); 540 | } else { 541 | labelFilter = BoxLabelFilter::create("labelFilter"); 542 | } 543 | 544 | if(!args_.excludeLabels.empty()) { 545 | labelFilter->attr("labels") = vector(args_.excludeLabels.begin(), 546 | args_.excludeLabels.end()); 547 | labelFilter->attr("labelFilterType") = LabelFilterType::EXCLUDE; 548 | } else if(!args_.includeLabels.empty()) { 549 | labelFilter->attr("labels") = vector(args_.includeLabels.begin(), 550 | args_.includeLabels.end()); 551 | labelFilter->attr("labelFilterType") = LabelFilterType::INCLUDE; 552 | } else { 553 | return nullptr; 554 | } 555 | 556 | return labelFilter; 557 | } 558 | 559 | PredictionToFeature::Ptr OpenSpaceNet::initPredictionToFeature() 560 | { 561 | auto predictionToFeature = PredictionToFeature::create("predToFeature"); 562 | predictionToFeature->attr("geometryType") = args_.geometryType; 563 | predictionToFeature->attr("pixelToProj") = pixelToProj_; 564 | predictionToFeature->attr("topNName") = "top_five"; 565 | predictionToFeature->attr("topNCategories") = 5; 566 | 567 | auto fields = predictionToFeature->attr("extraFields").cast>(); 568 | 569 | time_t currentTime = time(nullptr); 570 | struct tm* timeInfo = gmtime(¤tTime); 571 | time_t gmTimet = timegm(timeInfo); 572 | fields.emplace("date", Field(FieldType::DATE, gmTimet)); 573 | 574 | if(args_.producerInfo) { 575 | fields.emplace("username", Field(FieldType::STRING, loginUser())); 576 | fields.emplace("app", Field(FieldType::STRING, "OpenSpaceNet")); 577 | fields.emplace("app_ver", Field(FieldType::STRING, OPENSPACENET_VERSION_STRING)); 578 | } 579 | 580 | if (!args_.extraFields.empty()) { 581 | for(int i = 0; i < args_.extraFields.size(); i += 2) { 582 | fields.emplace(args_.extraFields[i], Field(FieldType::STRING, args_.extraFields[i + 1])); 583 | } 584 | } 585 | 586 | predictionToFeature->attr("extraFields") = fields; 587 | return predictionToFeature; 588 | } 589 | 590 | WfsFeatureFieldExtractor::Ptr OpenSpaceNet::initWfs() 591 | { 592 | if (args_.dgcsCatalogID || args_.evwhsCatalogID) { 593 | string baseUrl; 594 | if(args_.dgcsCatalogID) { 595 | OSN_LOG(info) << "Connecting to DGCS web feature service..."; 596 | baseUrl = "https://services.digitalglobe.com/catalogservice/wfsaccess"; 597 | } else if (args_.evwhsCatalogID) { 598 | OSN_LOG(info) << "Connecting to EVWHS web feature service..."; 599 | baseUrl = Url("https://evwhs.digitalglobe.com/catalogservice/wfsaccess"); 600 | } 601 | 602 | auto wfsCreds = args_.wfsCredentials; 603 | if (wfsCreds.empty()) { 604 | DG_CHECK(!args_.credentials.empty(), "No credentials specified for WFS service"); 605 | wfsCreds = args_.credentials; 606 | } 607 | 608 | DG_CHECK(!args_.token.empty(), "No token specified for WFS service"); 609 | 610 | map query; 611 | query["service"] = "wfs"; 612 | query["version"] = "1.1.0"; 613 | query["connectid"] = args_.token; 614 | query["request"] = "getFeature"; 615 | query["typeName"] = WFS_TYPENAME; 616 | query["srsName"] = "EPSG:3857"; 617 | 618 | vector splitCreds; 619 | split(splitCreds, wfsCreds, is_any_of(":")); 620 | auto url = Url(baseUrl); 621 | url.user = splitCreds[0]; 622 | url.password = splitCreds[1]; 623 | url.query = move(query); 624 | 625 | vector fieldNames = {"legacyId"}; 626 | Fields defaultFields = { {"legacyId", Field(FieldType::STRING, "uncataloged")}}; 627 | 628 | auto featureFieldExtractor = WfsFeatureFieldExtractor::create("fieldExtractor"); 629 | featureFieldExtractor->attr("inputSpatialReference") = imageSr_; 630 | featureFieldExtractor->attr("fieldNames") = fieldNames; 631 | featureFieldExtractor->attr("defaultFields") = defaultFields; 632 | featureFieldExtractor->attr("url") = url; 633 | return featureFieldExtractor; 634 | } 635 | 636 | return nullptr; 637 | } 638 | 639 | FileFeatureSink::Ptr OpenSpaceNet::initFeatureSink() 640 | { 641 | FieldDefinitions definitions = { 642 | { FieldType::STRING, "top_cat", 50 }, 643 | { FieldType::REAL, "top_score" }, 644 | { FieldType::DATE, "date" }, 645 | { FieldType::STRING, "top_five", 254 } 646 | }; 647 | 648 | if(args_.producerInfo) { 649 | definitions.emplace_back(FieldType::STRING, "username", 50); 650 | definitions.emplace_back(FieldType::STRING, "app", 50); 651 | definitions.emplace_back(FieldType::STRING, "app_ver", 50); 652 | } 653 | 654 | if(args_.dgcsCatalogID || args_.evwhsCatalogID) { 655 | definitions.emplace_back(FieldType::STRING, "catalog_id"); 656 | } 657 | 658 | for (int i = 0 ; i < args_.extraFields.size(); i+=2){ 659 | definitions.emplace_back(FieldType::STRING, args_.extraFields[i]); 660 | } 661 | 662 | VectorOpenMode openMode = args_.append ? APPEND : OVERWRITE; 663 | 664 | auto featureSink = FileFeatureSink::create("featureSink"); 665 | featureSink->attr("spatialReference") = imageSr_; 666 | featureSink->attr("outputSpatialReference") = sr_; 667 | featureSink->attr("geometryType") = args_.geometryType; 668 | featureSink->attr("path") = args_.outputPath; 669 | featureSink->attr("layerName") = args_.layerName; 670 | featureSink->attr("outputFormat") = args_.outputFormat; 671 | featureSink->attr("openMode") = openMode; 672 | featureSink->attr("fieldDefinitions") = definitions; 673 | 674 | return featureSink; 675 | } 676 | 677 | void OpenSpaceNet::printModel() 678 | { 679 | skipLine(); 680 | 681 | OSN_LOG(info) << "Model Name: " << metadata_->name() 682 | << "; Version: " << metadata_->version() 683 | << "; Created: " << to_simple_string(from_time_t(metadata_->timeCreated())); 684 | OSN_LOG(info) << "Description: " << metadata_->description(); 685 | OSN_LOG(info) << "Dimensions (pixels): " << metadata_->modelSize() 686 | << "; Color Mode: " << metadata_->colorMode(); 687 | OSN_LOG(info) << "Bounding box (lat/lon): " << metadata_->boundingBox(); 688 | OSN_LOG(info) << "Labels: " << join(metadata_->labels(), ", "); 689 | 690 | skipLine(); 691 | } 692 | 693 | void OpenSpaceNet::skipLine() const 694 | { 695 | if(!args_.quiet) { 696 | std::cout << std::endl; 697 | } 698 | } 699 | 700 | SizeSteps OpenSpaceNet::calcWindows() const 701 | { 702 | DG_CHECK(args_.windowSize.size() < 2 || args_.windowStep.size() < 2 || 703 | args_.windowSize.size() == args_.windowStep.size(), 704 | "Number of window sizes and window steps must match."); 705 | 706 | if(args_.windowSize.size() == args_.windowStep.size() && !args_.windowStep.empty()) { 707 | SizeSteps ret; 708 | for(const auto& c : boost::combine(args_.windowSize, args_.windowStep)) { 709 | int windowSize, windowStep; 710 | boost::tie(windowSize, windowStep) = c; 711 | ret.emplace_back(cv::Size {windowSize, (int) roundf(modelAspectRatio_ * windowSize)}, 712 | cv::Point {windowStep, (int) roundf(modelAspectRatio_ * windowStep)}); 713 | } 714 | return ret; 715 | } else if (args_.windowSize.size() > 1) { 716 | SizeSteps ret; 717 | for(const auto& c : args_.windowSize) { 718 | ret.emplace_back(cv::Size { c, (int) roundf(modelAspectRatio_ * c) }, primaryWindowStep_); 719 | } 720 | return ret; 721 | } else if (args_.windowStep.size() > 1) { 722 | SizeSteps ret; 723 | for(const auto& c : args_.windowStep) { 724 | ret.emplace_back(primaryWindowSize_, cv::Point { c, (int) roundf(modelAspectRatio_ * c) }); 725 | } 726 | return ret; 727 | } else { 728 | return { { primaryWindowSize_, primaryWindowStep_ } }; 729 | } 730 | } 731 | 732 | } } // namespace dg { namespace osn { 733 | -------------------------------------------------------------------------------- /common/version.h.in: -------------------------------------------------------------------------------- 1 | /******************************************************************************** 2 | * Copyright 2016 DigitalGlobe, Inc. 3 | * Author: Joe White 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a 6 | * copy of this software and associated documentation files (the "Software"), 7 | * to deal in the Software without restriction, including without limitation 8 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 | * and/or sell copies of the Software, and to permit persons to whom the 10 | * Software is furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included 13 | * in all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 16 | * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 21 | * DEALINGS IN THE SOFTWARE. 22 | ********************************************************************************/ 23 | 24 | #ifndef OPENSPACENET_VERSION_H 25 | #define OPENSPACENET_VERSION_H 26 | 27 | #define OPENSPACENET_VERSION_MAJOR @OpenSpaceNet_VERSION_MAJOR@ 28 | #define OPENSPACENET_VERSION_MINOR @OpenSpaceNet_VERSION_MINOR@ 29 | #define OPENSPACENET_VERSION_PATCH @OpenSpaceNet_VERSION_PATCH@ 30 | #define OPENSPACENET_VERSION_BUILD @OpenSpaceNet_VERSION_BUILD@ 31 | #define OPENSPACENET_VERSION_STRING "@OpenSpaceNet_VERSION_MAJOR@.@OpenSpaceNet_VERSION_MINOR@.@OpenSpaceNet_VERSION_PATCH@+@OpenSpaceNet_VERSION_BUILD@" 32 | 33 | #endif // OPENSPACENET_VERSION_H 34 | -------------------------------------------------------------------------------- /doc/BUILDING.md: -------------------------------------------------------------------------------- 1 | # Building OpenSpaceNet 2 | 3 | To build OpenSpaceNet DeepCore and DeepCoreDependencies libraries must be installed and configured. -------------------------------------------------------------------------------- /doc/REFERENCE.md: -------------------------------------------------------------------------------- 1 | # OpenSpaceNet User Reference Guide 2 | 3 | _OpenSpaceNet_ application to perform object or terrain detection against georegistered imagery using the DeepCore libraries. 4 | This application includes and is based on CUDA 8.0 and requires NVIDIA driver version 384 or higher to run using the GPU. 5 | 6 | ## Table Of Contents 7 | 8 | * [Command Line Arguments](#arguments) 9 | * [Feature Detection Options](#detect) 10 | * [Local Image Input Options](#image) 11 | * [Web Service Input Options](#service) 12 | * [Output Options](#output) 13 | * [Processing Options](#processing) 14 | * [Segmentation Options](#segmentation) 15 | * [Filtering Options](#filter) 16 | * [Logging Options](#logging) 17 | * [Further Details](#details) 18 | * [Image Input](#input) 19 | * [Size Parameters](#size) 20 | * [Using Configuration Files](#config) 21 | * [S3 Input Files](#s3) 22 | * [Usage Statement](#usage) 23 | 24 | 25 | 26 | ## Command Line Arguments 27 | 28 | 29 | 30 | ### Feature Detection Options 31 | 32 | ##### --confidence 33 | This option sets the minimum percent score for results to be included in the output. This should be a value between 0 and 100. 34 | The default value for this argument is 95%. 35 | 36 | i.e. `--confidence 99` sets the confidence to 99%. 37 | 38 | ##### --nms 39 | This option will cause _OpenSpaceNet_ to perform non-maximum suppression on the output. Adjacent detection boxes 40 | will be removed for each feature detected and only one detection box per object will be output. This option results in much 41 | better quality output. You can optionally specify the overlap threshold percentage for non-maximum suppression calculation. 42 | The default overlap is 30%. 43 | i.e. `--nms` will result in non-maximum suppression with 30% overlap, while `--nms 20` will result in non-maximum 44 | suppression with 20% overlap. 45 | 46 | 47 | 48 | ### Local Image Input Options 49 | 50 | ##### --image 51 | 52 | This argument is required for local image input and specifies the path to the image. For the advanced user, GDAL syntax 53 | may be used to load files from URLs or S3 buckets (see below for instructions on how to load imagery from S3). 54 | 55 | i.e. `--image /home/user/Pictures/my_image.tif` 56 | 57 | ##### --bbox 58 | 59 | The bounding box argument is optional for local image input. If the specified bounding box is not fully within the input 60 | image an intersection of the image bounding box and the specified bounding box will be used. If the specified bounding 61 | box does not intersect the image, an error will be displayed. 62 | 63 | 64 | 65 | ### Web Service Input Options 66 | 67 | ##### --service 68 | 69 | This argument is required for a web service input. If this argument is specified, the `--image` argument cannot be 70 | present. The following services are available: 71 | 72 | * 'dgcs' is the DigitalGlobe Cloud Services WMTS data source. This service requires both `--token` and `--credentials` to 73 | be set. The service's Web URL is http://services.digitalglobe.com/. 74 | 75 | * 'evwhs' is the DigitalGlobe Enhanced View Web Hosting Service WMTS data source. This service requires both `--token` 76 | and `--credentials` to be set. The service's web URL is http://evwhs.digitalglobe.com/. 77 | 78 | * 'maps-api' is the DigitalGlobe's MapsAPI service hosted by MapBox. This service only requires the `--token` to be set. 79 | In addition, the user can specify the `--map-id` argument to set the image source map ID. The service's web URL is 80 | http://mapsapi.digitalglobe.com/. 81 | 82 | * 'tile-json' is a format by MapBox for describing collections of tiles. This service requires both `--url` and accepts 83 | `--credentials` (for URL that require them). The format specification is available at https://github.com/mapbox/tilejson-spec. 84 | 85 | ##### --bbox 86 | 87 | The bounding box argument is required for all web service input sources. The coordinates are specified in the WGS84 88 | Lat/Lon coordinate system. The order of coordinates is west longitude, south latitude, east longitude, and north latitude. 89 | Note that for the landcover action the bounding box may be expanded so that the tiles downloaded from the service will 90 | be processed completely. 91 | 92 | i.e. `--bbox 125.541 38.866 125.562 38.881` specifies a bounding box for the Atlanta Hartsfield-Jackson Airport area. 93 | 94 | ##### --token 95 | 96 | This argument specifies the API token for the desired web service. This is required for WMTS and MapsAPI services. 97 | 98 | ##### --credentials 99 | 100 | This argument specifies the user name and password for the WMTS services, that is `dgcs` and `evwhs`. The format is 101 | `--credentials username:password`. If only the user name is specified like `--credentials username`, the user will be 102 | prompted for the password before _OpenSpaceNet_ will proceed. In addition, credentials can be specified by setting the 103 | `OSN_CREDENTIALS` environment variable. 104 | 105 | ##### --url 106 | 107 | TileJSON server URL. TileJSON server URL. This is only required for the tile-json service. 108 | 109 | TileJSON server URL should either directly point to the to the JSON definition of the 110 | TileJSON server, or if the JSON definition's file name matches the root tile directory 111 | name, we can just specify the root tile directory name. 112 | 113 | i.e. if the server 114 | URL is `https://my.server.com/tiles`, then _OpenSpaceNet_ will look for 115 | `https://my.server.com/tiles.json`. 116 | 117 | _OpenSpaceNet_ also supports reading tilesets locally. To do this, just specify 118 | a `file://` URL. 119 | 120 | i.e. 121 | 122 | If I have the following file structure: 123 | 124 | * /data/ 125 | * tiles.json 126 | * tiles/ 127 | * 14/ 128 | * 4348/ 129 | * 6564.jpg 130 | * 4349/ 131 | * 6564.jpg 132 | * 4350/ 133 | * 6564.jpg 134 | 135 | I can specify it as `file:///data/tiles`. 136 | 137 | ##### --use-tiles 138 | 139 | If set, the "tiles" field in TileJSON metadata will be used as the tile service 140 | address. The default behavior is to derive the service address from the provided URL. 141 | 142 | 143 | ##### --zoom 144 | 145 | This argument specifies the zoom level for the web service. For MapsAPI the zoom level is 0 to 22, while both DGCS and 146 | EVWHS zoom levels range from 0 to 20. The default zoom level is 18. 147 | 148 | ##### --map-id 149 | 150 | This argument is only valid for the MapsAPI, or `maps-api` service. The DigitalGlobe map IDs are listed on this web page: 151 | http://mapsapidocs.digitalglobe.com/docs/imagery-and-basemaps. 152 | 153 | ##### --max-connections 154 | 155 | This argument specifies the number of image tiles that will be downloaded simultaneously. Increasing the value of this 156 | argument can dramatically speed up downloads, but it can cause the service to crash or deny you access. The default value 157 | is 10. 158 | 159 | 160 | 161 | ### Output Options 162 | 163 | ##### --format 164 | 165 | This option specifies the output vector format. The default format is `shp`. 166 | The following formats are supported: 167 | 168 | * `csv` outputs to a CSV file. 169 | * `elasticsearch` writes the output to an Elastic Search database. 170 | * `geojson` outputs a GeoJSON file. 171 | * `kml` outputs a Google's Keyhole Markup Language format. 172 | * `postgis` writes the output to a Postgres SQL database with PostGIS extensions. 173 | * `shp` for ESRI Shapefile output. For this format, `--output-layer` is ignored. 174 | * `sqlite` writes the output to a SpatialLite SQLite database. 175 | 176 | ##### --output 177 | 178 | This option specifies the output path or connection settings on non-file output formats. See the GDAL documentation 179 | for information on how to specify non-file formats (http://www.gdal.org/ogr_formats.html). Only the formats listed 180 | above are supported. 181 | 182 | ##### --output-layer 183 | 184 | This option specifies the output layer name if the output format supports it. 185 | This option is ignored for `shp` output. 186 | 187 | ##### --type 188 | 189 | This option specifies the output geometry type. The supported types are: 190 | 191 | * `polygon` draws a polygon around each detected feature. 192 | * `point` draws a point in the centroid of each detected feature polygon. 193 | 194 | ##### --producer-info 195 | 196 | This option adds additional attributes to each vector feature, the attributes are: 197 | 198 | * `username` is the login user name on the machine that ran that _OpenSpaceNet_ job. 199 | * `app` is the name of the application, this is set to "OpenSpaceNet". 200 | * `version` is the application version, which is the _OpenSpaceNet_ version. 201 | 202 | 203 | ##### --dgcs-catalog-id 204 | 205 | Add `catalog_id` property to detected features by finding the most intersected 206 | `legacyId` from DGCS WFS data source `DigitalGlobe:FinishedFeature`. 207 | 208 | ##### --evwhs-catalog-id 209 | 210 | Add `catalog_id` property to detected features by finding the most intersected 211 | `legacyId` from EVWHS WFS data source `DigitalGlobe:FinishedFeature`. 212 | 213 | ##### --wfs-credentials 214 | 215 | Specifies credentials for the WFS service, if appending legacyId. The format is 216 | `--wfs-credentials username:password`. If not specified, credentials from the 217 | `--credentials` option will be used. 218 | 219 | #### --extra-fields 220 | 221 | Specifies extra metadata to be added to each vector feature. This is specified 222 | in key-value pairs, and must be strings. 223 | 224 | ##### --append 225 | This option will cause _OpenSpaceNet_ to append to the specified output. If the 226 | specified output is not found, it will be created. If this option is not 227 | specified and the output already exists, it will be overwritten. 228 | 229 | 230 | 231 | ### Processing Options 232 | 233 | Processing options contain the classifier configuration options that are common 234 | for both landcover and detect modes. 235 | 236 | ##### --cpu 237 | 238 | Specifying this option causes _OpenSpaceNet_ to use the fall-back CPU mode. This 239 | can be used on machines that don't have the supported GPU, but it is 240 | dramatically slower, so it's not recommended. It is recommended that a GPU be 241 | used to do CNN feature detection efficiently. 242 | 243 | ##### --max-utilization 244 | 245 | This option specifies how much GPU memory _OpenSpaceNet_ will try to use. The 246 | default value is 95%, which empirically shown to yield best performance. As 247 | utilization approaches 100%, it slows down drastically, so 95% is the 248 | recommended maximum. The valid values are between 5% and 100%. Values outside of 249 | this range will be clamped, and a warning will be shown. 250 | 251 | ##### --model 252 | 253 | This option specifies the path to a package GBDXM model file to use in processing. 254 | 255 | ##### --window-size 256 | 257 | This option sets the size of the window that is chipped from the source imagery. 258 | If multiple arguments are supplied, windows for each size are extracted in turn. 259 | Each window that is extracted will have the aspect ratio of the model and the 260 | width that is specified. 261 | 262 | If one window size and move than one window step is given, it is used for every 263 | window step. If more than one window size and more than one window step is 264 | specified, the number of window sizes must match the number of window steps. 265 | 266 | Unless `--resample-size` is supplied, each window size must be equal to or 267 | smaller than the model's size. If it is smaller, the chipped image will be 268 | padded with uniformly distributed white noise. The range of the noise is 269 | dependent on the image's datatype (specifically, 0 to 1 for floating point 270 | datatypes and the full representable range for integer datatypes). 271 | 272 | ##### --window-step 273 | 274 | This option sets the sliding window step. The default value is 20% of the 275 | model's size. Each window step will have the aspect ratio of the model and the 276 | _x_ step that is specified. 277 | 278 | If one window step and move than one window size is given, it is used for every 279 | window step. If more than one window size and more than one window step is 280 | specified, the number of window sizes must match the number of window steps. 281 | 282 | i.e. `--window-step 30` will result in the sliding window step being 30 in the 283 | _x_ direction and 30 in the _y_ direction. 284 | 285 | ##### --resampled-size 286 | 287 | This option resampled all chipped windows to a single size. The resampled 288 | images will have the aspect ratio of the model and the width that is specified. 289 | 290 | If given, the resampled size must be equal to or smaller than the model's size. 291 | If it is smaller, the chipped image will be padded with uniformly distributed 292 | white noise. The range of the noise is dependent on the image's datatype 293 | (specifically, 0 to 1 for floating point datatypes and the full representable 294 | range for integer datatypes). 295 | 296 | 297 | 298 | ### Segmentation Options 299 | 300 | Segmentation options contain options for raster-to-polygon conversion for 301 | segmentation models. These are not mandatory and are only applicable to 302 | segmentation models. If set for a different model type, a warning will be 303 | displayed. 304 | 305 | ##### --r2p-method METHOD 306 | This is the raster-to-polygon approximation method. 307 | Valid values are: `none`, `simple`, `tc89-l1`, `tc89-kcos`. The default value 308 | is `simple`. 309 | 310 | The methods are as follows: 311 | 312 | * `none`: Stores absolutely all the contour points. That is, any 2 subsequent 313 | points _(x1,y1)_ and _(x2,y2)_ of 314 | the contour will be either horizontal, vertical or diagonal neighbors, that is, 315 | max(|x1 - x2|, |y2-y1|) = 1. 316 | 317 | * `simple`: Compresses horizontal, vertical, and diagonal segments and leaves 318 | only their end points. For example, an up-right rectangular contour is encoded 319 | with 4 points. 320 | 321 | * `tc89-l1`: Applies the 1 curvature Teh-Chin chain approximation 322 | algorithm. 323 | 324 | * `tc89-kcos`: Applies the k cosine Teh-Chin chain approximation 325 | algorithm. 326 | 327 | 328 | The Teh-Chin approximations are as described in: 329 | 330 | > Teh, C-H and Chin, Roland T. On the detection of dominant points on digital curves.
331 | > IEEE Transactions on Pattern Analysis and Machine Intelligence, 1989, vol. 11, num. 8, pp. 859-872 332 | 333 | ##### --r2p-accuracy 334 | 335 | This argument specifies the polygon approximation accuracy. This is the maximum 336 | distance between the original curve and its approximation. The default value is 3. 337 | 338 | 339 | ##### --r2p-min-area 340 | 341 | This argument sets the minimum area of a polygon in pixels. The default value 342 | is 0. 343 | 344 | 345 | 346 | ### Filtering Options 347 | 348 | ##### --include-labels / --exclude-labels 349 | 350 | This option will cause _OpenSpaceNet_ to retain or remove labels in the output. It is invalid to include both an 351 | inclusion and an exclusion list at the same time. 352 | 353 | When specified in an environmental variable or configuration file, the input string will be tokenized. Quotes are 354 | required to keep label names with spaces together. 355 | 356 | ##### --include-region / --exclude-region / --region 357 | 358 | These options will cause _OpenSpaceNet_ to skip processing any windows which touch the excluded region. 359 | 360 | By default, no regions are excluded. If any of these options are given, a filter is built. If the 361 | first action is "exclude", the filter is initialized to include the bounding box. 362 | 363 | `--exclude-region` will subtract geometries contained within the supplied path(s) to the from the search region. 364 | 365 | `--include-region` will add the geometries contained within the supplied path(s) from the search region. 366 | 367 | `--region` can be used to chain together multiple includes and excludes. Parameters to this option are in the 368 | form `(exclude|include) path [path..]` This may be repeated any number of times after `--region`. 369 | 370 | _Note:_ Region filtering is designed to guide window selection and is not performed at the detection level. This will 371 | impact detection models, such as DetectNet, where one window may result in many detections. As a result, detects 372 | far from the include/exclude boundary may be included in the output. Post processing will be required to remove these 373 | detects. 374 | 375 | i.e `--region exclude northwest.shp northeast.shp include truenorth.shp` . In this example, "exclude" is first, 376 | so the search region is initialized to the bounding box. The geometries defined in northwest.shp and northeast.shp 377 | are excluded (through geometric union). The geometry in truenorth.shp added back. This way of specifying a region 378 | filter is exactly the same as `--exclude-region northwest.shp northeast.shp --include-region truenorth.shp`. 379 | 380 | 381 | 382 | 383 | ### Logging Options 384 | 385 | ##### --log 386 | 387 | This option specifies a log file that _OpenSpaceNet_ writes to. Optionally, a log level can be specified. Permitted 388 | log level values are `trace`, `debug`, `info`, `warning`, `error`, and `fatal`. The default log level is `debug`. 389 | 390 | When specified in an environmental variable or configuration file, the input string will be tokenized. Quotes are 391 | required to keep inputs with spaces together. 392 | 393 | Only one log file may only be specified. 394 | 395 | i.e. 396 | `--log log.txt` will create a log file with the log level of `info` 397 | `--log debug log.txt` will create a log file with the log level of `debug` 398 | 399 | ##### --quiet 400 | 401 | Normally `_OpenSpaceNet_` will output its status to the console even if a log file is specified. If 402 | this is not desired, console output can be suppressed by specifying this option. 403 | 404 | 405 | 406 | ## Further Details 407 | 408 | 409 | 410 | 411 | ### Image Input 412 | 413 | _OpenSpaceNet_ is able use either a geo-registered local image or one of number of web maping servcies as input. Depending on the 414 | input source, different command line arguments apply. To select which source to use, one of two options must be present: 415 | 416 | * `--service ` To load imagery from a web service. 417 | * `--image ` To load a local image file. 418 | 419 | Depending on which source you use, different arguments apply. 420 | 421 | | source | token | credentials | map-id | zoom | bbox | max-connections | url | use-tiles | 422 | |-----------------------|----------|-------------|----------|----------|----------|-----------------|----------|-----------| 423 | | `--service dgcs` | Required | Required | | Optional | Required | Optional | | | 424 | | `--service evwhs` | Required | Required | | Optional | Required | Optional | | | 425 | | `--service maps-api` | Required | | Optional | Optional | Required | Optional | | | 426 | | `--service tile-json` | | Optional | | Optional | Required | Optional | Required | Optional | 427 | | `--image ` | | | | | Optional | | | | 428 | 429 | 430 | 431 | ### Size Parameters 432 | 433 | Some models may require that the imagery be resized or padded to fit the target 434 | model. When performing these actions, use `--resampled-size` and `--window-size` 435 | to add padding and resizing, respectively. See the image below for guidance. 436 | 437 | ![Resizing parameters](./modelsize.png) 438 | 439 | 440 | 441 | 442 | ### Using Configuration Files 443 | 444 | When using _OpenSpaceNet_, some or all command line arguments can be put in configuration file(s) and passed through the `--config` command 445 | line option. Multiple files may be used to specify different options. 446 | 447 | #### Configuration File Syntax 448 | 449 | Configuration files are text files with each command line option specified in the following format: 450 | ``` 451 | name1=value1 452 | name2=value2 453 | ``` 454 | 455 | Option names are the same as the regular command line options without the preceding dashes. The action argument can be 456 | specified in a configuration file by using the following syntax: 457 | ``` 458 | action= 459 | ``` 460 | 461 | The `--nms` argument is different when using a configuration file in that the overlap percentage is not optional, this means that this 462 | statement is the same as just specifying `--nms` on the command line: 463 | ``` 464 | nms=30 465 | ``` 466 | 467 | If you try to specify ~~`nms`~~ or ~~`nms=`~~ an error will be displayed. 468 | 469 | If a configuration file contains an option that is also specified through a command line parameter, the command line parameter takes 470 | precedence. If multiple configuration files contain the same option, the option in the file specified last will be used. 471 | 472 | When specified in an environmental variable or configuration file, the input string will be tokenized. Quotes are 473 | required to keep inputs with spaces together. 474 | 475 | Configuration files may be included in the command line, environment, and other configuration files (and multiple times 476 | within each). 477 | 478 | 479 | #### Example Using a Configuration File for All Options 480 | 481 | In this example, we will use the following file: 482 | 483 | **dgcs_detect_atl.cfg** 484 | ```ini 485 | action=detect 486 | service=dgcs 487 | token=abcd-efgh-ijkl-mnop-qrst-uvxyz 488 | credentials=username:password 489 | bbox=-84.44579 33.63404 -84.40601 33.64853 490 | model=airliner.gbdxm 491 | output=atl_detected.shp 492 | confidence=99 493 | window-step=15 494 | max-connections=200 495 | nms=30 496 | ``` 497 | 498 | Running _OpenSpaceNet_ with this file 499 | ``` 500 | ./OpenSpaceNet --config dgcs_detect_atl.cfg 501 | ``` 502 | 503 | is the same as running _OpenSpaceNet_ with these options: 504 | ``` 505 | ./OpenSpaceNet detect --service dgcs --token abcd-efgh-ijkl-mnop-qrst-uvxyz --credentials username:password --bbox -84.44579 33.63404 -84.40601 33.64853 --model airliner.gbdxm --output atl_detected.shp --confidence 99 --window-step 15 --max-connections 200 --nms 506 | ``` 507 | #### Example Using Multiple Configuration Files 508 | 509 | As a use case for using multiple files, we'll use the fact that because _OpenSpaceNet_ can use different input sources for its input, 510 | it can be cumbersome to enter that particular service's token and credentials every time. We can create a configuration file with a 511 | service's credentials and use it with another configuration file that configures detection parameters. 512 | 513 | Let's use this file for configuring DGCS credentials: 514 | 515 | **dgcs.cfg** 516 | ```ini 517 | service=dgcs 518 | token=abcd-efgh-ijkl-mnop-qrst-uvxyz 519 | credentials=username:password 520 | max-connections=200 521 | ``` 522 | 523 | and this file for detection options: 524 | 525 | **detect_atl.cfg** 526 | ```ini 527 | action=detect 528 | service=dgcs 529 | token=abcd-efgh-ijkl-mnop-qrst-uvxyz 530 | credentials=username:password 531 | bbox=-84.44579 33.63404 -84.40601 33.64853 532 | model=airliner.gbdxm 533 | output=atl_detected.shp 534 | confidence=99 535 | window-step=15 536 | nms=30 537 | ``` 538 | 539 | We can now combine the two files and get the same result as our previous example, but with the flexibility of reusing our credentials file for other jobs: 540 | 541 | ``` 542 | ./OpenSpaceNet --config dgcs.cfg detect_atl.cfg 543 | ``` 544 | 545 | #### Example Using a Configuration File Combined with Command Line Options 546 | 547 | Alternatively, we can use the configuration file from previous example to run a landcover job against the same DGCS account: 548 | 549 | ``` 550 | ./OpenSpaceNet landcover --config dgcs.cfg --bbox -84.44579 33.63404 -84.40601 33.64853 --model landcover.gbdxm --output atl_detected.shp 551 | ``` 552 | 553 | 554 | 555 | ### Files on S3 (a variant of local files) 556 | 557 | Since _OpenSpaceNet_ uses GDAL to load local imagery, VSI sources are supported out of the box. Of 558 | particular interest is imagery stored on S3. 559 | 560 | i.e. 561 | 562 | ``` 563 | AWS_ACCESS_KEY_ID=[AWS_ACCESS_KEY_ID] AWS_SECRET_ACCESS_KEY=[AWS_SECRET_ACCESS_KEY] \ 564 | ./OpenSpaceNet --image /vsis3/bucket/path/to/image.tif --model /path/to/model.gbdxm \ 565 | --output foo.shp 566 | ``` 567 | 568 | #### VSI S3 Parameters 569 | * `AWS_SECRET_ACCESS_KEY`, `AWS_ACCESS_KEY_ID` (Required) define the access credentials 570 | * `AWS_SESSION_TOKEN` (Required for temporary credentials) 571 | * `AWS_REGION` (defaults to 'us-east-1') 572 | * `AWS_S3_ENDPOINT` (defaults to 's3.amazonaws.com') 573 | 574 | #### Additional HTTP Parameters 575 | * `GDAL_HTTP_PROXY`, `GDAL_HTTP_PROXYUSERPWD`, `GDAL_PROXY_AUTH` configuration options can be used to define a proxy server. 576 | 577 | 578 | 579 | ## Usage Statement 580 | 581 | This is the usage statement as it appears on the command line. 582 | 583 | ```text 584 | Radiant Solutions 585 | ____ _____ _ _ _ 586 | / __ \ / ____| | \ | | | | 587 | | | | |_ __ ___ _ __ | (___ _ __ __ _ ___ ___| \| | ___| |_ 588 | | | | | '_ \ / _ \ '_ \ \___ \| '_ \ / _` |/ __/ _ \ . ` |/ _ \ __| 589 | | |__| | |_) | __/ | | |____) | |_) | (_| | (_| __/ |\ | __/ |_ _ _ _ 590 | \____/| .__/ \___|_| |_|_____/| .__/ \__,_|\___\___|_| \_|\___|\__(_|_|_) 591 | | | | | 592 | |_| |_| 593 | 594 | Version: 1.1.0+SNAPSHOT 595 | DeepCore Version: 1.1.0+SNAPSHOT 596 | GBDXM Metadata Version: 3.0 597 | 598 | Usage: 599 | OpenSpaceNet 600 | OpenSpaceNet --config [other options] 601 | 602 | 603 | Feature Detection Options: 604 | --confidence PERCENT (=95) Minimum percent score for results to be 605 | included in the output. 606 | --nms PERCENT (=30) Perform non-maximum suppression on the 607 | output. You can optionally specify the 608 | overlap threshold percentage for 609 | non-maximum suppression calculation. 610 | 611 | Local Image Input Options: 612 | --image PATH If this is specified, the input will be 613 | taken from a local image. 614 | --bbox WEST SOUTH EAST NORTH Optional bounding box for image subset, 615 | optional for local images. Coordinates 616 | are specified in the following order: 617 | west longitude, south latitude, east 618 | longitude, and north latitude. 619 | 620 | Web Service Input Options: 621 | --service SERVICE Web service that will be the source of 622 | input. Valid values are: dgcs, evwhs, 623 | maps-api, and tile-json. 624 | --token TOKEN API token used for licensing. This is 625 | the connectId for WMTS services or the 626 | API key for the Web Maps API. 627 | --credentials USERNAME[:PASSWORD] Credentials for the map service. Not 628 | required for Web Maps API, optional for 629 | TileJSON. If password is not specified, 630 | you will be prompted to enter it. The 631 | credentials can also be set by setting 632 | the OSN_CREDENTIALS environment 633 | variable. 634 | --url URL TileJSON server URL. This is only 635 | required for the tile-json service. 636 | --use-tiles If set, the "tiles" field in TileJSON 637 | metadata will be used as the tile 638 | service address. The default behavioris 639 | to derive the service address from the 640 | provided URL. 641 | --zoom ZOOM (=18) Zoom level. 642 | --map-id MAPID (=digitalglobe.nal0g75k) 643 | MapsAPI map id to use. 644 | --max-connections NUM (=10) Used to speed up downloads by allowing 645 | multiple concurrent downloads to happen 646 | at once. 647 | --bbox WEST SOUTH EAST NORTH Bounding box for determining tiles 648 | specified in WGS84 Lat/Lon coordinate 649 | system. Coordinates are specified in 650 | the following order: west longitude, 651 | south latitude, east longitude, and 652 | north latitude. 653 | 654 | Output Options: 655 | --format FORMAT (=shp) Output file format for the results. 656 | Valid values are: csv, elasticsearch, 657 | geojson, kml, postgis, shp, sqlite. 658 | --output PATH Output location with file name and path 659 | or URL. 660 | --output-layer NAME (=osndetects) The output layer name, index name, or 661 | table name. 662 | --type TYPE (=polygon) Output geometry type. Currently only 663 | point and polygon are valid. 664 | --producer-info Add user name, application name, and 665 | application version to the output 666 | feature set. 667 | --dgcs-catalog-id Add catalog_id property to detected 668 | features, by finding the most 669 | intersected legacyId from DGCS WFS data 670 | source DigitalGlobe:FinishedFeature 671 | --evwhs-catalog-id Add catalog_id property to detected 672 | features, by finding the most 673 | intersected legacyId from EVWHS WFS 674 | data source DigitalGlobe:FinishedFeatur 675 | e 676 | --wfs-credentials USERNAME[:PASSWORD] Credentials for the WFS service, if 677 | appending legacyId. If not specified, 678 | credentials from the credentials option 679 | will be used. 680 | --append Append to an existing vector set. If 681 | the output does not exist, it will be 682 | created. 683 | 684 | Processing Options: 685 | --cpu Use the CPU for processing, the default 686 | is to use the GPU. 687 | --max-utilization PERCENT (=95) Maximum GPU utilization %. Minimum is 688 | 5, and maximum is 100. Not used if 689 | processing on CPU 690 | --model PATH Path to the the trained model. 691 | --window-size SIZE [SIZE...] Sliding window detection box sizes. 692 | The source image is chipped with boxes 693 | of the given sizes. If resampled-size 694 | is not specified, all windows must fit 695 | within the model.Default is the model 696 | size. 697 | --window-step STEP [STEP...] Sliding window step. Either a single 698 | step or a step for each window size may 699 | be given. Default is 20% of the model 700 | size. 701 | --resampled-size SIZE Resample window chips to a fixed size. 702 | This must fit within the model. 703 | --max-cache-size SIZE Maximum raster cache size. This can be 704 | specified as a memory amount, e.g. 16G, 705 | or as a percentage, e.g. 50%. 706 | Specifying 0 turns off raster cache 707 | size limiting. The default is 25% of 708 | the total physical RAM. 709 | 710 | Segmentation Options: 711 | --r2p-method METHOD (=simple) Raster-to-polygon approximation method. 712 | Valid values are: none, simple, 713 | tc89-l1, tc89-kcos. 714 | --r2p-accuracy EPSILON (=3) Approximation accuracy for the 715 | raster-to-polygon operation. 716 | --r2p-min-area AREA (=0) Minimum polygon area (in pixels). 717 | 718 | Filtering Options: 719 | --include-labels LABEL [LABEL...] Filter results to only include 720 | specified labels. 721 | --exclude-labels LABEL [LABEL...] Filter results to exclude specified 722 | labels. 723 | --include-region PATH [PATH...] Path to a file prescribing regions to 724 | include when filtering. 725 | --exclude-region PATH [PATH...] Path to a file prescribing regions to 726 | exclude when filtering. 727 | --region (include/exclude) PATH [PATH...] [(include/exclude) PATH [PATH...]...] 728 | Paths to files including and excluding 729 | regions. 730 | 731 | Logging Options: 732 | --log [LEVEL (=debug)] PATH Log to a file, a file name preceded by 733 | an optional log level must be 734 | specified. Permitted values for log 735 | level are: trace, debug, info, warning, 736 | error, fatal. 737 | --quiet If set, no output will be sent to 738 | console, only a log file, if specified. 739 | 740 | General Options: 741 | --config PATH Use options from a configuration file. 742 | --help Show this help message 743 | ``` 744 | -------------------------------------------------------------------------------- /doc/modelsize.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DigitalGlobe/GGD-OpenSpaceNet/ced25e3ebd68dad4a6d9b903f8190147d3d9bd30/doc/modelsize.png --------------------------------------------------------------------------------