├── .vs ├── slnx.sqlite ├── raster-deep-learning │ └── v17 │ │ └── .suo └── VSWorkspaceState.json ├── requirements.txt ├── Custom_Models ├── CustomInference.docx └── DraftTemplateEMD.emd ├── docs ├── img │ ├── keras_maskrcnn_housefootprintsexample.jpg │ ├── pytorch_FeatureClassifier_buildingdamage.jpg │ ├── cntk_fasterRCNN_coconuttreedetectionexample.jpg │ ├── tensorflow_deeplab_landclassificationexample.jpg │ ├── cntk_azurepixellevellandclassification_example.jpg │ ├── pytorch_fastaiSSD_swimmingpooldetectionexample.jpg │ ├── tensorflow_objectdetectionAPI_coconuttreeexample.png │ └── keras_objectclassification_housedamageclassfication.jpg ├── questions_and_answers.md ├── writing_model_definition.md └── writing_deep_learning_python_raster_functions.md ├── python_raster_functions ├── CNTK │ ├── utils │ │ ├── cython_modules │ │ │ ├── cpu_nms.cp36-win_amd64.pyd │ │ │ ├── gpu_nms.cp36-win_amd64.pyd │ │ │ └── cython_bbox.cp36-win_amd64.pyd │ │ ├── configs │ │ │ ├── Trees_config.py │ │ │ ├── AlexNet_config.py │ │ │ ├── Pascal_config.py │ │ │ ├── VGG16_config.py │ │ │ └── Grocery_config.py │ │ ├── rpn │ │ │ ├── cntk_smoothL1_loss.py │ │ │ ├── generate_anchors.py │ │ │ └── bbox_transform.py │ │ ├── config_helpers.py │ │ ├── Readme.md │ │ ├── annotations │ │ │ ├── LabelMeConverter.py │ │ │ ├── C2_AssignLabelsToBboxes.py │ │ │ ├── C1_DrawBboxesOnImages.py │ │ │ └── annotations_helper.py │ │ ├── caffe_layers │ │ │ ├── bbox_transform.py │ │ │ ├── proposal_layer.py │ │ │ └── proposal_target_layer.py │ │ ├── nms_wrapper.py │ │ ├── od_mb_source.py │ │ ├── od_utils.py │ │ └── map_helpers.py │ ├── AzurePixelLevelLandClassification.py │ └── FasterRCNN.py ├── fields.py ├── features.py ├── attribute_table.py ├── Templates │ ├── TemplateBaseDetector.py │ ├── TemplateBaseClassifier.py │ ├── ImageClassifierTemplate.py │ └── ObjectDetectorTemplate.py ├── PyTorch │ ├── FastaiSSD.py │ ├── FeatureClassifier.py │ └── model.py ├── TensorFlow │ ├── DeepLab.py │ └── ObjectDetectionAPI.py ├── Keras │ └── MaskRCNN.py ├── ImageClassifier.py ├── ObjectDetector.py └── ObjectClassifier.py ├── examples ├── pytorch │ ├── object_detection │ │ ├── pytorch_fastai_ssd.emd │ │ └── README.md │ └── object_classification │ │ ├── README.md │ │ └── woolseyFire_600_50.emd ├── cntk │ ├── object_detection │ │ └── coconut_tree │ │ │ ├── cntk_fasterrcnn_coconut_tree.emd │ │ │ └── README.md │ └── image_classification │ │ └── land_classification │ │ ├── README.md │ │ └── azure_pixel_level_land_classification.emd ├── tensorflow │ ├── object_detection │ │ └── coconut_tree_detection │ │ │ ├── tensorflow_objectdetectionapi_coconuttree.emd │ │ │ └── README.md │ └── image_classification │ │ └── land_cover_classification │ │ ├── README.md │ │ └── tensorflow_deeplab_landclassification.emd └── keras │ ├── mask_rcnn │ ├── mask_rcnn.emd │ ├── mrcnn │ │ ├── spacenet.py │ │ └── parallel_model.py │ └── README.md │ └── object_classification │ ├── HouseDamageClassifier_ProBuiltin.emd │ └── README.md ├── env_setup_server_tensorflow.bat ├── samples ├── TextSAM │ ├── TextSAM.emd │ └── README.md └── GroundingDINO │ ├── GroundingDINO.emd │ └── README.md ├── env_setup.bat ├── env_setup_Pro25.bat └── env_setup_server_108.bat /.vs/slnx.sqlite: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Esri/raster-deep-learning/HEAD/.vs/slnx.sqlite -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | tensorflow 2 | cntk-gpu 3 | keras 4 | scikit-image 5 | torch 6 | torchvision 7 | fastai 8 | Pillow 9 | -------------------------------------------------------------------------------- /.vs/raster-deep-learning/v17/.suo: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Esri/raster-deep-learning/HEAD/.vs/raster-deep-learning/v17/.suo -------------------------------------------------------------------------------- /Custom_Models/CustomInference.docx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Esri/raster-deep-learning/HEAD/Custom_Models/CustomInference.docx -------------------------------------------------------------------------------- /docs/img/keras_maskrcnn_housefootprintsexample.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Esri/raster-deep-learning/HEAD/docs/img/keras_maskrcnn_housefootprintsexample.jpg -------------------------------------------------------------------------------- /docs/img/pytorch_FeatureClassifier_buildingdamage.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Esri/raster-deep-learning/HEAD/docs/img/pytorch_FeatureClassifier_buildingdamage.jpg -------------------------------------------------------------------------------- /docs/img/cntk_fasterRCNN_coconuttreedetectionexample.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Esri/raster-deep-learning/HEAD/docs/img/cntk_fasterRCNN_coconuttreedetectionexample.jpg -------------------------------------------------------------------------------- /docs/img/tensorflow_deeplab_landclassificationexample.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Esri/raster-deep-learning/HEAD/docs/img/tensorflow_deeplab_landclassificationexample.jpg -------------------------------------------------------------------------------- /docs/img/cntk_azurepixellevellandclassification_example.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Esri/raster-deep-learning/HEAD/docs/img/cntk_azurepixellevellandclassification_example.jpg -------------------------------------------------------------------------------- /docs/img/pytorch_fastaiSSD_swimmingpooldetectionexample.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Esri/raster-deep-learning/HEAD/docs/img/pytorch_fastaiSSD_swimmingpooldetectionexample.jpg -------------------------------------------------------------------------------- /docs/img/tensorflow_objectdetectionAPI_coconuttreeexample.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Esri/raster-deep-learning/HEAD/docs/img/tensorflow_objectdetectionAPI_coconuttreeexample.png -------------------------------------------------------------------------------- /docs/img/keras_objectclassification_housedamageclassfication.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Esri/raster-deep-learning/HEAD/docs/img/keras_objectclassification_housedamageclassfication.jpg -------------------------------------------------------------------------------- /.vs/VSWorkspaceState.json: -------------------------------------------------------------------------------- 1 | { 2 | "ExpandedNodes": [ 3 | "" 4 | ], 5 | "SelectedNode": "\\C:\\Users\\sang4758\\Source\\Repos\\raster-deep-learning", 6 | "PreviewInSolutionExplorer": false 7 | } -------------------------------------------------------------------------------- /python_raster_functions/CNTK/utils/cython_modules/cpu_nms.cp36-win_amd64.pyd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Esri/raster-deep-learning/HEAD/python_raster_functions/CNTK/utils/cython_modules/cpu_nms.cp36-win_amd64.pyd -------------------------------------------------------------------------------- /python_raster_functions/CNTK/utils/cython_modules/gpu_nms.cp36-win_amd64.pyd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Esri/raster-deep-learning/HEAD/python_raster_functions/CNTK/utils/cython_modules/gpu_nms.cp36-win_amd64.pyd -------------------------------------------------------------------------------- /python_raster_functions/CNTK/utils/cython_modules/cython_bbox.cp36-win_amd64.pyd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Esri/raster-deep-learning/HEAD/python_raster_functions/CNTK/utils/cython_modules/cython_bbox.cp36-win_amd64.pyd -------------------------------------------------------------------------------- /examples/pytorch/object_detection/pytorch_fastai_ssd.emd: -------------------------------------------------------------------------------- 1 | { 2 | "Framework": "PyTorch", 3 | "ModelConfiguration": "FastaiSSD", 4 | "ModelFile": ".\\9.h5", 5 | "ModelType": "ObjectDetection", 6 | "ImageHeight": 224, 7 | "ImageWidth": 224, 8 | "ExtractBands": [ 0, 1, 2 ], 9 | "Classes": [ 10 | { 11 | "Value": 0, 12 | "Name": "Pool", 13 | "Color": [ 0, 255, 0 ] 14 | } 15 | ], 16 | "ModelName": "FastAI_SSD_PyTorch", 17 | "Version": "2019.02" 18 | } -------------------------------------------------------------------------------- /examples/cntk/object_detection/coconut_tree/cntk_fasterrcnn_coconut_tree.emd: -------------------------------------------------------------------------------- 1 | { 2 | "Framework": "CNTK", 3 | "ModelConfiguration": "FasterRCNN", 4 | "ModelType": "ObjectDetection", 5 | "ModelFile": ".\\CoconutTreeDetection.model", 6 | "ExtractBands": [ 0, 1, 2 ], 7 | "ImageHeight": 850, 8 | "ImageWidth": 850, 9 | "BatchSize": 1, 10 | "Classes": [ 11 | { 12 | "Value": 0, 13 | "Name": "CoconutTree", 14 | "Color": [ 0, 255, 0 ] 15 | } 16 | ], 17 | "ModelName": "CoconutTree_FasterRCNN_CNTK", 18 | "Version": "2019.02" 19 | } -------------------------------------------------------------------------------- /examples/tensorflow/object_detection/coconut_tree_detection/tensorflow_objectdetectionapi_coconuttree.emd: -------------------------------------------------------------------------------- 1 | { 2 | "Framework": "TensorFlow", 3 | "ModelConfiguration": "ObjectDetectionAPI", 4 | "ModelFile":".\\frozen_inference_graph.pb", 5 | "ModelType":"ObjectDetection", 6 | "ImageHeight":850, 7 | "ImageWidth":850, 8 | "ExtractBands":[0,1,2], 9 | 10 | "Classes" : [ 11 | { 12 | "Value": 1, 13 | "Name": "Tree", 14 | "Color": [0, 255, 0] 15 | } 16 | ], 17 | "ModelName": "CoconutTree_TF_ObjectDetectionAPI", 18 | "Version": "2019.02" 19 | } -------------------------------------------------------------------------------- /python_raster_functions/CNTK/utils/configs/Trees_config.py: -------------------------------------------------------------------------------- 1 | from easydict import EasyDict as edict 2 | 3 | __C = edict() 4 | __C.DATA = edict() 5 | cfg = __C 6 | 7 | __C.DATA.DATASET = "Trees" 8 | __C.DATA.MAP_FILE_PATH = "../../DataSets/Trees" 9 | __C.DATA.CLASS_MAP_FILE = "class_map.txt" 10 | __C.DATA.TRAIN_MAP_FILE = "train_img_file.txt" 11 | __C.DATA.TEST_MAP_FILE = "test_img_file.txt" 12 | __C.DATA.TRAIN_ROI_FILE = "train_roi_file.txt" 13 | __C.DATA.TEST_ROI_FILE = "test_roi_file.txt" 14 | __C.DATA.NUM_TRAIN_IMAGES = 31 15 | __C.DATA.NUM_TEST_IMAGES = 1 16 | __C.DATA.PROPOSAL_LAYER_SCALES = [8, 16, 32] -------------------------------------------------------------------------------- /examples/keras/mask_rcnn/mask_rcnn.emd: -------------------------------------------------------------------------------- 1 | { 2 | "Framework": "Keras", 3 | "ModelConfiguration": { 4 | "Name": "MaskRCNN", 5 | "Architecture": ".\\mrcnn\\spacenet", 6 | "Config": ".\\mrcnn\\spacenet" 7 | }, 8 | "ModelFile": ".\\mask_rcnn_spacenet_0053.h5", 9 | "ModelType": "ObjectDetection", 10 | "ImageHeight": 650, 11 | "ImageWidth": 650, 12 | "ExtractBands": [ 0, 1, 2 ], 13 | "Classes": [ 14 | { 15 | "Value": 1, 16 | "Name": "Building", 17 | "Color": [ 0, 255, 0 ] 18 | } 19 | ], 20 | "ModelName": "Keras_MaskRCNN", 21 | "Version": "2019.02" 22 | } 23 | -------------------------------------------------------------------------------- /Custom_Models/DraftTemplateEMD.emd: -------------------------------------------------------------------------------- 1 | { 2 | "ModelFile": "frozen_inference_graph.pb", 3 | "InferenceFunction": "DraftTemplateObjectDetector.py", 4 | "ModelType": "ObjectDetection", 5 | "ImageHeight": 512, 6 | "ImageWidth": 512, 7 | "Padding": 0, 8 | "Threshold": 0.5, 9 | "BatchSize": 64, 10 | "ExtractBands": [0,1,2], 11 | "DataRange":[0.1, 1.0], 12 | 13 | "Classes": [ 14 | { 15 | "Value": 1, 16 | "Name": "Airplane", 17 | "Color": [0, 255, 0] 18 | }, 19 | { 20 | "Value": 2, 21 | "Name": "Calf", 22 | "Color": [255, 255, 0] 23 | } 24 | ] 25 | } -------------------------------------------------------------------------------- /examples/keras/mask_rcnn/mrcnn/spacenet.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | sys.path.append(os.path.dirname(__file__)) 5 | from config import Config 6 | import model as modellib 7 | 8 | class SpaceNetConfig(Config): 9 | NAME = 'spacenet' 10 | IMAGES_PER_GPU = 1 11 | NUM_CLASSES = 1+1 12 | USE_MINI_MASK = False 13 | 14 | IMAGE_MIN_DIM = 1024 15 | IMAGE_MAX_DIM = 1024 16 | 17 | MEAN_PIXEL = [0.0, 0.0, 0.0] 18 | 19 | STEPS_PER_EPOCH = 500 20 | 21 | class InferenceConfig(SpaceNetConfig): 22 | IMAGES_PER_GPU = 1 23 | GPU_COUNT = 1 24 | 25 | config = InferenceConfig() 26 | 27 | model = modellib.MaskRCNN(mode='inference', config=config, model_dir='./') -------------------------------------------------------------------------------- /examples/keras/mask_rcnn/README.md: -------------------------------------------------------------------------------- 1 | # Keras Mask RCNN House Footprints Example in ArcGIS Pro 2 | Step 0. Download the test deep learning model and image [here](https://www.arcgis.com/home/item.html?id=646dae44d4334d5ba68c7541031d9edc). 3 | You can also use your own trained model and test image. 4 | 5 | Step 1. Open "Detect Object Using Deep Learning" 6 | 7 | Step 2. Fill in the parameters 8 | 9 | | Parameter | Value | 10 | | --------- | ----- | 11 | | Input Raster | "images\15OCT22183656-S2AS_R1C1-056155973040_01_P001.TIF" | 12 | | Input Model Definition File | mask_rcnn_spacenet.emd | 13 | | Arguments | padding:0 | 14 | 15 | Step 3. Run the tool. 16 | 17 | -------------------------------------------------------------------------------- /examples/pytorch/object_detection/README.md: -------------------------------------------------------------------------------- 1 | # PyTorch Swimming Pools Detection Example in ArcGIS Pro 2 | Step 0. Download the test deep learning model and image [here](https://www.arcgis.com/home/item.html?id=16fa8bab78d24832b4a7c2ecac835019). 3 | You can also use your own trained model and test image. 4 | 5 | Step 1. Open "Detect Object Using Deep Learning" geoprocessing tool. 6 | 7 | Step 2. Fill in the parameters. 8 | 9 | | Parameter | Value | 10 | | --------- | ----- | 11 | | Input Raster | "redlands_large_18_12_(2014).jpg" | 12 | | Input Model Definition File | pytorch_fastai_ssd.emd | 13 | | Arguments | padding:0, batch_size:1| 14 | Step 3. Run the GP tool. 15 | 16 | -------------------------------------------------------------------------------- /examples/tensorflow/image_classification/land_cover_classification/README.md: -------------------------------------------------------------------------------- 1 | # TensorFlow Tree Detection Example in ArcGIS Pro 2 | Step 0. Download the test deep learning model and image [here](https://www.arcgis.com/home/item.html?id=47631bf61aad46ae90f91f80707c129c). 3 | You can also use your own trained model and test image. 4 | 5 | Step 1. Open "Classify Pixels Using Deep Learning" geoprocessing tool. 6 | 7 | Step 2. Fill in the parameters. 8 | 9 | | Parameter | Value | 10 | | --------- | ----- | 11 | | Input Raster | "images\m_4712101_nw_10_1_20150928.tif" | 12 | | Input Model Definition File | tensorflow_deeplab_landclassification.emd | 13 | Step 3. Run the tool. 14 | 15 | 16 | -------------------------------------------------------------------------------- /examples/cntk/image_classification/land_classification/README.md: -------------------------------------------------------------------------------- 1 | # Azure Pixel-level Land Classification Example using CNTK in ArcGIS Pro 2 | Step 0. Download the test deep learning model and image [here](https://www.arcgis.com/home/item.html?id=e8bc272d1ce2456fa4b87c9af749a57f). 3 | You can also use your own trained model and test image. 4 | 5 | Step 1. Open "Classify Pixels Using Deep Learning" geoprocessing tool. 6 | 7 | Step 2. Fill in the parameters. 8 | 9 | | Parameter | Value | 10 | | --------- | ----- | 11 | | Input Raster | "images\m_4712101_nw_10_1_20150928.tif" | 12 | | Input Model Definition File | azure_pixel_level_land_classification.emd | 13 | 14 | Step 3. Run the tool. 15 | 16 | 17 | -------------------------------------------------------------------------------- /examples/cntk/object_detection/coconut_tree/README.md: -------------------------------------------------------------------------------- 1 | # CNTK Coconut Tree Detection Example in ArcGIS Pro 2 | Step 0. Download the test deep learning model and image [here](https://www.arcgis.com/home/item.html?id=a75a2b1086774525ac46370c5f4c1fdd). 3 | You can also use your own trained model and test image. 4 | 5 | Step 1. Open "Detect Object Using Deep Learning" geoprocessing tool. 6 | 7 | Step 2. Fill in the parameters. 8 | 9 | | Parameter | Value | 10 | | --------- | ----- | 11 | | Input Raster | "images\coconutTrees.tif" | 12 | | Input Model Definition File | cntk_fasterrcnn_cocnut_tree.emd | 13 | | Arguments | score_threshold:0.0, padding:0, nms_threshold:0.2| 14 | 15 | Step 3. Run the tool. 16 | 17 | 18 | -------------------------------------------------------------------------------- /examples/keras/object_classification/HouseDamageClassifier_ProBuiltin.emd: -------------------------------------------------------------------------------- 1 | { 2 | "Framework": "Keras", 3 | "ModelConfiguration": { 4 | "Name": "KerasClassifier" 5 | }, 6 | "ModelFile": ".\\Damage_Classification_Model_V7.h5", 7 | "ModelType": "ObjectClassification", 8 | "ImageHeight": 256, 9 | "ImageWidth": 256, 10 | "ExtractBands": [ 0, 1, 2 ], 11 | "CropSizeFixed": 1, 12 | "BlackenAroundFeature": 1, 13 | "ImageSpaceUsed": "MAP_SPACE", 14 | "Classes": [ 15 | { 16 | "Value": 0, 17 | "Name": "undamaged", 18 | "Color": [ 0, 255, 0 ] 19 | }, 20 | { 21 | "Value": 1, 22 | "Name": "damaged", 23 | "Color": [ 0, 255, 0 ] 24 | } 25 | ], 26 | "ModelName": "HouseDamageClassifier", 27 | "Version": "2019.02" 28 | } 29 | -------------------------------------------------------------------------------- /examples/tensorflow/object_detection/coconut_tree_detection/README.md: -------------------------------------------------------------------------------- 1 | # TensorFlow Coconut Tree Detection Example in ArcGIS Pro 2 | Step 0. Download the test deep learning model and image [here](https://www.arcgis.com/home/item.html?id=9e45c7ba200c4f1ea1379983c3ef23d8). 3 | You can also use your own trained model and test image. 4 | 5 | Step 1. Open "Detect Object Using Deep Learning" geoprocessing tool. 6 | 7 | Step 2. Fill in the parameters. 8 | 9 | | Parameter | Value | 10 | | --------- | ----- | 11 | | Input Raster | "images\Kolovai_UAV4R_Subset.tif" | 12 | | Input Model Definition File | tensorflow_objectdetectionapi_coconuttree.emd | 13 | | Arguments | padding:64,score_threshold:0.6,batch_size:1 | 14 | 15 | Step 3. Run the tool. 16 | 17 | 18 | -------------------------------------------------------------------------------- /examples/keras/object_classification/README.md: -------------------------------------------------------------------------------- 1 | # Keras House Damage Classification Example in ArcGIS Pro 2 | Step 0. Download test deep learning model, input image, input features from the link [here](https://www.arcgis.com/home/item.html?id=8549fbf2be404e678934d224f9804298). 3 | You can also use your own trained model and test image. 4 | 5 | Step 1. Open "Classify Object Using Deep Learning" 6 | 7 | Step 2. Fill in the parameters 8 | 9 | | Parameter | Value | 10 | | --------- | ----- | 11 | | Input Raster | inputs/keras_objectclassification_testimage.tif | 12 | | Input Features | inputs/data.gdb/Building_Footprints 13 | | Input Model Definition File | model/HouseDamageClassifier_ProBuiltin.emd | 14 | | Arguments | batch_size:64; resampled_image_size:64| 15 | 16 | Step 3. Run the tool. 17 | 18 | -------------------------------------------------------------------------------- /examples/pytorch/object_classification/README.md: -------------------------------------------------------------------------------- 1 | # PyTorch Building Damage Classification Example in ArcGIS Pro 2 | Step 0. Download test deep learning model, image and input feature class from the link [here](https://www.arcgis.com/home/item.html?id=d8b0584ff908448abfa095533484934d). 3 | You can use Train Deep Learning Model geoprocessing tool to train your own deep learning model, specify the model type as "FeatureClassifier". 4 | 5 | Step 1. Open "Classify Object Using Deep Learning" geoprocessing tool. 6 | 7 | Step 2. Fill in the parameters. 8 | 9 | | Parameter | Value | 10 | | --------- | ----- | 11 | | Input Raster | inputs/image.jpg | 12 | | Input Features | inputs/DemoProject.gdb/InputFeatures | 13 | | Input Model Definition File | model/woolseyFire_600_50.emd | 14 | | Arguments | batch_size:16| 15 | Step 3. Run the GP tool. 16 | 17 | -------------------------------------------------------------------------------- /env_setup_server_tensorflow.bat: -------------------------------------------------------------------------------- 1 | SETLOCAL 2 | FOR /F "usebackq tokens=2*" %%A IN (`reg query "HKLM\Software\Microsoft\Windows\CurrentVersion\Uninstall\ArcGIS Server 10.7" /v "InstallLocation"`) DO ( 3 | SET ARCGISSERVER_INSTALL_DIR=%%B 4 | ) 5 | SET SCRIPTS_DIR=%ARCGISSERVER_INSTALL_DIR%framework\runtime\ArcGIS\bin\Python\Scripts\ 6 | SET ENV_DIR=%ARCGISSERVER_INSTALL_DIR%framework\runtime\ArcGIS\bin\Python\envs\ 7 | SET ENV_NAME=TensorFlowEnvTest 8 | 9 | cd "%SCRIPTS_DIR%" 10 | CALL "%SCRIPTS_DIR%deactivate.bat" 11 | 12 | SET TF_PACKAGE=tensorflow-gpu 13 | 14 | ECHO INFO: Clone a new ArcGIS python environment %ENV_NAME% ... 15 | CALL "%SCRIPTS_DIR%conda.exe" create --name %ENV_NAME% --clone arcgispro-py3 16 | CALL "%SCRIPTS_DIR%activate.bat" %ENV_NAME% 17 | 18 | ECHO INFO: Install tensorflow-gpu ... 19 | CALL "%SCRIPTS_DIR%conda.exe" install -c anaconda %TF_PACKAGE%=1.13.1 --yes 20 | 21 | ECHO INFO: Run proswap to switch to the new env ... 22 | CALL "%SCRIPTS_DIR%proswap.bat" %ENV_NAME% 23 | 24 | EXIT -------------------------------------------------------------------------------- /examples/pytorch/object_classification/woolseyFire_600_50.emd: -------------------------------------------------------------------------------- 1 | { 2 | "ModelFile": "woolseyFire_600_50.pth", 3 | "ImageHeight": 600, 4 | "ImageWidth": 600, 5 | "ModelParameters": { 6 | "backbone": "resnet34" 7 | }, 8 | "LearningRate": "0.004365158322401656", 9 | "resize_to": null, 10 | "Framework": "PyTorch", 11 | "ModelConfiguration": "FeatureClassifier", 12 | "ModelType": "ObjectClassification", 13 | "ExtractBands": [ 14 | 0, 15 | 1, 16 | 2 17 | ], 18 | "CropSizeFixed": 1, 19 | "BlackenAroundFeature": 0, 20 | "ImageSpaceUsed": "MAP_SPACE", 21 | "Classes": [ 22 | { 23 | "Value": 1, 24 | "Name": "Damaged", 25 | "Color": [ 26 | 17, 27 | 157, 28 | 34 29 | ] 30 | }, 31 | { 32 | "Value": 2, 33 | "Name": "Undamaged", 34 | "Color": [ 35 | 158, 36 | 187, 37 | 171 38 | ] 39 | } 40 | ], 41 | "ModelName": "WoolseyFire_FeatureClassifier", 42 | "Version": "2019.02" 43 | } -------------------------------------------------------------------------------- /python_raster_functions/CNTK/utils/configs/AlexNet_config.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Microsoft. All rights reserved. 2 | 3 | # Licensed under the MIT license. See LICENSE.md file in the project root 4 | # for full license information. 5 | # ============================================================================== 6 | 7 | # `pip install easydict` if you don't have it 8 | from easydict import EasyDict as edict 9 | 10 | __C = edict() 11 | __C.MODEL = edict() 12 | cfg = __C 13 | 14 | # model config 15 | __C.MODEL.BASE_MODEL = "AlexNet" 16 | __C.MODEL.BASE_MODEL_FILE = "AlexNet_ImageNet_Caffe.model" 17 | __C.MODEL.IMG_PAD_COLOR = [114, 114, 114] 18 | __C.MODEL.FEATURE_NODE_NAME = "data" 19 | __C.MODEL.LAST_CONV_NODE_NAME = "relu5" 20 | __C.MODEL.START_TRAIN_CONV_NODE_NAME = __C.MODEL.FEATURE_NODE_NAME 21 | __C.MODEL.POOL_NODE_NAME = "pool5" 22 | __C.MODEL.LAST_HIDDEN_NODE_NAME = "drop7" 23 | __C.MODEL.FEATURE_STRIDE = 16 24 | __C.MODEL.RPN_NUM_CHANNELS = 256 25 | __C.MODEL.ROI_DIM = 6 26 | __C.MODEL.E2E_LR_FACTOR = 1.0 27 | __C.MODEL.RPN_LR_FACTOR = 1.0 28 | __C.MODEL.FRCN_LR_FACTOR = 1.0 29 | -------------------------------------------------------------------------------- /samples/TextSAM/TextSAM.emd: -------------------------------------------------------------------------------- 1 | { 2 | "ModelType": "ObjectDetection", 3 | "InferenceFunction": "TextSAM.py", 4 | "ExtractBands": [ 5 | 0, 6 | 1, 7 | 2 8 | ], 9 | "Classes": [ 10 | { 11 | "Value": 1, 12 | "Name": "Object", 13 | "Color": [ 14 | 186, 15 | 199, 16 | 208 17 | ] 18 | } 19 | ], 20 | "ModelFormat": "NCHW", 21 | "MinCellSize": { 22 | "x": 2.146743517369032e-06, 23 | "y": 2.155325191644639e-06, 24 | "spatialReference": { 25 | "wkid": 4326, 26 | "latestWkid": 4326 27 | } 28 | }, 29 | "MaxCellSize": { 30 | "x": 2.146743517369032e-06, 31 | "y": 2.155325191644639e-06, 32 | "spatialReference": { 33 | "wkid": 4326, 34 | "latestWkid": 4326 35 | } 36 | }, 37 | "SupportsVariableTileSize": true, 38 | "ImageHeight": 1024, 39 | "ImageWidth": 1024, 40 | "ImageSpaceUsed": "MAP_SPACE", 41 | "IsMultispectral": false 42 | } -------------------------------------------------------------------------------- /samples/GroundingDINO/GroundingDINO.emd: -------------------------------------------------------------------------------- 1 | { 2 | "ModelType": "ObjectDetection", 3 | "InferenceFunction": "GroundingDINO.py", 4 | "ExtractBands": [ 5 | 0, 6 | 1, 7 | 2 8 | ], 9 | "Classes": [ 10 | { 11 | "Value": 1, 12 | "Name": "Object", 13 | "Color": [ 14 | 186, 15 | 199, 16 | 208 17 | ] 18 | } 19 | ], 20 | "ModelFormat": "NCHW", 21 | "MinCellSize": { 22 | "x": 2.146743517369032e-06, 23 | "y": 2.155325191644639e-06, 24 | "spatialReference": { 25 | "wkid": 4326, 26 | "latestWkid": 4326 27 | } 28 | }, 29 | "MaxCellSize": { 30 | "x": 2.146743517369032e-06, 31 | "y": 2.155325191644639e-06, 32 | "spatialReference": { 33 | "wkid": 4326, 34 | "latestWkid": 4326 35 | } 36 | }, 37 | "SupportsVariableTileSize": true, 38 | "ImageHeight": 1024, 39 | "ImageWidth": 1024, 40 | "ImageSpaceUsed": "MAP_SPACE", 41 | "IsMultispectral": false 42 | } -------------------------------------------------------------------------------- /python_raster_functions/CNTK/utils/configs/Pascal_config.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Microsoft. All rights reserved. 2 | 3 | # Licensed under the MIT license. See LICENSE.md file in the project root 4 | # for full license information. 5 | # ============================================================================== 6 | 7 | # `pip install easydict` if you don't have it 8 | from easydict import EasyDict as edict 9 | 10 | __C = edict() 11 | __C.DATA = edict() 12 | cfg = __C 13 | 14 | # data set config 15 | __C.DATA.DATASET = "Pascal" 16 | __C.DATA.MAP_FILE_PATH = "../../DataSets/Pascal/mappings" 17 | __C.DATA.CLASS_MAP_FILE = "class_map.txt" 18 | __C.DATA.TRAIN_MAP_FILE = "trainval2007.txt" 19 | __C.DATA.TRAIN_ROI_FILE = "trainval2007_rois_abs-xyxy_noPad_skipDif.txt" 20 | __C.DATA.TEST_MAP_FILE = "test2007.txt" 21 | __C.DATA.TEST_ROI_FILE = "test2007_rois_abs-xyxy_noPad_skipDif.txt" 22 | __C.DATA.NUM_TRAIN_IMAGES = 5010 23 | __C.DATA.NUM_TEST_IMAGES = 4952 24 | __C.DATA.PROPOSAL_LAYER_SCALES = [8, 16, 32] 25 | 26 | __C.DATA.TRAIN_PRECOMPUTED_PROPOSALS_FILE = "trainval2007_proposals.txt" 27 | __C.DATA.TEST_PRECOMPUTED_PROPOSALS_FILE = "test2007_proposals.txt" 28 | -------------------------------------------------------------------------------- /examples/tensorflow/image_classification/land_cover_classification/tensorflow_deeplab_landclassification.emd: -------------------------------------------------------------------------------- 1 | { 2 | "Framework": "TensorFlow", 3 | "ModelConfiguration": "DeepLab", 4 | "ModelFile": ".\\frozen_inference_graph.pb", 5 | "ModelType": "ImageClassification", 6 | "ExtractBands": [ 0, 1, 2 ], 7 | "ImageWidth": 256, 8 | "ImageHeight": 256, 9 | "BatchSize": 1, 10 | "Classes": [ 11 | { 12 | "Value": 0, 13 | "Name": "Evergreen Forest", 14 | "Color": [ 0, 51, 0 ] 15 | }, 16 | { 17 | "Value": 1, 18 | "Name": "Grassland/Herbaceous", 19 | "Color": [ 241, 185, 137 ] 20 | }, 21 | { 22 | "Value": 2, 23 | "Name": "Bare Land", 24 | "Color": [ 236, 236, 0 ] 25 | }, 26 | { 27 | "Value": 3, 28 | "Name": "Open Water", 29 | "Color": [ 0, 0, 117 ] 30 | }, 31 | { 32 | "Value": 4, 33 | "Name": "Scrub/Shrub", 34 | "Color": [ 102, 102, 0 ] 35 | }, 36 | { 37 | "Value": 5, 38 | "Name": "Impervious Surface", 39 | "Color": [ 236, 236, 236 ] 40 | } 41 | ], 42 | "ModelName": "LandClassification_DeepLab_TF", 43 | "Version": "2019.02" 44 | } -------------------------------------------------------------------------------- /python_raster_functions/CNTK/utils/configs/VGG16_config.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Microsoft. All rights reserved. 2 | 3 | # Licensed under the MIT license. See LICENSE.md file in the project root 4 | # for full license information. 5 | # ============================================================================== 6 | 7 | # `pip install easydict` if you don't have it 8 | from easydict import EasyDict as edict 9 | 10 | __C = edict() 11 | __C.MODEL = edict() 12 | cfg = __C 13 | 14 | # model config 15 | __C.MODEL.BASE_MODEL = "VGG16" 16 | __C.MODEL.BASE_MODEL_FILE = "VGG16_ImageNet_Caffe.model" 17 | __C.MODEL.IMG_PAD_COLOR = [103, 116, 123] 18 | __C.MODEL.FEATURE_NODE_NAME = "data" 19 | __C.MODEL.LAST_CONV_NODE_NAME = "relu5_3" 20 | __C.MODEL.START_TRAIN_CONV_NODE_NAME = "pool2" # __C.MODEL.FEATURE_NODE_NAME 21 | __C.MODEL.POOL_NODE_NAME = "pool5" 22 | __C.MODEL.LAST_HIDDEN_NODE_NAME = "drop7" 23 | __C.MODEL.FEATURE_STRIDE = 16 24 | __C.MODEL.RPN_NUM_CHANNELS = 512 25 | __C.MODEL.ROI_DIM = 7 26 | ## Try changing `LR_FACTOR` parameters, if the training does not converge. 27 | ## Ex.) For Grocery dataset, it may be better to set it to 0.1 28 | __C.MODEL.E2E_LR_FACTOR = 1.0 29 | __C.MODEL.RPN_LR_FACTOR = 1.0 30 | __C.MODEL.FRCN_LR_FACTOR = 1.0 31 | 32 | -------------------------------------------------------------------------------- /examples/cntk/image_classification/land_classification/azure_pixel_level_land_classification.emd: -------------------------------------------------------------------------------- 1 | { 2 | "Framework": "CNTK", 3 | "ModelConfiguration": "AzurePixelLevelLandClassification", 4 | "ModelType": "ImageClassification", 5 | "ModelFile": ".\\trained.model", 6 | "ExtractBands": [ 0, 1, 2, 3 ], 7 | "DataRange": [ 0.0, 1.0 ], 8 | "ImageHeight": 256, 9 | "ImageWidth": 256, 10 | "ModelPadding": 64, 11 | 12 | "Classes": [ 13 | { 14 | "Value": 0, 15 | "Name": "Evergreen Forest", 16 | "Color": [ 0, 51, 0 ] 17 | }, 18 | { 19 | "Value": 1, 20 | "Name": "Grassland/Herbaceous", 21 | "Color": [ 241, 185, 137 ] 22 | }, 23 | { 24 | "Value": 2, 25 | "Name": "Bare Land", 26 | "Color": [ 236, 236, 0 ] 27 | }, 28 | { 29 | "Value": 3, 30 | "Name": "Open Water", 31 | "Color": [ 0, 0, 117 ] 32 | }, 33 | { 34 | "Value": 4, 35 | "Name": "Scrub/Shrub", 36 | "Color": [ 102, 102, 0 ] 37 | }, 38 | { 39 | "Value": 5, 40 | "Name": "Impervious Surface", 41 | "Color": [ 236, 236, 236 ] 42 | } 43 | ], 44 | "ModelName": "AzurePixelLevelLandClassification_CNTK", 45 | "Version": "2019.02" 46 | } -------------------------------------------------------------------------------- /python_raster_functions/fields.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Copyright 2018 Esri 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | 6 | you may not use this file except in compliance with the License. 7 | 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | 16 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | 18 | See the License for the specific language governing permissions and 19 | 20 | limitations under the License.​ 21 | ''' 22 | 23 | fields = { 24 | 'fields': [ 25 | { 26 | 'name': 'OID', 27 | 'type': 'esriFieldTypeOID', 28 | 'alias': 'OID' 29 | }, 30 | { 31 | 'name': 'Class', 32 | 'type': 'esriFieldTypeString', 33 | 'alias': 'Class' 34 | }, 35 | { 36 | 'name': 'Confidence', 37 | 'type': 'esriFieldTypeDouble', 38 | 'alias': 'Confidence' 39 | }, 40 | { 41 | 'name': 'Shape', 42 | 'type': 'esriFieldTypeGeometry', 43 | 'alias': 'Shape' 44 | } 45 | ] 46 | } 47 | -------------------------------------------------------------------------------- /env_setup.bat: -------------------------------------------------------------------------------- 1 | SETLOCAL 2 | FOR /F "usebackq tokens=2*" %%A IN (`reg query "HKLM\Software\Microsoft\Windows\CurrentVersion\Uninstall\ArcGISPro" /v "InstallLocation"`) DO ( 3 | SET ARCGISPRO_INSTALL_DIR=%%B 4 | ) 5 | SET SCRIPTS_DIR=%ARCGISPRO_INSTALL_DIR%bin\Python\Scripts\ 6 | SET ENV_DIR=%ARCGISPRO_INSTALL_DIR%bin\Python\envs\ 7 | SET ENV_NAME=DeepLearningEnvTest 8 | 9 | cd "%SCRIPTS_DIR%" 10 | CALL "%SCRIPTS_DIR%deactivate.bat" 11 | 12 | SET TF_PACKAGE=tensorflow-gpu 13 | SET CTNK_PACKAGE=cntk-gpu 14 | 15 | ECHO INFO: Clone a new ArcGIS python environment %ENV_NAME% ... 16 | CALL "%SCRIPTS_DIR%conda.exe" create --name %ENV_NAME% --clone arcgispro-py3 17 | CALL "%SCRIPTS_DIR%activate.bat" %ENV_NAME% 18 | 19 | ECHO INFO: Install tensorflow-gpu, keras, and scikit-image ... 20 | CALL "%SCRIPTS_DIR%conda.exe" install -c anaconda %TF_PACKAGE%=1.14.0 keras=2.2.4 scikit-image=0.15.0 --yes 21 | 22 | ECHO INFO: Install cntk-gpu and Pillow ... 23 | CALL "%ENV_DIR%%ENV_NAME%\Scripts\pip.exe" install %CTNK_PACKAGE% Pillow==6.1.0 24 | 25 | ECHO INFO: Install Env for ArcGIS API For Python ... 26 | CALL "%SCRIPTS_DIR%conda.exe" install -c fastai -c pytorch fastai=1.0.54 pytorch=1.1.0 torchvision=0.3.0 --yes 27 | 28 | ECHO INFO: Run proswap to switch to the new env ... 29 | CALL "%SCRIPTS_DIR%proswap.bat" %ENV_NAME% 30 | 31 | EXIT -------------------------------------------------------------------------------- /env_setup_Pro25.bat: -------------------------------------------------------------------------------- 1 | SETLOCAL 2 | FOR /F "usebackq tokens=2*" %%A IN (`reg query "HKLM\Software\Microsoft\Windows\CurrentVersion\Uninstall\ArcGISPro" /v "InstallLocation"`) DO ( 3 | SET ARCGISPRO_INSTALL_DIR=%%B 4 | ) 5 | SET SCRIPTS_DIR=%ARCGISPRO_INSTALL_DIR%bin\Python\Scripts\ 6 | SET ENV_DIR=%ARCGISPRO_INSTALL_DIR%bin\Python\envs\ 7 | SET ENV_NAME=DeepLearningEnvTest1 8 | 9 | cd "%SCRIPTS_DIR%" 10 | CALL "%SCRIPTS_DIR%deactivate.bat" 11 | 12 | SET TF_PACKAGE=tensorflow-gpu 13 | SET KERAS_PACKAGE=keras-gpu 14 | 15 | ECHO INFO: Clone a new ArcGIS python environment %ENV_NAME% ... 16 | CALL "%SCRIPTS_DIR%conda.exe" create --name %ENV_NAME% --clone arcgispro-py3 17 | CALL "%SCRIPTS_DIR%activate.bat" %ENV_NAME% 18 | 19 | ECHO INFO: Install tensorflow-gpu, keras, and scikit-image ... 20 | CALL "%SCRIPTS_DIR%conda.exe" install -c anaconda %TF_PACKAGE%=1.14.0 %KERAS_PACKAGE%=2.2.4 scikit-image=0.15.0 --yes 21 | CALL "%SCRIPTS_DIR%conda.exe" install -c anaconda Pillow=6.1.0 --yes 22 | 23 | 24 | ECHO INFO: Install Env for ArcGIS API For Python ... 25 | CALL "%SCRIPTS_DIR%conda.exe" install fastai=1.0.54 --yes 26 | CALL "%SCRIPTS_DIR%conda.exe" install pytorch=1.1.0 --yes 27 | CALL "%SCRIPTS_DIR%conda.exe" install libtiff=4.0.10 --no-deps --yes 28 | 29 | ECHO INFO: Run proswap to switch to the new env ... 30 | CALL "%SCRIPTS_DIR%proswap.bat" %ENV_NAME% 31 | 32 | EXIT -------------------------------------------------------------------------------- /python_raster_functions/features.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Copyright 2018 Esri 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | 6 | you may not use this file except in compliance with the License. 7 | 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | 16 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | 18 | See the License for the specific language governing permissions and 19 | 20 | limitations under the License.​ 21 | ''' 22 | 23 | features = { 24 | 'displayFieldName': '', 25 | 'fieldAliases': { 26 | 'FID': 'FID', 27 | 'Class': 'Class', 28 | 'Confidence': 'Confidence' 29 | }, 30 | 'geometryType': 'esriGeometryPolygon', 31 | 'fields': [ 32 | { 33 | 'name': 'FID', 34 | 'type': 'esriFieldTypeOID', 35 | 'alias': 'FID' 36 | }, 37 | { 38 | 'name': 'Class', 39 | 'type': 'esriFieldTypeString', 40 | 'alias': 'Class' 41 | }, 42 | { 43 | 'name': 'Confidence', 44 | 'type': 'esriFieldTypeDouble', 45 | 'alias': 'Confidence' 46 | } 47 | ], 48 | 'features': [] 49 | } 50 | -------------------------------------------------------------------------------- /env_setup_server_108.bat: -------------------------------------------------------------------------------- 1 | SETLOCAL 2 | FOR /F "usebackq tokens=2*" %%A IN (`reg query "HKLM\Software\Microsoft\Windows\CurrentVersion\Uninstall\ArcGIS Server 10.8" /v "InstallLocation"`) DO ( 3 | SET ARCGISSERVER_INSTALL_DIR=%%B 4 | ) 5 | SET SCRIPTS_DIR=%ARCGISSERVER_INSTALL_DIR%framework\runtime\ArcGIS\bin\Python\Scripts\ 6 | SET ENV_DIR=%ARCGISSERVER_INSTALL_DIR%framework\runtime\ArcGIS\bin\Python\envs\ 7 | SET ENV_NAME=ServerDLEnvTest 8 | 9 | cd "%SCRIPTS_DIR%" 10 | CALL "%SCRIPTS_DIR%deactivate.bat" 11 | 12 | SET TF_PACKAGE=tensorflow-gpu 13 | SET KERAS_PACKAGE=keras-gpu 14 | 15 | ECHO INFO: Clone a new ArcGIS python environment %ENV_NAME% ... 16 | CALL "%SCRIPTS_DIR%conda.exe" create --name %ENV_NAME% --clone arcgispro-py3 17 | CALL "%SCRIPTS_DIR%activate.bat" %ENV_NAME% 18 | 19 | ECHO INFO: Install tensorflow-gpu, keras, and scikit-image ... 20 | CALL "%SCRIPTS_DIR%conda.exe" install -c anaconda %TF_PACKAGE%=1.14.0 %KERAS_PACKAGE%=2.2.4 scikit-image=0.15.0 --yes 21 | CALL "%SCRIPTS_DIR%conda.exe" install -c anaconda Pillow=6.1.0 --yes 22 | 23 | ECHO INFO: Install Env for ArcGIS API For Python ... 24 | CALL "%SCRIPTS_DIR%conda.exe" install fastai=1.0.54 --yes 25 | CALL "%SCRIPTS_DIR%conda.exe" install pytorch=1.1.0 --yes 26 | CALL "%SCRIPTS_DIR%conda.exe" install libtiff=4.0.10 --no-deps --yes 27 | 28 | ECHO INFO: Run proswap to switch to the new env ... 29 | CALL "%SCRIPTS_DIR%proswap.bat" %ENV_NAME% 30 | 31 | EXIT -------------------------------------------------------------------------------- /python_raster_functions/CNTK/utils/rpn/cntk_smoothL1_loss.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Microsoft. All rights reserved. 2 | 3 | # Licensed under the MIT license. See LICENSE.md file in the project root 4 | # for full license information. 5 | # ============================================================================== 6 | 7 | import numpy as np 8 | import cntk as C 9 | 10 | def SmoothL1Loss(sigma, bbox_pred, bbox_targets, bbox_inside_weights, bbox_outside_weights): 11 | """ 12 | From https://github.com/smallcorgi/Faster-RCNN_TF/blob/master/lib/fast_rcnn/train.py 13 | 14 | ResultLoss = outside_weights * SmoothL1(inside_weights * (bbox_pred - bbox_targets)) 15 | SmoothL1(x) = 0.5 * (sigma * x)^2, if |x| < 1 / sigma^2 16 | |x| - 0.5 / sigma^2, otherwise 17 | """ 18 | sigma2 = sigma * sigma 19 | 20 | inside_mul_abs = C.abs(C.element_times(bbox_inside_weights, C.minus(bbox_pred, bbox_targets))) 21 | 22 | smooth_l1_sign = C.less(inside_mul_abs, 1.0 / sigma2) 23 | smooth_l1_option1 = C.element_times(C.element_times(inside_mul_abs, inside_mul_abs), 0.5 * sigma2) 24 | smooth_l1_option2 = C.minus(inside_mul_abs, 0.5 / sigma2) 25 | smooth_l1_result = C.plus(C.element_times(smooth_l1_option1, smooth_l1_sign), 26 | C.element_times(smooth_l1_option2, C.minus(1.0, smooth_l1_sign))) 27 | 28 | return C.element_times(bbox_outside_weights, smooth_l1_result) 29 | -------------------------------------------------------------------------------- /docs/questions_and_answers.md: -------------------------------------------------------------------------------- 1 | # Frequently Asked Questions 2 | 3 | 1. What is python raster function? 4 | 5 | Python raster function allows developers to implement custom image processing algorithms in ArcGIS 6 | with Python. Since many deep learning developers and data scientists use Python to train, interpret and deploy 7 | deep learning models, we build our deep learning platform on top of existing Python raster function framework to allow an 8 | integrated deep learning workflow in ArcGIS. You can get more information on Python raster function from Esri's help 9 | documentation and the Esri Raster Function Github repository (Link: https://github.com/Esri/raster-functions). 10 | 11 | 2. Which model frameworks are supported for integration? 12 | 13 | The following model frameworks have built-in Python raster function in ArcGIS Pro 2.3 and ArcGIS Enterprise 10.7: 14 | - TensorFlow 15 | - Keras 16 | - CNTK 17 | 18 | However, if you use a different deep learning framework, you can write your own deep learning Python raster function and 19 | point to this Python raster function in your .emd file next to the "InferenceFunction" parameter. For more information, 20 | see [Writng deep learning Python raster function](writing_deep_learning_python_raster_functions.md). 21 | 22 | 3. What license do I need? 23 | 24 | Deep learning inference tools in the ArcGIS platform require the ArcGIS Image Analyst license. 25 | 26 | -------------------------------------------------------------------------------- /python_raster_functions/CNTK/utils/configs/Grocery_config.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Microsoft. All rights reserved. 2 | 3 | # Licensed under the MIT license. See LICENSE.md file in the project root 4 | # for full license information. 5 | # ============================================================================== 6 | 7 | # `pip install easydict` if you don't have it 8 | from easydict import EasyDict as edict 9 | 10 | __C = edict() 11 | __C.DATA = edict() 12 | cfg = __C 13 | 14 | # data set config 15 | __C.DATA.DATASET = "Grocery" 16 | __C.DATA.MAP_FILE_PATH = "../../DataSets/Grocery" 17 | __C.DATA.CLASS_MAP_FILE = "class_map.txt" 18 | __C.DATA.TRAIN_MAP_FILE = "train_img_file.txt" 19 | __C.DATA.TRAIN_ROI_FILE = "train_roi_file.txt" 20 | __C.DATA.TEST_MAP_FILE = "test_img_file.txt" 21 | __C.DATA.TEST_ROI_FILE = "test_roi_file.txt" 22 | __C.DATA.NUM_TRAIN_IMAGES = 20 23 | __C.DATA.NUM_TEST_IMAGES = 5 24 | __C.DATA.PROPOSAL_LAYER_SCALES = [4, 8, 12] 25 | 26 | # overwriting proposal parameters for Fast R-CNN 27 | # minimum relative width/height of an ROI 28 | __C.roi_min_side_rel = 0.04 29 | # maximum relative width/height of an ROI 30 | __C.roi_max_side_rel = 0.4 31 | # minimum relative area of an ROI 32 | __C.roi_min_area_rel = 2 * __C.roi_min_side_rel * __C.roi_min_side_rel 33 | # maximum relative area of an ROI 34 | __C.roi_max_area_rel = 0.33 * __C.roi_max_side_rel * __C.roi_max_side_rel 35 | # maximum aspect ratio of an ROI vertically and horizontally 36 | __C.roi_max_aspect_ratio = 4.0 37 | 38 | # For this data set use the following lr factor for Fast R-CNN: 39 | # __C.CNTK.LR_FACTOR = 10.0 40 | -------------------------------------------------------------------------------- /python_raster_functions/attribute_table.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Copyright 2018 Esri 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | 6 | you may not use this file except in compliance with the License. 7 | 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | 16 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | 18 | See the License for the specific language governing permissions and 19 | 20 | limitations under the License.​ 21 | ''' 22 | 23 | attribute_table = { 24 | 'displayFieldName': '', 25 | 'fieldAliases': { 26 | 'OID': 'OID', 27 | 'Value': 'Value', 28 | 'Class': 'Class', 29 | 'Red': 'Red', 30 | 'Green': 'Green', 31 | 'Blue': 'Blue' 32 | }, 33 | 'fields': [ 34 | { 35 | 'name': 'OID', 36 | 'type': 'esriFieldTypeOID', 37 | 'alias': 'OID' 38 | }, 39 | { 40 | 'name': 'Value', 41 | 'type': 'esriFieldTypeInteger', 42 | 'alias': 'Value' 43 | }, 44 | { 45 | 'name': 'Class', 46 | 'type': 'esriFieldTypeString', 47 | 'alias': 'Class' 48 | }, 49 | { 50 | 'name': 'Red', 51 | 'type': 'esriFieldTypeInteger', 52 | 'alias': 'Red' 53 | }, 54 | { 55 | 'name': 'Green', 56 | 'type': 'esriFieldTypeInteger', 57 | 'alias': 'Green' 58 | }, 59 | { 60 | 'name': 'Blue', 61 | 'type': 'esriFieldTypeInteger', 62 | 'alias': 'Blue' 63 | } 64 | ], 65 | 'features': [] 66 | } 67 | -------------------------------------------------------------------------------- /python_raster_functions/CNTK/utils/config_helpers.py: -------------------------------------------------------------------------------- 1 | from easydict import EasyDict 2 | 3 | def merge_configs(config_list): 4 | if config_list == None or len(config_list) == 0: 5 | return None 6 | 7 | base_config = config_list[0] 8 | if type(base_config) is dict: 9 | base_config = EasyDict(base_config) 10 | 11 | if type(base_config) is not EasyDict: 12 | print("The argument given to 'merge_configs' have to be of type dict or EasyDict.") 13 | return None 14 | 15 | for i in range(len(config_list) - 1): 16 | config_to_merge = config_list[i+1] 17 | if type(config_to_merge) is dict: 18 | config_to_merge = EasyDict(config_to_merge) 19 | _merge_add_a_into_b(config_to_merge, base_config) 20 | return base_config 21 | 22 | 23 | def _merge_add_a_into_b(a, b): 24 | """ 25 | Merge config dictionary a into config dictionary b, 26 | clobbering the options in b whenever they are also specified in a. 27 | New options that are only in a will be added to b. 28 | """ 29 | if type(a) is not EasyDict: 30 | return 31 | 32 | for k, v in a.items(): 33 | # if the key from a is new to b simply add it 34 | if not k in b: 35 | b[k] = v 36 | continue 37 | 38 | # the types must match 39 | old_type = type(b[k]) 40 | if old_type is not type(v): 41 | if isinstance(b[k], np.ndarray): 42 | v = np.array(v, dtype=b[k].dtype) 43 | else: 44 | raise ValueError(('Type mismatch ({} vs. {}) for config key: {}').format(type(b[k]), type(v), k)) 45 | 46 | # recursively merge dicts 47 | if type(v) is EasyDict: 48 | try: 49 | _merge_add_a_into_b(a[k], b[k]) 50 | except: 51 | print('Error under config key: {}'.format(k)) 52 | raise 53 | else: 54 | b[k] = v 55 | -------------------------------------------------------------------------------- /python_raster_functions/CNTK/utils/Readme.md: -------------------------------------------------------------------------------- 1 | ## Detection utils 2 | 3 | This folder contains Python modules that are utilities for object detection networks. 4 | 5 | ### Cython modules 6 | 7 | To use the rpn component you need precompiled cython modules for nms (at least cpu_nms.cpXX-win_amd64.pyd for Windows or cpu_nms.cpython-XXm.so for Linux) and bbox (cython_bbox.cpXX-win_amd64.pyd for Windows or cython_bbox.cpython-XXm.so for Linux). 8 | To compile the cython modules for windows see (https://github.com/MrGF/py-faster-rcnn-windows): 9 | ``` 10 | git clone https://github.com/MrGF/py-faster-rcnn-windows 11 | cd $FRCN_ROOT/lib 12 | python setup.py build_ext --inplace 13 | ``` 14 | For Linux see (https://github.com/rbgirshick/py-faster-rcnn): 15 | ``` 16 | git clone https://github.com/rbgirshick/py-faster-rcnn 17 | cd $FRCN_ROOT/lib 18 | python setup.py build_ext --inplace 19 | ``` 20 | Copy the compiled `.pyd` (Windows) or `.so` (Linux) files into the `cython_modules` subfolder of this utils folder. 21 | 22 | ### `rpn` module overview 23 | 24 | The rpn module contains helper methods and required layers to generate region proposal networks for object detection. 25 | 26 | ##### `rpn_helpers` 27 | 28 | Contains helper methods to create a region proposal network (rpn) and a proposal target layer for training the rpn. 29 | 30 | ##### `generate_anchors.py` 31 | 32 | Generates a regular grid of multi-scale, multi-aspect anchor boxes. 33 | 34 | ##### `proposal_layer.py` 35 | 36 | Converts RPN outputs (per-anchor scores and bbox regression estimates) into object proposals. 37 | 38 | ##### `anchor_target_layer.py` 39 | 40 | Generates training targets/labels for each anchor. Classification labels are 1 (object), 0 (not object) or -1 (ignore). 41 | Bbox regression targets are specified when the classification label is > 0. 42 | 43 | ##### `proposal_target_layer.py` 44 | 45 | Generates training targets/labels for each object proposal: classification labels 0 - K (bg or object class 1, ... , K) 46 | and bbox regression targets in that case that the label is > 0. 47 | -------------------------------------------------------------------------------- /python_raster_functions/CNTK/utils/annotations/LabelMeConverter.py: -------------------------------------------------------------------------------- 1 | import os 2 | import xml.etree.ElementTree as ET 3 | import csv 4 | 5 | filepath = "C:/Your/Folder/Labelme/Files/" # set path of Labelme XML Files here include slash at end of path 6 | 7 | for filename in os.listdir(filepath): 8 | try: 9 | file = filepath + filename 10 | 11 | tree = ET.parse(file) 12 | root = tree.getroot() 13 | 14 | outputpath = filepath + "Parsed/" 15 | 16 | if not os.path.exists(outputpath): 17 | os.makedirs(outputpath) 18 | 19 | imagename = os.path.splitext(filename)[0] 20 | 21 | ## create output files 22 | outputFile_label = outputpath + imagename + ".bboxes.labels.tsv" 23 | outputFile_ROI = outputpath + imagename + ".bboxes.tsv" 24 | 25 | labelFile = open(outputFile_label, 'w') 26 | ROIFile = open(outputFile_ROI, 'w') 27 | 28 | # loop through to get objects 29 | for child in root: 30 | if str(child.tag) == 'object': 31 | 32 | label = "" 33 | xlist = [] 34 | ylist = [] 35 | 36 | # loop through to get name and BBox values from object 37 | for child in child: 38 | if str(child.tag) == 'name': 39 | label = child.text 40 | if str(child.tag) == 'polygon' or str(child.tag) == 'segm': 41 | for child in child: 42 | if str(child.tag) == 'box' or str(child.tag) == 'pt': 43 | for child in child: 44 | if str(child.tag) == 'xmin' or str(child.tag) == 'xmax' or str(child.tag) == 'x': 45 | xlist.append(int(child.text)) 46 | if str(child.tag) == 'ymin' or str(child.tag) == 'ymax' or str(child.tag) == 'y': 47 | ylist.append(int(child.text)) 48 | 49 | xmin = min(xlist) 50 | xmax = max(xlist) 51 | 52 | ymin = min(ylist) 53 | ymax = max(ylist) 54 | 55 | # output object roi based on cntk format of xmin ymin xmax ymax 56 | obj_ROI = str(xmin) + "\t" + str(ymin) + "\t" +str(xmax) + "\t" + str(ymax) 57 | 58 | labelFile.write(label + '\n') 59 | ROIFile.write(obj_ROI + '\n') 60 | 61 | labelFile.close() 62 | ROIFile.close() 63 | 64 | except Exception: 65 | pass 66 | 67 | print("Done") 68 | -------------------------------------------------------------------------------- /python_raster_functions/CNTK/utils/rpn/generate_anchors.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Microsoft. All rights reserved. 2 | 3 | # Licensed under the MIT license. See LICENSE.md file in the project root 4 | # for full license information. 5 | # ============================================================================== 6 | 7 | import numpy as np 8 | 9 | def generate_anchors(base_size=16, ratios=[0.5, 1, 2], 10 | scales=2**np.arange(3, 6)): 11 | """ 12 | Generate anchor (reference) windows by enumerating aspect ratios X 13 | scales wrt a reference (0, 0, 15, 15) window. 14 | """ 15 | 16 | base_anchor = np.array([1, 1, base_size, base_size]) - 1 17 | ratio_anchors = _ratio_enum(base_anchor, ratios) 18 | anchors = np.vstack([_scale_enum(ratio_anchors[i, :], scales) 19 | for i in range(ratio_anchors.shape[0])]) # was xrange 20 | return anchors 21 | 22 | def _whctrs(anchor): 23 | """ 24 | Return width, height, x center, and y center for an anchor (window). 25 | """ 26 | 27 | w = anchor[2] - anchor[0] + 1 28 | h = anchor[3] - anchor[1] + 1 29 | x_ctr = anchor[0] + 0.5 * (w - 1) 30 | y_ctr = anchor[1] + 0.5 * (h - 1) 31 | return w, h, x_ctr, y_ctr 32 | 33 | def _mkanchors(ws, hs, x_ctr, y_ctr): 34 | """ 35 | Given a vector of widths (ws) and heights (hs) around a center 36 | (x_ctr, y_ctr), output a set of anchors (windows). 37 | """ 38 | 39 | ws = ws[:, np.newaxis] 40 | hs = hs[:, np.newaxis] 41 | anchors = np.hstack((x_ctr - 0.5 * (ws - 1), 42 | y_ctr - 0.5 * (hs - 1), 43 | x_ctr + 0.5 * (ws - 1), 44 | y_ctr + 0.5 * (hs - 1))) 45 | return anchors 46 | 47 | def _ratio_enum(anchor, ratios): 48 | """ 49 | Enumerate a set of anchors for each aspect ratio wrt an anchor. 50 | """ 51 | 52 | w, h, x_ctr, y_ctr = _whctrs(anchor) 53 | size = w * h 54 | size_ratios = size / ratios 55 | ws = np.round(np.sqrt(size_ratios)) 56 | hs = np.round(ws * ratios) 57 | anchors = _mkanchors(ws, hs, x_ctr, y_ctr) 58 | return anchors 59 | 60 | def _scale_enum(anchor, scales): 61 | """ 62 | Enumerate a set of anchors for each scale wrt an anchor. 63 | """ 64 | 65 | w, h, x_ctr, y_ctr = _whctrs(anchor) 66 | ws = w * scales 67 | hs = h * scales 68 | anchors = _mkanchors(ws, hs, x_ctr, y_ctr) 69 | return anchors 70 | 71 | if __name__ == '__main__': 72 | import time 73 | t = time.time() 74 | a = generate_anchors() 75 | print (time.time() - t) 76 | print (a) 77 | from IPython import embed; embed() 78 | -------------------------------------------------------------------------------- /python_raster_functions/CNTK/utils/caffe_layers/bbox_transform.py: -------------------------------------------------------------------------------- 1 | # -------------------------------------------------------- 2 | # Fast R-CNN 3 | # Copyright (c) 2015 Microsoft 4 | # Licensed under The MIT License [see LICENSE for details] 5 | # Written by Ross Girshick 6 | # -------------------------------------------------------- 7 | 8 | import numpy as np 9 | 10 | def bbox_transform(ex_rois, gt_rois): 11 | ex_widths = ex_rois[:, 2] - ex_rois[:, 0] + 1.0 12 | ex_heights = ex_rois[:, 3] - ex_rois[:, 1] + 1.0 13 | ex_ctr_x = ex_rois[:, 0] + 0.5 * ex_widths 14 | ex_ctr_y = ex_rois[:, 1] + 0.5 * ex_heights 15 | 16 | gt_widths = gt_rois[:, 2] - gt_rois[:, 0] + 1.0 17 | gt_heights = gt_rois[:, 3] - gt_rois[:, 1] + 1.0 18 | gt_ctr_x = gt_rois[:, 0] + 0.5 * gt_widths 19 | gt_ctr_y = gt_rois[:, 1] + 0.5 * gt_heights 20 | 21 | targets_dx = (gt_ctr_x - ex_ctr_x) / ex_widths 22 | targets_dy = (gt_ctr_y - ex_ctr_y) / ex_heights 23 | targets_dw = np.log(gt_widths / ex_widths) 24 | targets_dh = np.log(gt_heights / ex_heights) 25 | 26 | targets = np.vstack( 27 | (targets_dx, targets_dy, targets_dw, targets_dh)).transpose() 28 | return targets 29 | 30 | def bbox_transform_inv(boxes, deltas): 31 | if boxes.shape[0] == 0: 32 | return np.zeros((0, deltas.shape[1]), dtype=deltas.dtype) 33 | 34 | boxes = boxes.astype(deltas.dtype, copy=False) 35 | 36 | widths = boxes[:, 2] - boxes[:, 0] + 1.0 37 | heights = boxes[:, 3] - boxes[:, 1] + 1.0 38 | ctr_x = boxes[:, 0] + 0.5 * widths 39 | ctr_y = boxes[:, 1] + 0.5 * heights 40 | 41 | dx = deltas[:, 0::4] 42 | dy = deltas[:, 1::4] 43 | dw = deltas[:, 2::4] 44 | dh = deltas[:, 3::4] 45 | 46 | pred_ctr_x = dx * widths[:, np.newaxis] + ctr_x[:, np.newaxis] 47 | pred_ctr_y = dy * heights[:, np.newaxis] + ctr_y[:, np.newaxis] 48 | pred_w = np.exp(dw) * widths[:, np.newaxis] 49 | pred_h = np.exp(dh) * heights[:, np.newaxis] 50 | 51 | pred_boxes = np.zeros(deltas.shape, dtype=deltas.dtype) 52 | # x1 53 | pred_boxes[:, 0::4] = pred_ctr_x - 0.5 * pred_w 54 | # y1 55 | pred_boxes[:, 1::4] = pred_ctr_y - 0.5 * pred_h 56 | # x2 57 | pred_boxes[:, 2::4] = pred_ctr_x + 0.5 * pred_w 58 | # y2 59 | pred_boxes[:, 3::4] = pred_ctr_y + 0.5 * pred_h 60 | 61 | return pred_boxes 62 | 63 | def clip_boxes(boxes, im_shape): 64 | """ 65 | Clip boxes to image boundaries. 66 | """ 67 | 68 | # x1 >= 0 69 | boxes[:, 0::4] = np.maximum(np.minimum(boxes[:, 0::4], im_shape[1] - 1), 0) 70 | # y1 >= 0 71 | boxes[:, 1::4] = np.maximum(np.minimum(boxes[:, 1::4], im_shape[0] - 1), 0) 72 | # x2 < im_shape[1] 73 | boxes[:, 2::4] = np.maximum(np.minimum(boxes[:, 2::4], im_shape[1] - 1), 0) 74 | # y2 < im_shape[0] 75 | boxes[:, 3::4] = np.maximum(np.minimum(boxes[:, 3::4], im_shape[0] - 1), 0) 76 | return boxes 77 | -------------------------------------------------------------------------------- /samples/GroundingDINO/README.md: -------------------------------------------------------------------------------- 1 | # GroundingDINO DLPK For ArcGIS Pro 2 | 3 | This sample showcases a deep learning package (DLPK) to detect object using text prompts in ArcGIS Pro. It is achieved by using GroundingDINO. Grounding DINO is an open set object detector that can find objects given a text prompt. The bounding boxes representing the detected objects are converted to polygons by this model and returned as GIS features. 4 | 5 | ## Prerequisites 6 | 1. ArcGIS Pro 2.3 or later 7 | 8 | 2. An ArcGIS Image Analyst license is required to run inferencing tools. 9 | 10 | 3. CPU or NVIDIA GPU + CUDA CuDNN 11 | 12 | 4. Set up the [Python deep learning environment in ArcGIS](https://developers.arcgis.com/python/guide/deep-learning/). 13 | 14 | 5. Get familiar with the format and requirements of the *[Esri model definition file (emd)](../../docs/writing_model_definition.md)*. 15 | 16 | 6. (Optional) Get familiar with *Python raster function*. 17 | *[Anatomy of a Python Raster Function](https://github.com/Esri/raster-functions/wiki/PythonRasterFunction#anatomy-of-a-python-raster-function)*. 18 | 19 | 7. (Optional) If you are interested in writing custom deep learning Python raster function to integrate additional deep learning 20 | models into ArcGIS, the 21 | *[Custom Python raster function guide](../../docs/writing_deep_learning_python_raster_functions.md)* provides details 22 | on the necessary functions and how to implement model inference calls using Python raster function. 23 | 24 | ## Steps for creating DLPK 25 | 1. Clone GroundingDINO repository: 26 | ``` 27 | git clone https://github.com/giswqs/GroundingDINO.git 28 | ``` 29 | 30 | 2. Install the GroundingDINO from the repository downloaded above by running the command below: 31 | ``` 32 | pip install --no-deps -e . 33 | ``` 34 | 35 | 3. Install the supervision package from pypi using the command: 36 | ``` 37 | pip install --no-deps supervision==0.6.0 38 | ``` 39 | 40 | 4. Download the GroundingDINO checkpoint and Config file from the [repo](https://github.com/IDEA-Research/GroundingDINO?tab=readme-ov-file#luggage-checkpoints) and point to it in the [GroundingDINO.py](GroundingDINO.py) file. 41 | 42 | 7. Finally, to create a standalone DLPK, you may create a 7zip archive containing the model weights, source code of grounding_dino along with the EMD and .py inference function from this repo. The archive should be given a DLPK extension. 43 | 44 | ## Issues 45 | 46 | Did you find a bug or do you want to request a new feature? Please let us know by submitting an issue. 47 | 48 | ## Contributing 49 | 50 | Esri welcomes contributions from anyone and everyone. Please see our [guidelines for contributing](https://github.com/esri/contributing). 51 | 52 | ## Licensing 53 | Copyright 2019 Esri 54 | 55 | Licensed under the Apache License, Version 2.0 (the "License"); 56 | you may not use this file except in compliance with the License. 57 | You may obtain a copy of the License at 58 | 59 | http://www.apache.org/licenses/LICENSE-2.0 60 | 61 | Unless required by applicable law or agreed to in writing, software 62 | distributed under the License is distributed on an "AS IS" BASIS, 63 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 64 | See the License for the specific language governing permissions and 65 | limitations under the License. 66 | 67 | A copy of the license is available in the repository's [license.txt](../../license.txt) file. -------------------------------------------------------------------------------- /python_raster_functions/Templates/TemplateBaseDetector.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Copyright 2018 Esri 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | 6 | you may not use this file except in compliance with the License. 7 | 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | 16 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | 18 | See the License for the specific language governing permissions and 19 | 20 | limitations under the License.​ 21 | ''' 22 | 23 | import json 24 | import os 25 | import sys 26 | 27 | prf_root_dir = os.path.join(os.path.dirname(__file__), os.pardir) 28 | sys.path.append(prf_root_dir) 29 | import prf_utils 30 | 31 | class TemplateBaseDetector: 32 | def initialize(self, model, model_as_file): 33 | if model_as_file: 34 | with open(model, 'r') as f: 35 | self.json_info = json.load(f) 36 | else: 37 | self.json_info = json.loads(model) 38 | 39 | model_path = self.json_info['ModelFile'] 40 | if model_as_file and not os.path.isabs(model_path): 41 | model_path = os.path.abspath(os.path.join(os.path.dirname(model), model_path)) 42 | 43 | # load model 44 | self.load_model(model_path) 45 | 46 | def getConfiguration(self, **scalars): 47 | if 'BatchSize' not in self.json_info and 'batch_size' not in scalars: 48 | self.batch_size = 1 49 | elif 'BatchSize' not in self.json_info and 'batch_size' in scalars: 50 | self.batch_size = int(scalars['batch_size']) 51 | else: 52 | self.batch_size = int(self.json_info['BatchSize']) 53 | 54 | self.padding = int(scalars['padding']) 55 | self.score_threshold = float(scalars['score_threshold']) 56 | 57 | self.scalars = scalars 58 | 59 | self.rectangle_height, self.rectangle_width = prf_utils.calculate_rectangle_size_from_batch_size(self.batch_size) 60 | ty, tx = prf_utils.get_tile_size(self.json_info['ImageHeight'], self.json_info['ImageWidth'], 61 | self.padding, self.rectangle_height, self.rectangle_width) 62 | 63 | return { 64 | 'extractBands': tuple(self.json_info['ExtractBands']), 65 | 'padding': int(scalars['padding']), 66 | 'tx': tx, 67 | 'ty': ty, 68 | 'fixedTileSize': 0 69 | } 70 | 71 | def vectorize(self, **pixelBlocks): 72 | input_image = pixelBlocks['raster_pixels'] 73 | _, height, width = input_image.shape 74 | batch, batch_height, batch_width = \ 75 | prf_utils.tile_to_batch(input_image, 76 | self.json_info['ImageHeight'], 77 | self.json_info['ImageWidth'], 78 | self.padding, 79 | fixed_tile_size=False) 80 | 81 | batch_bounding_boxes, batch_scores, batch_classes = self.inference(batch, **self.scalars) 82 | return prf_utils.batch_detection_results_to_tile_results( 83 | batch_bounding_boxes, 84 | batch_scores, 85 | batch_classes, 86 | self.json_info['ImageHeight'], 87 | self.json_info['ImageWidth'], 88 | self.padding, 89 | batch_width 90 | ) 91 | -------------------------------------------------------------------------------- /python_raster_functions/CNTK/utils/annotations/C2_AssignLabelsToBboxes.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Microsoft. All rights reserved. 2 | 3 | # Licensed under the MIT license. See LICENSE.md file in the project root 4 | # for full license information. 5 | # ============================================================================== 6 | 7 | from __future__ import print_function 8 | try: 9 | # for Python2 10 | from Tkinter import * 11 | except ImportError: 12 | # for Python3 13 | from tkinter import * 14 | from PIL import ImageTk 15 | from cntk_helpers import * 16 | 17 | 18 | #################################### 19 | # Parameters 20 | #################################### 21 | imgDir = "C:/Users/chazhang/Desktop/newImgs/" 22 | classes = ("avocado", "orange", "butter", "champagne", "cheese", "eggBox", "gerkin", "joghurt", "ketchup", 23 | "orangeJuice", "onion", "pepper", "sausage", "tomato", "water", "apple", "milk", 24 | "tabasco", "soySauce", "mustard", "beer") 25 | 26 | #no need to change these 27 | drawingImgSize = 1000 28 | boxWidth = 10 29 | boxHeight = 2 30 | 31 | 32 | #################################### 33 | # Main 34 | #################################### 35 | # define callback function for tk button 36 | def buttonPressedCallback(s): 37 | global global_lastButtonPressed 38 | global_lastButtonPressed = s 39 | 40 | # create UI 41 | objectNames = np.sort(classes).tolist() 42 | objectNames += ["UNDECIDED", "EXCLUDE"] 43 | tk = Tk() 44 | w = Canvas(tk, width=len(objectNames) * boxWidth, height=len(objectNames) * boxHeight, bd = boxWidth, bg = 'white') 45 | w.grid(row = len(objectNames), column = 0, columnspan = 2) 46 | for objectIndex,objectName in enumerate(objectNames): 47 | b = Button(width=boxWidth, height=boxHeight, text=objectName, command=lambda s = objectName: buttonPressedCallback(s)) 48 | b.grid(row = objectIndex, column = 0) 49 | 50 | # loop over all images 51 | imgFilenames = getFilesInDirectory(imgDir, ".jpg") 52 | for imgIndex, imgFilename in enumerate(imgFilenames): 53 | print (imgIndex, imgFilename) 54 | labelsPath = os.path.join(imgDir, imgFilename[:-4] + ".bboxes.labels.tsv") 55 | if os.path.exists(labelsPath): 56 | print ("Skipping image {:3} ({}) since annotation file already exists: {}".format(imgIndex, imgFilename, labelsPath)) 57 | continue 58 | 59 | # load image and ground truth rectangles 60 | img = imread(os.path.join(imgDir,imgFilename)) 61 | rectsPath = os.path.join(imgDir, imgFilename[:-4] + ".bboxes.tsv") 62 | rects = [ToIntegers(rect) for rect in readTable(rectsPath)] 63 | 64 | # annotate each rectangle in turn 65 | labels = [] 66 | for rectIndex,rect in enumerate(rects): 67 | imgCopy = img.copy() 68 | drawRectangles(imgCopy, [rect], thickness = 15) 69 | 70 | # draw image in tk window 71 | imgTk, _ = imresizeMaxDim(imgCopy, drawingImgSize, boUpscale = True) 72 | imgTk = ImageTk.PhotoImage(imconvertCv2Pil(imgTk)) 73 | label = Label(tk, image=imgTk) 74 | label.grid(row=0, column=1, rowspan=drawingImgSize) 75 | tk.update_idletasks() 76 | tk.update() 77 | 78 | # busy-wait until button pressed 79 | global_lastButtonPressed = None 80 | while not global_lastButtonPressed: 81 | tk.update_idletasks() 82 | tk.update() 83 | 84 | # store result 85 | print ("Button pressed = ", global_lastButtonPressed) 86 | labels.append(global_lastButtonPressed) 87 | 88 | writeFile(labelsPath, labels) 89 | tk.destroy() 90 | print ("DONE.") -------------------------------------------------------------------------------- /python_raster_functions/CNTK/AzurePixelLevelLandClassification.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Copyright 2018 Esri 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | 6 | you may not use this file except in compliance with the License. 7 | 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | 16 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | 18 | See the License for the specific language governing permissions and 19 | 20 | limitations under the License.​ 21 | ''' 22 | 23 | import os 24 | import sys 25 | 26 | import cntk as C 27 | import numpy as np 28 | 29 | prf_root_dir = os.path.join(os.path.dirname(__file__), os.pardir) 30 | sys.path.append(prf_root_dir) 31 | from Templates.TemplateBaseClassifier import TemplateBaseClassifier 32 | 33 | class ChildImageClassifier(TemplateBaseClassifier): 34 | # template method to fill in 35 | def load_model(self, model_path): 36 | ''' 37 | Fill this method to write your own model loading python code 38 | save it in self object if you would like to reference it later. 39 | 40 | Tips: you can access emd information through self.json_info. 41 | ''' 42 | # Todo: fill in this method to load your model 43 | self.model = C.load_model(model_path) 44 | 45 | def getParameterInfo(self, required_parameters): 46 | required_parameters.extend( 47 | [ 48 | # Todo: add your inference parameters here 49 | # https://github.com/Esri/raster-functions/wiki/PythonRasterFunction#getparameterinfo 50 | ] 51 | ) 52 | return required_parameters 53 | 54 | # template method to fill in 55 | def inference(self, batch, **kwargs): 56 | ''' 57 | Fill this method to write your own inference python code, you can refer to the model instance that is created 58 | in the load_model method. Expected results format is described in the returns as below. 59 | 60 | Tips: you can access emd information through self.json_info. 61 | 62 | :param batch: numpy array with shape (B, H, W, D), B is batch size, H, W is specified and equal to 63 | ImageHeight and ImageWidth in the emd file and D is the number of bands and equal to the length 64 | of ExtractBands in the emd. 65 | :param kwargs: inference parameters, accessed by the parameter name, 66 | i.e. score_threshold=float(kwargs['score_threshold']). If you want to have more inference 67 | parameters, add it to the list of the following getParameterInfo method. 68 | :return: semantic segmentation, numpy array in the shape [B, 1, H, W] and type np.uint8, B is the batch size, 69 | H and W are the tile size, equal to ImageHeight and ImageWidth in the emd file respectively if Padding 70 | is not set 71 | ''' 72 | # Todo: fill in this method to inference your model and return bounding boxes, scores and classes 73 | batch=batch.astype(np.float32) 74 | 75 | output = self.model.eval( 76 | { 77 | self.model.arguments[0]: batch 78 | } 79 | ) 80 | semantic_predictions = np.argmax(output, axis=1) 81 | semantic_predictions = np.expand_dims(semantic_predictions, axis=1) 82 | 83 | return semantic_predictions 84 | -------------------------------------------------------------------------------- /python_raster_functions/Templates/TemplateBaseClassifier.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Copyright 2018 Esri 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | 6 | you may not use this file except in compliance with the License. 7 | 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | 16 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | 18 | See the License for the specific language governing permissions and 19 | 20 | limitations under the License.​ 21 | ''' 22 | 23 | import json 24 | import os 25 | import sys 26 | 27 | prf_root_dir = os.path.join(os.path.dirname(__file__), os.pardir) 28 | sys.path.append(prf_root_dir) 29 | import prf_utils 30 | 31 | class TemplateBaseClassifier: 32 | def initialize(self, model, model_as_file): 33 | if model_as_file: 34 | with open(model, 'r') as f: 35 | self.json_info = json.load(f) 36 | else: 37 | self.json_info = json.loads(model) 38 | 39 | model_path = self.json_info['ModelFile'] 40 | if model_as_file and not os.path.isabs(model_path): 41 | model_path = os.path.abspath(os.path.join(os.path.dirname(model), model_path)) 42 | 43 | # load model 44 | self.load_model(model_path) 45 | 46 | def getConfiguration(self, **scalars): 47 | if 'BatchSize' not in self.json_info and 'batch_size' not in scalars: 48 | self.batch_size = 1 49 | elif 'BatchSize' not in self.json_info and 'batch_size' in scalars: 50 | self.batch_size = int(scalars['batch_size']) 51 | else: 52 | self.batch_size = int(self.json_info['BatchSize']) 53 | 54 | if 'ModelPadding' in self.json_info: 55 | self.model_padding = self.json_info['ModelPadding'] 56 | self.padding = self.model_padding 57 | else: 58 | self.model_padding = 0 59 | self.padding = int(scalars['padding']) 60 | 61 | self.scalars = scalars 62 | 63 | self.rectangle_height, self.rectangle_width = prf_utils.calculate_rectangle_size_from_batch_size(self.batch_size) 64 | ty, tx = prf_utils.get_tile_size(self.json_info['ImageHeight'], self.json_info['ImageWidth'], 65 | self.padding, self.rectangle_height, self.rectangle_width) 66 | 67 | return { 68 | 'extractBands': tuple(self.json_info['ExtractBands']), 69 | 'padding': self.padding, 70 | 'tx': tx, 71 | 'ty': ty, 72 | 'fixedTileSize': 1 73 | } 74 | 75 | def updatePixels(self, tlc, shape, props, **pixelBlocks): 76 | input_image = pixelBlocks['raster_pixels'] 77 | _, height, width = input_image.shape 78 | batch, batch_height, batch_width = \ 79 | prf_utils.tile_to_batch(input_image, 80 | self.json_info['ImageHeight'], 81 | self.json_info['ImageWidth'], 82 | self.padding, 83 | fixed_tile_size=True, 84 | batch_height=self.rectangle_height, 85 | batch_width=self.rectangle_width) 86 | 87 | semantic_predictions = self.inference(batch, **self.scalars) 88 | 89 | height, width = semantic_predictions.shape[2], semantic_predictions.shape[3] 90 | if self.model_padding == 0 and self.padding != 0: 91 | semantic_predictions = semantic_predictions[:, :, self.padding:height - self.padding, 92 | self.padding:width - self.padding] 93 | 94 | semantic_predictions = prf_utils.batch_to_tile(semantic_predictions, batch_height, batch_width) 95 | 96 | return semantic_predictions 97 | -------------------------------------------------------------------------------- /samples/TextSAM/README.md: -------------------------------------------------------------------------------- 1 | # TextSAM DLPK For ArcGIS Pro 2 | 3 | This sample showcases a deep learning package (DLPK) to generate masks for an object using text prompts in ArcGIS Pro. It is achieved by using Segment Anything Model (SAM) and GroundingDINO. Grounding DINO is an open set object detector that can find objects given a text prompt. The bounding boxes representing the detected objects are then fed into Segment Anything Model as prompts to generate masks for those. Finally, the masks are converted to polygons by this model and returned as GIS features. Both models are called sequentially within this deep learning package. 4 | 5 | ## Prerequisites 6 | 1. ArcGIS Pro 2.3 or later 7 | 8 | 2. An ArcGIS Image Analyst license is required to run inferencing tools. 9 | 10 | 3. CPU or NVIDIA GPU + CUDA CuDNN 11 | 12 | 4. Set up the [Python deep learning environment in ArcGIS](https://developers.arcgis.com/python/guide/deep-learning/). 13 | 14 | 5. Get familiar with the format and requirements of the *[Esri model definition file (emd)](../../docs/writing_model_definition.md)*. 15 | 16 | 6. (Optional) Get familiar with *Python raster function*. 17 | *[Anatomy of a Python Raster Function](https://github.com/Esri/raster-functions/wiki/PythonRasterFunction#anatomy-of-a-python-raster-function)*. 18 | 19 | 7. (Optional) If you are interested in writing custom deep learning Python raster function to integrate additional deep learning 20 | models into ArcGIS, the 21 | *[Custom Python raster function guide](../../docs/writing_deep_learning_python_raster_functions.md)* provides details 22 | on the necessary functions and how to implement model inference calls using Python raster function. 23 | 24 | ## Steps for creating DLPK 25 | 1. Clone GroundingDINO repository: 26 | ``` 27 | git clone https://github.com/giswqs/GroundingDINO.git 28 | ``` 29 | 30 | 2. Install the GroundingDINO from the repository downloaded above by running the command below: 31 | ``` 32 | pip install --no-deps -e . 33 | ``` 34 | 35 | 3. Install the supervision package from pypi using the command: 36 | ``` 37 | pip install --no-deps supervision==0.6.0 38 | ``` 39 | 40 | 4. Install the segment anything from pypi using the command: 41 | ``` 42 | pip install segment-anything 43 | ``` 44 | 45 | 5. Download the Segment Anything Model (SAM) checkpoint from the [repo](https://github.com/facebookresearch/segment-anything?tab=readme-ov-file#model-checkpoints) and point to it in the [TextSAM.py](TextSAM.py) file. 46 | 47 | 6. Download the GroundingDINO checkpoint and Config file from the [repo](https://github.com/IDEA-Research/GroundingDINO?tab=readme-ov-file#luggage-checkpoints) and point to it in the [TextSAM.py](TextSAM.py) file. 48 | 49 | 7. Finally, to create a standalone DLPK, you may create a 7zip archive containing the model weights, source code of grounding_dino and segment_anything, along with the EMD and .py inference function from this repo. The archive should be given a DLPK extension. 50 | 51 | ## Issues 52 | 53 | Did you find a bug or do you want to request a new feature? Please let us know by submitting an issue. 54 | 55 | ## Contributing 56 | 57 | Esri welcomes contributions from anyone and everyone. Please see our [guidelines for contributing](https://github.com/esri/contributing). 58 | 59 | ## Licensing 60 | Copyright 2019 Esri 61 | 62 | Licensed under the Apache License, Version 2.0 (the "License"); 63 | you may not use this file except in compliance with the License. 64 | You may obtain a copy of the License at 65 | 66 | http://www.apache.org/licenses/LICENSE-2.0 67 | 68 | Unless required by applicable law or agreed to in writing, software 69 | distributed under the License is distributed on an "AS IS" BASIS, 70 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 71 | See the License for the specific language governing permissions and 72 | limitations under the License. 73 | 74 | A copy of the license is available in the repository's [license.txt](../../license.txt) file. -------------------------------------------------------------------------------- /python_raster_functions/PyTorch/FastaiSSD.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Copyright 2018 Esri 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | 6 | you may not use this file except in compliance with the License. 7 | 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | 16 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | 18 | See the License for the specific language governing permissions and 19 | 20 | limitations under the License.​ 21 | ''' 22 | 23 | import os 24 | import sys 25 | 26 | prf_root_dir = os.path.join(os.path.dirname(__file__), os.pardir) 27 | sys.path.append(prf_root_dir) 28 | from Templates.TemplateBaseDetector import TemplateBaseDetector 29 | 30 | sys.path.append(os.path.dirname(__file__)) 31 | 32 | # from util import load_model, norm, denorm, export_img, get_tile_images, get_img, \ 33 | # get_cropped_tiles, predict_, detect_objects, suppress_close_pools, overlap, predict_classf, \ 34 | # get_nms_preds 35 | import util 36 | from model import ConvnetBuilder, SSD_MultiHead, resnet34, k 37 | 38 | class ChildObjectDetector(TemplateBaseDetector): 39 | def load_model(self, model_path): 40 | ''' 41 | Fill this method to write your own model loading python code 42 | save it self object if you would like to reference it later. 43 | 44 | Tips: you can access emd information through self.json_info. 45 | ''' 46 | # Todo: fill in this method to load your model 47 | f_model = resnet34 48 | 49 | head_reg = SSD_MultiHead(k, -4.) 50 | models = ConvnetBuilder(f_model, 0, 0, 0, custom_head=head_reg) 51 | self.model = models.model 52 | self.model = util.load_model(self.model, model_path) 53 | self.model.eval() 54 | 55 | def getParameterInfo(self, required_parameters): 56 | required_parameters.extend( 57 | [ 58 | # Todo: add your inference parameters here 59 | # https://github.com/Esri/raster-functions/wiki/PythonRasterFunction#getparameterinfo 60 | ] 61 | ) 62 | return required_parameters 63 | 64 | def inference(self, batch, **scalars): 65 | ''' 66 | Fill this method to write your own inference python code, you can refer to the model instance that is created 67 | in the load_model method. Expected results format is described in the returns as below. 68 | 69 | :param batch: numpy array with shape (B, H, W, D), B is batch size, H, W is specified and equal to 70 | ImageHeight and ImageWidth in the emd file and D is the number of bands and equal to the length 71 | of ExtractBands in the emd. If BatchInference is set to False in emd, B is constant 1. 72 | :param scalars: inference parameters, accessed by the parameter name, 73 | i.e. score_threshold=float(kwargs['score_threshold']). If you want to have more inference 74 | parameters, add it to the list of the following getParameterInfo method. 75 | :return: bounding boxes, python list representing bounding boxes whose length is equal to B, each element is 76 | [N,4] numpy array representing [ymin, xmin, ymax, xmax] with respect to the upper left 77 | corner of the image tile. 78 | scores, python list representing the score of each bounding box whose length is equal to B, each element 79 | is [N,] numpy array 80 | classes, python list representing the class of each bounding box whose length is equal to B, each element 81 | is [N,] numpy array and its dype is np.uint8 82 | ''' 83 | #Todo: fill in this method to inference your model and return bounding boxes, scores and classes 84 | return util.detect_objects_image_space(self.model, batch, self.score_threshold) 85 | -------------------------------------------------------------------------------- /python_raster_functions/TensorFlow/DeepLab.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Copyright 2018 Esri 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | 6 | you may not use this file except in compliance with the License. 7 | 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | 16 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | 18 | See the License for the specific language governing permissions and 19 | 20 | limitations under the License.​ 21 | ''' 22 | 23 | import os 24 | import sys 25 | 26 | import numpy as np 27 | import tensorflow as tf 28 | 29 | prf_root_dir = os.path.join(os.path.dirname(__file__), os.pardir) 30 | sys.path.append(prf_root_dir) 31 | from Templates.TemplateBaseClassifier import TemplateBaseClassifier 32 | 33 | class ChildImageClassifier(TemplateBaseClassifier): 34 | # template method to fill in 35 | def load_model(self, model_path): 36 | ''' 37 | Fill this method to write your own model loading python code 38 | save it in self object if you would like to reference it later. 39 | 40 | Tips: you can access emd information through self.json_info. 41 | ''' 42 | # Todo: fill in this method to load your model 43 | self.classification_graph = tf.Graph() 44 | with self.classification_graph.as_default(): 45 | od_graph_def = tf.GraphDef() 46 | with tf.gfile.GFile(model_path, 'rb') as fid: 47 | serialized_graph = fid.read() 48 | od_graph_def.ParseFromString(serialized_graph) 49 | tf.import_graph_def(od_graph_def, name='') 50 | 51 | def getParameterInfo(self, required_parameters): 52 | required_parameters.extend( 53 | [ 54 | # Todo: add your inference parameters here 55 | # https://github.com/Esri/raster-functions/wiki/PythonRasterFunction#getparameterinfo 56 | ] 57 | ) 58 | required_parameters[:] = [d for d in required_parameters if not d.get('name') == 'batch_size'] 59 | 60 | return required_parameters 61 | 62 | # template method to fill in 63 | def inference(self, batch, **scalars): 64 | ''' 65 | Fill this method to write your own inference python code, you can refer to the model instance that is created 66 | in the load_model method. Expected results format is described in the returns as below. 67 | 68 | Tips: you can access emd information through self.json_info. 69 | 70 | :param batch: numpy array with shape (B, H, W, D), B is batch size, H, W is specified and equal to 71 | ImageHeight and ImageWidth in the emd file and D is the number of bands and equal to the length 72 | of ExtractBands in the emd. 73 | :param scalars: inference parameters, accessed by the parameter name, 74 | i.e. score_threshold=float(kwargs['score_threshold']). If you want to have more inference 75 | parameters, add it to the list of the following getParameterInfo method. 76 | :return: semantic segmentation, numpy array in the shape [B, 1, H, W] and type np.uint8, B is the batch size, 77 | H and W are the tile size, equal to ImageHeight and ImageWidth in the emd file respectively. 78 | ''' 79 | #Todo: fill in this method to inference your model and return bounding boxes, scores and classes 80 | batch = np.transpose(batch, [0, 2, 3, 1]) 81 | 82 | config = tf.ConfigProto() 83 | if 'PerProcessGPUMemoryFraction' in self.json_info: 84 | config.gpu_options.per_process_gpu_memory_fraction = float(self.json_info['PerProcessGPUMemoryFraction']) 85 | 86 | with self.classification_graph.as_default(): 87 | with tf.Session(config=config) as sess: 88 | feed_dict = { 89 | 'ImageTensor:0': batch 90 | } 91 | fetches = { 92 | 'SemanticPredictions': 'SemanticPredictions:0' 93 | } 94 | 95 | output_dict = sess.run(fetches, feed_dict=feed_dict) 96 | 97 | semantic_predictions = output_dict['SemanticPredictions'] 98 | return np.expand_dims(semantic_predictions, axis=1) 99 | -------------------------------------------------------------------------------- /python_raster_functions/CNTK/utils/nms_wrapper.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Microsoft. All rights reserved. 2 | 3 | # Licensed under the MIT license. See LICENSE.md file in the project root 4 | # for full license information. 5 | # ============================================================================== 6 | 7 | import numpy as np 8 | from utils.cython_modules.cpu_nms import cpu_nms 9 | try: 10 | from utils.cython_modules.gpu_nms import gpu_nms 11 | gpu_nms_available = True 12 | except ImportError: 13 | gpu_nms_available = False 14 | 15 | def nms(dets, thresh, use_gpu_nms=True, device_id=0): 16 | ''' 17 | Dispatches the call to either CPU or GPU NMS implementations 18 | ''' 19 | if dets.shape[0] == 0: 20 | return [] 21 | if gpu_nms_available and use_gpu_nms: 22 | return gpu_nms(dets, thresh, device_id=device_id) 23 | else: 24 | return cpu_nms(dets, thresh) 25 | 26 | def apply_nms_to_single_image_results(coords, labels, scores, use_gpu_nms, device_id, nms_threshold=0.5, conf_threshold=0.0): 27 | ''' 28 | Applies nms to the results for a single image. 29 | 30 | Args: 31 | coords: (x_min, y_min, x_max, y_max) coordinates for n rois. shape = (n, 4) 32 | labels: the predicted label per roi. shape = (n, 1) 33 | scores: the predicted score per roi. shape = (n, 1) 34 | nms_threshold: the threshold for discarding overlapping ROIs in nms 35 | conf_threshold: a minimum value for the score of an ROI. ROIs with lower score will be discarded 36 | 37 | Returns: 38 | nmsKeepIndices - the indices of the ROIs to keep after nms 39 | ''' 40 | 41 | # generate input for nms 42 | allIndices = [] 43 | nmsRects = [[[]] for _ in range(max(labels) + 1)] 44 | coordsWithScores = np.hstack((coords, np.array([scores]).T)) 45 | for i in range(max(labels) + 1): 46 | indices = np.where(np.array(labels) == i)[0] 47 | nmsRects[i][0] = coordsWithScores[indices,:] 48 | allIndices.append(indices) 49 | 50 | # call nms 51 | _, nmsKeepIndicesList = apply_nms_to_test_set_results(nmsRects, nms_threshold, conf_threshold, use_gpu_nms, device_id) 52 | 53 | # map back to original roi indices 54 | nmsKeepIndices = [] 55 | for i in range(max(labels) + 1): 56 | for keepIndex in nmsKeepIndicesList[i][0]: 57 | nmsKeepIndices.append(allIndices[i][keepIndex]) # for keepIndex in nmsKeepIndicesList[i][0]] 58 | assert (len(nmsKeepIndices) == len(set(nmsKeepIndices))) # check if no roi indices was added >1 times 59 | return nmsKeepIndices 60 | 61 | def apply_nms_to_test_set_results(all_boxes, nms_threshold, conf_threshold, use_gpu_nms, device_id): 62 | ''' 63 | Applies nms to the results of multiple images. 64 | 65 | Args: 66 | all_boxes: shape of all_boxes: e.g. 21 classes x 4952 images x 58 rois x 5 coords+score 67 | nms_threshold: the threshold for discarding overlapping ROIs in nms 68 | conf_threshold: a minimum value for the score of an ROI. ROIs with lower score will be discarded 69 | 70 | Returns: 71 | nms_boxes - the reduced set of rois after nms 72 | nmsKeepIndices - the indices of the ROIs to keep after nms 73 | ''' 74 | 75 | num_classes = len(all_boxes) 76 | num_images = len(all_boxes[0]) 77 | nms_boxes = [[[] for _ in range(num_images)] 78 | for _ in range(num_classes)] 79 | nms_keepIndices = [[[] for _ in range(num_images)] 80 | for _ in range(num_classes)] 81 | for cls_ind in range(num_classes): 82 | for im_ind in range(num_images): 83 | dets = all_boxes[cls_ind][im_ind] 84 | if len(dets) == 0: 85 | continue 86 | if len(dets) == 1: 87 | keep = [0] 88 | else: 89 | keep = nms(dets.astype(np.float32), nms_threshold, use_gpu_nms, device_id) 90 | 91 | # also filter out low confidences 92 | if conf_threshold > 0: 93 | keep_conf_idx = np.where(dets[:, -1] > conf_threshold) 94 | keep = list(set(keep_conf_idx[0]).intersection(keep)) 95 | 96 | if len(keep) == 0: 97 | continue 98 | nms_boxes[cls_ind][im_ind] = dets[keep, :].copy() 99 | nms_keepIndices[cls_ind][im_ind] = keep 100 | return nms_boxes, nms_keepIndices 101 | 102 | -------------------------------------------------------------------------------- /python_raster_functions/CNTK/utils/annotations/C1_DrawBboxesOnImages.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Microsoft. All rights reserved. 2 | 3 | # Licensed under the MIT license. See LICENSE.md file in the project root 4 | # for full license information. 5 | # ============================================================================== 6 | 7 | from __future__ import print_function 8 | from builtins import chr 9 | import os, sys, importlib, shutil 10 | from cntk_helpers import * 11 | 12 | 13 | #################################### 14 | # Parameters 15 | #################################### 16 | imgDir = "C:/Users/chazhang/Desktop/newImgs/" 17 | 18 | # no need to change these params 19 | drawingImgSize = 1000.0 20 | 21 | 22 | #################################### 23 | # Functions 24 | #################################### 25 | def event_cv2_drawRectangles(event, x, y, flags, param): 26 | global global_image 27 | global global_bboxes 28 | global global_leftButtonDownPoint 29 | 30 | # draw all previous bounding boxes, and the most recent box in a different color 31 | imgCopy = global_image.copy() 32 | drawRectangles(imgCopy, global_bboxes) 33 | if len(global_bboxes)>0: 34 | drawRectangles(imgCopy, [global_bboxes[-1]], color = (255, 0, 0)) 35 | 36 | # handle mouse events 37 | if event == cv2.EVENT_LBUTTONDOWN: 38 | global_leftButtonDownPoint = (x, y) 39 | 40 | elif event == cv2.EVENT_LBUTTONUP: 41 | pt1 = global_leftButtonDownPoint 42 | pt2 = (x, y) 43 | minPt = (min(pt1[0], pt2[0]), min(pt1[1], pt2[1])) 44 | maxPt = (max(pt1[0], pt2[0]), max(pt1[1], pt2[1])) 45 | imgWidth, imgHeight = imArrayWidthHeight(global_image) 46 | minPt = ptClip(minPt, imgWidth, imgHeight) 47 | maxPt = ptClip(maxPt, imgWidth, imgHeight) 48 | global_bboxes.append(minPt + maxPt) 49 | 50 | elif flags == cv2.EVENT_FLAG_LBUTTON: #if left mouse button is held down 51 | cv2.rectangle(imgCopy, global_leftButtonDownPoint, (x, y), (255, 255, 0), 1) 52 | 53 | else: 54 | drawCrossbar(imgCopy, (x, y)) 55 | cv2.imshow("AnnotationWindow", imgCopy) 56 | 57 | def scaleCropBboxes(rectsIn, scaleFactor, imgWidth, imgHeight): 58 | if len(rectsIn) <= 0: 59 | return rectsIn 60 | else: 61 | rects = [ [int(round(rect[i]/scaleFactor)) for i in range(4)] 62 | for rect in rectsIn] 63 | rects = [Bbox(*rect).crop(imgWidth, imgHeight).rect() for rect in rects] 64 | for rect in rects: 65 | assert (Bbox(*rect).isValid()) 66 | return rects 67 | 68 | 69 | #################################### 70 | # Main 71 | #################################### 72 | imgFilenames = [f for f in os.listdir(imgDir) if f.lower().endswith(".jpg")] 73 | 74 | # loop over each image and get annotation 75 | for imgFilenameIndex,imgFilename in enumerate(imgFilenames): 76 | print (imgFilenameIndex, imgFilename) 77 | imgPath = os.path.join(imgDir, imgFilename) 78 | bBoxPath = imgPath[:-4] + ".bboxes.tsv" 79 | 80 | # skip image if ground truth already exists 81 | if os.path.exists(bBoxPath): 82 | print ("Skipping image {0} since ground truth already exists".format(imgFilename)) 83 | continue 84 | else: 85 | print ("Processing image {0} of {1}: {2}".format(imgFilenameIndex, len(imgFilenames), imgPath)) 86 | 87 | # prepare image window and callback 88 | global_bboxes = [] 89 | global_image, scaleFactor = imresizeMaxDim(imread(imgPath), drawingImgSize) 90 | cv2.namedWindow("AnnotationWindow") 91 | cv2.setMouseCallback("AnnotationWindow", event_cv2_drawRectangles) 92 | cv2.imshow("AnnotationWindow", global_image) 93 | 94 | # process user input 95 | while True: 96 | key = chr(cv2.waitKey()) 97 | 98 | # undo/remove last rectangle 99 | if key == "u": 100 | if len(global_bboxes) >= 1: 101 | global_bboxes = global_bboxes[:-1] 102 | imgCopy = global_image.copy() 103 | drawRectangles(imgCopy, global_bboxes) 104 | cv2.imshow("AnnotationWindow", imgCopy) 105 | 106 | # skip image 107 | elif key == "s": 108 | if os.path.exists(bBoxPath): 109 | print ("Skipping image hence deleting existing bbox file: " + bBoxPath) 110 | os.remove(bBoxPath) 111 | break 112 | 113 | # next image 114 | elif key == "n": 115 | bboxes = scaleCropBboxes(global_bboxes, scaleFactor, imWidth(imgPath), imHeight(imgPath)) 116 | writeTable(bBoxPath, bboxes) 117 | break 118 | 119 | # quit 120 | elif key == "q": 121 | sys.exit() 122 | 123 | cv2.destroyAllWindows() 124 | -------------------------------------------------------------------------------- /python_raster_functions/Templates/ImageClassifierTemplate.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Copyright 2018 Esri 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | 6 | you may not use this file except in compliance with the License. 7 | 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | 16 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | 18 | See the License for the specific language governing permissions and 19 | 20 | limitations under the License.​ 21 | ''' 22 | 23 | import os 24 | import sys 25 | 26 | prf_root_dir = os.path.join(os.path.dirname(__file__), os.pardir) 27 | sys.path.append(prf_root_dir) 28 | from Templates.TemplateBaseClassifier import TemplateBaseClassifier 29 | 30 | class ChildImageClassifier(TemplateBaseClassifier): 31 | # template method to fill in 32 | def load_model(self, model_path): 33 | ''' 34 | Fill this method to write your own model loading python code 35 | save it in self object if you would like to reference it later. 36 | 37 | Tips: you can access emd information through self.json_info. 38 | 39 | TensorFlow example to import graph def from frozen pb file: 40 | 41 | self.classification_graph = tf.Graph() 42 | with self.classification_graph.as_default(): 43 | od_graph_def = tf.GraphDef() 44 | with tf.gfile.GFile(model_path, 'rb') as fid: 45 | serialized_graph = fid.read() 46 | od_graph_def.ParseFromString(serialized_graph) 47 | tf.import_graph_def(od_graph_def, name='') 48 | ''' 49 | #Todo: fill in this method to load your model 50 | pass 51 | 52 | def getParameterInfo(self, required_parameters): 53 | required_parameters.extend( 54 | [ 55 | # Todo: add your inference parameters here 56 | # https://github.com/Esri/raster-functions/wiki/PythonRasterFunction#getparameterinfo 57 | ''' 58 | Example: 59 | { 60 | 'name': 'nms_threshold', 61 | 'dataType': 'numeric', 62 | 'value': 0.2, 63 | 'required': True, 64 | 'displayName': 'nms threshold', 65 | 'description': 'non maximum suppression(nms) threshold' 66 | }, 67 | ''' 68 | ] 69 | ) 70 | return required_parameters 71 | 72 | # template method to fill in 73 | def inference(self, batch, **scalars): 74 | ''' 75 | Fill this method to write your own inference python code, you can refer to the model instance that is created 76 | in the load_model method. Expected results format is described in the returns as below. 77 | 78 | Tips: you can access emd information through self.json_info. 79 | 80 | :param batch: numpy array with shape (B, H, W, D), B is batch size, H, W is specified and equal to 81 | ImageHeight and ImageWidth in the emd file and D is the number of bands and equal to the length 82 | of ExtractBands in the emd. 83 | :param scalars: inference parameters, accessed by the parameter name, 84 | i.e. score_threshold=float(scalars['score_threshold']). If you want to have more inference 85 | parameters, add it to the list of the following getParameterInfo method. 86 | :return: semantic segmentation, numpy array in the shape [B, 1, H, W] and type np.uint8, B is the batch size, 87 | H and W are the tile size, equal to ImageHeight and ImageWidth in the emd file respectively if Padding 88 | is not set 89 | 90 | Tensorflow Example: 91 | batch = np.transpose(batch, [0, 2, 3, 1]) 92 | 93 | config = tf.ConfigProto() 94 | if 'PerProcessGPUMemoryFraction' in self.json_info: 95 | config.gpu_options.per_process_gpu_memory_fraction = float(self.json_info['PerProcessGPUMemoryFraction']) 96 | 97 | with self.classification_graph.as_default(): 98 | with tf.Session(config=config) as sess: 99 | feed_dict = { 100 | 'ImageTensor:0': batch 101 | } 102 | fetches = { 103 | 'SemanticPredictions': 'SemanticPredictions:0' 104 | } 105 | 106 | output_dict = sess.run(fetches, feed_dict=feed_dict) 107 | 108 | semantic_predictions = output_dict['SemanticPredictions'] 109 | return np.expand_dims(semantic_predictions, axis=1) 110 | ''' 111 | #Todo: fill in this method to inference your model and return bounding boxes, scores and classes 112 | pass 113 | -------------------------------------------------------------------------------- /python_raster_functions/CNTK/utils/rpn/bbox_transform.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Microsoft. All rights reserved. 2 | 3 | # Licensed under the MIT license. See LICENSE.md file in the project root 4 | # for full license information. 5 | # ============================================================================== 6 | 7 | import numpy as np 8 | 9 | # compute example and gt width ctr, width and height 10 | # and returns optimal target deltas 11 | def bbox_transform(ex_rois, gt_rois): 12 | ex_widths = ex_rois[:, 2] - ex_rois[:, 0] + 1.0 13 | ex_heights = ex_rois[:, 3] - ex_rois[:, 1] + 1.0 14 | ex_ctr_x = ex_rois[:, 0] + 0.5 * ex_widths 15 | ex_ctr_y = ex_rois[:, 1] + 0.5 * ex_heights 16 | 17 | gt_widths = gt_rois[:, 2] - gt_rois[:, 0] + 1.0 18 | gt_heights = gt_rois[:, 3] - gt_rois[:, 1] + 1.0 19 | gt_ctr_x = gt_rois[:, 0] + 0.5 * gt_widths 20 | gt_ctr_y = gt_rois[:, 1] + 0.5 * gt_heights 21 | 22 | targets_dx = (gt_ctr_x - ex_ctr_x) / ex_widths 23 | targets_dy = (gt_ctr_y - ex_ctr_y) / ex_heights 24 | targets_dw = np.log(gt_widths / ex_widths) 25 | targets_dh = np.log(gt_heights / ex_heights) 26 | 27 | targets = np.vstack( 28 | (targets_dx, targets_dy, targets_dw, targets_dh)).transpose() 29 | return targets 30 | 31 | # gets 32 | # - boxes (n, 4) as [x_low, y_low, x_high, y_high] 33 | # - deltas (n, 4) as [dx, dy, dw, dh] 34 | # returns 35 | # - pred_boxes (n, 4) as [x_low, y_low, x_high, y_high] 36 | # where 37 | # pred_ctr_x = dx * widths + ctr_x 38 | # --> pred_x_low = pred_ctr_x - 0.5 * pred_w 39 | # and 40 | # pred_w = np.exp(dw) * widths 41 | def bbox_transform_inv(boxes, deltas): 42 | if boxes.shape[0] == 0: 43 | return np.zeros((0, deltas.shape[1]), dtype=deltas.dtype) 44 | 45 | boxes = boxes.astype(deltas.dtype, copy=False) 46 | 47 | widths = boxes[:, 2] - boxes[:, 0] + 1.0 48 | heights = boxes[:, 3] - boxes[:, 1] + 1.0 49 | ctr_x = boxes[:, 0] + 0.5 * widths 50 | ctr_y = boxes[:, 1] + 0.5 * heights 51 | 52 | dx = deltas[:, 0::4] 53 | dy = deltas[:, 1::4] 54 | dw = deltas[:, 2::4] 55 | dh = deltas[:, 3::4] 56 | 57 | pred_ctr_x = dx * widths[:, np.newaxis] + ctr_x[:, np.newaxis] 58 | pred_ctr_y = dy * heights[:, np.newaxis] + ctr_y[:, np.newaxis] 59 | pred_w = np.exp(dw) * widths[:, np.newaxis] 60 | pred_h = np.exp(dh) * heights[:, np.newaxis] 61 | 62 | pred_boxes = np.zeros(deltas.shape, dtype=deltas.dtype) 63 | # x1 64 | pred_boxes[:, 0::4] = pred_ctr_x - 0.5 * pred_w 65 | # y1 66 | pred_boxes[:, 1::4] = pred_ctr_y - 0.5 * pred_h 67 | # x2 68 | pred_boxes[:, 2::4] = pred_ctr_x + 0.5 * pred_w 69 | # y2 70 | pred_boxes[:, 3::4] = pred_ctr_y + 0.5 * pred_h 71 | 72 | return pred_boxes 73 | 74 | def clip_boxes(boxes, im_info): 75 | ''' 76 | Clip boxes to image boundaries. 77 | :param boxes: boxes 78 | :param im_info: (pad_width, pad_height, scaled_image_width, scaled_image_height, orig_img_width, orig_img_height) 79 | e.g.(1000, 1000, 1000, 600, 500, 300) for an original image of 600x300 that is scaled and padded to 1000x1000 80 | ''' 81 | 82 | im_info.shape = (6) 83 | padded_wh = im_info[0:2] 84 | scaled_wh = im_info[2:4] 85 | xy_offset = (padded_wh - scaled_wh) / 2 86 | xy_min = xy_offset 87 | xy_max = xy_offset + scaled_wh 88 | 89 | # x_min <= x1 <= x_max 90 | boxes[:, 0::4] = np.maximum(np.minimum(boxes[:, 0::4], xy_max[0] - 1), xy_min[0]) 91 | # y_min <= y1 <= y_max 92 | boxes[:, 1::4] = np.maximum(np.minimum(boxes[:, 1::4], xy_max[1] - 1), xy_min[1]) 93 | # x_min <= x2 <= x_max 94 | boxes[:, 2::4] = np.maximum(np.minimum(boxes[:, 2::4], xy_max[0] - 1), xy_min[0]) 95 | # y_min <= y2 <= y_max 96 | boxes[:, 3::4] = np.maximum(np.minimum(boxes[:, 3::4], xy_max[1] - 1), xy_min[1]) 97 | return boxes 98 | 99 | def regress_rois(roi_proposals, roi_regression_factors, labels, dims_input): 100 | for i in range(len(labels)): 101 | label = labels[i] 102 | if label > 0: 103 | deltas = roi_regression_factors[i:i+1,label*4:(label+1)*4] 104 | roi_coords = roi_proposals[i:i+1,:] 105 | regressed_rois = bbox_transform_inv(roi_coords, deltas) 106 | roi_proposals[i,:] = regressed_rois 107 | 108 | if dims_input is not None: 109 | # dims_input -- (pad_width, pad_height, scaled_image_width, scaled_image_height, orig_img_width, orig_img_height) 110 | pad_width, pad_height, scaled_image_width, scaled_image_height, _, _ = dims_input 111 | left = (pad_width - scaled_image_width) / 2 112 | right = pad_width - left - 1 113 | top = (pad_height - scaled_image_height) / 2 114 | bottom = pad_height - top - 1 115 | 116 | roi_proposals[:,0] = roi_proposals[:,0].clip(left, right) 117 | roi_proposals[:,1] = roi_proposals[:,1].clip(top, bottom) 118 | roi_proposals[:,2] = roi_proposals[:,2].clip(left, right) 119 | roi_proposals[:,3] = roi_proposals[:,3].clip(top, bottom) 120 | 121 | return roi_proposals 122 | -------------------------------------------------------------------------------- /python_raster_functions/Keras/MaskRCNN.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Copyright 2018 Esri 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | 6 | you may not use this file except in compliance with the License. 7 | 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | 16 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | 18 | See the License for the specific language governing permissions and 19 | 20 | limitations under the License.​ 21 | ''' 22 | 23 | import json 24 | import os 25 | import sys 26 | 27 | import numpy as np 28 | 29 | sys.path.append(os.path.dirname(__file__)) 30 | import importlib 31 | from skimage.measure import find_contours 32 | import keras.backend as K 33 | import tensorflow as tf 34 | 35 | class MatterMaskRCNN: 36 | def initialize(self, model, model_as_file): 37 | K.clear_session() 38 | 39 | if model_as_file: 40 | with open(model, 'r') as f: 41 | self.json_info = json.load(f) 42 | else: 43 | self.json_info = json.loads(model) 44 | 45 | model_path = self.json_info['ModelFile'] 46 | if model_as_file and not os.path.isabs(model_path): 47 | model_path = os.path.abspath(os.path.join(os.path.dirname(model), model_path)) 48 | 49 | config_module = self.json_info['ModelConfiguration']['Config'] 50 | if not os.path.isabs(config_module): 51 | config_module = os.path.abspath(os.path.join(os.path.dirname(model), config_module)) 52 | 53 | sys.path.append(os.path.dirname(config_module)) 54 | config_module_name = os.path.basename(config_module) 55 | 56 | if config_module_name in sys.modules: 57 | del sys.modules[config_module_name] 58 | 59 | self.config = getattr(importlib.import_module(config_module_name), 'config') 60 | 61 | architecture_module = self.json_info['ModelConfiguration']['Architecture'] 62 | if not os.path.isabs(architecture_module): 63 | architecture_module = os.path.abspath(os.path.join(os.path.dirname(model), architecture_module)) 64 | 65 | sys.path.append(os.path.dirname(architecture_module)) 66 | architecture_module_name = os.path.basename(architecture_module) 67 | 68 | if (architecture_module_name != config_module_name) and (architecture_module_name in sys.modules): 69 | del sys.modules[architecture_module_name] 70 | 71 | self.model = getattr(importlib.import_module(architecture_module_name), 'model') 72 | 73 | self.model.load_weights(model_path, by_name=True) 74 | 75 | self.graph = tf.get_default_graph() 76 | 77 | def getParameterInfo(self, required_parameters): 78 | return required_parameters 79 | 80 | def getConfiguration(self, **scalars): 81 | self.padding = int(scalars['padding']) 82 | 83 | return { 84 | 'extractBands': tuple(self.json_info['ExtractBands']), 85 | 'padding': int(scalars['padding']), 86 | 'tx': self.json_info['ImageWidth'] - 2 * self.padding, 87 | 'ty': self.json_info['ImageHeight'] - 2 * self.padding 88 | } 89 | 90 | class ChildImageClassifier(MatterMaskRCNN): 91 | def updatePixels(self, tlc, shape, props, **pixelBlocks): 92 | image = pixelBlocks['raster_pixels'] 93 | _, height, width = image.shape 94 | image = np.transpose(image, [1,2,0]) 95 | 96 | with self.graph.as_default(): 97 | results = self.model.detect([image], verbose=1) 98 | 99 | masks = results[0]['masks'] 100 | class_ids = results[0]['class_ids'] 101 | output_raster = np.zeros((masks.shape[0], masks.shape[1], 1), dtype=props['pixelType']) 102 | mask_count = masks.shape[-1] 103 | for i in range(mask_count): 104 | mask = masks[:, :, i] 105 | output_raster[np.where(mask==True)] = class_ids[i] 106 | 107 | return np.transpose(output_raster, [2,0,1]) 108 | 109 | 110 | class ChildObjectDetector(MatterMaskRCNN): 111 | def vectorize(self, **pixelBlocks): 112 | image = pixelBlocks['raster_pixels'] 113 | _, height, width = image.shape 114 | image = np.transpose(image, [1,2,0]) 115 | 116 | with self.graph.as_default(): 117 | results = self.model.detect([image], verbose=1) 118 | 119 | masks = results[0]['masks'] 120 | mask_count = masks.shape[-1] 121 | coord_list = [] 122 | for m in range(mask_count): 123 | mask = masks[:, :, m] 124 | padded_mask = np.zeros((mask.shape[0]+2, mask.shape[1]+2), dtype=np.uint8) 125 | padded_mask[1:-1, 1:-1] = mask 126 | contours = find_contours(padded_mask, 0.5, fully_connected='high') 127 | 128 | if len(contours) != 0: 129 | verts = contours[0] - 1 130 | coord_list.append(verts) 131 | 132 | if self.padding != 0: 133 | coord_list[:] = [item - self.padding for item in coord_list] 134 | 135 | return coord_list, results[0]['scores'], results[0]['class_ids'] -------------------------------------------------------------------------------- /python_raster_functions/PyTorch/FeatureClassifier.py: -------------------------------------------------------------------------------- 1 | from __future__ import division 2 | import os 3 | import sys 4 | import json 5 | import warnings 6 | from fastai.vision import * 7 | from torchvision import models as torchvision_models 8 | import arcgis 9 | from arcgis.learn import FeatureClassifier 10 | import arcpy 11 | import torch 12 | from fastai.metrics import accuracy 13 | import tempfile 14 | from pathlib import Path 15 | prf_root_dir = os.path.join(os.path.dirname(__file__), os.pardir) 16 | sys.path.append(prf_root_dir) 17 | import numpy as np 18 | 19 | imagenet_stats = ([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) 20 | 21 | imagenet_mean = 255 * np.array(imagenet_stats[0], dtype=np.float32) 22 | imagenet_std = 255 * np.array(imagenet_stats[1], dtype=np.float32) 23 | 24 | def norm(x, mean=imagenet_mean, std=imagenet_std): 25 | return (x - mean)/std 26 | 27 | def denorm(x, mean=imagenet_mean, std=imagenet_std): 28 | return x * std + mean 29 | 30 | class ChildObjectDetector: 31 | 32 | def initialize(self, model, model_as_file): 33 | if model_as_file: 34 | with open(model, 'r') as f: 35 | self.emd = json.load(f) 36 | else: 37 | self.emd = json.loads(model) 38 | 39 | if arcpy.env.processorType == "GPU" and torch.cuda.is_available(): 40 | self.device = torch.device('cuda') 41 | arcgis.env._processorType = "GPU" 42 | else: 43 | self.device = torch.device('cpu') 44 | arcgis.env._processorType = "CPU" 45 | 46 | # Using arcgis.learn FeatureClassifer from_model function. 47 | self.cf = FeatureClassifier.from_model(emd_path=model) 48 | self.model = self.cf.learn.model 49 | self.model.eval() 50 | 51 | def getParameterInfo(self, required_parameters): 52 | return required_parameters 53 | 54 | def getConfiguration(self, **scalars): 55 | 56 | if 'BatchSize' not in self.emd and 'batch_size' not in scalars: 57 | self.batch_size = 1 58 | elif 'BatchSize' not in self.emd and 'batch_size' in scalars: 59 | self.batch_size = int(scalars['batch_size']) 60 | else: 61 | self.batch_size = int(self.emd['BatchSize']) 62 | 63 | return { 64 | # CropSizeFixed is a boolean value parameter (1 or 0) in the emd file, representing whether the size of 65 | # tile cropped around the feature is fixed or not. 66 | # 1 -- fixed tile size, crop fixed size tiles centered on the feature. The tile can be bigger or smaller 67 | # than the feature; 68 | # 0 -- Variable tile size, crop out the feature using the smallest fitting rectangle. This results in tiles 69 | # of varying size, both in x and y. the ImageWidth and ImageHeight in the emd file are still passed and used 70 | # as a maximum size. If the feature is bigger than the defined ImageWidth/ImageHeight, the tiles are cropped 71 | # the same way as in the fixed tile size option using the maximum size. 72 | 'CropSizeFixed': int(self.emd['CropSizeFixed']), 73 | 74 | # BlackenAroundFeature is a boolean value paramater (1 or 0) in the emd file, representing whether blacken 75 | # the pixels outside the feature in each image tile. 76 | # 1 -- Blacken 77 | # 0 -- Not blacken 78 | 'BlackenAroundFeature': int(self.emd['BlackenAroundFeature']), 79 | 80 | 'extractBands': tuple(self.emd['ExtractBands']), 81 | 'tx': self.emd['ImageWidth'], 82 | 'ty': self.emd['ImageHeight'], 83 | 'batch_size': self.batch_size 84 | } 85 | 86 | def vectorize(self, **pixelBlocks): 87 | 88 | # Get pixel blocks - tuple of 3-d rasters: ([bands,height,width],[bands,height.width],...) 89 | # Convert tuple to 4-d numpy array 90 | batch_images = np.asarray(pixelBlocks['rasters_pixels']) 91 | 92 | # Get the shape of the 4-d numpy array 93 | batch, bands, height, width = batch_images.shape 94 | 95 | # Transpose the image dimensions to [batch, height, width, bands] 96 | batch_images = np.transpose(batch_images, [0, 2, 3, 1]) 97 | 98 | rings = [] 99 | labels, confidences = [], [] 100 | 101 | # Convert to torch tensor and transpose the dimensions to [batch, bands, height, width] 102 | batch_images = torch.tensor(norm(batch_images).transpose(0, 3, 1, 2)).to(self.device) 103 | 104 | # the second element in the passed tuple is hardcoded to make fastai's pred_batch work 105 | predictions = self.cf.learn.pred_batch(batch=(batch_images, torch.tensor([40]).to(self.device))) 106 | 107 | # torch.max returns the max value and the index of the max as a tuple 108 | confidences, class_idxs = torch.max(predictions, dim=1) 109 | 110 | # Using emd to map the class 111 | class_map = [c['Name'] for c in self.emd["Classes"]] 112 | labels = [class_map[c] for c in class_idxs] 113 | 114 | # Appending this ring for all the features in the batch 115 | rings = [[[[0, 0], [0, width - 1], [height - 1, width - 1], [height - 1, 0]]] for i in range(self.batch_size)] 116 | 117 | return rings, confidences.tolist(), labels 118 | -------------------------------------------------------------------------------- /python_raster_functions/CNTK/utils/annotations/annotations_helper.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Microsoft. All rights reserved. 2 | 3 | # Licensed under the MIT license. See LICENSE.md file in the project root 4 | # for full license information. 5 | # ============================================================================== 6 | 7 | import numpy as np 8 | import os 9 | 10 | def _getFilesInDirectory(directory, postfix = ""): 11 | fileNames = [s for s in os.listdir(directory) if not os.path.isdir(os.path.join(directory, s))] 12 | if not postfix or postfix == "": 13 | return fileNames 14 | else: 15 | return [s for s in fileNames if s.lower().endswith(postfix)] 16 | 17 | def _get_image_paths(img_dir, training_set): 18 | if training_set: 19 | subDirs = ['positive', 'negative'] 20 | else: 21 | subDirs = ['testImages'] 22 | 23 | image_paths = [] 24 | for subdir in subDirs: 25 | sub_dir_path = os.path.join(img_dir, subdir) 26 | imgFilenames = _getFilesInDirectory(sub_dir_path, ".jpg") 27 | for img in imgFilenames: 28 | image_paths.append("{}/{}".format(subdir, img)) 29 | 30 | return image_paths 31 | 32 | def _removeLineEndCharacters(line): 33 | if line.endswith(b'\r\n'): 34 | return line[:-2] 35 | elif line.endswith(b'\n'): 36 | return line[:-1] 37 | else: 38 | return line 39 | 40 | def _load_annotation(imgPath, class_dict): 41 | bboxesPaths = imgPath[:-4] + ".bboxes.tsv" 42 | labelsPaths = imgPath[:-4] + ".bboxes.labels.tsv" 43 | # if no ground truth annotations are available, return None 44 | if not os.path.exists(bboxesPaths) or not os.path.exists(labelsPaths): 45 | return None 46 | bboxes = np.loadtxt(bboxesPaths, np.float32) 47 | 48 | # in case there's only one annotation and numpy read the array as single array, 49 | # we need to make sure the input is treated as a multi dimensional array instead of a list/ 1D array 50 | if len(bboxes.shape) == 1: 51 | bboxes = np.array([bboxes]) 52 | 53 | with open(labelsPaths, 'rb') as f: 54 | lines = f.readlines() 55 | labels = [_removeLineEndCharacters(s) for s in lines] 56 | 57 | label_idxs = np.asarray([class_dict[l.decode('utf-8')] for l in labels]) 58 | label_idxs.shape = label_idxs.shape + (1,) 59 | annotations = np.hstack((bboxes, label_idxs)) 60 | 61 | return annotations 62 | 63 | def create_map_files(data_folder, class_dict, training_set): 64 | # get relative paths for map files 65 | img_file_paths = _get_image_paths(data_folder, training_set) 66 | 67 | out_map_file_path = os.path.join(data_folder, "{}_img_file.txt".format("train" if training_set else "test")) 68 | roi_file_path = os.path.join(data_folder, "{}_roi_file.txt".format("train" if training_set else "test")) 69 | 70 | counter = 0 71 | with open(out_map_file_path, 'w') as img_file: 72 | with open(roi_file_path, 'w') as roi_file: 73 | for img_path in img_file_paths: 74 | abs_img_path = os.path.join(data_folder, img_path) 75 | gt_annotations = _load_annotation(abs_img_path, class_dict) 76 | if gt_annotations is None: 77 | continue 78 | 79 | img_line = "{}\t{}\t0\n".format(counter, img_path) 80 | img_file.write(img_line) 81 | 82 | roi_line = "{} |roiAndLabel".format(counter) 83 | for val in gt_annotations.flatten(): 84 | roi_line += " {}".format(val) 85 | 86 | roi_file.write(roi_line + "\n") 87 | counter += 1 88 | if counter % 500 == 0: 89 | print("Processed {} images".format(counter)) 90 | 91 | def create_class_dict(data_folder): 92 | # get relative paths for map files 93 | img_file_paths = _get_image_paths(data_folder, True) 94 | train_classes = ["__background__"] 95 | 96 | for img_path in img_file_paths: 97 | abs_img_path = os.path.join(data_folder, img_path) 98 | labelsPaths = abs_img_path[:-4] + ".bboxes.labels.tsv" 99 | if not os.path.exists(labelsPaths): 100 | continue 101 | with open(labelsPaths, 'rb') as f: 102 | lines = f.readlines() 103 | labels = [_removeLineEndCharacters(s).decode('utf-8') for s in lines] 104 | 105 | for label in labels: 106 | if not label in train_classes: 107 | train_classes.append(label) 108 | 109 | class_dict = {k: v for v, k in enumerate(train_classes)} 110 | class_list = [None]*len(class_dict) 111 | for k in class_dict: 112 | class_list[class_dict[k]]=k 113 | class_map_file_path = os.path.join(data_folder, "class_map.txt") 114 | with open(class_map_file_path, 'w') as class_map_file: 115 | for i in range(len(class_list)): 116 | class_map_file.write("{}\t{}\n".format(class_list[i], i)) 117 | 118 | return class_dict 119 | 120 | def parse_class_map_file(class_map_file): 121 | with open(class_map_file, "r") as f: 122 | lines = f.readlines() 123 | class_list = [None]*len(lines) 124 | for line in lines: 125 | tab_pos = line.find('\t') 126 | class_name = line[:tab_pos] 127 | class_id = int(line[tab_pos+1:-1]) 128 | class_list[class_id] = class_name 129 | 130 | return class_list 131 | 132 | if __name__ == '__main__': 133 | abs_path = os.path.dirname(os.path.abspath(__file__)) 134 | data_set_path = os.path.join(abs_path, "../../../DataSets/Trees") 135 | 136 | class_dict = create_class_dict(data_set_path) 137 | create_map_files(data_set_path, class_dict, training_set=True) 138 | create_map_files(data_set_path, class_dict, training_set=False) 139 | -------------------------------------------------------------------------------- /python_raster_functions/CNTK/utils/od_mb_source.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Microsoft. All rights reserved. 2 | 3 | # Licensed under the MIT license. See LICENSE.md file in the project root 4 | # for full license information. 5 | # ============================================================================== 6 | 7 | from cntk.io import UserMinibatchSource, StreamInformation, MinibatchData 8 | from cntk.core import Value 9 | from utils.od_reader import ObjectDetectionReader 10 | import numpy as np 11 | 12 | class ObjectDetectionMinibatchSource(UserMinibatchSource): 13 | def __init__(self, img_map_file, roi_map_file, num_classes, 14 | max_annotations_per_image, pad_width, pad_height, pad_value, 15 | randomize, use_flipping, proposal_provider, proposal_iou_threshold=0.5, 16 | provide_targets=False, normalize_means=None, normalize_stds=None, max_images=None): 17 | 18 | self.image_si = StreamInformation("image", 0, 'dense', np.float32, (3, pad_height, pad_width,)) 19 | self.roi_si = StreamInformation("annotation", 1, 'dense', np.float32, (max_annotations_per_image, 5,)) 20 | self.dims_si = StreamInformation("dims", 1, 'dense', np.float32, (4,)) 21 | 22 | if proposal_provider is not None: 23 | num_proposals = proposal_provider.num_proposals() 24 | self.proposals_si = StreamInformation("proposals", 1, 'dense', np.float32, (num_proposals, 4)) 25 | self.label_targets_si = StreamInformation("label_targets", 1, 'dense', np.float32, (num_proposals, num_classes)) 26 | self.bbox_targets_si = StreamInformation("bbox_targets", 1, 'dense', np.float32, (num_proposals, num_classes*4)) 27 | self.bbiw_si = StreamInformation("bbiw", 1, 'dense', np.float32, (num_proposals, num_classes*4)) 28 | else: 29 | self.proposals_si = None 30 | 31 | self.od_reader = ObjectDetectionReader(img_map_file, roi_map_file, num_classes, 32 | max_annotations_per_image, pad_width, pad_height, pad_value, 33 | randomize, use_flipping, proposal_provider, proposal_iou_threshold, 34 | provide_targets, normalize_means, normalize_stds, max_images) 35 | 36 | super(ObjectDetectionMinibatchSource, self).__init__() 37 | 38 | def stream_infos(self): 39 | if self.proposals_si is None: 40 | return [self.image_si, self.roi_si, self.dims_si] 41 | else: 42 | return [self.image_si, self.roi_si, self.dims_si, self.proposals_si, self.label_targets_si, self.bbox_targets_si, self.bbiw_si] 43 | 44 | def image_si(self): 45 | return self.image_si 46 | 47 | def roi_si(self): 48 | return self.roi_si 49 | 50 | def dims_si(self): 51 | return self.dims_si 52 | 53 | def proposals_si(self): 54 | return self.proposals_si 55 | 56 | def label_targets_si(self): 57 | return self.label_targets_si 58 | 59 | def bbox_targets_si(self): 60 | return self.bbox_targets_si 61 | 62 | def bbiw_si(self): 63 | return self.bbiw_si 64 | 65 | def next_minibatch(self, num_samples, number_of_workers=1, worker_rank=1, device=None, input_map=None): 66 | if num_samples > 1: 67 | print("Only single item mini batches are supported currently by od_mb_source.py") 68 | exit(1) 69 | 70 | img_data, roi_data, img_dims, proposals, label_targets, bbox_targets, bbox_inside_weights = self.od_reader.get_next_input() 71 | sweep_end = self.od_reader.sweep_end() 72 | 73 | if input_map is None: 74 | result = { 75 | self.image_si: MinibatchData(Value(batch=img_data), 1, 1, sweep_end), 76 | self.roi_si: MinibatchData(Value(batch=roi_data), 1, 1, sweep_end), 77 | self.dims_si: MinibatchData(Value(batch=np.asarray(img_dims, dtype=np.float32)), 1, 1, sweep_end), 78 | self.proposals_si: MinibatchData(Value(batch=np.asarray(proposals, dtype=np.float32)), 1, 1, sweep_end), 79 | self.label_targets_si: MinibatchData(Value(batch=np.asarray(label_targets, dtype=np.float32)), 1, 1, sweep_end), 80 | self.bbox_targets_si: MinibatchData(Value(batch=np.asarray(bbox_targets, dtype=np.float32)), 1, 1, sweep_end), 81 | self.bbiw_si: MinibatchData(Value(batch=np.asarray(bbox_inside_weights, dtype=np.float32)), 1, 1, sweep_end), 82 | } 83 | else: 84 | result = { 85 | input_map[self.image_si]: MinibatchData(Value(batch=np.asarray(img_data, dtype=np.float32)), 1, 1, sweep_end) 86 | } 87 | if self.roi_si in input_map: 88 | result[input_map[self.roi_si]] = MinibatchData(Value(batch=np.asarray(roi_data, dtype=np.float32)), 1, 1, sweep_end) 89 | if self.dims_si in input_map: 90 | result[input_map[self.dims_si]] = MinibatchData(Value(batch=np.asarray(img_dims, dtype=np.float32)), 1, 1, sweep_end) 91 | if self.proposals_si in input_map: 92 | result[input_map[self.proposals_si]] = MinibatchData(Value(batch=np.asarray(proposals, dtype=np.float32)), 1, 1, sweep_end) 93 | if self.label_targets_si in input_map: 94 | result[input_map[self.label_targets_si]] = MinibatchData(Value(batch=np.asarray(label_targets, dtype=np.float32)), 1, 1, sweep_end) 95 | if self.bbox_targets_si in input_map: 96 | result[input_map[self.bbox_targets_si]] = MinibatchData(Value(batch=np.asarray(bbox_targets, dtype=np.float32)), 1, 1, sweep_end) 97 | if self.bbiw_si in input_map: 98 | result[input_map[self.bbiw_si]] = MinibatchData(Value(batch=np.asarray(bbox_inside_weights, dtype=np.float32)), 1, 1, sweep_end) 99 | 100 | return result 101 | -------------------------------------------------------------------------------- /python_raster_functions/TensorFlow/ObjectDetectionAPI.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Copyright 2018 Esri 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | 6 | you may not use this file except in compliance with the License. 7 | 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | 16 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | 18 | See the License for the specific language governing permissions and 19 | 20 | limitations under the License.​ 21 | ''' 22 | 23 | import os 24 | import sys 25 | 26 | import numpy as np 27 | import tensorflow as tf 28 | 29 | prf_root_dir = os.path.join(os.path.dirname(__file__), os.pardir) 30 | sys.path.append(prf_root_dir) 31 | from Templates.TemplateBaseDetector import TemplateBaseDetector 32 | 33 | class ChildObjectDetector(TemplateBaseDetector): 34 | def load_model(self, model_path): 35 | ''' 36 | Fill this method to write your own model loading python code 37 | save it self object if you would like to reference it later. 38 | 39 | Tips: you can access emd information through self.json_info. 40 | 41 | TensorFlow example to import graph def from frozen pb file: 42 | self.detection_graph = tf.Graph() 43 | with self.detection_graph.as_default(): 44 | od_graph_def = tf.GraphDef() 45 | with tf.gfile.GFile(model_path, 'rb') as fid: 46 | serialized_graph = fid.read() 47 | od_graph_def.ParseFromString(serialized_graph) 48 | tf.import_graph_def(od_graph_def, name='') 49 | ''' 50 | # Todo: fill in this method to load your model 51 | self.detection_graph = tf.Graph() 52 | with self.detection_graph.as_default(): 53 | od_graph_def = tf.GraphDef() 54 | with tf.gfile.GFile(model_path, 'rb') as fid: 55 | serialized_graph = fid.read() 56 | od_graph_def.ParseFromString(serialized_graph) 57 | tf.import_graph_def(od_graph_def, name='') 58 | 59 | def getParameterInfo(self, required_parameters): 60 | required_parameters.extend( 61 | [ 62 | # Todo: add your inference parameters here 63 | # https://github.com/Esri/raster-functions/wiki/PythonRasterFunction#getparameterinfo 64 | ] 65 | ) 66 | return required_parameters 67 | 68 | def inference(self, batch, **scalars): 69 | ''' 70 | Fill this method to write your own inference python code, you can refer to the model instance that is created 71 | in the load_model method. Expected results format is described in the returns as below. 72 | 73 | :param batch: numpy array with shape (B, H, W, D), B is batch size, H, W is specified and equal to 74 | ImageHeight and ImageWidth in the emd file and D is the number of bands and equal to the length 75 | of ExtractBands in the emd. If BatchInference is set to False in emd, B is constant 1. 76 | :param scalars: inference parameters, accessed by the parameter name, 77 | i.e. score_threshold=float(kwargs['score_threshold']). If you want to have more inference 78 | parameters, add it to the list of the following getParameterInfo method. 79 | :return: bounding boxes, python list representing bounding boxes whose length is equal to B, each element is 80 | [N,4] numpy array representing [ymin, xmin, ymax, xmax] with respect to the upper left 81 | corner of the image tile. 82 | scores, python list representing the score of each bounding box whose length is equal to B, each element 83 | is [N,] numpy array 84 | classes, python list representing the class of each bounding box whose length is equal to B, each element 85 | is [N,] numpy array and its dype is np.uint8 86 | ''' 87 | #Todo: fill in this method to inference your model and return bounding boxes, scores and classes 88 | score_threshold = float(scalars['score_threshold']) 89 | 90 | config = tf.ConfigProto() 91 | if 'PerProcessGPUMemoryFraction' in self.json_info: 92 | config.gpu_options.per_process_gpu_memory_fraction = float(self.json_info['PerProcessGPUMemoryFraction']) 93 | 94 | batch = np.transpose(batch, (0, 2, 3, 1)) 95 | with self.detection_graph.as_default(): 96 | with tf.Session(config=config) as sess: 97 | feed_dict = { 98 | 'image_tensor:0': batch 99 | } 100 | fetches = { 101 | 'boundingboxes': 'detection_boxes:0', 102 | 'scores': 'detection_scores:0', 103 | 'classes': 'detection_classes:0' 104 | } 105 | 106 | output_dict = sess.run(fetches, feed_dict=feed_dict) 107 | 108 | bounding_boxes = output_dict['boundingboxes'] 109 | scores = output_dict['scores'] 110 | classes = output_dict['classes'] 111 | 112 | bounding_boxes[:, :, [0, 2]] = bounding_boxes[:, :, [0, 2]] * self.json_info['ImageHeight'] 113 | bounding_boxes[:, :, [1, 3]] = bounding_boxes[:, :, [1, 3]] * self.json_info['ImageWidth'] 114 | 115 | batch_bounding_boxes, batch_scores, batch_classes = [], [], [] 116 | 117 | batch_size = bounding_boxes.shape[0] 118 | for batch_idx in range(batch_size): 119 | keep_indices = np.where(scores[batch_idx] > score_threshold) 120 | batch_bounding_boxes.append(bounding_boxes[batch_idx][keep_indices]) 121 | batch_scores.append(scores[batch_idx][keep_indices]) 122 | batch_classes.append(classes[batch_idx][keep_indices]) 123 | 124 | return batch_bounding_boxes, batch_scores, batch_classes 125 | -------------------------------------------------------------------------------- /python_raster_functions/CNTK/FasterRCNN.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Copyright 2018 Esri 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | 6 | you may not use this file except in compliance with the License. 7 | 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | 16 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | 18 | See the License for the specific language governing permissions and 19 | 20 | limitations under the License.​ 21 | ''' 22 | 23 | import os 24 | import sys 25 | 26 | import cntk as C 27 | import numpy as np 28 | 29 | sys.path.append(os.path.dirname(__file__)) 30 | from utils.rpn.bbox_transform import regress_rois 31 | from utils.nms_wrapper import apply_nms_to_single_image_results 32 | 33 | prf_root_dir = os.path.join(os.path.dirname(__file__), os.pardir) 34 | sys.path.append(prf_root_dir) 35 | from Templates.TemplateBaseDetector import TemplateBaseDetector 36 | 37 | class ChildObjectDetector(TemplateBaseDetector): 38 | def load_model(self, model_path): 39 | ''' 40 | Fill this method to write your own model loading python code 41 | save it self object if you would like to reference it later. 42 | 43 | Tips: you can access emd information through self.json_info. 44 | ''' 45 | #Todo: fill in this method to load your model 46 | self.model = C.load_model(model_path) 47 | 48 | 49 | def getParameterInfo(self, required_parameters): 50 | required_parameters.extend( 51 | [ 52 | # Todo: add your inference parameters here 53 | # https://github.com/Esri/raster-functions/wiki/PythonRasterFunction#getparameterinfo 54 | { 55 | 'name': 'nms_threshold', 56 | 'dataType': 'numeric', 57 | 'value': 0.2, 58 | 'required': True, 59 | 'displayName': 'nms threshold', 60 | 'description': 'non maximum suppression(nms) threshold' 61 | }, 62 | ] 63 | ) 64 | required_parameters[:] = [d for d in required_parameters if not d.get('name') == 'batch_size'] 65 | 66 | return required_parameters 67 | 68 | def inference(self, batch, **scalars): 69 | ''' 70 | Fill this method to write your own inference python code, you can refer to the model instance that is created 71 | in the load_model method. Expected results format is described in the returns as below. 72 | 73 | :param batch: numpy array with shape (B, H, W, D), B is batch size, H, W is specified and equal to 74 | ImageHeight and ImageWidth in the emd file and D is the number of bands and equal to the length 75 | of ExtractBands in the emd. If BatchInference is set to False in emd, B is constant 1. 76 | :param scalars: inference parameters, accessed by the parameter name, 77 | i.e. score_threshold=float(scalars['score_threshold']). If you want to have more inference 78 | parameters, add it to the list of the following getParameterInfo method. 79 | :return: bounding boxes, python list representing bounding boxes whose length is equal to B, each element is 80 | [N,4] numpy array representing [ymin, xmin, ymax, xmax] with respect to the upper left 81 | corner of the image tile. 82 | scores, python list representing the score of each bounding box whose length is equal to B, each element 83 | is [N,] numpy array 84 | classes, python list representing the class of each bounding box whose length is equal to B, each element 85 | is [N,] numpy array and its dype is np.uint8 86 | ''' 87 | # Todo: fill in this method to inference your model and return bounding boxes, scores and classes 88 | nms_threshold = float(scalars['nms_threshold']) 89 | 90 | batch = batch.astype(np.float32) 91 | batch_size, bands, height, width = batch.shape 92 | 93 | dims = np.array([width, height, 94 | width, height, 95 | width, height], np.float32) 96 | output = self.model.eval( 97 | { 98 | self.model.arguments[0]: batch, 99 | self.model.arguments[1]: [dims] 100 | } 101 | ) 102 | out_dict = dict([(k.name, k) for k in output]) 103 | 104 | out_cls_pred = output[out_dict['cls_pred']][0] 105 | out_rpn_rois = output[out_dict['rpn_rois']][0] 106 | out_bbox_regr = output[out_dict['bbox_regr']][0] 107 | 108 | labels = out_cls_pred.argmax(axis=1) 109 | scores = out_cls_pred.max(axis=1) 110 | regressed_rois = regress_rois(out_rpn_rois, out_bbox_regr, labels, dims) 111 | 112 | nmsKeepIndices = apply_nms_to_single_image_results(regressed_rois, labels, scores, 113 | False, 114 | 0, 115 | nms_threshold=nms_threshold, 116 | conf_threshold=self.score_threshold) 117 | 118 | filtered_bboxes = regressed_rois[nmsKeepIndices] 119 | filtered_scores = scores[nmsKeepIndices] 120 | filtered_labels = labels[nmsKeepIndices] 121 | 122 | positive_label_indices = np.where(filtered_labels > 0) 123 | filtered_bboxes = filtered_bboxes[positive_label_indices] 124 | filtered_scores = filtered_scores[positive_label_indices] 125 | filtered_labels = filtered_labels[positive_label_indices] 126 | 127 | filtered_bboxes = filtered_bboxes[:, [1, 0, 3, 2]] 128 | 129 | return [filtered_bboxes], [filtered_scores], [filtered_labels] 130 | -------------------------------------------------------------------------------- /python_raster_functions/Templates/ObjectDetectorTemplate.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Copyright 2018 Esri 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | 6 | you may not use this file except in compliance with the License. 7 | 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | 16 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | 18 | See the License for the specific language governing permissions and 19 | 20 | limitations under the License.​ 21 | ''' 22 | 23 | import os 24 | import sys 25 | 26 | prf_root_dir = os.path.join(os.path.dirname(__file__), os.pardir) 27 | sys.path.append(prf_root_dir) 28 | from Templates.TemplateBaseDetector import TemplateBaseDetector 29 | 30 | class ChildObjectDetector(TemplateBaseDetector): 31 | # template method to fill in 32 | def load_model(self, model_path): 33 | ''' 34 | Fill this method to write your own model loading python code 35 | save it self object if you would like to reference it later. 36 | 37 | Tips: you can access emd information through self.json_info. 38 | 39 | TensorFlow example to import graph def from frozen pb file: 40 | self.detection_graph = tf.Graph() 41 | with self.detection_graph.as_default(): 42 | od_graph_def = tf.GraphDef() 43 | with tf.gfile.GFile(model_path, 'rb') as fid: 44 | serialized_graph = fid.read() 45 | od_graph_def.ParseFromString(serialized_graph) 46 | tf.import_graph_def(od_graph_def, name='') 47 | ''' 48 | #Todo: fill in this method to load your model 49 | pass 50 | 51 | def getParameterInfo(self, required_parameters): 52 | required_parameters.extend( 53 | [ 54 | # Todo: add your inference parameters here 55 | # https://github.com/Esri/raster-functions/wiki/PythonRasterFunction#getparameterinfo 56 | ''' 57 | Example: 58 | { 59 | 'name': 'nms_threshold', 60 | 'dataType': 'numeric', 61 | 'value': 0.2, 62 | 'required': True, 63 | 'displayName': 'nms threshold', 64 | 'description': 'non maximum suppression(nms) threshold' 65 | }, 66 | ''' 67 | ] 68 | ) 69 | return required_parameters 70 | 71 | # template method to fill in 72 | def inference(self, batch, **scalars): 73 | ''' 74 | Fill this method to write your own inference python code, you can refer to the model instance that is created 75 | in the load_model method. Expected results format is described in the returns as below. 76 | 77 | :param batch: numpy array with shape (B, H, W, D), B is batch size, H, W is specified and equal to 78 | ImageHeight and ImageWidth in the emd file and D is the number of bands and equal to the length 79 | of ExtractBands in the emd. If BatchInference is set to False in emd, B is constant 1. 80 | :param scalars: inference parameters, accessed by the parameter name, 81 | i.e. score_threshold=float(scalars['score_threshold']). If you want to have more inference 82 | parameters, add it to the list of the following getParameterInfo method. 83 | :return: bounding boxes, python list representing bounding boxes whose length is equal to B, each element is 84 | [N,4] numpy array representing [ymin, xmin, ymax, xmax] with respect to the upper left 85 | corner of the image tile. 86 | scores, python list representing the score of each bounding box whose length is equal to B, each element 87 | is [N,] numpy array 88 | classes, python list representing the class of each bounding box whose length is equal to B, each element 89 | is [N,] numpy array and its dtype is np.uint8 90 | 91 | Tensorflow Example: 92 | score_threshold = float(scalars['score_threshold']) 93 | 94 | config = tf.ConfigProto() 95 | if 'PerProcessGPUMemoryFraction' in self.json_info: 96 | config.gpu_options.per_process_gpu_memory_fraction = float(self.json_info['PerProcessGPUMemoryFraction']) 97 | 98 | with self.detection_graph.as_default(): 99 | with tf.Session(config=config) as sess: 100 | feed_dict = { 101 | 'image_tensor:0': batch 102 | } 103 | fetches = { 104 | 'boundingboxes': 'detection_boxes:0', 105 | 'scores': 'detection_scores:0', 106 | 'classes': 'detection_classes:0' 107 | } 108 | 109 | output_dict = sess.run(fetches, feed_dict=feed_dict) 110 | 111 | bounding_boxes = output_dict['boundingboxes'] 112 | scores = output_dict['scores'] 113 | classes = output_dict['classes'] 114 | 115 | bounding_boxes[:, :, [0, 2]] = bounding_boxes[:, :, [0, 2]] * self.json_info['ImageHeight'] 116 | bounding_boxes[:, :, [1, 3]] = bounding_boxes[:, :, [1, 3]] * self.json_info['ImageWidth'] 117 | 118 | batch_bounding_boxes, batch_scores, batch_classes = [], [], [] 119 | 120 | batch_size = bounding_boxes.shape[0] 121 | for batch_idx in range(batch_size): 122 | keep_indices = np.where(scores[batch_idx]>score_threshold) 123 | batch_bounding_boxes.append(bounding_boxes[batch_idx][keep_indices]) 124 | batch_scores.append(scores[batch_idx][keep_indices]) 125 | batch_classes.append(classes[batch_idx][keep_indices]) 126 | 127 | return batch_bounding_boxes, batch_scores, batch_classes 128 | ''' 129 | #Todo: fill in this method to inference your model and return bounding boxes, scores and classes 130 | pass 131 | -------------------------------------------------------------------------------- /docs/writing_model_definition.md: -------------------------------------------------------------------------------- 1 | # Esri Model Definition file 2 | 3 | The Esri Model Definition file (EMD) is a configuration file in JSON format that provides the parameters to use 4 | for deep learning model inference in ArcGIS. Importantly, the EMD contains a parameter 5 | called "InferenceFunction" which specifies the custom Python raster function you want to use. Therefore, the EMD also 6 | serves as an argument (as file path) to the Python raster function. The EMD includes model properties and metadata 7 | that can be accessed from the Python raster function (for example, the location of the trained model file). 8 | 9 | In the built-in Python raster functions examples, we include the height and width of the training image chips 10 | in the EMD file. Then, the raster functions can access this information through the getConfiguration method 11 | to ensure that ArcGIS delivers the pixel blocks of the correct size. 12 | 13 | ```json 14 | { 15 | "Framework": "TensorFlow", 16 | "ModelConfiguration": "ObjectDetectionAPI", 17 | "ModelFile":".\\frozen_inference_graph.pb", 18 | "ModelType":"ObjectionDetection", 19 | "InferenceFunction":".\\CustomObjectDetector.py", 20 | "ImageHeight":850, 21 | "ImageWidth":850, 22 | "ExtractBands":[0,1,2], 23 | 24 | "Classes" : [ 25 | { 26 | "Value": 0, 27 | "Name": "Tree", 28 | "Color": [0, 255, 0] 29 | } 30 | ], 31 | "ModelName": "ModelNameofYourChoice" 32 | "Version": "2024.9" 33 | } 34 | ``` 35 | ## Keywords supported in built-in Python raster functions: 36 | - "Framework" 37 | 38 | Type: String. 39 | 40 | The name of a deep learning framework used to train your model, e.g. "TensorFlow", "CNTK", "PyTorch", and etc. Use 41 | "Templates" if you use built-in templates and fill in your own python code. 42 | 43 | - "ModelConfiguration" 44 | 45 | Type: String or JSON object. 46 | 47 | The name of a model configuration. A model configuration specifies a model trained by a well-known python implementation. 48 | There are many existing open source deep learning projects that define "standard" inputs and outputs, 49 | and inference logic. ArcGIS built-in Python raster functions support a set of predefined well-known 50 | configurations, and we are continuing to expand the list of supported model configurations. 51 | 52 | The current supported model configurations are listed below along 53 | with a link to their project pages: 54 | 55 | **TensorFlow** 56 | - "ObjectDetectionAPI", https://github.com/tensorflow/models/tree/master/research/object_detection 57 | - "DeepLab", https://github.com/tensorflow/models/tree/master/research/deeplab 58 | 59 | **PyTorch** 60 | - 'FastaiSSD', https://github.com/Esri/arcgis-python-api/tree/master/talks/uc2018/Plenary/pools 61 | 62 | **CNTK** 63 | - "FasterRCNN", https://docs.microsoft.com/en-us/cognitive-toolkit/object-detection-using-faster-r-cnn 64 | - "AzurePixelLevelLandClassification", https://github.com/Azure/pixel_level_land_classification 65 | 66 | **Keras** 67 | - "MaskRCNN", https://github.com/matterport/Mask_RCNN 68 | 69 | In this MaskRCNN configuration case, you also need to specify the "Architecture" python module where a python 70 | MaskRCNN object with name "model" should be declared and also the "Config" python module where a python 71 | configuration object with name "conig" is declared. See [Mask RCNN example](../examples/keras/mask_rcnn/README.md) 72 | for more details. 73 | ```json 74 | "ModelConfiguration": { 75 | "Name":"MaskRCNN", 76 | "Architecture":".\\mrcnn\\spacenet", 77 | "Config":".\\mrcnn\\spacenet" 78 | }, 79 | ``` 80 | 81 | - "ModelFile" 82 | 83 | Type: String. 84 | 85 | The path of a trained deep learning model file. 86 | 87 | - "ImageHeight" 88 | 89 | Type: Int. 90 | 91 | The image height of the deep learning model. 92 | 93 | - "ImageWidth" 94 | 95 | Type: Int. 96 | 97 | The image width of the deep learning model. 98 | 99 | - "ExtractBands" 100 | 101 | Type: Array of ints or strings. 102 | 103 | The array of band indexes or band names to extract from the input raster/imagery. 104 | 105 | - "DataRange" 106 | 107 | Type: Array of two elements. 108 | 109 | This represents the minimum and maximum value the deep learning model expects. ArcGIS will 110 | rescale the input raster tile to this data range if this is defined by the actual range of the input raster. 111 | 112 | - "BatchSize" 113 | 114 | Type: Integer. 115 | 116 | Specify the batch size you want to inference the model. If missing, a parameter batch_size is added to the python 117 | raster function so the batch size is specified every time the tool runs. 118 | 119 | - "ModelPadding" 120 | 121 | Type: Integer. 122 | 123 | This is only used in pixel-level classification. If the model has a padding itself, for example, if a model outputs 124 | the center 128x128 segmentation pixels given an input tile 256x256, the model has a padding of 64. 125 | 126 | - "ModelType" 127 | 128 | Type: String. 129 | 130 | The type of the model: "ImageClassification" for classifying pixels, and "ObjectDetection" for detecting objects. 131 | 132 | - "InferenceFunction" 133 | 134 | Type: String. 135 | 136 | The path of a custom inference Python raster function. If not specified, the default built-in Python raster function 137 | will be used. 138 | 139 | - Templates 140 | 141 | --"ImageClassifierTemplate" If you fill in the image classifier template with your own python code. 142 | 143 | --"ObjectDetectorTemplate" If your fill in the object detector template with your own python code. 144 | 145 | Deep learning Python raster function templates are also provided to help you writing your custom deep learning 146 | Python raster function. 147 | 148 | - ModelName 149 | 150 | Type: String. 151 | 152 | Optional. Example format: "ModelNameofYourChoice". This is the model name of user's choice. 153 | 154 | - Version 155 | 156 | Type: String. 157 | 158 | Optional. Example format: "YYYY.MM". This is the model's version by year and month. 159 | 160 | ## EMD for custom Python raster functions 161 | If you find that the sample model definition files and built-in Python raster function cannot describe your deep learning 162 | model architecture/properties, or if you use a deep learning framework other than TensorFlow, Keras, CNTK, or PyTorch, you 163 | can write your own deep learning Python raster function and reference the Python raster function in the EMD file next 164 | to the "InferenceFunction" parameter. The built-in Python raster functions are good references for formatting your custom 165 | deep learning Python raster functions. 166 | 167 | 168 | -------------------------------------------------------------------------------- /python_raster_functions/ImageClassifier.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Copyright 2018 Esri 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | 6 | you may not use this file except in compliance with the License. 7 | 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | 16 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | 18 | See the License for the specific language governing permissions and 19 | 20 | limitations under the License.​ 21 | ''' 22 | 23 | import importlib 24 | import json 25 | import os 26 | import sys 27 | 28 | import numpy as np 29 | 30 | sys.path.append(os.path.dirname(__file__)) 31 | from attribute_table import attribute_table 32 | import prf_utils 33 | 34 | class ImageClassifier: 35 | def __init__(self): 36 | self.name = 'Image Classifier' 37 | self.description = 'Image classification python raster function to inference a tensorflow ' \ 38 | 'deep learning model' 39 | 40 | def initialize(self, **kwargs): 41 | if 'model' not in kwargs: 42 | return 43 | 44 | model = kwargs['model'] 45 | model_as_file = True 46 | try: 47 | with open(model, 'r') as f: 48 | self.json_info = json.load(f) 49 | except FileNotFoundError: 50 | try: 51 | self.json_info = json.loads(model) 52 | model_as_file = False 53 | except json.decoder.JSONDecodeError: 54 | raise Exception("Invalid model argument") 55 | 56 | if 'device' in kwargs: 57 | device = kwargs['device'] 58 | if device < -1: 59 | os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" 60 | device = prf_utils.get_available_device() 61 | os.environ['CUDA_VISIBLE_DEVICES'] = str(device) 62 | else: 63 | os.environ['CUDA_VISIBLE_DEVICES'] = '-1' 64 | 65 | sys.path.append(os.path.dirname(__file__)) 66 | framework = self.json_info['Framework'] 67 | if 'ModelConfiguration' in self.json_info: 68 | if isinstance(self.json_info['ModelConfiguration'], str): 69 | ChildImageClassifier = getattr(importlib.import_module( 70 | '{}.{}'.format(framework, self.json_info['ModelConfiguration'])), 'ChildImageClassifier') 71 | else: 72 | ChildImageClassifier = getattr(importlib.import_module( 73 | '{}.{}'.format(framework, self.json_info['ModelConfiguration']['Name'])), 'ChildImageClassifier') 74 | else: 75 | raise Exception("Invalid model configuration") 76 | 77 | self.child_image_classifier = ChildImageClassifier() 78 | self.child_image_classifier.initialize(model, model_as_file) 79 | 80 | def getParameterInfo(self): 81 | required_parameters = [ 82 | { 83 | 'name': 'raster', 84 | 'dataType': 'raster', 85 | 'required': True, 86 | 'displayName': 'Raster', 87 | 'description': 'Input Raster' 88 | }, 89 | { 90 | 'name': 'model', 91 | 'dataType': 'string', 92 | 'required': True, 93 | 'displayName': 'Input Model Definition (EMD) File', 94 | 'description': 'Input model definition (EMD) JSON file' 95 | }, 96 | { 97 | 'name': 'device', 98 | 'dataType': 'numeric', 99 | 'required': False, 100 | 'displayName': 'Device ID', 101 | 'description': 'Device ID' 102 | }, 103 | ] 104 | 105 | if 'ModelPadding' not in self.json_info: 106 | required_parameters.append( 107 | { 108 | 'name': 'padding', 109 | 'dataType': 'numeric', 110 | 'value': 0, 111 | 'required': False, 112 | 'displayName': 'Padding', 113 | 'description': 'Padding' 114 | }, 115 | ) 116 | 117 | if 'BatchSize' not in self.json_info: 118 | required_parameters.append( 119 | { 120 | 'name': 'batch_size', 121 | 'dataType': 'numeric', 122 | 'required': False, 123 | 'value': 1, 124 | 'displayName': 'Batch Size', 125 | 'description': 'Batch Size' 126 | }, 127 | ) 128 | 129 | return self.child_image_classifier.getParameterInfo(required_parameters) 130 | 131 | def getConfiguration(self, **scalars): 132 | configuration = self.child_image_classifier.getConfiguration(**scalars) 133 | if 'DataRange' in self.json_info: 134 | configuration['dataRange'] = tuple(self.json_info['DataRange']) 135 | configuration['inheritProperties'] = 2|4|8 136 | configuration['inputMask'] = True 137 | return configuration 138 | 139 | def updateRasterInfo(self, **kwargs): 140 | kwargs['output_info']['bandCount'] = 1 141 | #todo: type is determined by the value range of classes in the json file 142 | kwargs['output_info']['pixelType'] = 'i4' 143 | class_info = self.json_info['Classes'] 144 | attribute_table['features'] = [] 145 | for i, c in enumerate(class_info): 146 | attribute_table['features'].append( 147 | { 148 | 'attributes':{ 149 | 'OID':i+1, 150 | 'Value':c['Value'], 151 | 'Class':c['Name'], 152 | 'Red':c['Color'][0], 153 | 'Green':c['Color'][1], 154 | 'Blue':c['Color'][2] 155 | } 156 | } 157 | ) 158 | kwargs['output_info']['rasterAttributeTable'] = json.dumps(attribute_table) 159 | 160 | return kwargs 161 | 162 | def updatePixels(self, tlc, shape, props, **pixelBlocks): 163 | # set pixel values in invalid areas to 0 164 | raster_mask = pixelBlocks['raster_mask'] 165 | raster_pixels = pixelBlocks['raster_pixels'] 166 | raster_pixels[np.where(raster_mask == 0)] = 0 167 | pixelBlocks['raster_pixels'] = raster_pixels 168 | 169 | pixelBlocks['output_pixels'] = self.child_image_classifier.updatePixels(tlc, shape, props, **pixelBlocks).astype(props['pixelType'], copy=False) 170 | return pixelBlocks 171 | -------------------------------------------------------------------------------- /python_raster_functions/CNTK/utils/od_utils.py: -------------------------------------------------------------------------------- 1 | import easydict 2 | from utils.nms_wrapper import apply_nms_to_single_image_results 3 | 4 | def train_object_detector(cfg): 5 | """ 6 | Trains an object detector as specified in the configuration 7 | :param cfg: the configuration 8 | :return: the eval model of the trained detector 9 | """ 10 | 11 | detector_name = _get_detector_name(cfg) 12 | eval_model = None 13 | print("training {}".format(detector_name)) 14 | if detector_name == 'FastRCNN': 15 | from FastRCNN.FastRCNN_train import prepare, train_fast_rcnn 16 | prepare(cfg, use_arg_parser=False) 17 | eval_model = train_fast_rcnn(cfg) 18 | elif detector_name == 'FasterRCNN': 19 | from FasterRCNN.FasterRCNN_train import prepare, train_faster_rcnn 20 | prepare(cfg, use_arg_parser=False) 21 | eval_model = train_faster_rcnn(cfg) 22 | else: 23 | print('Unknown detector: {}'.format(detector_name)) 24 | 25 | return eval_model 26 | 27 | def evaluate_test_set(model, cfg): 28 | """ 29 | Evaluates the given model on the test set as specified in the configuration 30 | :param model: the model 31 | :param cfg: the configuration 32 | :return: AP (average precision) per class 33 | """ 34 | 35 | detector_name = _get_detector_name(cfg) 36 | aps = None 37 | print("evaluating {}".format(detector_name)) 38 | if detector_name == 'FastRCNN': 39 | from FastRCNN.FastRCNN_eval import compute_test_set_aps 40 | aps = compute_test_set_aps(model, cfg) 41 | elif detector_name == 'FasterRCNN': 42 | from FasterRCNN.FasterRCNN_eval import compute_test_set_aps 43 | aps = compute_test_set_aps(model, cfg) 44 | else: 45 | print('Unknown detector: {}'.format(detector_name)) 46 | 47 | return aps 48 | 49 | def evaluate_single_image(model, img_path, cfg): 50 | """ 51 | Computes detection results for the given model on the provided image 52 | :param model: the model 53 | :param img_path: the path to the image 54 | :param cfg: the configuration 55 | :return: 56 | regressed_rois - the predicted bounding boxes 57 | cls_probs - class probabilities per bounding box 58 | """ 59 | 60 | detector_name = _get_detector_name(cfg) 61 | regressed_rois = None 62 | cls_probs = None 63 | print("detecting objects in image {}".format(img_path)) 64 | if detector_name == 'FastRCNN': 65 | from FastRCNN.FastRCNN_eval import FastRCNN_Evaluator 66 | evaluator = FastRCNN_Evaluator(model, cfg) 67 | regressed_rois, cls_probs = evaluator.process_image(img_path) 68 | elif detector_name == 'FasterRCNN': 69 | from FasterRCNN.FasterRCNN_eval import FasterRCNN_Evaluator 70 | evaluator = FasterRCNN_Evaluator(model, cfg) 71 | regressed_rois, cls_probs = evaluator.process_image(img_path) 72 | else: 73 | print('Unknown detector: {}'.format(detector_name)) 74 | 75 | return regressed_rois, cls_probs 76 | 77 | def filter_results(regressed_rois, cls_probs, cfg): 78 | """ 79 | Filters the provided results by performing NMS (non maximum suppression) 80 | :param regressed_rois: the predicted bounding boxes 81 | :param cls_probs: class probabilities per bounding box 82 | :param cfg: the configuration 83 | :return: 84 | bboxes - the filtered list of bounding boxes 85 | labels - the single class label per bounding box 86 | scores - the probability for the assigned class label per bounding box 87 | """ 88 | 89 | labels = cls_probs.argmax(axis=1) 90 | scores = cls_probs.max(axis=1) 91 | nmsKeepIndices = apply_nms_to_single_image_results( 92 | regressed_rois, labels, scores, 93 | use_gpu_nms=cfg.USE_GPU_NMS, 94 | device_id=cfg.GPU_ID, 95 | nms_threshold=cfg.RESULTS_NMS_THRESHOLD, 96 | conf_threshold=cfg.RESULTS_NMS_CONF_THRESHOLD) 97 | 98 | filtered_bboxes = regressed_rois[nmsKeepIndices] 99 | filtered_labels = labels[nmsKeepIndices] 100 | filtered_scores = scores[nmsKeepIndices] 101 | 102 | return filtered_bboxes, filtered_labels, filtered_scores 103 | 104 | def visualize_results(img_path, bboxes, labels, scores, cfg, store_to_path=None): 105 | """ 106 | Renders the detection results (bboxes and labels) onto the image. 107 | :param img_path: the path to the image 108 | :param bboxes: the predicted bounding boxes 109 | :param labels: the single class label per bounding box 110 | :param scores: the probability for the assigned class label per bounding box 111 | :param cfg: the configuration 112 | :param store_to_path: optional: a path where to store the rendered image. 113 | If set to 'None' the image will be displayed on screen. 114 | :return: 115 | """ 116 | 117 | from matplotlib.pyplot import imsave, imshow, show 118 | from utils.plot_helpers import visualize_detections 119 | img = visualize_detections(img_path, bboxes, labels, scores, 120 | cfg.IMAGE_WIDTH, cfg.IMAGE_HEIGHT, 121 | classes = cfg["DATA"].CLASSES, 122 | draw_negative_rois = cfg.DRAW_NEGATIVE_ROIS) 123 | 124 | if store_to_path is not None: 125 | imsave(store_to_path, img) 126 | else: 127 | imshow(img) 128 | show() 129 | 130 | def _get_detector_name(cfg): 131 | try: 132 | detector = cfg['DETECTOR'] 133 | except: 134 | print("Please specify a 'DETECTOR' in your configuration.") 135 | detector = None 136 | return detector 137 | 138 | def measure_inference_time(model, img_path, cfg, num_repetitions=100): 139 | """ 140 | Computes detection results for the given model on the provided image 141 | :param model: the model 142 | :param img_path: the path to the image 143 | :param cfg: the configuration 144 | :return: 145 | regressed_rois - the predicted bounding boxes 146 | cls_probs - class probabilities per bounding box 147 | """ 148 | 149 | detector_name = _get_detector_name(cfg) 150 | print("Measuring inference time (seconds per image) as average over {} runs".format(num_repetitions)) 151 | if detector_name == 'FastRCNN': 152 | from FastRCNN.FastRCNN_eval import FastRCNN_Evaluator 153 | evaluator = FastRCNN_Evaluator(model, cfg) 154 | elif detector_name == 'FasterRCNN': 155 | from FasterRCNN.FasterRCNN_eval import FasterRCNN_Evaluator 156 | evaluator = FasterRCNN_Evaluator(model, cfg) 157 | else: 158 | print('Unknown detector: {}'.format(detector_name)) 159 | return 160 | 161 | from time import time 162 | start = time() 163 | for i in range(num_repetitions): 164 | _,_ = evaluator.process_image(img_path) 165 | total = time() - start 166 | print("seconds per image: {:2f} (total for {} images: {:2f})".format(total/num_repetitions, num_repetitions, total)) 167 | 168 | -------------------------------------------------------------------------------- /python_raster_functions/ObjectDetector.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Copyright 2018 Esri 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | 6 | you may not use this file except in compliance with the License. 7 | 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | 16 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | 18 | See the License for the specific language governing permissions and 19 | 20 | limitations under the License.​ 21 | ''' 22 | 23 | import importlib 24 | import json 25 | import os 26 | import sys 27 | 28 | sys.path.append(os.path.dirname(__file__)) 29 | from fields import fields 30 | from features import features 31 | import numpy as np 32 | import prf_utils 33 | 34 | class GeometryType: 35 | Point = 1 36 | Multipoint = 2 37 | Polyline = 3 38 | Polygon = 4 39 | 40 | 41 | class ObjectDetector: 42 | def __init__(self): 43 | self.name = 'Object Detector' 44 | self.description = 'This python raster function applies deep learning model to detect objects in imagery' 45 | 46 | def initialize(self, **kwargs): 47 | if 'model' not in kwargs: 48 | return 49 | 50 | model = kwargs['model'] 51 | model_as_file = True 52 | try: 53 | with open(model, 'r') as f: 54 | self.json_info = json.load(f) 55 | except FileNotFoundError: 56 | try: 57 | self.json_info = json.loads(model) 58 | model_as_file = False 59 | except json.decoder.JSONDecodeError: 60 | raise Exception("Invalid model argument") 61 | 62 | if 'device' in kwargs: 63 | device = kwargs['device'] 64 | if device < -1: 65 | os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" 66 | device = prf_utils.get_available_device() 67 | os.environ['CUDA_VISIBLE_DEVICES'] = str(device) 68 | else: 69 | os.environ['CUDA_VISIBLE_DEVICES'] = '-1' 70 | 71 | sys.path.append(os.path.dirname(__file__)) 72 | framework = self.json_info['Framework'] 73 | if 'ModelConfiguration' in self.json_info: 74 | if isinstance(self.json_info['ModelConfiguration'], str): 75 | ChildModelDetector = getattr(importlib.import_module( 76 | '{}.{}'.format(framework, self.json_info['ModelConfiguration'])), 'ChildObjectDetector') 77 | else: 78 | ChildModelDetector = getattr(importlib.import_module( 79 | '{}.{}'.format(framework, self.json_info['ModelConfiguration']['Name'])), 'ChildObjectDetector') 80 | else: 81 | raise Exception("Invalid model configuration") 82 | 83 | self.child_object_detector = ChildModelDetector() 84 | self.child_object_detector.initialize(model, model_as_file) 85 | 86 | def getParameterInfo(self): 87 | required_parameters = [ 88 | { 89 | 'name': 'raster', 90 | 'dataType': 'raster', 91 | 'required': True, 92 | 'displayName': 'Raster', 93 | 'description': 'Input Raster' 94 | }, 95 | { 96 | 'name': 'model', 97 | 'dataType': 'string', 98 | 'required': True, 99 | 'displayName': 'Input Model Definition (EMD) File', 100 | 'description': 'Input model definition (EMD) JSON file' 101 | }, 102 | { 103 | 'name': 'device', 104 | 'dataType': 'numeric', 105 | 'required': False, 106 | 'displayName': 'Device ID', 107 | 'description': 'Device ID' 108 | }, 109 | { 110 | 'name': 'padding', 111 | 'dataType': 'numeric', 112 | 'value': 0, 113 | 'required': False, 114 | 'displayName': 'Padding', 115 | 'description': 'Padding' 116 | }, 117 | { 118 | 'name': 'score_threshold', 119 | 'dataType': 'numeric', 120 | 'value': 0.6, 121 | 'required': False, 122 | 'displayName': 'Confidence Score Threshold [0.0, 1.0]', 123 | 'description': 'Confidence score threshold value [0.0, 1.0]' 124 | }, 125 | ] 126 | 127 | if 'BatchSize' not in self.json_info: 128 | required_parameters.append( 129 | { 130 | 'name': 'batch_size', 131 | 'dataType': 'numeric', 132 | 'required': False, 133 | 'value': 1, 134 | 'displayName': 'Batch Size', 135 | 'description': 'Batch Size' 136 | }, 137 | ) 138 | 139 | return self.child_object_detector.getParameterInfo(required_parameters) 140 | 141 | def getConfiguration(self, **scalars): 142 | configuration = self.child_object_detector.getConfiguration(**scalars) 143 | if 'DataRange' in self.json_info: 144 | configuration['dataRange'] = tuple(self.json_info['DataRange']) 145 | configuration['inheritProperties'] = 2|4|8 146 | configuration['inputMask'] = True 147 | return configuration 148 | 149 | def getFields(self): 150 | return json.dumps(fields) 151 | 152 | def getGeometryType(self): 153 | return GeometryType.Polygon 154 | 155 | def vectorize(self, **pixelBlocks): 156 | # set pixel values in invalid areas to 0 157 | raster_mask = pixelBlocks['raster_mask'] 158 | raster_pixels = pixelBlocks['raster_pixels'] 159 | raster_pixels[np.where(raster_mask == 0)] = 0 160 | pixelBlocks['raster_pixels'] = raster_pixels 161 | 162 | polygon_list, scores, classes = self.child_object_detector.vectorize(**pixelBlocks) 163 | 164 | # bounding_boxes = bounding_boxes.tolist() 165 | scores = scores.tolist() 166 | classes = classes.tolist() 167 | features['features'] = [] 168 | 169 | for i in range(len(polygon_list)): 170 | rings = [[]] 171 | for j in range(polygon_list[i].shape[0]): 172 | rings[0].append( 173 | [ 174 | polygon_list[i][j][1], 175 | polygon_list[i][j][0] 176 | ] 177 | ) 178 | 179 | features['features'].append({ 180 | 'attributes': { 181 | 'OID': i + 1, 182 | 'Class': self.json_info['Classes'][classes[i] - 1]['Name'], 183 | 'Confidence': scores[i] 184 | }, 185 | 'geometry': { 186 | 'rings': rings 187 | } 188 | }) 189 | return {'output_vectors': json.dumps(features)} 190 | -------------------------------------------------------------------------------- /python_raster_functions/CNTK/utils/map_helpers.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Microsoft. All rights reserved. 2 | 3 | # Licensed under the MIT license. See LICENSE.md file in the project root 4 | # for full license information. 5 | # ============================================================================== 6 | 7 | import numpy as np 8 | from utils.nms_wrapper import apply_nms_to_test_set_results 9 | 10 | def evaluate_detections(all_boxes, all_gt_infos, classes, 11 | use_gpu_nms, device_id, 12 | apply_mms=True, nms_threshold=0.5, conf_threshold=0.0, 13 | use_07_metric=False): 14 | ''' 15 | Computes per-class average precision. 16 | 17 | Args: 18 | all_boxes: shape of all_boxes: e.g. 21 classes x 4952 images x 58 rois x 5 coords+score 19 | all_gt_infos: a dictionary that contains all ground truth annoations in the following form: 20 | {'class_A': [{'bbox': array([[ 376., 210., 456., 288., 10.]], dtype=float32), 'det': [False], 'difficult': [False]}, ... ]} 21 | 'class_B': [ ], } 22 | classes: a list of class name, e.g. ['__background__', 'avocado', 'orange', 'butter'] 23 | use_07_metric: whether to use VOC07's 11 point AP computation (default False) 24 | apply_mms: whether to apply non maximum suppression before computing average precision values 25 | nms_threshold: the threshold for discarding overlapping ROIs in nms 26 | conf_threshold: a minimum value for the score of an ROI. ROIs with lower score will be discarded 27 | 28 | Returns: 29 | aps - average precision value per class in a dictionary {classname: ap} 30 | ''' 31 | 32 | if apply_mms: 33 | print ("Number of rois before non-maximum suppression: %d" % sum([len(all_boxes[i][j]) for i in range(len(all_boxes)) for j in range(len(all_boxes[0]))])) 34 | nms_dets,_ = apply_nms_to_test_set_results(all_boxes, nms_threshold, conf_threshold, use_gpu_nms, device_id) 35 | print ("Number of rois after non-maximum suppression: %d" % sum([len(nms_dets[i][j]) for i in range(len(all_boxes)) for j in range(len(all_boxes[0]))])) 36 | else: 37 | print ("Skipping non-maximum suppression") 38 | nms_dets = all_boxes 39 | 40 | aps = {} 41 | for classIndex, className in enumerate(classes): 42 | if className != '__background__': 43 | rec, prec, ap = _evaluate_detections(classIndex, nms_dets, all_gt_infos[className], use_07_metric=use_07_metric) 44 | aps[className] = ap 45 | 46 | return aps 47 | 48 | def _evaluate_detections(classIndex, all_boxes, gtInfos, overlapThreshold=0.5, use_07_metric=False): 49 | ''' 50 | Top level function that does the PASCAL VOC evaluation. 51 | ''' 52 | 53 | # parse detections for this class 54 | # shape of all_boxes: e.g. 21 classes x 4952 images x 58 rois x 5 coords+score 55 | num_images = len(all_boxes[0]) 56 | detBboxes = [] 57 | detImgIndices = [] 58 | detConfidences = [] 59 | for imgIndex in range(num_images): 60 | dets = all_boxes[classIndex][imgIndex] 61 | if dets != []: 62 | for k in range(dets.shape[0]): 63 | detImgIndices.append(imgIndex) 64 | detConfidences.append(dets[k, -1]) 65 | # the VOCdevkit expects 1-based indices 66 | detBboxes.append([dets[k, 0] + 1, dets[k, 1] + 1, dets[k, 2] + 1, dets[k, 3] + 1]) 67 | detBboxes = np.array(detBboxes) 68 | detConfidences = np.array(detConfidences) 69 | 70 | # compute precision / recall / ap 71 | rec, prec, ap = _voc_computePrecisionRecallAp( 72 | class_recs=gtInfos, 73 | confidence=detConfidences, 74 | image_ids=detImgIndices, 75 | BB=detBboxes, 76 | ovthresh=overlapThreshold, 77 | use_07_metric=use_07_metric) 78 | return rec, prec, ap 79 | 80 | def computeAveragePrecision(recalls, precisions, use_07_metric=False): 81 | ''' 82 | Computes VOC AP given precision and recall. 83 | ''' 84 | if use_07_metric: 85 | # 11 point metric 86 | ap = 0. 87 | for t in np.arange(0., 1.1, 0.1): 88 | if np.sum(recalls >= t) == 0: 89 | p = 0 90 | else: 91 | p = np.max(precisions[recalls >= t]) 92 | ap = ap + p / 11. 93 | else: 94 | # correct AP calculation 95 | # first append sentinel values at the end 96 | mrecalls = np.concatenate(([0.], recalls, [1.])) 97 | mprecisions = np.concatenate(([0.], precisions, [0.])) 98 | 99 | # compute the precision envelope 100 | for i in range(mprecisions.size - 1, 0, -1): 101 | mprecisions[i - 1] = np.maximum(mprecisions[i - 1], mprecisions[i]) 102 | 103 | # to calculate area under PR curve, look for points 104 | # where X axis (recall) changes value 105 | i = np.where(mrecalls[1:] != mrecalls[:-1])[0] 106 | 107 | # and sum (\Delta recall) * prec 108 | ap = np.sum((mrecalls[i + 1] - mrecalls[i]) * mprecisions[i + 1]) 109 | return ap 110 | 111 | def _voc_computePrecisionRecallAp(class_recs, confidence, image_ids, BB, ovthresh=0.5, use_07_metric=False): 112 | ''' 113 | Computes precision, recall. and average precision 114 | ''' 115 | if len(BB) == 0: 116 | return 0.0, 0.0, 0.0 117 | 118 | # sort by confidence 119 | sorted_ind = np.argsort(-confidence) 120 | 121 | BB = BB[sorted_ind, :] 122 | image_ids = [image_ids[x] for x in sorted_ind] 123 | 124 | # go down dets and mark TPs and FPs 125 | nd = len(image_ids) 126 | tp = np.zeros(nd) 127 | fp = np.zeros(nd) 128 | for d in range(nd): 129 | R = class_recs[image_ids[d]] 130 | bb = BB[d, :].astype(float) 131 | ovmax = -np.inf 132 | BBGT = R['bbox'].astype(float) 133 | 134 | if BBGT.size > 0: 135 | # compute overlaps 136 | ixmin = np.maximum(BBGT[:, 0], bb[0]) 137 | iymin = np.maximum(BBGT[:, 1], bb[1]) 138 | ixmax = np.minimum(BBGT[:, 2], bb[2]) 139 | iymax = np.minimum(BBGT[:, 3], bb[3]) 140 | iw = np.maximum(ixmax - ixmin + 1., 0.) 141 | ih = np.maximum(iymax - iymin + 1., 0.) 142 | inters = iw * ih 143 | 144 | # union 145 | uni = ((bb[2] - bb[0] + 1.) * (bb[3] - bb[1] + 1.) + 146 | (BBGT[:, 2] - BBGT[:, 0] + 1.) * 147 | (BBGT[:, 3] - BBGT[:, 1] + 1.) - inters) 148 | 149 | overlaps = inters / uni 150 | ovmax = np.max(overlaps) 151 | jmax = np.argmax(overlaps) 152 | 153 | if ovmax > ovthresh: 154 | if not R['difficult'][jmax]: 155 | if not R['det'][jmax]: 156 | tp[d] = 1. 157 | R['det'][jmax] = 1 158 | else: 159 | fp[d] = 1. 160 | else: 161 | fp[d] = 1. 162 | 163 | # compute precision recall 164 | npos = sum([len(cr['bbox']) for cr in class_recs]) 165 | fp = np.cumsum(fp) 166 | tp = np.cumsum(tp) 167 | rec = tp / float(npos) 168 | # avoid divide by zero in case the first detection matches a difficult ground truth 169 | prec = tp / np.maximum(tp + fp, np.finfo(np.float64).eps) 170 | ap = computeAveragePrecision(rec, prec, use_07_metric) 171 | return rec, prec, ap 172 | -------------------------------------------------------------------------------- /docs/writing_deep_learning_python_raster_functions.md: -------------------------------------------------------------------------------- 1 | # Writing deep learning Python raster function 2 | If you find that the sample model definition files and built-in Python raster functions cannot describe your deep learning 3 | model architecture/properties, or if you use a deep learning framework other than TensorFlow, Keras, CNTK, or PyTorch, 4 | you can write your own deep learning Python raster function and reference the Python raster function in the .emd file next 5 | to the "InferenceFunction" parameter. 6 | 7 | Help documentation for working with Python raster functions in ArcGIS can be found on the 8 | [Python raster function wiki page](https://github.com/Esri/raster-functions/wiki). 9 | The [anatomy of a Python raster function](https://github.com/Esri/raster-functions/wiki/PythonRasterFunction#anatomy-of-a-python-raster-function) 10 | is an especially useful document if you want to get familiar with the components of a Python raster function. 11 | In this page, we will show you what a deep learning Python raster function 12 | looks like and how to write your own functions more efficiently. 13 | 14 | ## `.initialize(self,**kwargs)` 15 | This method is called at the beginning of your Python raster function. 16 | kwargs\['model'\] is the model definition file. This method requires the following steps: 17 | 1. Load the model information your Esri model definition file (EMD). 18 | 2. Load your deep learning model and keep a handle to the model. 19 | 20 | e.g. 21 | ```python 22 | def initialize(self, **kwargs): 23 | if 'model' not in kwargs: 24 | return 25 | 26 | json_file = kwargs['model'] 27 | with open(json_file, 'r') as f: 28 | self.json_info = json.load(f) 29 | 30 | # access the model path in the model definition file 31 | model_path = json_info['ModelFile'] 32 | # load your model and keep and instance for the model 33 | # self.model = load_your_model(model_path) 34 | ``` 35 | 36 | ## `.getParameterInfo(self)` 37 | This is called after initialize(), and it is where you define your parameters. The first two parameters are 38 | mandatory as they define the input raster and the input EMD. You can copy the following code snippet directly to your 39 | Python raster function: 40 | 41 | ```python 42 | { 43 | 'name': 'raster', 44 | 'dataType': 'raster', 45 | 'required': True, 46 | 'displayName': 'Raster', 47 | 'description': 'Input Raster' 48 | }, 49 | { 50 | 'name': 'model', 51 | 'dataType': 'string', 52 | 'required': True, 53 | 'displayName': 'Input Model Description (EMD) File', 54 | 'description': 'Input model description (EMD) JSON file' 55 | } 56 | ``` 57 | 58 | ## `.getConfiguration(self, **scalars)` 59 | This method is used to set the input bands, padding and tile size. 60 | The *sclars* value contains all the parameter values that can be accessed by the parameter name. 61 | Remember to save the parameter values here if you want to use the parameter values in other methods. 62 | 63 | e.g. 64 | ```python 65 | def getConfiguration(self, **scalars): 66 | if 'score_threshold' in scalars: 67 | self.score_threshold = float(scalars['score_threshold']) 68 | if 'padding' in scalars: 69 | self.padding = int(scalars['padding']) 70 | 71 | return { 72 | 'extractBands': tuple(self.json_info['ExtractBands']), 73 | 'padding': self.padding, 74 | 'tx': self.json_info['ImageWidth'] - 2*self.padding, 75 | 'ty': self.json_info['ImageHeight'] - 2*self.padding 76 | } 77 | ``` 78 | 79 | ## `.getFields(self)` 80 | Use this method to return the JSON string fields of the output feature class. 81 | 82 | e.g. 83 | ```python 84 | def getFields(self): 85 | fields = { 86 | 'fields': [ 87 | { 88 | 'name': 'OID', 89 | 'type': 'esriFieldTypeOID', 90 | 'alias': 'OID' 91 | }, 92 | { 93 | 'name': 'Class', 94 | 'type': 'esriFieldTypeString', 95 | 'alias': 'Class' 96 | }, 97 | { 98 | 'name': 'Confidence', 99 | 'type': 'esriFieldTypeDouble', 100 | 'alias': 'Confidence' 101 | }, 102 | { 103 | 'name': 'Shape', 104 | 'type': 'esriFieldTypeGeometry', 105 | 'alias': 'Shape' 106 | } 107 | ] 108 | } 109 | return json.dumps(fields) 110 | ``` 111 | 112 | ## `.getGeometryType(self)` 113 | Use this method if you use the Detect Objects Using Deep Learning tool and you want to declare the feature geometry type of 114 | the output detected objects. Typically, the output is a polygon feature class if the model is to draw bounding boxes around objects. 115 | 116 | e.g. 117 | ```python 118 | class GeometryType: 119 | Point = 1 120 | Multipoint = 2 121 | Polyline = 3 122 | Polygon = 4 123 | 124 | def getGeometryType(self): 125 | return GeometryType.Polygon 126 | ``` 127 | 128 | ## `.vectorize(self, **pixelBlocks)` 129 | Use this method if you use the Detect Objects Using Deep Learning tool. This method returns a dictionary in which 130 | the "output_vectors" property is a string of features in image space in JSON format. A typical workflow is below: 131 | 1. Obtain the input image from *pixelBlocks* and transform to the shape of the model's input. 132 | 2. Run the deep learning model on the input image tile. 133 | 3. Post-process the model's output as necessary. 134 | 4. Generate a feature JSON object, wrap it as a string in a dictionary and return the dictionary. 135 | 136 | e.g. 137 | ```python 138 | def vectorize(self, **pixelBlocks): 139 | # obtain the input image 140 | input_image = pixelBlocks['raster_pixels'] 141 | 142 | # Todo: transform the input image to the shape of the model's input 143 | # input_image = np.transform(input_image, ....) 144 | 145 | # Todo: run the model on the transformed input image, something like 146 | # model_output = self.model.run(input_image) 147 | 148 | # Todo: create feature json object and fill out the geometry 149 | # features geometry and properties are filled by model_output 150 | 151 | # Todo: wrap the json object as a string in dictionary 152 | # return {'output_vectors': json.dumps(features)} 153 | ``` 154 | 155 | 156 | ## `.updatePixels(self, tlc, shape, props, **pixelBlocks)` 157 | Use this method if you use the Classify Pixels Using Deep Learning tool for semantic segmentation. 158 | This method returns the classified raster wrapped in a dictionary. The typical workflow is below: 159 | 160 | 1. Obtain the input image from *pixelBlocks* and transform to the shape of the model's input. 161 | 2. Run the deep learning model on the input image tile. 162 | 3. Post-process the model's output as necessary. 163 | 4. Generate a classified raster, wrap it in a dictionary and return the dictionary. 164 | 165 | e.g. 166 | ```python 167 | def updatePixels(self, tlc, shape, props, **pixelBlocks): 168 | # obtain the input image 169 | input_image = pixelBlocks['raster_pixels'] 170 | 171 | # Todo: transform the input image to the shape of the model's input 172 | # input_image = np.transform(input_image, ....) 173 | 174 | # Todo: run the model on the transformed input image, something like 175 | # model_output = self.model.run(input_image) 176 | 177 | # Todo: wrap the classified raster in dictonary, something like 178 | # pixelBlocks['output_pixels'] = model_output.astype(props['pixelType'], copy=False) 179 | 180 | # Return the dict 181 | # return pixelBlocks 182 | 183 | return pixelBlocks 184 | ``` 185 | -------------------------------------------------------------------------------- /examples/keras/mask_rcnn/mrcnn/parallel_model.py: -------------------------------------------------------------------------------- 1 | """ 2 | Mask R-CNN 3 | Multi-GPU Support for Keras. 4 | 5 | Copyright (c) 2017 Matterport, Inc. 6 | Licensed under the MIT License (see LICENSE for details) 7 | Written by Waleed Abdulla 8 | 9 | Ideas and a small code snippets from these sources: 10 | https://github.com/fchollet/keras/issues/2436 11 | https://medium.com/@kuza55/transparent-multi-gpu-training-on-tensorflow-with-keras-8b0016fd9012 12 | https://github.com/avolkov1/keras_experiments/blob/master/keras_exp/multigpu/ 13 | https://github.com/fchollet/keras/blob/master/keras/utils/training_utils.py 14 | """ 15 | 16 | import keras.backend as K 17 | import keras.layers as KL 18 | import keras.models as KM 19 | import tensorflow as tf 20 | 21 | 22 | class ParallelModel(KM.Model): 23 | """Subclasses the standard Keras Model and adds multi-GPU support. 24 | It works by creating a copy of the model on each GPU. Then it slices 25 | the inputs and sends a slice to each copy of the model, and then 26 | merges the outputs together and applies the loss on the combined 27 | outputs. 28 | """ 29 | 30 | def __init__(self, keras_model, gpu_count): 31 | """Class constructor. 32 | keras_model: The Keras model to parallelize 33 | gpu_count: Number of GPUs. Must be > 1 34 | """ 35 | self.inner_model = keras_model 36 | self.gpu_count = gpu_count 37 | merged_outputs = self.make_parallel() 38 | super(ParallelModel, self).__init__(inputs=self.inner_model.inputs, 39 | outputs=merged_outputs) 40 | 41 | def __getattribute__(self, attrname): 42 | """Redirect loading and saving methods to the inner model. That's where 43 | the weights are stored.""" 44 | if 'load' in attrname or 'save' in attrname: 45 | return getattr(self.inner_model, attrname) 46 | return super(ParallelModel, self).__getattribute__(attrname) 47 | 48 | def summary(self, *args, **kwargs): 49 | """Override summary() to display summaries of both, the wrapper 50 | and inner models.""" 51 | super(ParallelModel, self).summary(*args, **kwargs) 52 | self.inner_model.summary(*args, **kwargs) 53 | 54 | def make_parallel(self): 55 | """Creates a new wrapper model that consists of multiple replicas of 56 | the original model placed on different GPUs. 57 | """ 58 | # Slice inputs. Slice inputs on the CPU to avoid sending a copy 59 | # of the full inputs to all GPUs. Saves on bandwidth and memory. 60 | input_slices = {name: tf.split(x, self.gpu_count) 61 | for name, x in zip(self.inner_model.input_names, 62 | self.inner_model.inputs)} 63 | 64 | output_names = self.inner_model.output_names 65 | outputs_all = [] 66 | for i in range(len(self.inner_model.outputs)): 67 | outputs_all.append([]) 68 | 69 | # Run the model call() on each GPU to place the ops there 70 | for i in range(self.gpu_count): 71 | with tf.device('/gpu:%d' % i): 72 | with tf.name_scope('tower_%d' % i): 73 | # Run a slice of inputs through this replica 74 | zipped_inputs = zip(self.inner_model.input_names, 75 | self.inner_model.inputs) 76 | inputs = [ 77 | KL.Lambda(lambda s: input_slices[name][i], 78 | output_shape=lambda s: (None,) + s[1:])(tensor) 79 | for name, tensor in zipped_inputs] 80 | # Create the model replica and get the outputs 81 | outputs = self.inner_model(inputs) 82 | if not isinstance(outputs, list): 83 | outputs = [outputs] 84 | # Save the outputs for merging back together later 85 | for l, o in enumerate(outputs): 86 | outputs_all[l].append(o) 87 | 88 | # Merge outputs on CPU 89 | with tf.device('/cpu:0'): 90 | merged = [] 91 | for outputs, name in zip(outputs_all, output_names): 92 | # Concatenate or average outputs? 93 | # Outputs usually have a batch dimension and we concatenate 94 | # across it. If they don't, then the output is likely a loss 95 | # or a metric value that gets averaged across the batch. 96 | # Keras expects losses and metrics to be scalars. 97 | if K.int_shape(outputs[0]) == (): 98 | # Average 99 | m = KL.Lambda(lambda o: tf.add_n(o) / len(outputs), name=name)(outputs) 100 | else: 101 | # Concatenate 102 | m = KL.Concatenate(axis=0, name=name)(outputs) 103 | merged.append(m) 104 | return merged 105 | 106 | 107 | if __name__ == "__main__": 108 | # Testing code below. It creates a simple model to train on MNIST and 109 | # tries to run it on 2 GPUs. It saves the graph so it can be viewed 110 | # in TensorBoard. Run it as: 111 | # 112 | # python3 parallel_model.py 113 | 114 | import os 115 | import numpy as np 116 | import keras.optimizers 117 | from keras.datasets import mnist 118 | from keras.preprocessing.image import ImageDataGenerator 119 | 120 | GPU_COUNT = 2 121 | 122 | # Root directory of the project 123 | ROOT_DIR = os.path.abspath("../") 124 | 125 | # Directory to save logs and trained model 126 | MODEL_DIR = os.path.join(ROOT_DIR, "logs") 127 | 128 | def build_model(x_train, num_classes): 129 | # Reset default graph. Keras leaves old ops in the graph, 130 | # which are ignored for execution but clutter graph 131 | # visualization in TensorBoard. 132 | tf.reset_default_graph() 133 | 134 | inputs = KL.Input(shape=x_train.shape[1:], name="input_image") 135 | x = KL.Conv2D(32, (3, 3), activation='relu', padding="same", 136 | name="conv1")(inputs) 137 | x = KL.Conv2D(64, (3, 3), activation='relu', padding="same", 138 | name="conv2")(x) 139 | x = KL.MaxPooling2D(pool_size=(2, 2), name="pool1")(x) 140 | x = KL.Flatten(name="flat1")(x) 141 | x = KL.Dense(128, activation='relu', name="dense1")(x) 142 | x = KL.Dense(num_classes, activation='softmax', name="dense2")(x) 143 | 144 | return KM.Model(inputs, x, "digit_classifier_model") 145 | 146 | # Load MNIST Data 147 | (x_train, y_train), (x_test, y_test) = mnist.load_data() 148 | x_train = np.expand_dims(x_train, -1).astype('float32') / 255 149 | x_test = np.expand_dims(x_test, -1).astype('float32') / 255 150 | 151 | print('x_train shape:', x_train.shape) 152 | print('x_test shape:', x_test.shape) 153 | 154 | # Build data generator and model 155 | datagen = ImageDataGenerator() 156 | model = build_model(x_train, 10) 157 | 158 | # Add multi-GPU support. 159 | model = ParallelModel(model, GPU_COUNT) 160 | 161 | optimizer = keras.optimizers.SGD(lr=0.01, momentum=0.9, clipnorm=5.0) 162 | 163 | model.compile(loss='sparse_categorical_crossentropy', 164 | optimizer=optimizer, metrics=['accuracy']) 165 | 166 | model.summary() 167 | 168 | # Train 169 | model.fit_generator( 170 | datagen.flow(x_train, y_train, batch_size=64), 171 | steps_per_epoch=50, epochs=10, verbose=1, 172 | validation_data=(x_test, y_test), 173 | callbacks=[keras.callbacks.TensorBoard(log_dir=MODEL_DIR, 174 | write_graph=True)] 175 | ) 176 | -------------------------------------------------------------------------------- /python_raster_functions/CNTK/utils/caffe_layers/proposal_layer.py: -------------------------------------------------------------------------------- 1 | # -------------------------------------------------------- 2 | # Faster R-CNN 3 | # Copyright (c) 2015 Microsoft 4 | # Licensed under The MIT License [see LICENSE for details] 5 | # Written by Ross Girshick and Sean Bell 6 | # -------------------------------------------------------- 7 | 8 | #import caffe 9 | import numpy as np 10 | import yaml 11 | from utils.caffe_layers.default_config import cfg 12 | from utils.rpn.generate_anchors import generate_anchors 13 | from utils.caffe_layers.bbox_transform import bbox_transform_inv, clip_boxes 14 | from utils.nms_wrapper import nms 15 | 16 | DEBUG = False 17 | 18 | class ProposalLayer: #(caffe.Layer): 19 | """ 20 | Outputs object detection proposals by applying estimated bounding-box 21 | transformations to a set of regular boxes (called "anchors"). 22 | """ 23 | 24 | def set_param_str(self, param_str): 25 | self.param_str_ = param_str 26 | 27 | def setup(self, bottom, top): 28 | # parse the layer parameter string, which must be valid YAML 29 | layer_params = yaml.load(self.param_str_) 30 | 31 | self._feat_stride = layer_params['feat_stride'] 32 | anchor_scales = layer_params.get('scales', (8, 16, 32)) 33 | self._anchors = generate_anchors(scales=np.array(anchor_scales)) 34 | self._num_anchors = self._anchors.shape[0] 35 | self.phase = "TEST" 36 | 37 | #if DEBUG: 38 | #print 'feat_stride: {}'.format(self._feat_stride) 39 | #print 'anchors:' 40 | #print self._anchors 41 | 42 | # rois blob: holds R regions of interest, each is a 5-tuple 43 | # (n, x1, y1, x2, y2) specifying an image batch index n and a 44 | # rectangle (x1, y1, x2, y2) 45 | #top[0].reshape(1, 5) 46 | 47 | # scores blob: holds scores for R regions of interest 48 | #if len(top) > 1: 49 | # top[1].reshape(1, 1, 1, 1) 50 | 51 | def forward(self, bottom, top): 52 | # Algorithm: 53 | # 54 | # for each (H, W) location i 55 | # generate A anchor boxes centered on cell i 56 | # apply predicted bbox deltas at cell i to each of the A anchors 57 | # clip predicted boxes to image 58 | # remove predicted boxes with either height or width < threshold 59 | # sort all (proposal, score) pairs by score from highest to lowest 60 | # take top pre_nms_topN proposals before NMS 61 | # apply NMS with threshold 0.7 to remaining proposals 62 | # take after_nms_topN proposals after NMS 63 | # return the top proposals (-> RoIs top, scores top) 64 | 65 | assert bottom[0].shape[0] == 1, \ 66 | 'Only single item batches are supported' 67 | 68 | cfg_key = str(self.phase) # either 'TRAIN' or 'TEST' 69 | pre_nms_topN = cfg[cfg_key].RPN_PRE_NMS_TOP_N 70 | post_nms_topN = cfg[cfg_key].RPN_POST_NMS_TOP_N 71 | nms_thresh = cfg[cfg_key].RPN_NMS_THRESH 72 | min_size = cfg[cfg_key].RPN_MIN_SIZE 73 | 74 | # the first set of _num_anchors channels are bg probs 75 | # the second set are the fg probs, which we want 76 | scores = bottom[0][:, self._num_anchors:, :, :] 77 | bbox_deltas = bottom[1] 78 | im_info = bottom[2][0, :] 79 | 80 | #if DEBUG: 81 | # print 'im_size: ({}, {})'.format(im_info[0], im_info[1]) 82 | # print 'scale: {}'.format(im_info[2]) 83 | 84 | # 1. Generate proposals from bbox deltas and shifted anchors 85 | height, width = scores.shape[-2:] 86 | 87 | #if DEBUG: 88 | # print 'score map size: {}'.format(scores.shape) 89 | 90 | # Enumerate all shifts 91 | shift_x = np.arange(0, width) * self._feat_stride 92 | shift_y = np.arange(0, height) * self._feat_stride 93 | shift_x, shift_y = np.meshgrid(shift_x, shift_y) 94 | shifts = np.vstack((shift_x.ravel(), shift_y.ravel(), 95 | shift_x.ravel(), shift_y.ravel())).transpose() 96 | 97 | # Enumerate all shifted anchors: 98 | # 99 | # add A anchors (1, A, 4) to 100 | # cell K shifts (K, 1, 4) to get 101 | # shift anchors (K, A, 4) 102 | # reshape to (K*A, 4) shifted anchors 103 | A = self._num_anchors 104 | K = shifts.shape[0] 105 | anchors = self._anchors.reshape((1, A, 4)) + \ 106 | shifts.reshape((1, K, 4)).transpose((1, 0, 2)) 107 | anchors = anchors.reshape((K * A, 4)) 108 | 109 | # Transpose and reshape predicted bbox transformations to get them 110 | # into the same order as the anchors: 111 | # 112 | # bbox deltas will be (1, 4 * A, H, W) format 113 | # transpose to (1, H, W, 4 * A) 114 | # reshape to (1 * H * W * A, 4) where rows are ordered by (h, w, a) 115 | # in slowest to fastest order 116 | bbox_deltas = bbox_deltas.transpose((0, 2, 3, 1)).reshape((-1, 4)) 117 | 118 | # Same story for the scores: 119 | # 120 | # scores are (1, A, H, W) format 121 | # transpose to (1, H, W, A) 122 | # reshape to (1 * H * W * A, 1) where rows are ordered by (h, w, a) 123 | scores = scores.transpose((0, 2, 3, 1)).reshape((-1, 1)) 124 | 125 | # Convert anchors into proposals via bbox transformations 126 | proposals = bbox_transform_inv(anchors, bbox_deltas) 127 | 128 | # 2. clip predicted boxes to image 129 | proposals = clip_boxes(proposals, im_info[:2]) 130 | 131 | # 3. remove predicted boxes with either height or width < threshold 132 | # (NOTE: convert min_size to input image scale stored in im_info[2]) 133 | keep = _filter_boxes(proposals, min_size * im_info[2]) 134 | proposals = proposals[keep, :] 135 | scores = scores[keep] 136 | 137 | # 4. sort all (proposal, score) pairs by score from highest to lowest 138 | # 5. take top pre_nms_topN (e.g. 6000) 139 | order = scores.ravel().argsort(kind='mergesort')[::-1] 140 | if pre_nms_topN > 0: 141 | order = order[:pre_nms_topN] 142 | proposals = proposals[order, :] 143 | scores = scores[order] 144 | 145 | # 6. apply nms (e.g. threshold = 0.7) 146 | # 7. take after_nms_topN (e.g. 300) 147 | # 8. return the top proposals (-> RoIs top) 148 | keep = nms(np.hstack((proposals, scores)), nms_thresh) 149 | if post_nms_topN > 0: 150 | keep = keep[:post_nms_topN] 151 | proposals = proposals[keep, :] 152 | scores = scores[keep] 153 | 154 | # Output rois blob 155 | # Our RPN implementation only supports a single input image, so all 156 | # batch inds are 0 157 | batch_inds = np.zeros((proposals.shape[0], 1), dtype=np.float32) 158 | blob = np.hstack((batch_inds, proposals.astype(np.float32, copy=False))) 159 | 160 | return blob 161 | #top[0].reshape(*(blob.shape)) 162 | #top[0].data[...] = blob 163 | 164 | # [Optional] output scores blob 165 | #if len(top) > 1: 166 | # top[1].reshape(*(scores.shape)) 167 | # top[1].data[...] = scores 168 | 169 | def backward(self, top, propagate_down, bottom): 170 | """This layer does not propagate gradients.""" 171 | pass 172 | 173 | def reshape(self, bottom, top): 174 | """Reshaping happens during the call to forward.""" 175 | pass 176 | 177 | def _filter_boxes(boxes, min_size): 178 | """Remove all boxes with any side smaller than min_size.""" 179 | ws = boxes[:, 2] - boxes[:, 0] + 1 180 | hs = boxes[:, 3] - boxes[:, 1] + 1 181 | keep = np.where((ws >= min_size) & (hs >= min_size))[0] 182 | return keep 183 | -------------------------------------------------------------------------------- /python_raster_functions/ObjectClassifier.py: -------------------------------------------------------------------------------- 1 | import importlib 2 | from importlib import reload, import_module 3 | import json 4 | import os 5 | import sys 6 | import arcpy 7 | sys.path.append(os.path.dirname(__file__)) 8 | from fields import fields 9 | from features import features 10 | import numpy as np 11 | import prf_utils 12 | 13 | class GeometryType: 14 | Point = 1 15 | Multipoint = 2 16 | Polyline = 3 17 | Polygon = 4 18 | 19 | 20 | class ObjectClassifier: 21 | def __init__(self): 22 | self.name = 'Object classifier' 23 | self.description = 'This python raster function applies deep learning model to ' \ 24 | 'classify objects from overlaid imagery' 25 | 26 | def initialize(self, **kwargs): 27 | 28 | if 'model' not in kwargs: 29 | return 30 | 31 | # Read esri model definition (emd) file 32 | model = kwargs['model'] 33 | model_as_file = True 34 | 35 | try: 36 | with open(model, 'r') as f: 37 | self.json_info = json.load(f) 38 | except FileNotFoundError: 39 | try: 40 | self.json_info = json.loads(model) 41 | model_as_file = False 42 | except json.decoder.JSONDecodeError: 43 | raise Exception("Invalid model argument") 44 | 45 | sys.path.append(os.path.dirname(__file__)) 46 | 47 | framework = self.json_info['Framework'] 48 | if 'ModelConfiguration' in self.json_info: 49 | modelconfig = self.json_info['ModelConfiguration'] 50 | if isinstance(modelconfig, str): 51 | if modelconfig not in sys.modules: 52 | ChildModelDetector = getattr(import_module( 53 | '{}.{}'.format(framework, modelconfig)), 'ChildObjectDetector') 54 | else: 55 | ChildModelDetector = getattr(reload( 56 | '{}.{}'.format(framework, modelconfig)), 'ChildObjectDetector') 57 | else: 58 | modelconfig = self.json_info['ModelConfiguration']['Name'] 59 | ChildModelDetector = getattr(importlib.import_module( 60 | '{}.{}'.format(framework, modelconfig)), 'ChildObjectDetector') 61 | else: 62 | raise Exception("Invalid model configuration") 63 | 64 | # if 'device' in kwargs: 65 | # device = kwargs['device'] 66 | # if device < -1: 67 | # os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" 68 | # device = prf_utils.get_available_device() 69 | # os.environ['CUDA_VISIBLE_DEVICES'] = str(device) 70 | # else: 71 | # os.environ['CUDA_VISIBLE_DEVICES'] = '-1' 72 | 73 | device = None 74 | if 'device' in kwargs: 75 | device = kwargs['device'] 76 | if device == -2: 77 | device = prf_utils.get_available_device() 78 | 79 | if device is not None: 80 | if device >= 0: 81 | os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" 82 | os.environ['CUDA_VISIBLE_DEVICES'] = str(device) 83 | arcpy.env.processorType = "GPU" 84 | arcpy.env.gpuId = str(device) 85 | else: 86 | arcpy.env.processorType = "CPU" 87 | else: 88 | arcpy.env.processorType = "CPU" 89 | 90 | self.child_object_detector = ChildModelDetector() 91 | self.child_object_detector.initialize(model, model_as_file) 92 | 93 | 94 | def getParameterInfo(self): 95 | 96 | # PRF needs values of these parameters from gp tool user, 97 | # either from gp tool UI or emd (a json) file. 98 | required_parameters = [ 99 | { 100 | # To support mini batch, it is required that Classify Objects Using Deep Learning geoprocessing Tool 101 | # passes down a stack of raster tiles to PRF for model inference, the keyword required here is 'rasters'. 102 | 'name': 'rasters', 103 | 'dataType': 'rasters', 104 | 'value': None, 105 | 'required': True, 106 | 'displayName': "Rasters", 107 | 'description': 'The collection of overlapping rasters to objects to be classified' 108 | }, 109 | { 110 | 'name': 'model', 111 | 'dataType': 'string', 112 | 'required': True, 113 | 'displayName': 'Input Model Definition (EMD) File', 114 | 'description': 'Input model definition (EMD) JSON file' 115 | }, 116 | { 117 | 'name': 'device', 118 | 'dataType': 'numeric', 119 | 'required': False, 120 | 'displayName': 'Device ID', 121 | 'description': 'Device ID' 122 | } 123 | ] 124 | 125 | if 'BatchSize' not in self.json_info: 126 | required_parameters.append( 127 | { 128 | 'name': 'batch_size', 129 | 'dataType': 'numeric', 130 | 'required': False, 131 | 'value': 4, 132 | 'displayName': 'Batch Size', 133 | 'description': 'Batch Size' 134 | } 135 | ) 136 | 137 | return self.child_object_detector.getParameterInfo(required_parameters) 138 | 139 | 140 | def getConfiguration(self, **scalars): 141 | 142 | # The information PRF returns to the GP tool, 143 | # the information is either from emd or defined in getConfiguration method. 144 | 145 | configuration = self.child_object_detector.getConfiguration(**scalars) 146 | 147 | if 'DataRange' in self.json_info: 148 | configuration['dataRange'] = tuple(self.json_info['DataRange']) 149 | 150 | configuration['inheritProperties'] = 2|4|8 151 | configuration['inputMask'] = True 152 | 153 | return configuration 154 | 155 | def getFields(self): 156 | 157 | if 'Label' not in fields['fields']: 158 | fields['fields'].append( 159 | { 160 | 'name': 'Label', 161 | 'type': 'esriFieldTypeString', 162 | 'alias': 'Label' 163 | } 164 | ) 165 | return json.dumps(fields) 166 | 167 | def getGeometryType(self): 168 | return GeometryType.Polygon 169 | 170 | def vectorize(self, **pixelBlocks): 171 | 172 | # set pixel values in invalid areas to 0 173 | rasters_mask = pixelBlocks['rasters_mask'] 174 | rasters_pixels = pixelBlocks['rasters_pixels'] 175 | 176 | for i in range(0, len(rasters_pixels)): 177 | rasters_pixels[i][np.where(rasters_mask[i] == 0)] = 0 178 | 179 | pixelBlocks['rasters_pixels'] = rasters_pixels 180 | 181 | polygon_list, scores, labels = self.child_object_detector.vectorize(**pixelBlocks) 182 | 183 | features['features'] = [] 184 | 185 | features['fieldAliases'].update({ 186 | 'Label': 'Label' 187 | }) 188 | features['fields'].append( 189 | { 190 | 'name': 'Label', 191 | 'type': 'esriFieldTypeString', 192 | 'alias': 'Label' 193 | } 194 | ) 195 | for i in range(len(polygon_list)): 196 | 197 | rings = [[]] 198 | for j in range(len(polygon_list[i])): 199 | rings[0].append( 200 | [ 201 | polygon_list[i][j][1], 202 | polygon_list[i][j][0] 203 | ] 204 | ) 205 | 206 | features['features'].append({ 207 | 'attributes': { 208 | 'OID': i + 1, 209 | 'Confidence': str(scores[i]), 210 | 'Label': labels[i] 211 | }, 212 | 'geometry': { 213 | 'rings': rings 214 | } 215 | }) 216 | 217 | return {'output_vectors': json.dumps(features)} 218 | -------------------------------------------------------------------------------- /python_raster_functions/PyTorch/model.py: -------------------------------------------------------------------------------- 1 | import os 2 | # torch.cuda.set_device(0) 3 | from distutils.version import LooseVersion 4 | 5 | import numpy as np 6 | import torch 7 | import torch.nn.functional as F 8 | from torch import nn 9 | from torch.autograd import Variable 10 | from torch.nn.init import kaiming_normal 11 | from torchvision.models import * 12 | 13 | total_classes = 2 14 | k = 9 15 | 16 | def children(m): return m if isinstance(m, (list, tuple)) else list(m.children()) 17 | 18 | def apply_init(m, init_fn): 19 | m.apply(lambda x: cond_init(x, init_fn)) 20 | 21 | def cond_init(m, init_fn): 22 | if not isinstance(m, (nn.BatchNorm1d,nn.BatchNorm2d,nn.BatchNorm3d)): 23 | if hasattr(m, 'weight'): init_fn(m.weight) 24 | if hasattr(m, 'bias') and hasattr(m.bias, 'data'): m.bias.data.fill_(0.) 25 | 26 | class StdConv(nn.Module): 27 | def __init__(self, nin, nout, stride=2, drop=0.1): 28 | super().__init__() 29 | self.conv = nn.Conv2d(nin, nout, 3, stride=stride, padding=1) 30 | self.bn = nn.BatchNorm2d(nout) 31 | self.drop = nn.Dropout(drop) 32 | 33 | def forward(self, x): return self.drop(self.bn(F.relu(self.conv(x)))) 34 | 35 | def flatten_conv(x,k): 36 | bs,nf,gx,gy = x.size() # batch size, num filters, width, height 37 | x = x.permute(0,3,2,1).contiguous() 38 | return x.view(bs,-1,nf//k) 39 | 40 | class OutConv(nn.Module): 41 | def __init__(self, k, nin, bias): 42 | super().__init__() 43 | self.k = k 44 | self.oconv1 = nn.Conv2d(nin, (total_classes + 1)*k, 3, padding=1) # nclasses 45 | self.oconv2 = nn.Conv2d(nin, 4*k, 3, padding=1) # bboxes 46 | self.oconv1.bias.data.zero_().add_(bias) 47 | 48 | def forward(self, x): 49 | return [flatten_conv(self.oconv1(x), self.k), 50 | flatten_conv(self.oconv2(x), self.k)] 51 | 52 | drop=0.4 53 | 54 | class SSD_MultiHead(nn.Module): 55 | def __init__(self, k, bias): 56 | super().__init__() 57 | self.drop = nn.Dropout(drop) 58 | self.sconv0 = StdConv(512,256, stride=1, drop=drop) 59 | self.sconv1 = StdConv(256,256, drop=drop) 60 | self.sconv2 = StdConv(256,256, drop=drop) 61 | self.sconv3 = StdConv(256,256, drop=drop) 62 | self.out0 = OutConv(k, 256, bias) 63 | self.out1 = OutConv(k, 256, bias) 64 | self.out2 = OutConv(k, 256, bias) 65 | self.out3 = OutConv(k, 256, bias) 66 | 67 | def forward(self, x): 68 | x = self.drop(F.relu(x)) 69 | x = self.sconv0(x) 70 | x = self.sconv1(x) 71 | o1c,o1l = self.out1(x) 72 | x = self.sconv2(x) 73 | o2c,o2l = self.out2(x) 74 | x = self.sconv3(x) 75 | o3c,o3l = self.out3(x) 76 | return [torch.cat([o1c,o2c,o3c], dim=1), 77 | torch.cat([o1l,o2l,o3l], dim=1)] 78 | 79 | 80 | 81 | IS_TORCH_04 = LooseVersion(torch.__version__) >= LooseVersion('0.4') 82 | 83 | model_meta = { 84 | resnet18:[8,6], resnet34:[8,6], resnet50:[8,6], resnet101:[8,6], resnet152:[8,6], 85 | vgg16:[0,22], vgg19:[0,22] 86 | } 87 | 88 | requires_grad = False 89 | #if 'CUDA_VISIBLE_DEVICES' not in os.environ or int(os.environ['CUDA_VISIBLE_DEVICES']) >= 0: 90 | USE_GPU = False 91 | if 'CUDA_VISIBLE_DEVICES' in os.environ: 92 | if int(os.environ['CUDA_VISIBLE_DEVICES']) >= 0: 93 | USE_GPU = True 94 | 95 | def map_over(x, f): return [f(o) for o in x] if is_listy(x) else f(x) 96 | def V (x, requires_grad=False, volatile=False): return map_over(x, lambda o: V_(o, requires_grad, volatile)) 97 | def V_(x, requires_grad=False, volatile=False): return create_variable(x, volatile, requires_grad) 98 | def is_listy(x): return isinstance(x, (list,tuple)) 99 | def A(*a): return np.array(a[0]) if len(a)==1 else [np.array(o) for o in a] 100 | 101 | def create_variable(x, volatile, requires_grad=False): 102 | if type (x) != Variable: 103 | if IS_TORCH_04: x = Variable(T(x), requires_grad=requires_grad) 104 | else: x = Variable(T(x), requires_grad=requires_grad, volatile=volatile) 105 | return x 106 | 107 | def T(a, half=False, cuda=True): 108 | if not torch.is_tensor(a): 109 | a = np.array(np.ascontiguousarray(a)) 110 | if a.dtype in (np.int8, np.int16, np.int32, np.int64): 111 | a = torch.LongTensor(a.astype(np.int64)) 112 | elif a.dtype in (np.float32, np.float64): 113 | a = torch.cuda.HalfTensor(a) if half else torch.FloatTensor(a) 114 | else: raise NotImplementedError(a.dtype) 115 | if cuda: a = to_gpu(a, async=True) 116 | return a 117 | 118 | def to_gpu(x, *args, **kwargs): 119 | return x.cuda(*args, **kwargs) if USE_GPU else x 120 | 121 | def to_np(v): 122 | if isinstance(v, (np.ndarray, np.generic)): return v 123 | if isinstance(v, (list,tuple)): return [to_np(o) for o in v] 124 | if isinstance(v, Variable): v=v.data 125 | if USE_GPU: 126 | if isinstance(v, torch.cuda.HalfTensor): v=v.float() 127 | else: 128 | if isinstance(v, torch.HalfTensor): v=v.float() 129 | return v.cpu().numpy() 130 | 131 | def cut_model(m, cut): 132 | return list(m.children())[:cut] if cut else [m] 133 | 134 | class AdaptiveConcatPool2d(nn.Module): 135 | def __init__(self, sz=None): 136 | super().__init__() 137 | sz = sz or (1,1) 138 | self.ap = nn.AdaptiveAvgPool2d(sz) 139 | self.mp = nn.AdaptiveMaxPool2d(sz) 140 | def forward(self, x): return torch.cat([self.mp(x), self.ap(x)], 1) 141 | 142 | class Flatten(nn.Module): 143 | def __init__(self): super().__init__() 144 | def forward(self, x): return x.view(x.size(0), -1) 145 | 146 | def num_features(m): 147 | c=children(m) 148 | if len(c)==0: return None 149 | for l in reversed(c): 150 | if hasattr(l, 'num_features'): return l.num_features 151 | res = num_features(l) 152 | if res is not None: return res 153 | 154 | class ConvnetBuilder(): 155 | 156 | def __init__(self, f, c, is_multi, is_reg, ps=None, xtra_fc=None, xtra_cut=0, custom_head=None, pretrained=True): 157 | self.f,self.c,self.is_multi,self.is_reg,self.xtra_cut = f,c,is_multi,is_reg,xtra_cut 158 | if xtra_fc is None: xtra_fc = [512] 159 | if ps is None: ps = [0.25]*len(xtra_fc) + [0.5] 160 | self.ps,self.xtra_fc = ps,xtra_fc 161 | 162 | if f in model_meta: cut,self.lr_cut = model_meta[f] 163 | else: cut,self.lr_cut = 0,0 164 | cut-=xtra_cut 165 | layers = cut_model(f(pretrained), cut) 166 | self.nf = num_features(layers)*2 167 | if not custom_head: layers += [AdaptiveConcatPool2d(), Flatten()] 168 | self.top_model = nn.Sequential(*layers) 169 | n_fc = len(self.xtra_fc)+1 170 | if not isinstance(self.ps, list): self.ps = [self.ps]*n_fc 171 | 172 | if custom_head: fc_layers = [custom_head] 173 | else: fc_layers = self.get_fc_layers() 174 | self.n_fc = len(fc_layers) 175 | self.fc_model = to_gpu(nn.Sequential(*fc_layers)) 176 | if not custom_head: apply_init(self.fc_model, kaiming_normal) 177 | self.model = to_gpu(nn.Sequential(*(layers+fc_layers))) 178 | 179 | @property 180 | def name(self): return f'{self.f.__name__}_{self.xtra_cut}' 181 | 182 | def create_fc_layer(self, ni, nf, p, actn=None): 183 | res=[nn.BatchNorm1d(num_features=ni)] 184 | if p: res.append(nn.Dropout(p=p)) 185 | res.append(nn.Linear(in_features=ni, out_features=nf)) 186 | if actn: res.append(actn) 187 | return res 188 | 189 | def get_fc_layers(self): 190 | res=[] 191 | ni=self.nf 192 | for i,nf in enumerate(self.xtra_fc): 193 | res += self.create_fc_layer(ni, nf, p=self.ps[i], actn=nn.ReLU()) 194 | ni=nf 195 | final_actn = nn.Sigmoid() if self.is_multi else nn.LogSoftmax(1) 196 | if self.is_reg: final_actn = None 197 | res += self.create_fc_layer(ni, self.c, p=self.ps[-1], actn=final_actn) 198 | return res 199 | 200 | def get_layer_groups(self, do_fc=False): 201 | if do_fc: 202 | return [self.fc_model] 203 | idxs = [self.lr_cut] 204 | c = children(self.top_model) 205 | if len(c)==3: c = children(c[0])+c[1:] 206 | lgs = list(split_by_idxs(c,idxs)) 207 | return lgs+[self.fc_model] -------------------------------------------------------------------------------- /python_raster_functions/CNTK/utils/caffe_layers/proposal_target_layer.py: -------------------------------------------------------------------------------- 1 | # -------------------------------------------------------- 2 | # Faster R-CNN 3 | # Copyright (c) 2015 Microsoft 4 | # Licensed under The MIT License [see LICENSE for details] 5 | # Written by Ross Girshick and Sean Bell 6 | # -------------------------------------------------------- 7 | 8 | #import caffe 9 | import yaml 10 | import numpy as np 11 | import numpy.random as npr 12 | from utils.caffe_layers.default_config import cfg 13 | from utils.rpn.bbox_transform import bbox_transform 14 | from utils.cython_modules.cython_bbox import bbox_overlaps 15 | 16 | DEBUG = False 17 | 18 | class ProposalTargetLayer(): #caffe.Layer): 19 | """ 20 | Assign object detection proposals to ground-truth targets. Produces proposal 21 | classification labels and bounding-box regression targets. 22 | """ 23 | 24 | def set_param_str(self, param_str): 25 | self.param_str_ = param_str 26 | 27 | def set_deterministic_mode(self, mode = True): 28 | self._determininistic_mode = mode 29 | 30 | def setup(self, bottom, top): 31 | layer_params = yaml.load(self.param_str_) 32 | self._num_classes = layer_params['num_classes'] 33 | self._determininistic_mode = False 34 | 35 | # sampled rois (0, x1, y1, x2, y2) 36 | #top[0].reshape(1, 5) 37 | # labels 38 | #top[1].reshape(1, 1) 39 | # bbox_targets 40 | #top[2].reshape(1, self._num_classes * 4) 41 | # bbox_inside_weights 42 | #top[3].reshape(1, self._num_classes * 4) 43 | # bbox_outside_weights 44 | #top[4].reshape(1, self._num_classes * 4) 45 | 46 | def forward(self, bottom, top): 47 | # Proposal ROIs (0, x1, y1, x2, y2) coming from RPN 48 | # (i.e., rpn.proposal_layer.ProposalLayer), or any other source 49 | all_rois = bottom[0] #.data 50 | # GT boxes (x1, y1, x2, y2, label) 51 | # TODO(rbg): it's annoying that sometimes I have extra info before 52 | # and other times after box coordinates -- normalize to one format 53 | gt_boxes = bottom[1] #.data 54 | 55 | # Include ground-truth boxes in the set of candidate rois 56 | zeros = np.zeros((gt_boxes.shape[0], 1), dtype=gt_boxes.dtype) 57 | all_rois = np.vstack( 58 | (all_rois, np.hstack((zeros, gt_boxes[:, :-1]))) 59 | ) 60 | 61 | # Sanity check: single batch only 62 | assert np.all(all_rois[:, 0] == 0), \ 63 | 'Only single item batches are supported' 64 | 65 | #num_images = 1 66 | #rois_per_image = int(cfg.TRAIN.BATCH_SIZE / num_images) 67 | rois_per_image = cfg.TRAIN.BATCH_SIZE 68 | fg_rois_per_image = np.round(cfg.TRAIN.FG_FRACTION * rois_per_image).astype(int) 69 | 70 | # Sample rois with classification labels and bounding box regression 71 | # targets 72 | labels, rois, bbox_targets, bbox_inside_weights = _sample_rois( 73 | all_rois, gt_boxes, fg_rois_per_image, 74 | rois_per_image, self._num_classes, 75 | deterministic=self._determininistic_mode) 76 | 77 | if DEBUG: 78 | print('num fg: {}'.format((labels > 0).sum())) 79 | print('num bg: {}'.format((labels == 0).sum())) 80 | self._count += 1 81 | self._fg_num += (labels > 0).sum() 82 | self._bg_num += (labels == 0).sum() 83 | print('num fg avg: {}'.format(self._fg_num / self._count)) 84 | print('num bg avg: {}'.format(self._bg_num / self._count)) 85 | print('ratio: {:.3f}'.format(float(self._fg_num) / float(self._bg_num))) 86 | 87 | return rois, labels, bbox_targets, bbox_inside_weights 88 | 89 | # sampled rois 90 | #top[0].reshape(*rois.shape) 91 | #top[0].data[...] = rois 92 | 93 | # classification labels 94 | #top[1].reshape(*labels.shape) 95 | #top[1].data[...] = labels 96 | 97 | # bbox_targets 98 | #top[2].reshape(*bbox_targets.shape) 99 | #top[2].data[...] = bbox_targets 100 | 101 | # bbox_inside_weights 102 | #top[3].reshape(*bbox_inside_weights.shape) 103 | #top[3].data[...] = bbox_inside_weights 104 | 105 | # bbox_outside_weights 106 | #top[4].reshape(*bbox_inside_weights.shape) 107 | #top[4].data[...] = np.array(bbox_inside_weights > 0).astype(np.float32) 108 | 109 | def backward(self, top, propagate_down, bottom): 110 | """This layer does not propagate gradients.""" 111 | pass 112 | 113 | def reshape(self, bottom, top): 114 | """Reshaping happens during the call to forward.""" 115 | pass 116 | 117 | 118 | def _get_bbox_regression_labels(bbox_target_data, num_classes): 119 | """Bounding-box regression targets (bbox_target_data) are stored in a 120 | compact form N x (class, tx, ty, tw, th) 121 | 122 | This function expands those targets into the 4-of-4*K representation used 123 | by the network (i.e. only one class has non-zero targets). 124 | 125 | Returns: 126 | bbox_target (ndarray): N x 4K blob of regression targets 127 | bbox_inside_weights (ndarray): N x 4K blob of loss weights 128 | """ 129 | 130 | clss = bbox_target_data[:, 0].astype(int) 131 | bbox_targets = np.zeros((clss.size, 4 * num_classes), dtype=np.float32) 132 | bbox_inside_weights = np.zeros(bbox_targets.shape, dtype=np.float32) 133 | inds = np.where(clss > 0)[0] 134 | for ind in inds: 135 | cls = clss[ind] 136 | start = 4 * cls 137 | end = start + 4 138 | bbox_targets[ind, start:end] = bbox_target_data[ind, 1:] 139 | bbox_inside_weights[ind, start:end] = cfg.TRAIN.BBOX_INSIDE_WEIGHTS 140 | return bbox_targets, bbox_inside_weights 141 | 142 | 143 | def _compute_targets(ex_rois, gt_rois, labels): 144 | """Compute bounding-box regression targets for an image.""" 145 | 146 | assert ex_rois.shape[0] == gt_rois.shape[0] 147 | assert ex_rois.shape[1] == 4 148 | assert gt_rois.shape[1] == 4 149 | 150 | targets = bbox_transform(ex_rois, gt_rois) 151 | if cfg.TRAIN.BBOX_NORMALIZE_TARGETS_PRECOMPUTED: 152 | # Optionally normalize targets by a precomputed mean and stdev 153 | targets = ((targets - np.array(cfg.TRAIN.BBOX_NORMALIZE_MEANS)) 154 | / np.array(cfg.TRAIN.BBOX_NORMALIZE_STDS)) 155 | return np.hstack( 156 | (labels[:, np.newaxis], targets)).astype(np.float32, copy=False) 157 | 158 | def _sample_rois(all_rois, gt_boxes, fg_rois_per_image, rois_per_image, num_classes, deterministic=False): 159 | """Generate a random sample of RoIs comprising foreground and background 160 | examples. 161 | """ 162 | # overlaps: (rois x gt_boxes) 163 | overlaps = bbox_overlaps( 164 | np.ascontiguousarray(all_rois[:, 1:5], dtype=np.float), 165 | np.ascontiguousarray(gt_boxes[:, :4], dtype=np.float)) 166 | gt_assignment = overlaps.argmax(axis=1) 167 | max_overlaps = overlaps.max(axis=1) 168 | labels = gt_boxes[gt_assignment, 4] 169 | 170 | # Select foreground RoIs as those with >= FG_THRESH overlap 171 | fg_inds = np.where(max_overlaps >= cfg.TRAIN.FG_THRESH)[0] 172 | # Guard against the case when an image has fewer than fg_rois_per_image 173 | # foreground RoIs 174 | fg_rois_per_this_image = min(fg_rois_per_image, fg_inds.size) 175 | 176 | # Sample foreground regions without replacement 177 | if fg_inds.size > 0: 178 | if deterministic: 179 | fg_inds = fg_inds[:fg_rois_per_this_image] 180 | else: 181 | fg_inds = npr.choice(fg_inds, size=fg_rois_per_this_image, replace=False) 182 | 183 | # Select background RoIs as those within [BG_THRESH_LO, BG_THRESH_HI) 184 | bg_inds = np.where((max_overlaps < cfg.TRAIN.BG_THRESH_HI) & 185 | (max_overlaps >= cfg.TRAIN.BG_THRESH_LO))[0] 186 | # Compute number of background RoIs to take from this image (guarding 187 | # against there being fewer than desired) 188 | bg_rois_per_this_image = rois_per_image - fg_rois_per_this_image 189 | bg_rois_per_this_image = min(bg_rois_per_this_image, bg_inds.size) 190 | # Sample background regions without replacement 191 | if bg_inds.size > 0: 192 | if deterministic: 193 | bg_inds = bg_inds[:bg_rois_per_this_image] 194 | else: 195 | bg_inds = npr.choice(bg_inds, size=bg_rois_per_this_image, replace=False) 196 | 197 | # The indices that we're selecting (both fg and bg) 198 | keep_inds = np.append(fg_inds, bg_inds) 199 | # Select sampled values from various arrays: 200 | labels = labels[keep_inds] 201 | # Clamp labels for the background RoIs to 0 202 | labels[fg_rois_per_this_image:] = 0 203 | rois = all_rois[keep_inds] 204 | 205 | bbox_target_data = _compute_targets( 206 | rois[:, 1:5], gt_boxes[gt_assignment[keep_inds], :4], labels) 207 | 208 | bbox_targets, bbox_inside_weights = \ 209 | _get_bbox_regression_labels(bbox_target_data, num_classes) 210 | 211 | return labels, rois, bbox_targets, bbox_inside_weights 212 | --------------------------------------------------------------------------------