├── .gitignore ├── LICENSE ├── README.md ├── ade20k_labels.py ├── caffe-tensorflow ├── .gitignore ├── .pylintrc ├── .style.yapf ├── LICENSE.md ├── README.md ├── convert.py ├── examples │ ├── imagenet │ │ ├── README.md │ │ ├── classify.py │ │ ├── dataset.py │ │ ├── imagenet-classes.txt │ │ ├── models │ │ │ ├── __init__.py │ │ │ ├── alexnet.py │ │ │ ├── caffenet.py │ │ │ ├── googlenet.py │ │ │ ├── helper.py │ │ │ ├── nin.py │ │ │ ├── resnet.py │ │ │ └── vgg.py │ │ └── validate.py │ └── mnist │ │ ├── README.md │ │ ├── finetune_mnist.py │ │ ├── lenet.prototxt │ │ └── lenet_iter_10000.caffemodel └── kaffe │ ├── __init__.py │ ├── caffe │ ├── __init__.py │ ├── caffepb.py │ └── resolver.py │ ├── errors.py │ ├── graph.py │ ├── layers.py │ ├── shapes.py │ ├── tensorflow │ ├── __init__.py │ ├── network.py │ └── transformer.py │ └── transformers.py ├── cityscapes_labels.py ├── drawImage ├── __init__.py ├── __init__.pyc ├── drawModule.py └── drawModule.pyc ├── example_images ├── ade20k.jpg ├── pascal_voc.jpg └── test_pycaffe.jpg ├── example_results ├── ade20k_probs.jpg ├── ade20k_seg.jpg ├── ade20k_seg_blended.jpg ├── ade20k_seg_pycaffe.jpg ├── cityscapes_probs.jpg ├── cityscapes_seg.jpg ├── cityscapes_seg_blended.jpg ├── pascal_voc_probs.jpg ├── pascal_voc_seg.jpg └── pascal_voc_seg_blended.jpg ├── layers_builder.py ├── pascal_voc_labels.py ├── pspnet-video.py ├── pspnet.py ├── requirements.txt ├── utils.py ├── utils ├── color150 │ ├── airplane.jpg │ ├── animal.jpg │ ├── apparel.jpg │ ├── arcade machine.jpg │ ├── armchair.jpg │ ├── ashcan.jpg │ ├── awning.jpg │ ├── bag.jpg │ ├── ball.jpg │ ├── bannister.jpg │ ├── bar.jpg │ ├── barrel.jpg │ ├── base.jpg │ ├── basket.jpg │ ├── bathtub.jpg │ ├── bed.jpg │ ├── bench.jpg │ ├── bicycle.jpg │ ├── bike.jpg │ ├── blanket.jpg │ ├── blind.jpg │ ├── boat.jpg │ ├── book.jpg │ ├── bookcase.jpg │ ├── booth.jpg │ ├── bottle.jpg │ ├── box.jpg │ ├── bridge.jpg │ ├── buffet.jpg │ ├── building.jpg │ ├── bulletin board.jpg │ ├── bus.jpg │ ├── cabinet.jpg │ ├── canopy.jpg │ ├── car.jpg │ ├── case.jpg │ ├── ceiling.jpg │ ├── chair.jpg │ ├── chandelier.jpg │ ├── chest of drawers.jpg │ ├── clock.jpg │ ├── coffee table.jpg │ ├── column.jpg │ ├── computer.jpg │ ├── conveyer belt.jpg │ ├── counter.jpg │ ├── countertop.jpg │ ├── cradle.jpg │ ├── crt screen.jpg │ ├── curtain.jpg │ ├── cushion.jpg │ ├── desk.jpg │ ├── dirt track.jpg │ ├── dishwasher.jpg │ ├── door.jpg │ ├── earth.jpg │ ├── escalator.jpg │ ├── fan.jpg │ ├── fence.jpg │ ├── field.jpg │ ├── fireplace.jpg │ ├── flag.jpg │ ├── floor.jpg │ ├── flower.jpg │ ├── food.jpg │ ├── fountain.jpg │ ├── furniture.jpg │ ├── glass.jpg │ ├── grandstand.jpg │ ├── grass.jpg │ ├── hill.jpg │ ├── hood.jpg │ ├── house.jpg │ ├── houseware.jpg │ ├── hovel.jpg │ ├── kitchen island.jpg │ ├── lake.jpg │ ├── lamp.jpg │ ├── land.jpg │ ├── light.jpg │ ├── microwave.jpg │ ├── minibike.jpg │ ├── mirror.jpg │ ├── monitor.jpg │ ├── mountain.jpg │ ├── ottoman.jpg │ ├── oven.jpg │ ├── painting.jpg │ ├── palm.jpg │ ├── path.jpg │ ├── person.jpg │ ├── pier.jpg │ ├── pillow.jpg │ ├── plant.jpg │ ├── plate.jpg │ ├── plaything.jpg │ ├── pole.jpg │ ├── pool table.jpg │ ├── poster.jpg │ ├── pot.jpg │ ├── radiator.jpg │ ├── railing.jpg │ ├── refrigerator.jpg │ ├── river.jpg │ ├── road.jpg │ ├── rock.jpg │ ├── rug.jpg │ ├── runway.jpg │ ├── sand.jpg │ ├── sconce.jpg │ ├── screen door.jpg │ ├── screen.jpg │ ├── sculpture.jpg │ ├── sea.jpg │ ├── seat.jpg │ ├── shelf.jpg │ ├── ship.jpg │ ├── shower.jpg │ ├── sidewalk.jpg │ ├── signboard.jpg │ ├── sink.jpg │ ├── sky.jpg │ ├── skyscraper.jpg │ ├── sofa.jpg │ ├── stage.jpg │ ├── stairs.jpg │ ├── stairway.jpg │ ├── step.jpg │ ├── stool.jpg │ ├── stove.jpg │ ├── streetlight.jpg │ ├── swimming pool.jpg │ ├── swivel chair.jpg │ ├── table.jpg │ ├── tank.jpg │ ├── television receiver.jpg │ ├── tent.jpg │ ├── toilet.jpg │ ├── towel.jpg │ ├── tower.jpg │ ├── trade name.jpg │ ├── traffic light.jpg │ ├── tray.jpg │ ├── tree.jpg │ ├── truck.jpg │ ├── van.jpg │ ├── vase.jpg │ ├── wall.jpg │ ├── wardrobe.jpg │ ├── washer.jpg │ ├── water.jpg │ ├── waterfall.jpg │ └── windowpane.jpg ├── colorization │ ├── color150.mat │ └── objectName150.mat └── model │ └── pspnet.prototxt ├── weight_converter.py └── weights ├── caffe └── Put caffemodels & prototxts here ├── keras └── Put keras weights here └── npy └── Put the npy weights here /.gitignore: -------------------------------------------------------------------------------- 1 | *.npy 2 | *.pyc 3 | *.png 4 | *.caffemodel 5 | *.prototxt 6 | *.h5 7 | *.json 8 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 VladKry 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Keras implementation of [PSPNet(caffe)](https://github.com/hszhao/PSPNet) 2 | 3 | Implemented Architecture of Pyramid Scene Parsing Network in Keras. 4 | 5 | ### Setup 6 | 1. Install dependencies: 7 | * Tensorflow (-gpu) 8 | * Keras 9 | * numpy 10 | * scipy 11 | * pycaffe(PSPNet)(optional for converting the weights) 12 | ```bash 13 | pip install -r requirements.txt --upgrade 14 | ``` 15 | 2. Converted trained weights are needed to run the network. 16 | Weights(in ```.npy``` format) have to be downloaded and placed into directory ``` weights/npy ``` 17 | 18 | 19 | Already converted weights can be downloaded here: 20 | 21 | [pspnet50_ade20k.npy](https://www.dropbox.com/s/ms8afun494dlh1t/pspnet50_ade20k.npy?dl=0) 22 | [pspnet101_cityscapes.npy](https://www.dropbox.com/s/b21j6hi6qql90l0/pspnet101_cityscapes.npy?dl=0) 23 | [pspnet101_voc2012.npy](https://www.dropbox.com/s/xkjmghsbn6sfj9k/pspnet101_voc2012.npy?dl=0) 24 | 25 | Weights are **automatically** converted to the ```json/h5``` format on **first start** 26 | 27 | (*optional*)If you want to convert by yourself original caffemodel, it can be converted with ```weight_converter.py``` 28 | 29 | Running this needs the compiled original PSPNet caffe code and pycaffe. 30 | 31 | ```bash 32 | python weight_converter.py 33 | ``` 34 | 35 | ## Usage: 36 | 37 | ```bash 38 | python pspnet.py -m -i -o 39 | python pspnet.py -m pspnet101_cityscapes -i example_images/cityscapes.png -o example_results/cityscapes.jpg 40 | python pspnet.py -m pspnet101_voc2012 -i example_images/pascal_voc.jpg -o example_results/pascal_voc.jpg 41 | ``` 42 | List of arguments: 43 | ```bash 44 | -m --model - which model to use: 'pspnet50_ade20k', 'pspnet101_cityscapes', 'pspnet101_voc2012' 45 | --id - (int) GPU Device id. Default 0 46 | -s --sliding - Use sliding window 47 | -f --flip - Additional prediction of flipped image 48 | -ms --multi_scale - Predict on multiscale images 49 | ``` 50 | ## Keras results: 51 | ![Original](example_images/ade20k.jpg) 52 | ![New](example_results/ade20k_seg.jpg) 53 | ![New](example_results/ade20k_seg_blended.jpg) 54 | ![New](example_results/ade20k_probs.jpg) 55 | 56 | ![Original](example_images/cityscapes.png) 57 | ![New](example_results/cityscapes_seg.jpg) 58 | ![New](example_results/cityscapes_seg_blended.jpg) 59 | ![New](example_results/cityscapes_probs.jpg) 60 | 61 | ![Original](example_images/pascal_voc.jpg) 62 | ![New](example_results/pascal_voc_seg.jpg) 63 | ![New](example_results/pascal_voc_seg_blended.jpg) 64 | ![New](example_results/pascal_voc_probs.jpg) 65 | 66 | 67 | ## Implementation details 68 | * The interpolation layer is implemented as custom layer "Interp" 69 | * Forward step takes about ~1 sec on single image 70 | * Memory usage can be optimized with: 71 | ```python 72 | config = tf.ConfigProto() 73 | config.gpu_options.per_process_gpu_memory_fraction = 0.3 74 | sess = tf.Session(config=config) 75 | ``` 76 | * ```ndimage.zoom``` can take a long time 77 | 78 | 79 | 80 | 81 | 82 | -------------------------------------------------------------------------------- /ade20k_labels.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # 3 | # ADE20k labels 4 | # 5 | 6 | from collections import namedtuple 7 | import scipy.io 8 | 9 | 10 | Label = namedtuple('Label', [ 11 | 12 | 'name', 13 | 'id', 14 | 'color' 15 | ]) 16 | 17 | labels = [Label('wall', 0, (120, 120, 120)), 18 | Label('building', 1, (180, 120, 120)), 19 | Label('sky', 2, (6, 230, 230)), 20 | Label('floor', 3, (80, 50, 50)), 21 | Label('tree', 4, (4, 200, 3)), 22 | Label('ceiling', 5, (120, 120, 80)), 23 | Label('road', 6, (140, 140, 140)), 24 | Label('bed', 7, (204, 5, 255)), 25 | Label('windowpane', 8, (230, 230, 230)), 26 | Label('grass', 9, (4, 250, 7)), 27 | Label('cabinet', 10, (224, 5, 255)), 28 | Label('sidewalk', 11, (235, 255, 7)), 29 | Label('person', 12, (150, 5, 61)), 30 | Label('earth', 13, (120, 120, 70)), 31 | Label('door', 14, (8, 255, 51)), 32 | Label('table', 15, (255, 6, 82)), 33 | Label('mountain', 16, (143, 255, 140)), 34 | Label('plant', 17, (204, 255, 4)), 35 | Label('curtain', 18, (255, 51, 7)), 36 | Label('chair', 19, (204, 70, 3)), 37 | Label('car', 20, (0, 102, 200)), 38 | Label('water', 21, (61, 230, 250)), 39 | Label('painting', 22, (255, 6, 51)), 40 | Label('sofa', 23, (11, 102, 255)), 41 | Label('shelf', 24, (255, 7, 71)), 42 | Label('house', 25, (255, 9, 224)), 43 | Label('sea', 26, (9, 7, 230)), 44 | Label('mirror', 27, (220, 220, 220)), 45 | Label('rug', 28, (255, 9, 92)), 46 | Label('field', 29, (112, 9, 255)), 47 | Label('armchair', 30, (8, 255, 214)), 48 | Label('seat', 31, (7, 255, 224)), 49 | Label('fence', 32, (255, 184, 6)), 50 | Label('desk', 33, (10, 255, 71)), 51 | Label('rock', 34, (255, 41, 10)), 52 | Label('wardrobe', 35, (7, 255, 255)), 53 | Label('lamp', 36, (224, 255, 8)), 54 | Label('bathtub', 37, (102, 8, 255)), 55 | Label('railing', 38, (255, 61, 6)), 56 | Label('cushion', 39, (255, 194, 7)), 57 | Label('base', 40, (255, 122, 8)), 58 | Label('box', 41, (0, 255, 20)), 59 | Label('column', 42, (255, 8, 41)), 60 | Label('signboard', 43, (255, 5, 153)), 61 | Label('chest of drawers', 44, (6, 51, 255)), 62 | Label('counter', 45, (235, 12, 255)), 63 | Label('sand', 46, (160, 150, 20)), 64 | Label('sink', 47, (0, 163, 255)), 65 | Label('skyscraper', 48, (140, 140, 140)), 66 | Label('fireplace', 49, (250, 10, 15)), 67 | Label('refrigerator', 50, (20, 255, 0)), 68 | Label('grandstand', 51, (31, 255, 0)), 69 | Label('path', 52, (255, 31, 0)), 70 | Label('stairs', 53, (255, 224, 0)), 71 | Label('runway', 54, (153, 255, 0)), 72 | Label('case', 55, (0, 0, 255)), 73 | Label('pool table', 56, (255, 71, 0)), 74 | Label('pillow', 57, (0, 235, 255)), 75 | Label('screen door', 58, (0, 173, 255)), 76 | Label('stairway', 59, (31, 0, 255)), 77 | Label('river', 60, (11, 200, 200)), 78 | Label('bridge', 61, (255, 82, 0)), 79 | Label('bookcase', 62, (0, 255, 245)), 80 | Label('blind', 63, (0, 61, 255)), 81 | Label('coffee table', 64, (0, 255, 112)), 82 | Label('toilet', 65, (0, 255, 133)), 83 | Label('flower', 66, (255, 0, 0)), 84 | Label('book', 67, (255, 163, 0)), 85 | Label('hill', 68, (255, 102, 0)), 86 | Label('bench', 69, (194, 255, 0)), 87 | Label('countertop', 70, (0, 143, 255)), 88 | Label('stove', 71, (51, 255, 0)), 89 | Label('palm', 72, (0, 82, 255)), 90 | Label('kitchen island', 73, (0, 255, 41)), 91 | Label('computer', 74, (0, 255, 173)), 92 | Label('swivel chair', 75, (10, 0, 255)), 93 | Label('boat', 76, (173, 255, 0)), 94 | Label('bar', 77, (0, 255, 153)), 95 | Label('arcade machine', 78, (255, 92, 0)), 96 | Label('hovel', 79, (255, 0, 255)), 97 | Label('bus', 80, (255, 0, 245)), 98 | Label('towel', 81, (255, 0, 102)), 99 | Label('light', 82, (255, 173, 0)), 100 | Label('truck', 83, (255, 0, 20)), 101 | Label('tower', 84, (255, 184, 184)), 102 | Label('chandelier', 85, (0, 31, 255)), 103 | Label('awning', 86, (0, 255, 61)), 104 | Label('streetlight', 87, (0, 71, 255)), 105 | Label('booth', 88, (255, 0, 204)), 106 | Label('television receiver', 89, (0, 255, 194)), 107 | Label('airplane', 90, (0, 255, 82)), 108 | Label('dirt track', 91, (0, 10, 255)), 109 | Label('apparel', 92, (0, 112, 255)), 110 | Label('pole', 93, (51, 0, 255)), 111 | Label('land', 94, (0, 194, 255)), 112 | Label('bannister', 95, (0, 122, 255)), 113 | Label('escalator', 96, (0, 255, 163)), 114 | Label('ottoman', 97, (255, 153, 0)), 115 | Label('bottle', 98, (0, 255, 10)), 116 | Label('buffet', 99, (255, 112, 0)), 117 | Label('poster', 100, (143, 255, 0)), 118 | Label('stage', 101, (82, 0, 255)), 119 | Label('van', 102, (163, 255, 0)), 120 | Label('ship', 103, (255, 235, 0)), 121 | Label('fountain', 104, (8, 184, 170)), 122 | Label('conveyer belt', 105, (133, 0, 255)), 123 | Label('canopy', 106, (0, 255, 92)), 124 | Label('washer', 107, (184, 0, 255)), 125 | Label('plaything', 108, (255, 0, 31)), 126 | Label('swimming pool', 109, (0, 184, 255)), 127 | Label('stool', 110, (0, 214, 255)), 128 | Label('barrel', 111, (255, 0, 112)), 129 | Label('basket', 112, (92, 255, 0)), 130 | Label('waterfall', 113, (0, 224, 255)), 131 | Label('tent', 114, (112, 224, 255)), 132 | Label('bag', 115, (70, 184, 160)), 133 | Label('minibike', 116, (163, 0, 255)), 134 | Label('cradle', 117, (153, 0, 255)), 135 | Label('oven', 118, (71, 255, 0)), 136 | Label('ball', 119, (255, 0, 163)), 137 | Label('food', 120, (255, 204, 0)), 138 | Label('step', 121, (255, 0, 143)), 139 | Label('tank', 122, (0, 255, 235)), 140 | Label('trade name', 123, (133, 255, 0)), 141 | Label('microwave', 124, (255, 0, 235)), 142 | Label('pot', 125, (245, 0, 255)), 143 | Label('animal', 126, (255, 0, 122)), 144 | Label('bicycle', 127, (255, 245, 0)), 145 | Label('lake', 128, (10, 190, 212)), 146 | Label('dishwasher', 129, (214, 255, 0)), 147 | Label('screen', 130, (0, 204, 255)), 148 | Label('blanket', 131, (20, 0, 255)), 149 | Label('sculpture', 132, (255, 255, 0)), 150 | Label('hood', 133, (0, 153, 255)), 151 | Label('sconce', 134, (0, 41, 255)), 152 | Label('vase', 135, (0, 255, 204)), 153 | Label('traffic light', 136, (41, 0, 255)), 154 | Label('tray', 137, (41, 255, 0)), 155 | Label('ashcan', 138, (173, 0, 255)), 156 | Label('fan', 139, (0, 245, 255)), 157 | Label('pier', 140, (71, 0, 255)), 158 | Label('crt screen', 141, (122, 0, 255)), 159 | Label('plate', 142, (0, 255, 184)), 160 | Label('monitor', 143, (0, 92, 255)), 161 | Label('bulletin board', 144, (184, 255, 0)), 162 | Label('shower', 145, (0, 133, 255)), 163 | Label('radiator', 146, (255, 214, 0)), 164 | Label('glass', 147, (25, 194, 194)), 165 | Label('clock', 148, (102, 255, 0)), 166 | Label('flag', 149, (92, 0, 255))] 167 | 168 | ade20k_id2label = {label.id: label for label in labels} 169 | 170 | 171 | def import_labels_from_mat(): 172 | colors = scipy.io.loadmat('utils/colorization/color150.mat') 173 | object_names = scipy.io.loadmat('utils/colorization/objectName150.mat') 174 | for id in range(150): 175 | color = colors['colors'][id] 176 | name = object_names['objectNames'][id][0][0] 177 | labels.append(Label(name, id, color)) 178 | print("Label(\'%s\', %i, (%i, %i, %i))," % (name, id, color[0], color[1], color[2])) 179 | -------------------------------------------------------------------------------- /caffe-tensorflow/.gitignore: -------------------------------------------------------------------------------- 1 | # OS X temporary metadata 2 | ._* 3 | *.DS_Store 4 | 5 | # Extracted parameters 6 | *.params 7 | 8 | # Python cache 9 | *.pyc 10 | 11 | -------------------------------------------------------------------------------- /caffe-tensorflow/.pylintrc: -------------------------------------------------------------------------------- 1 | [MASTER] 2 | ignore=caffepb.py 3 | 4 | [MESSAGES CONTROL] 5 | disable=missing-docstring,invalid-name,wildcard-import,unused-wildcard-import,bad-builtin,no-self-use,locally-disabled 6 | 7 | [MISCELLANEOUS] 8 | # Exclude TODOs 9 | notes= 10 | 11 | [TYPECHECK] 12 | ignored-classes=numpy,cv2,NodeKind,LayerType,NetParameter,NpzFile 13 | 14 | [DESIGN] 15 | # Maximum number of arguments for function / method 16 | max-args=20 17 | # Maximum number of locals for function / method body 18 | max-locals=30 19 | # Maximum number of return / yield for function / method body 20 | max-returns=10 21 | # Maximum number of branch for function / method body 22 | max-branches=12 23 | # Maximum number of statements in function / method body 24 | max-statements=200 25 | # Maximum number of attributes for a class (see R0902). 26 | max-attributes=100 27 | # Maximum number of public methods for a class (see R0904). 28 | max-public-methods=200 29 | # Maximum number of boolean expressions in a if statement 30 | max-bool-expr=10 31 | -------------------------------------------------------------------------------- /caffe-tensorflow/.style.yapf: -------------------------------------------------------------------------------- 1 | [style] 2 | based_on_style = chromium 3 | column_limit = 100 4 | indent_width = 4 5 | -------------------------------------------------------------------------------- /caffe-tensorflow/LICENSE.md: -------------------------------------------------------------------------------- 1 | # License 2 | 3 | Each contributor holds copyright over their contributions to Caffe-Tensorflow. In particular: 4 | 5 | - Any included network model is provided under its original license. 6 | 7 | - Any portion derived from Caffe is provided under its original license. 8 | 9 | - Caffe-tensorflow is provided under the MIT license, as specified below. 10 | 11 | # The MIT License (MIT) 12 | 13 | Copyright (c) 2016 Saumitro Dasgupta 14 | 15 | Permission is hereby granted, free of charge, to any person obtaining a copy 16 | of this software and associated documentation files (the "Software"), to deal 17 | in the Software without restriction, including without limitation the rights 18 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 19 | copies of the Software, and to permit persons to whom the Software is 20 | furnished to do so, subject to the following conditions: 21 | 22 | The above copyright notice and this permission notice shall be included in all 23 | copies or substantial portions of the Software. 24 | 25 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 26 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 27 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 28 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 29 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 30 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 | SOFTWARE. 32 | -------------------------------------------------------------------------------- /caffe-tensorflow/README.md: -------------------------------------------------------------------------------- 1 | # Caffe to TensorFlow 2 | 3 | Convert [Caffe](https://github.com/BVLC/caffe/) models to [TensorFlow](https://github.com/tensorflow/tensorflow). 4 | 5 | ## Usage 6 | 7 | Run `convert.py` to convert an existing Caffe model to TensorFlow. 8 | 9 | Make sure you're using the latest Caffe format (see the notes section for more info). 10 | 11 | The output consists of two files: 12 | 13 | 1. A data file (in NumPy's native format) containing the model's learned parameters. 14 | 2. A Python class that constructs the model's graph. 15 | 16 | ### Examples 17 | 18 | See the [examples](examples/) folder for more details. 19 | 20 | ## Verification 21 | 22 | The following converted models have been verified on the ILSVRC2012 validation set using 23 | [validate.py](examples/imagenet/validate.py). 24 | 25 | | Model | Top 5 Accuracy | 26 | |:------------------------------------------------------|---------------:| 27 | | [ResNet 152](http://arxiv.org/abs/1512.03385) | 92.92% | 28 | | [ResNet 101](http://arxiv.org/abs/1512.03385) | 92.63% | 29 | | [ResNet 50](http://arxiv.org/abs/1512.03385) | 92.02% | 30 | | [VGG 16](http://arxiv.org/abs/1409.1556) | 89.88% | 31 | | [GoogLeNet](http://arxiv.org/abs/1409.4842) | 89.06% | 32 | | [Network in Network](http://arxiv.org/abs/1312.4400) | 81.21% | 33 | | [CaffeNet](http://arxiv.org/abs/1408.5093) | 79.93% | 34 | | [AlexNet](http://goo.gl/3BilWd) | 79.84% | 35 | 36 | ## Notes 37 | 38 | - Only the new Caffe model format is supported. If you have an old model, use the `upgrade_net_proto_text` and `upgrade_net_proto_binary` tools that ship with Caffe to upgrade them first. Also make sure you're using a fairly recent version of Caffe. 39 | 40 | - It appears that Caffe and TensorFlow cannot be concurrently invoked (CUDA conflicts - even with `set_mode_cpu`). This makes it a two-stage process: first extract the parameters with `convert.py`, then import it into TensorFlow. 41 | 42 | - Caffe is not strictly required. If PyCaffe is found in your `PYTHONPATH`, and the `USE_PYCAFFE` environment variable is set, it will be used. Otherwise, a fallback will be used. However, the fallback uses the pure Python-based implementation of protobuf, which is astoundingly slow (~1.5 minutes to parse the VGG16 parameters). The experimental CPP protobuf backend doesn't particularly help here, since it runs into the file size limit (Caffe gets around this by overriding this limit in C++). A cleaner solution here would be to implement the loader as a C++ module. 43 | 44 | - Only a subset of Caffe layers and accompanying parameters are currently supported. 45 | 46 | - Not all Caffe models can be converted to TensorFlow. For instance, Caffe supports arbitrary padding whereas TensorFlow's support is currently restricted to `SAME` and `VALID`. 47 | 48 | - The border values are handled differently by Caffe and TensorFlow. However, these don't appear to affect things too much. 49 | 50 | - Image rescaling can affect the ILSVRC2012 top 5 accuracy listed above slightly. VGG16 expects isotropic rescaling (anisotropic reduces accuracy to 88.45%) whereas BVLC's implementation of GoogLeNet expects anisotropic (isotropic reduces accuracy to 87.7%). 51 | 52 | - The support class `kaffe.tensorflow.Network` has no internal dependencies. It can be safely extracted and deployed without the rest of this library. 53 | 54 | - The ResNet model uses 1x1 convolutions with a stride of 2. This is currently only supported in the master branch of TensorFlow (the latest release at time of writing being v0.8.0, which does not support it). 55 | -------------------------------------------------------------------------------- /caffe-tensorflow/convert.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import os 4 | import sys 5 | import numpy as np 6 | import argparse 7 | from kaffe import KaffeError, print_stderr 8 | from kaffe.tensorflow import TensorFlowTransformer 9 | 10 | 11 | def fatal_error(msg): 12 | print_stderr(msg) 13 | exit(-1) 14 | 15 | 16 | def validate_arguments(args): 17 | if (args.data_output_path is not None) and (args.caffemodel is None): 18 | fatal_error('No input data path provided.') 19 | if (args.caffemodel is not None) and (args.data_output_path is None): 20 | fatal_error('No output data path provided.') 21 | if (args.code_output_path is None) and (args.data_output_path is None): 22 | fatal_error('No output path specified.') 23 | 24 | 25 | def convert(def_path, caffemodel_path, data_output_path, code_output_path, phase): 26 | try: 27 | transformer = TensorFlowTransformer(def_path, caffemodel_path, phase=phase) 28 | print_stderr('Converting data...') 29 | if caffemodel_path is not None: 30 | data = transformer.transform_data() 31 | print_stderr('Saving data...') 32 | with open(data_output_path, 'wb') as data_out: 33 | np.save(data_out, data) 34 | if code_output_path: 35 | print_stderr('Saving source...') 36 | with open(code_output_path, 'wb') as src_out: 37 | src_out.write(transformer.transform_source()) 38 | print_stderr('Done.') 39 | except KaffeError as err: 40 | fatal_error('Error encountered: {}'.format(err)) 41 | 42 | 43 | def main(): 44 | parser = argparse.ArgumentParser() 45 | parser.add_argument('def_path', help='Model definition (.prototxt) path') 46 | parser.add_argument('--caffemodel', help='Model data (.caffemodel) path') 47 | parser.add_argument('--data-output-path', help='Converted data output path') 48 | parser.add_argument('--code-output-path', help='Save generated source to this path') 49 | parser.add_argument('-p', 50 | '--phase', 51 | default='test', 52 | help='The phase to convert: test (default) or train') 53 | args = parser.parse_args() 54 | validate_arguments(args) 55 | convert(args.def_path, args.caffemodel, args.data_output_path, args.code_output_path, 56 | args.phase) 57 | 58 | 59 | if __name__ == '__main__': 60 | main() 61 | -------------------------------------------------------------------------------- /caffe-tensorflow/examples/imagenet/README.md: -------------------------------------------------------------------------------- 1 | # ImageNet Examples 2 | 3 | This folder contains two examples that demonstrate how to use converted networks for 4 | image classification. Also included are sample converted models and helper scripts. 5 | 6 | ## 1. Image Classification 7 | 8 | `classify.py` uses a GoogleNet trained on ImageNet, converted to TensorFlow, for classifying images. 9 | 10 | The architecture used is defined in `models/googlenet.py` (which was auto-generated). You will need 11 | to download and convert the weights from Caffe to run the example. The download link for the 12 | corresponding weights can be found in Caffe's `models/bvlc_googlenet/` folder. 13 | 14 | You can run this example like so: 15 | 16 | $ ./classify.py /path/to/googlenet.npy ~/pics/kitty.png ~/pics/woof.jpg 17 | 18 | You should expect to see an output similar to this: 19 | 20 | Image Classified As Confidence 21 | ---------------------------------------------------------------------- 22 | kitty.png Persian cat 99.75 % 23 | woof.jpg Bernese mountain dog 82.02 % 24 | 25 | 26 | ## 2. ImageNet Validation 27 | 28 | `validate.py` evaluates a converted model against the ImageNet (ILSVRC12) validation set. To run 29 | this script, you will need a copy of the ImageNet validation set. You can run it as follows: 30 | 31 | $ ./validate.py alexnet.npy val.txt imagenet-val/ --model AlexNet 32 | 33 | The validation results specified in the main readme were generated using this script. 34 | 35 | ## Helper Scripts 36 | 37 | In addition to the examples above, this folder includes a few additional files: 38 | 39 | - `dataset.py` : helper script for loading, pre-processing, and iterating over images 40 | - `models/` : contains converted models (auto-generated) 41 | - `models/helper.py` : describes how the data should be preprocessed for each model 42 | -------------------------------------------------------------------------------- /caffe-tensorflow/examples/imagenet/classify.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import argparse 3 | import numpy as np 4 | import tensorflow as tf 5 | import os.path as osp 6 | 7 | import models 8 | import dataset 9 | 10 | 11 | def display_results(image_paths, probs): 12 | '''Displays the classification results given the class probability for each image''' 13 | # Get a list of ImageNet class labels 14 | with open('imagenet-classes.txt', 'rb') as infile: 15 | class_labels = map(str.strip, infile.readlines()) 16 | # Pick the class with the highest confidence for each image 17 | class_indices = np.argmax(probs, axis=1) 18 | # Display the results 19 | print('\n{:20} {:30} {}'.format('Image', 'Classified As', 'Confidence')) 20 | print('-' * 70) 21 | for img_idx, image_path in enumerate(image_paths): 22 | img_name = osp.basename(image_path) 23 | class_name = class_labels[class_indices[img_idx]] 24 | confidence = round(probs[img_idx, class_indices[img_idx]] * 100, 2) 25 | print('{:20} {:30} {} %'.format(img_name, class_name, confidence)) 26 | 27 | 28 | def classify(model_data_path, image_paths): 29 | '''Classify the given images using GoogleNet.''' 30 | 31 | # Get the data specifications for the GoogleNet model 32 | spec = models.get_data_spec(model_class=models.GoogleNet) 33 | 34 | # Create a placeholder for the input image 35 | input_node = tf.placeholder(tf.float32, 36 | shape=(None, spec.crop_size, spec.crop_size, spec.channels)) 37 | 38 | # Construct the network 39 | net = models.GoogleNet({'data': input_node}) 40 | 41 | # Create an image producer (loads and processes images in parallel) 42 | image_producer = dataset.ImageProducer(image_paths=image_paths, data_spec=spec) 43 | 44 | with tf.Session() as sesh: 45 | # Start the image processing workers 46 | coordinator = tf.train.Coordinator() 47 | threads = image_producer.start(session=sesh, coordinator=coordinator) 48 | 49 | # Load the converted parameters 50 | print('Loading the model') 51 | net.load(model_data_path, sesh) 52 | 53 | # Load the input image 54 | print('Loading the images') 55 | indices, input_images = image_producer.get(sesh) 56 | 57 | # Perform a forward pass through the network to get the class probabilities 58 | print('Classifying') 59 | probs = sesh.run(net.get_output(), feed_dict={input_node: input_images}) 60 | display_results([image_paths[i] for i in indices], probs) 61 | 62 | # Stop the worker threads 63 | coordinator.request_stop() 64 | coordinator.join(threads, stop_grace_period_secs=2) 65 | 66 | def main(): 67 | # Parse arguments 68 | parser = argparse.ArgumentParser() 69 | parser.add_argument('model_path', help='Converted parameters for the GoogleNet model') 70 | parser.add_argument('image_paths', nargs='+', help='One or more images to classify') 71 | args = parser.parse_args() 72 | 73 | # Classify the image 74 | classify(args.model_path, args.image_paths) 75 | 76 | 77 | if __name__ == '__main__': 78 | main() 79 | -------------------------------------------------------------------------------- /caffe-tensorflow/examples/imagenet/dataset.py: -------------------------------------------------------------------------------- 1 | '''Utility functions and classes for handling image datasets.''' 2 | 3 | import os.path as osp 4 | import numpy as np 5 | import tensorflow as tf 6 | 7 | 8 | def process_image(img, scale, isotropic, crop, mean): 9 | '''Crops, scales, and normalizes the given image. 10 | scale : The image wil be first scaled to this size. 11 | If isotropic is true, the smaller side is rescaled to this, 12 | preserving the aspect ratio. 13 | crop : After scaling, a central crop of this size is taken. 14 | mean : Subtracted from the image 15 | ''' 16 | # Rescale 17 | if isotropic: 18 | img_shape = tf.to_float(tf.shape(img)[:2]) 19 | min_length = tf.minimum(img_shape[0], img_shape[1]) 20 | new_shape = tf.to_int32((scale / min_length) * img_shape) 21 | else: 22 | new_shape = tf.pack([scale, scale]) 23 | img = tf.image.resize_images(img, new_shape[0], new_shape[1]) 24 | # Center crop 25 | # Use the slice workaround until crop_to_bounding_box supports deferred tensor shapes 26 | # See: https://github.com/tensorflow/tensorflow/issues/521 27 | offset = (new_shape - crop) / 2 28 | img = tf.slice(img, begin=tf.pack([offset[0], offset[1], 0]), size=tf.pack([crop, crop, -1])) 29 | # Mean subtraction 30 | return tf.to_float(img) - mean 31 | 32 | 33 | class ImageProducer(object): 34 | ''' 35 | Loads and processes batches of images in parallel. 36 | ''' 37 | 38 | def __init__(self, image_paths, data_spec, num_concurrent=4, batch_size=None, labels=None): 39 | # The data specifications describe how to process the image 40 | self.data_spec = data_spec 41 | # A list of full image paths 42 | self.image_paths = image_paths 43 | # An optional list of labels corresponding to each image path 44 | self.labels = labels 45 | # A boolean flag per image indicating whether its a JPEG or PNG 46 | self.extension_mask = self.create_extension_mask(self.image_paths) 47 | # Create the loading and processing operations 48 | self.setup(batch_size=batch_size, num_concurrent=num_concurrent) 49 | 50 | def setup(self, batch_size, num_concurrent): 51 | # Validate the batch size 52 | num_images = len(self.image_paths) 53 | batch_size = min(num_images, batch_size or self.data_spec.batch_size) 54 | if num_images % batch_size != 0: 55 | raise ValueError( 56 | 'The total number of images ({}) must be divisible by the batch size ({}).'.format( 57 | num_images, batch_size)) 58 | self.num_batches = num_images / batch_size 59 | 60 | # Create a queue that will contain image paths (and their indices and extension indicator) 61 | self.path_queue = tf.FIFOQueue(capacity=num_images, 62 | dtypes=[tf.int32, tf.bool, tf.string], 63 | name='path_queue') 64 | 65 | # Enqueue all image paths, along with their indices 66 | indices = tf.range(num_images) 67 | self.enqueue_paths_op = self.path_queue.enqueue_many([indices, self.extension_mask, 68 | self.image_paths]) 69 | # Close the path queue (no more additions) 70 | self.close_path_queue_op = self.path_queue.close() 71 | 72 | # Create an operation that dequeues a single path and returns a processed image 73 | (idx, processed_image) = self.process() 74 | 75 | # Create a queue that will contain the processed images (and their indices) 76 | image_shape = (self.data_spec.crop_size, self.data_spec.crop_size, self.data_spec.channels) 77 | processed_queue = tf.FIFOQueue(capacity=int(np.ceil(num_images / float(num_concurrent))), 78 | dtypes=[tf.int32, tf.float32], 79 | shapes=[(), image_shape], 80 | name='processed_queue') 81 | 82 | # Enqueue the processed image and path 83 | enqueue_processed_op = processed_queue.enqueue([idx, processed_image]) 84 | 85 | # Create a dequeue op that fetches a batch of processed images off the queue 86 | self.dequeue_op = processed_queue.dequeue_many(batch_size) 87 | 88 | # Create a queue runner to perform the processing operations in parallel 89 | num_concurrent = min(num_concurrent, num_images) 90 | self.queue_runner = tf.train.QueueRunner(processed_queue, 91 | [enqueue_processed_op] * num_concurrent) 92 | 93 | def start(self, session, coordinator, num_concurrent=4): 94 | '''Start the processing worker threads.''' 95 | # Queue all paths 96 | session.run(self.enqueue_paths_op) 97 | # Close the path queue 98 | session.run(self.close_path_queue_op) 99 | # Start the queue runner and return the created threads 100 | return self.queue_runner.create_threads(session, coord=coordinator, start=True) 101 | 102 | def get(self, session): 103 | ''' 104 | Get a single batch of images along with their indices. If a set of labels were provided, 105 | the corresponding labels are returned instead of the indices. 106 | ''' 107 | (indices, images) = session.run(self.dequeue_op) 108 | if self.labels is not None: 109 | labels = [self.labels[idx] for idx in indices] 110 | return (labels, images) 111 | return (indices, images) 112 | 113 | def batches(self, session): 114 | '''Yield a batch until no more images are left.''' 115 | for _ in xrange(self.num_batches): 116 | yield self.get(session=session) 117 | 118 | def load_image(self, image_path, is_jpeg): 119 | # Read the file 120 | file_data = tf.read_file(image_path) 121 | # Decode the image data 122 | img = tf.cond( 123 | is_jpeg, 124 | lambda: tf.image.decode_jpeg(file_data, channels=self.data_spec.channels), 125 | lambda: tf.image.decode_png(file_data, channels=self.data_spec.channels)) 126 | if self.data_spec.expects_bgr: 127 | # Convert from RGB channel ordering to BGR 128 | # This matches, for instance, how OpenCV orders the channels. 129 | img = tf.reverse(img, [False, False, True]) 130 | return img 131 | 132 | def process(self): 133 | # Dequeue a single image path 134 | idx, is_jpeg, image_path = self.path_queue.dequeue() 135 | # Load the image 136 | img = self.load_image(image_path, is_jpeg) 137 | # Process the image 138 | processed_img = process_image(img=img, 139 | scale=self.data_spec.scale_size, 140 | isotropic=self.data_spec.isotropic, 141 | crop=self.data_spec.crop_size, 142 | mean=self.data_spec.mean) 143 | # Return the processed image, along with its index 144 | return (idx, processed_img) 145 | 146 | @staticmethod 147 | def create_extension_mask(paths): 148 | 149 | def is_jpeg(path): 150 | extension = osp.splitext(path)[-1].lower() 151 | if extension in ('.jpg', '.jpeg'): 152 | return True 153 | if extension != '.png': 154 | raise ValueError('Unsupported image format: {}'.format(extension)) 155 | return False 156 | 157 | return [is_jpeg(p) for p in paths] 158 | 159 | def __len__(self): 160 | return len(self.image_paths) 161 | 162 | 163 | class ImageNetProducer(ImageProducer): 164 | 165 | def __init__(self, val_path, data_path, data_spec): 166 | # Read in the ground truth labels for the validation set 167 | # The get_ilsvrc_aux.sh in Caffe's data/ilsvrc12 folder can fetch a copy of val.txt 168 | gt_lines = open(val_path).readlines() 169 | gt_pairs = [line.split() for line in gt_lines] 170 | # Get the full image paths 171 | # You will need a copy of the ImageNet validation set for this. 172 | image_paths = [osp.join(data_path, p[0]) for p in gt_pairs] 173 | # The corresponding ground truth labels 174 | labels = np.array([int(p[1]) for p in gt_pairs]) 175 | # Initialize base 176 | super(ImageNetProducer, self).__init__(image_paths=image_paths, 177 | data_spec=data_spec, 178 | labels=labels) 179 | -------------------------------------------------------------------------------- /caffe-tensorflow/examples/imagenet/models/__init__.py: -------------------------------------------------------------------------------- 1 | from helper import * 2 | -------------------------------------------------------------------------------- /caffe-tensorflow/examples/imagenet/models/alexnet.py: -------------------------------------------------------------------------------- 1 | from kaffe.tensorflow import Network 2 | 3 | class AlexNet(Network): 4 | def setup(self): 5 | (self.feed('data') 6 | .conv(11, 11, 96, 4, 4, padding='VALID', name='conv1') 7 | .lrn(2, 2e-05, 0.75, name='norm1') 8 | .max_pool(3, 3, 2, 2, padding='VALID', name='pool1') 9 | .conv(5, 5, 256, 1, 1, group=2, name='conv2') 10 | .lrn(2, 2e-05, 0.75, name='norm2') 11 | .max_pool(3, 3, 2, 2, padding='VALID', name='pool2') 12 | .conv(3, 3, 384, 1, 1, name='conv3') 13 | .conv(3, 3, 384, 1, 1, group=2, name='conv4') 14 | .conv(3, 3, 256, 1, 1, group=2, name='conv5') 15 | .max_pool(3, 3, 2, 2, padding='VALID', name='pool5') 16 | .fc(4096, name='fc6') 17 | .fc(4096, name='fc7') 18 | .fc(1000, relu=False, name='fc8') 19 | .softmax(name='prob')) 20 | -------------------------------------------------------------------------------- /caffe-tensorflow/examples/imagenet/models/caffenet.py: -------------------------------------------------------------------------------- 1 | from kaffe.tensorflow import Network 2 | 3 | class CaffeNet(Network): 4 | def setup(self): 5 | (self.feed('data') 6 | .conv(11, 11, 96, 4, 4, padding='VALID', name='conv1') 7 | .max_pool(3, 3, 2, 2, padding='VALID', name='pool1') 8 | .lrn(2, 2e-05, 0.75, name='norm1') 9 | .conv(5, 5, 256, 1, 1, group=2, name='conv2') 10 | .max_pool(3, 3, 2, 2, padding='VALID', name='pool2') 11 | .lrn(2, 2e-05, 0.75, name='norm2') 12 | .conv(3, 3, 384, 1, 1, name='conv3') 13 | .conv(3, 3, 384, 1, 1, group=2, name='conv4') 14 | .conv(3, 3, 256, 1, 1, group=2, name='conv5') 15 | .max_pool(3, 3, 2, 2, padding='VALID', name='pool5') 16 | .fc(4096, name='fc6') 17 | .fc(4096, name='fc7') 18 | .fc(1000, relu=False, name='fc8') 19 | .softmax(name='prob')) 20 | -------------------------------------------------------------------------------- /caffe-tensorflow/examples/imagenet/models/googlenet.py: -------------------------------------------------------------------------------- 1 | from kaffe.tensorflow import Network 2 | 3 | class GoogleNet(Network): 4 | def setup(self): 5 | (self.feed('data') 6 | .conv(7, 7, 64, 2, 2, name='conv1_7x7_s2') 7 | .max_pool(3, 3, 2, 2, name='pool1_3x3_s2') 8 | .lrn(2, 2e-05, 0.75, name='pool1_norm1') 9 | .conv(1, 1, 64, 1, 1, name='conv2_3x3_reduce') 10 | .conv(3, 3, 192, 1, 1, name='conv2_3x3') 11 | .lrn(2, 2e-05, 0.75, name='conv2_norm2') 12 | .max_pool(3, 3, 2, 2, name='pool2_3x3_s2') 13 | .conv(1, 1, 64, 1, 1, name='inception_3a_1x1')) 14 | 15 | (self.feed('pool2_3x3_s2') 16 | .conv(1, 1, 96, 1, 1, name='inception_3a_3x3_reduce') 17 | .conv(3, 3, 128, 1, 1, name='inception_3a_3x3')) 18 | 19 | (self.feed('pool2_3x3_s2') 20 | .conv(1, 1, 16, 1, 1, name='inception_3a_5x5_reduce') 21 | .conv(5, 5, 32, 1, 1, name='inception_3a_5x5')) 22 | 23 | (self.feed('pool2_3x3_s2') 24 | .max_pool(3, 3, 1, 1, name='inception_3a_pool') 25 | .conv(1, 1, 32, 1, 1, name='inception_3a_pool_proj')) 26 | 27 | (self.feed('inception_3a_1x1', 28 | 'inception_3a_3x3', 29 | 'inception_3a_5x5', 30 | 'inception_3a_pool_proj') 31 | .concat(3, name='inception_3a_output') 32 | .conv(1, 1, 128, 1, 1, name='inception_3b_1x1')) 33 | 34 | (self.feed('inception_3a_output') 35 | .conv(1, 1, 128, 1, 1, name='inception_3b_3x3_reduce') 36 | .conv(3, 3, 192, 1, 1, name='inception_3b_3x3')) 37 | 38 | (self.feed('inception_3a_output') 39 | .conv(1, 1, 32, 1, 1, name='inception_3b_5x5_reduce') 40 | .conv(5, 5, 96, 1, 1, name='inception_3b_5x5')) 41 | 42 | (self.feed('inception_3a_output') 43 | .max_pool(3, 3, 1, 1, name='inception_3b_pool') 44 | .conv(1, 1, 64, 1, 1, name='inception_3b_pool_proj')) 45 | 46 | (self.feed('inception_3b_1x1', 47 | 'inception_3b_3x3', 48 | 'inception_3b_5x5', 49 | 'inception_3b_pool_proj') 50 | .concat(3, name='inception_3b_output') 51 | .max_pool(3, 3, 2, 2, name='pool3_3x3_s2') 52 | .conv(1, 1, 192, 1, 1, name='inception_4a_1x1')) 53 | 54 | (self.feed('pool3_3x3_s2') 55 | .conv(1, 1, 96, 1, 1, name='inception_4a_3x3_reduce') 56 | .conv(3, 3, 208, 1, 1, name='inception_4a_3x3')) 57 | 58 | (self.feed('pool3_3x3_s2') 59 | .conv(1, 1, 16, 1, 1, name='inception_4a_5x5_reduce') 60 | .conv(5, 5, 48, 1, 1, name='inception_4a_5x5')) 61 | 62 | (self.feed('pool3_3x3_s2') 63 | .max_pool(3, 3, 1, 1, name='inception_4a_pool') 64 | .conv(1, 1, 64, 1, 1, name='inception_4a_pool_proj')) 65 | 66 | (self.feed('inception_4a_1x1', 67 | 'inception_4a_3x3', 68 | 'inception_4a_5x5', 69 | 'inception_4a_pool_proj') 70 | .concat(3, name='inception_4a_output') 71 | .conv(1, 1, 160, 1, 1, name='inception_4b_1x1')) 72 | 73 | (self.feed('inception_4a_output') 74 | .conv(1, 1, 112, 1, 1, name='inception_4b_3x3_reduce') 75 | .conv(3, 3, 224, 1, 1, name='inception_4b_3x3')) 76 | 77 | (self.feed('inception_4a_output') 78 | .conv(1, 1, 24, 1, 1, name='inception_4b_5x5_reduce') 79 | .conv(5, 5, 64, 1, 1, name='inception_4b_5x5')) 80 | 81 | (self.feed('inception_4a_output') 82 | .max_pool(3, 3, 1, 1, name='inception_4b_pool') 83 | .conv(1, 1, 64, 1, 1, name='inception_4b_pool_proj')) 84 | 85 | (self.feed('inception_4b_1x1', 86 | 'inception_4b_3x3', 87 | 'inception_4b_5x5', 88 | 'inception_4b_pool_proj') 89 | .concat(3, name='inception_4b_output') 90 | .conv(1, 1, 128, 1, 1, name='inception_4c_1x1')) 91 | 92 | (self.feed('inception_4b_output') 93 | .conv(1, 1, 128, 1, 1, name='inception_4c_3x3_reduce') 94 | .conv(3, 3, 256, 1, 1, name='inception_4c_3x3')) 95 | 96 | (self.feed('inception_4b_output') 97 | .conv(1, 1, 24, 1, 1, name='inception_4c_5x5_reduce') 98 | .conv(5, 5, 64, 1, 1, name='inception_4c_5x5')) 99 | 100 | (self.feed('inception_4b_output') 101 | .max_pool(3, 3, 1, 1, name='inception_4c_pool') 102 | .conv(1, 1, 64, 1, 1, name='inception_4c_pool_proj')) 103 | 104 | (self.feed('inception_4c_1x1', 105 | 'inception_4c_3x3', 106 | 'inception_4c_5x5', 107 | 'inception_4c_pool_proj') 108 | .concat(3, name='inception_4c_output') 109 | .conv(1, 1, 112, 1, 1, name='inception_4d_1x1')) 110 | 111 | (self.feed('inception_4c_output') 112 | .conv(1, 1, 144, 1, 1, name='inception_4d_3x3_reduce') 113 | .conv(3, 3, 288, 1, 1, name='inception_4d_3x3')) 114 | 115 | (self.feed('inception_4c_output') 116 | .conv(1, 1, 32, 1, 1, name='inception_4d_5x5_reduce') 117 | .conv(5, 5, 64, 1, 1, name='inception_4d_5x5')) 118 | 119 | (self.feed('inception_4c_output') 120 | .max_pool(3, 3, 1, 1, name='inception_4d_pool') 121 | .conv(1, 1, 64, 1, 1, name='inception_4d_pool_proj')) 122 | 123 | (self.feed('inception_4d_1x1', 124 | 'inception_4d_3x3', 125 | 'inception_4d_5x5', 126 | 'inception_4d_pool_proj') 127 | .concat(3, name='inception_4d_output') 128 | .conv(1, 1, 256, 1, 1, name='inception_4e_1x1')) 129 | 130 | (self.feed('inception_4d_output') 131 | .conv(1, 1, 160, 1, 1, name='inception_4e_3x3_reduce') 132 | .conv(3, 3, 320, 1, 1, name='inception_4e_3x3')) 133 | 134 | (self.feed('inception_4d_output') 135 | .conv(1, 1, 32, 1, 1, name='inception_4e_5x5_reduce') 136 | .conv(5, 5, 128, 1, 1, name='inception_4e_5x5')) 137 | 138 | (self.feed('inception_4d_output') 139 | .max_pool(3, 3, 1, 1, name='inception_4e_pool') 140 | .conv(1, 1, 128, 1, 1, name='inception_4e_pool_proj')) 141 | 142 | (self.feed('inception_4e_1x1', 143 | 'inception_4e_3x3', 144 | 'inception_4e_5x5', 145 | 'inception_4e_pool_proj') 146 | .concat(3, name='inception_4e_output') 147 | .max_pool(3, 3, 2, 2, name='pool4_3x3_s2') 148 | .conv(1, 1, 256, 1, 1, name='inception_5a_1x1')) 149 | 150 | (self.feed('pool4_3x3_s2') 151 | .conv(1, 1, 160, 1, 1, name='inception_5a_3x3_reduce') 152 | .conv(3, 3, 320, 1, 1, name='inception_5a_3x3')) 153 | 154 | (self.feed('pool4_3x3_s2') 155 | .conv(1, 1, 32, 1, 1, name='inception_5a_5x5_reduce') 156 | .conv(5, 5, 128, 1, 1, name='inception_5a_5x5')) 157 | 158 | (self.feed('pool4_3x3_s2') 159 | .max_pool(3, 3, 1, 1, name='inception_5a_pool') 160 | .conv(1, 1, 128, 1, 1, name='inception_5a_pool_proj')) 161 | 162 | (self.feed('inception_5a_1x1', 163 | 'inception_5a_3x3', 164 | 'inception_5a_5x5', 165 | 'inception_5a_pool_proj') 166 | .concat(3, name='inception_5a_output') 167 | .conv(1, 1, 384, 1, 1, name='inception_5b_1x1')) 168 | 169 | (self.feed('inception_5a_output') 170 | .conv(1, 1, 192, 1, 1, name='inception_5b_3x3_reduce') 171 | .conv(3, 3, 384, 1, 1, name='inception_5b_3x3')) 172 | 173 | (self.feed('inception_5a_output') 174 | .conv(1, 1, 48, 1, 1, name='inception_5b_5x5_reduce') 175 | .conv(5, 5, 128, 1, 1, name='inception_5b_5x5')) 176 | 177 | (self.feed('inception_5a_output') 178 | .max_pool(3, 3, 1, 1, name='inception_5b_pool') 179 | .conv(1, 1, 128, 1, 1, name='inception_5b_pool_proj')) 180 | 181 | (self.feed('inception_5b_1x1', 182 | 'inception_5b_3x3', 183 | 'inception_5b_5x5', 184 | 'inception_5b_pool_proj') 185 | .concat(3, name='inception_5b_output') 186 | .avg_pool(7, 7, 1, 1, padding='VALID', name='pool5_7x7_s1') 187 | .fc(1000, relu=False, name='loss3_classifier') 188 | .softmax(name='prob')) 189 | -------------------------------------------------------------------------------- /caffe-tensorflow/examples/imagenet/models/helper.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os.path as osp 3 | import numpy as np 4 | 5 | # Add the kaffe module to the import path 6 | sys.path.append(osp.realpath(osp.join(osp.dirname(__file__), '../../../'))) 7 | 8 | from googlenet import GoogleNet 9 | from vgg import VGG16 10 | from alexnet import AlexNet 11 | from caffenet import CaffeNet 12 | from nin import NiN 13 | from resnet import ResNet50, ResNet101, ResNet152 14 | 15 | 16 | class DataSpec(object): 17 | '''Input data specifications for an ImageNet model.''' 18 | 19 | def __init__(self, 20 | batch_size, 21 | scale_size, 22 | crop_size, 23 | isotropic, 24 | channels=3, 25 | mean=None, 26 | bgr=True): 27 | # The recommended batch size for this model 28 | self.batch_size = batch_size 29 | # The image should be scaled to this size first during preprocessing 30 | self.scale_size = scale_size 31 | # Whether the model expects the rescaling to be isotropic 32 | self.isotropic = isotropic 33 | # A square crop of this dimension is expected by this model 34 | self.crop_size = crop_size 35 | # The number of channels in the input image expected by this model 36 | self.channels = channels 37 | # The mean to be subtracted from each image. By default, the per-channel ImageNet mean. 38 | # The values below are ordered BGR, as many Caffe models are trained in this order. 39 | # Some of the earlier models (like AlexNet) used a spatial three-channeled mean. 40 | # However, using just the per-channel mean values instead doesn't affect things too much. 41 | self.mean = mean if mean is not None else np.array([104., 117., 124.]) 42 | # Whether this model expects images to be in BGR order 43 | self.expects_bgr = True 44 | 45 | 46 | def alexnet_spec(batch_size=500): 47 | '''Parameters used by AlexNet and its variants.''' 48 | return DataSpec(batch_size=batch_size, scale_size=256, crop_size=227, isotropic=False) 49 | 50 | 51 | def std_spec(batch_size, isotropic=True): 52 | '''Parameters commonly used by "post-AlexNet" architectures.''' 53 | return DataSpec(batch_size=batch_size, scale_size=256, crop_size=224, isotropic=isotropic) 54 | 55 | # Collection of sample auto-generated models 56 | MODELS = (AlexNet, CaffeNet, GoogleNet, NiN, ResNet50, ResNet101, ResNet152, VGG16) 57 | 58 | # The corresponding data specifications for the sample models 59 | # These specifications are based on how the models were trained. 60 | # The recommended batch size is based on a Titan X (12GB). 61 | MODEL_DATA_SPECS = { 62 | AlexNet: alexnet_spec(), 63 | CaffeNet: alexnet_spec(), 64 | GoogleNet: std_spec(batch_size=200, isotropic=False), 65 | ResNet50: std_spec(batch_size=25), 66 | ResNet101: std_spec(batch_size=25), 67 | ResNet152: std_spec(batch_size=25), 68 | NiN: std_spec(batch_size=500), 69 | VGG16: std_spec(batch_size=25) 70 | } 71 | 72 | 73 | def get_models(): 74 | '''Returns a tuple of sample models.''' 75 | return MODELS 76 | 77 | 78 | def get_data_spec(model_instance=None, model_class=None): 79 | '''Returns the data specifications for the given network.''' 80 | model_class = model_class or model_instance.__class__ 81 | return MODEL_DATA_SPECS[model_class] 82 | -------------------------------------------------------------------------------- /caffe-tensorflow/examples/imagenet/models/nin.py: -------------------------------------------------------------------------------- 1 | from kaffe.tensorflow import Network 2 | 3 | class NiN(Network): 4 | def setup(self): 5 | (self.feed('data') 6 | .conv(11, 11, 96, 4, 4, padding='VALID', name='conv1') 7 | .conv(1, 1, 96, 1, 1, name='cccp1') 8 | .conv(1, 1, 96, 1, 1, name='cccp2') 9 | .max_pool(3, 3, 2, 2, name='pool1') 10 | .conv(5, 5, 256, 1, 1, name='conv2') 11 | .conv(1, 1, 256, 1, 1, name='cccp3') 12 | .conv(1, 1, 256, 1, 1, name='cccp4') 13 | .max_pool(3, 3, 2, 2, padding='VALID', name='pool2') 14 | .conv(3, 3, 384, 1, 1, name='conv3') 15 | .conv(1, 1, 384, 1, 1, name='cccp5') 16 | .conv(1, 1, 384, 1, 1, name='cccp6') 17 | .max_pool(3, 3, 2, 2, padding='VALID', name='pool3') 18 | .conv(3, 3, 1024, 1, 1, name='conv4-1024') 19 | .conv(1, 1, 1024, 1, 1, name='cccp7-1024') 20 | .conv(1, 1, 1000, 1, 1, name='cccp8-1024') 21 | .avg_pool(6, 6, 1, 1, padding='VALID', name='pool4') 22 | .softmax(name='prob')) 23 | -------------------------------------------------------------------------------- /caffe-tensorflow/examples/imagenet/models/vgg.py: -------------------------------------------------------------------------------- 1 | from kaffe.tensorflow import Network 2 | 3 | class VGG16(Network): 4 | def setup(self): 5 | (self.feed('data') 6 | .conv(3, 3, 64, 1, 1, name='conv1_1') 7 | .conv(3, 3, 64, 1, 1, name='conv1_2') 8 | .max_pool(2, 2, 2, 2, name='pool1') 9 | .conv(3, 3, 128, 1, 1, name='conv2_1') 10 | .conv(3, 3, 128, 1, 1, name='conv2_2') 11 | .max_pool(2, 2, 2, 2, name='pool2') 12 | .conv(3, 3, 256, 1, 1, name='conv3_1') 13 | .conv(3, 3, 256, 1, 1, name='conv3_2') 14 | .conv(3, 3, 256, 1, 1, name='conv3_3') 15 | .max_pool(2, 2, 2, 2, name='pool3') 16 | .conv(3, 3, 512, 1, 1, name='conv4_1') 17 | .conv(3, 3, 512, 1, 1, name='conv4_2') 18 | .conv(3, 3, 512, 1, 1, name='conv4_3') 19 | .max_pool(2, 2, 2, 2, name='pool4') 20 | .conv(3, 3, 512, 1, 1, name='conv5_1') 21 | .conv(3, 3, 512, 1, 1, name='conv5_2') 22 | .conv(3, 3, 512, 1, 1, name='conv5_3') 23 | .max_pool(2, 2, 2, 2, name='pool5') 24 | .fc(4096, name='fc6') 25 | .fc(4096, name='fc7') 26 | .fc(1000, relu=False, name='fc8') 27 | .softmax(name='prob')) 28 | -------------------------------------------------------------------------------- /caffe-tensorflow/examples/imagenet/validate.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | '''Validates a converted ImageNet model against the ILSVRC12 validation set.''' 3 | 4 | import argparse 5 | import numpy as np 6 | import tensorflow as tf 7 | import os.path as osp 8 | 9 | import models 10 | import dataset 11 | 12 | 13 | def load_model(name): 14 | '''Creates and returns an instance of the model given its class name. 15 | The created model has a single placeholder node for feeding images. 16 | ''' 17 | # Find the model class from its name 18 | all_models = models.get_models() 19 | lut = {model.__name__: model for model in all_models} 20 | if name not in lut: 21 | print('Invalid model index. Options are:') 22 | # Display a list of valid model names 23 | for model in all_models: 24 | print('\t* {}'.format(model.__name__)) 25 | return None 26 | NetClass = lut[name] 27 | 28 | # Create a placeholder for the input image 29 | spec = models.get_data_spec(model_class=NetClass) 30 | data_node = tf.placeholder(tf.float32, 31 | shape=(None, spec.crop_size, spec.crop_size, spec.channels)) 32 | 33 | # Construct and return the model 34 | return NetClass({'data': data_node}) 35 | 36 | 37 | def validate(net, model_path, image_producer, top_k=5): 38 | '''Compute the top_k classification accuracy for the given network and images.''' 39 | # Get the data specifications for given network 40 | spec = models.get_data_spec(model_instance=net) 41 | # Get the input node for feeding in the images 42 | input_node = net.inputs['data'] 43 | # Create a placeholder for the ground truth labels 44 | label_node = tf.placeholder(tf.int32) 45 | # Get the output of the network (class probabilities) 46 | probs = net.get_output() 47 | # Create a top_k accuracy node 48 | top_k_op = tf.nn.in_top_k(probs, label_node, top_k) 49 | # The number of images processed 50 | count = 0 51 | # The number of correctly classified images 52 | correct = 0 53 | # The total number of images 54 | total = len(image_producer) 55 | 56 | with tf.Session() as sesh: 57 | coordinator = tf.train.Coordinator() 58 | # Load the converted parameters 59 | net.load(data_path=model_path, session=sesh) 60 | # Start the image processing workers 61 | threads = image_producer.start(session=sesh, coordinator=coordinator) 62 | # Iterate over and classify mini-batches 63 | for (labels, images) in image_producer.batches(sesh): 64 | correct += np.sum(sesh.run(top_k_op, 65 | feed_dict={input_node: images, 66 | label_node: labels})) 67 | count += len(labels) 68 | cur_accuracy = float(correct) * 100 / count 69 | print('{:>6}/{:<6} {:>6.2f}%'.format(count, total, cur_accuracy)) 70 | # Stop the worker threads 71 | coordinator.request_stop() 72 | coordinator.join(threads, stop_grace_period_secs=2) 73 | print('Top {} Accuracy: {}'.format(top_k, float(correct) / total)) 74 | 75 | 76 | 77 | def main(): 78 | # Parse arguments 79 | parser = argparse.ArgumentParser() 80 | parser.add_argument('model_path', help='Path to the converted model parameters (.npy)') 81 | parser.add_argument('val_gt', help='Path to validation set ground truth (.txt)') 82 | parser.add_argument('imagenet_data_dir', help='ImageNet validation set images directory path') 83 | parser.add_argument('--model', default='GoogleNet', help='The name of the model to evaluate') 84 | args = parser.parse_args() 85 | 86 | # Load the network 87 | net = load_model(args.model) 88 | if net is None: 89 | exit(-1) 90 | 91 | # Load the dataset 92 | data_spec = models.get_data_spec(model_instance=net) 93 | image_producer = dataset.ImageNetProducer(val_path=args.val_gt, 94 | data_path=args.imagenet_data_dir, 95 | data_spec=data_spec) 96 | 97 | # Evaluate its performance on the ILSVRC12 validation set 98 | validate(net, args.model_path, image_producer) 99 | 100 | 101 | if __name__ == '__main__': 102 | main() 103 | -------------------------------------------------------------------------------- /caffe-tensorflow/examples/mnist/README.md: -------------------------------------------------------------------------------- 1 | ### LeNet Example 2 | 3 | _Thanks to @Russell91 for this example_ 4 | 5 | This example showns you how to finetune code from the [Caffe MNIST tutorial](http://caffe.berkeleyvision.org/gathered/examples/mnist.html) using Tensorflow. 6 | First, you can convert a prototxt model to tensorflow code: 7 | 8 | $ ./convert.py examples/mnist/lenet.prototxt --code-output-path=mynet.py 9 | 10 | This produces tensorflow code for the LeNet network in `mynet.py`. The code can be imported as described below in the Inference section. Caffe-tensorflow also lets you convert `.caffemodel` weight files to `.npy` files that can be directly loaded from tensorflow: 11 | 12 | $ ./convert.py examples/mnist/lenet.prototxt --caffemodel examples/mnist/lenet_iter_10000.caffemodel --data-output-path=mynet.npy 13 | 14 | The above command will generate a weight file named `mynet.npy`. 15 | 16 | #### Inference: 17 | 18 | Once you have generated both the code weight files for LeNet, you can finetune LeNet using tensorflow with 19 | 20 | $ ./examples/mnist/finetune_mnist.py 21 | 22 | At a high level, `finetune_mnist.py` works as follows: 23 | 24 | ```python 25 | # Import the converted model's class 26 | from mynet import MyNet 27 | 28 | # Create an instance, passing in the input data 29 | net = MyNet({'data':my_input_data}) 30 | 31 | with tf.Session() as sesh: 32 | # Load the data 33 | net.load('mynet.npy', sesh) 34 | # Forward pass 35 | output = sesh.run(net.get_output(), ...) 36 | ``` 37 | -------------------------------------------------------------------------------- /caffe-tensorflow/examples/mnist/finetune_mnist.py: -------------------------------------------------------------------------------- 1 | # Import the converted model's class 2 | import numpy as np 3 | import random 4 | import tensorflow as tf 5 | from tensorflow.examples.tutorials.mnist import input_data 6 | 7 | from mynet import LeNet as MyNet 8 | 9 | mnist = input_data.read_data_sets('MNIST_data', one_hot=True) 10 | batch_size = 32 11 | 12 | def gen_data(source): 13 | while True: 14 | indices = range(len(source.images)) 15 | random.shuffle(indices) 16 | for i in indices: 17 | image = np.reshape(source.images[i], (28, 28, 1)) 18 | label = source.labels[i] 19 | yield image, label 20 | 21 | def gen_data_batch(source): 22 | data_gen = gen_data(source) 23 | while True: 24 | image_batch = [] 25 | label_batch = [] 26 | for _ in range(batch_size): 27 | image, label = next(data_gen) 28 | image_batch.append(image) 29 | label_batch.append(label) 30 | yield np.array(image_batch), np.array(label_batch) 31 | 32 | 33 | images = tf.placeholder(tf.float32, [batch_size, 28, 28, 1]) 34 | labels = tf.placeholder(tf.float32, [batch_size, 10]) 35 | net = MyNet({'data': images}) 36 | 37 | ip2 = net.layers['ip2'] 38 | pred = tf.nn.softmax(ip2) 39 | 40 | loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(ip2, labels), 0) 41 | opt = tf.train.RMSPropOptimizer(0.001) 42 | train_op = opt.minimize(loss) 43 | 44 | with tf.Session() as sess: 45 | # Load the data 46 | sess.run(tf.initialize_all_variables()) 47 | net.load('mynet.npy', sess) 48 | 49 | data_gen = gen_data_batch(mnist.train) 50 | for i in range(1000): 51 | np_images, np_labels = next(data_gen) 52 | feed = {images: np_images, labels: np_labels} 53 | 54 | np_loss, np_pred, _ = sess.run([loss, pred, train_op], feed_dict=feed) 55 | if i % 10 == 0: 56 | print('Iteration: ', i, np_loss) 57 | -------------------------------------------------------------------------------- /caffe-tensorflow/examples/mnist/lenet.prototxt: -------------------------------------------------------------------------------- 1 | name: "LeNet" 2 | layer { 3 | name: "data" 4 | type: "Input" 5 | top: "data" 6 | input_param { shape: { dim: 64 dim: 1 dim: 28 dim: 28 } } 7 | } 8 | layer { 9 | name: "conv1" 10 | type: "Convolution" 11 | bottom: "data" 12 | top: "conv1" 13 | param { 14 | lr_mult: 1 15 | } 16 | param { 17 | lr_mult: 2 18 | } 19 | convolution_param { 20 | num_output: 20 21 | kernel_size: 5 22 | stride: 1 23 | weight_filler { 24 | type: "xavier" 25 | } 26 | bias_filler { 27 | type: "constant" 28 | } 29 | } 30 | } 31 | layer { 32 | name: "pool1" 33 | type: "Pooling" 34 | bottom: "conv1" 35 | top: "pool1" 36 | pooling_param { 37 | pool: MAX 38 | kernel_size: 2 39 | stride: 2 40 | } 41 | } 42 | layer { 43 | name: "conv2" 44 | type: "Convolution" 45 | bottom: "pool1" 46 | top: "conv2" 47 | param { 48 | lr_mult: 1 49 | } 50 | param { 51 | lr_mult: 2 52 | } 53 | convolution_param { 54 | num_output: 50 55 | kernel_size: 5 56 | stride: 1 57 | weight_filler { 58 | type: "xavier" 59 | } 60 | bias_filler { 61 | type: "constant" 62 | } 63 | } 64 | } 65 | layer { 66 | name: "pool2" 67 | type: "Pooling" 68 | bottom: "conv2" 69 | top: "pool2" 70 | pooling_param { 71 | pool: MAX 72 | kernel_size: 2 73 | stride: 2 74 | } 75 | } 76 | layer { 77 | name: "ip1" 78 | type: "InnerProduct" 79 | bottom: "pool2" 80 | top: "ip1" 81 | param { 82 | lr_mult: 1 83 | } 84 | param { 85 | lr_mult: 2 86 | } 87 | inner_product_param { 88 | num_output: 500 89 | weight_filler { 90 | type: "xavier" 91 | } 92 | bias_filler { 93 | type: "constant" 94 | } 95 | } 96 | } 97 | layer { 98 | name: "relu1" 99 | type: "ReLU" 100 | bottom: "ip1" 101 | top: "ip1" 102 | } 103 | layer { 104 | name: "ip2" 105 | type: "InnerProduct" 106 | bottom: "ip1" 107 | top: "ip2" 108 | param { 109 | lr_mult: 1 110 | } 111 | param { 112 | lr_mult: 2 113 | } 114 | inner_product_param { 115 | num_output: 10 116 | weight_filler { 117 | type: "xavier" 118 | } 119 | bias_filler { 120 | type: "constant" 121 | } 122 | } 123 | } 124 | layer { 125 | name: "prob" 126 | type: "Softmax" 127 | bottom: "ip2" 128 | top: "prob" 129 | } 130 | -------------------------------------------------------------------------------- /caffe-tensorflow/examples/mnist/lenet_iter_10000.caffemodel: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/caffe-tensorflow/examples/mnist/lenet_iter_10000.caffemodel -------------------------------------------------------------------------------- /caffe-tensorflow/kaffe/__init__.py: -------------------------------------------------------------------------------- 1 | from .graph import GraphBuilder, NodeMapper 2 | from .errors import KaffeError, print_stderr 3 | 4 | from . import tensorflow 5 | -------------------------------------------------------------------------------- /caffe-tensorflow/kaffe/caffe/__init__.py: -------------------------------------------------------------------------------- 1 | from .resolver import get_caffe_resolver, has_pycaffe 2 | -------------------------------------------------------------------------------- /caffe-tensorflow/kaffe/caffe/resolver.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | SHARED_CAFFE_RESOLVER = None 4 | 5 | class CaffeResolver(object): 6 | def __init__(self): 7 | self.import_caffe() 8 | 9 | def import_caffe(self): 10 | self.caffe = None 11 | try: 12 | # Try to import PyCaffe first 13 | import caffe 14 | self.caffe = caffe 15 | except ImportError: 16 | # Fall back to the protobuf implementation 17 | from . import caffepb 18 | self.caffepb = caffepb 19 | show_fallback_warning() 20 | if self.caffe: 21 | # Use the protobuf code from the imported distribution. 22 | # This way, Caffe variants with custom layers will work. 23 | self.caffepb = self.caffe.proto.caffe_pb2 24 | self.NetParameter = self.caffepb.NetParameter 25 | 26 | def has_pycaffe(self): 27 | return self.caffe is not None 28 | 29 | def get_caffe_resolver(): 30 | global SHARED_CAFFE_RESOLVER 31 | if SHARED_CAFFE_RESOLVER is None: 32 | SHARED_CAFFE_RESOLVER = CaffeResolver() 33 | return SHARED_CAFFE_RESOLVER 34 | 35 | def has_pycaffe(): 36 | return get_caffe_resolver().has_pycaffe() 37 | 38 | def show_fallback_warning(): 39 | msg = ''' 40 | ------------------------------------------------------------ 41 | WARNING: PyCaffe not found! 42 | Falling back to a pure protocol buffer implementation. 43 | * Conversions will be drastically slower. 44 | * This backend is UNTESTED! 45 | ------------------------------------------------------------ 46 | 47 | ''' 48 | sys.stderr.write(msg) 49 | -------------------------------------------------------------------------------- /caffe-tensorflow/kaffe/errors.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | class KaffeError(Exception): 4 | pass 5 | 6 | def print_stderr(msg): 7 | sys.stderr.write('%s\n' % msg) 8 | -------------------------------------------------------------------------------- /caffe-tensorflow/kaffe/graph.py: -------------------------------------------------------------------------------- 1 | from google.protobuf import text_format 2 | 3 | from .caffe import get_caffe_resolver 4 | from .errors import KaffeError, print_stderr 5 | from .layers import LayerAdapter, LayerType, NodeKind, NodeDispatch 6 | from .shapes import TensorShape 7 | 8 | class Node(object): 9 | 10 | def __init__(self, name, kind, layer=None): 11 | self.name = name 12 | self.kind = kind 13 | self.layer = LayerAdapter(layer, kind) if layer else None 14 | self.parents = [] 15 | self.children = [] 16 | self.data = None 17 | self.output_shape = None 18 | self.metadata = {} 19 | 20 | def add_parent(self, parent_node): 21 | assert parent_node not in self.parents 22 | self.parents.append(parent_node) 23 | if self not in parent_node.children: 24 | parent_node.children.append(self) 25 | 26 | def add_child(self, child_node): 27 | assert child_node not in self.children 28 | self.children.append(child_node) 29 | if self not in child_node.parents: 30 | child_node.parents.append(self) 31 | 32 | def get_only_parent(self): 33 | if len(self.parents) != 1: 34 | raise KaffeError('Node (%s) expected to have 1 parent. Found %s.' % 35 | (self, len(self.parents))) 36 | return self.parents[0] 37 | 38 | @property 39 | def parameters(self): 40 | if self.layer is not None: 41 | return self.layer.parameters 42 | return None 43 | 44 | def __str__(self): 45 | return '[%s] %s' % (self.kind, self.name) 46 | 47 | def __repr__(self): 48 | return '%s (0x%x)' % (self.name, id(self)) 49 | 50 | 51 | class Graph(object): 52 | 53 | def __init__(self, nodes=None, name=None): 54 | self.nodes = nodes or [] 55 | self.node_lut = {node.name: node for node in self.nodes} 56 | self.name = name 57 | 58 | def add_node(self, node): 59 | self.nodes.append(node) 60 | self.node_lut[node.name] = node 61 | 62 | def get_node(self, name): 63 | try: 64 | return self.node_lut[name] 65 | except KeyError: 66 | raise KaffeError('Layer not found: %s' % name) 67 | 68 | def get_input_nodes(self): 69 | return [node for node in self.nodes if len(node.parents) == 0] 70 | 71 | def get_output_nodes(self): 72 | return [node for node in self.nodes if len(node.children) == 0] 73 | 74 | def topologically_sorted(self): 75 | sorted_nodes = [] 76 | unsorted_nodes = list(self.nodes) 77 | temp_marked = set() 78 | perm_marked = set() 79 | 80 | def visit(node): 81 | if node in temp_marked: 82 | raise KaffeError('Graph is not a DAG.') 83 | if node in perm_marked: 84 | return 85 | temp_marked.add(node) 86 | for child in node.children: 87 | visit(child) 88 | perm_marked.add(node) 89 | temp_marked.remove(node) 90 | sorted_nodes.insert(0, node) 91 | 92 | while len(unsorted_nodes): 93 | visit(unsorted_nodes.pop()) 94 | return sorted_nodes 95 | 96 | def compute_output_shapes(self): 97 | sorted_nodes = self.topologically_sorted() 98 | for node in sorted_nodes: 99 | node.output_shape = TensorShape(*NodeKind.compute_output_shape(node)) 100 | 101 | def replaced(self, new_nodes): 102 | return Graph(nodes=new_nodes, name=self.name) 103 | 104 | def transformed(self, transformers): 105 | graph = self 106 | for transformer in transformers: 107 | graph = transformer(graph) 108 | if graph is None: 109 | raise KaffeError('Transformer failed: {}'.format(transformer)) 110 | assert isinstance(graph, Graph) 111 | return graph 112 | 113 | def __contains__(self, key): 114 | return key in self.node_lut 115 | 116 | def __str__(self): 117 | hdr = '{:<20} {:<30} {:>20} {:>20}'.format('Type', 'Name', 'Param', 'Output') 118 | s = [hdr, '-' * 94] 119 | for node in self.topologically_sorted(): 120 | # If the node has learned parameters, display the first one's shape. 121 | # In case of convolutions, this corresponds to the weights. 122 | data_shape = node.data[0].shape if node.data else '--' 123 | out_shape = node.output_shape or '--' 124 | s.append('{:<20} {:<30} {:>20} {:>20}'.format(node.kind, node.name, data_shape, 125 | tuple(out_shape))) 126 | return '\n'.join(s) 127 | 128 | 129 | class GraphBuilder(object): 130 | '''Constructs a model graph from a Caffe protocol buffer definition.''' 131 | 132 | def __init__(self, def_path, phase='test'): 133 | ''' 134 | def_path: Path to the model definition (.prototxt) 135 | data_path: Path to the model data (.caffemodel) 136 | phase: Either 'test' or 'train'. Used for filtering phase-specific nodes. 137 | ''' 138 | self.def_path = def_path 139 | self.phase = phase 140 | self.load() 141 | 142 | def load(self): 143 | '''Load the layer definitions from the prototxt.''' 144 | self.params = get_caffe_resolver().NetParameter() 145 | with open(self.def_path, 'rb') as def_file: 146 | text_format.Merge(def_file.read(), self.params) 147 | 148 | def filter_layers(self, layers): 149 | '''Filter out layers based on the current phase.''' 150 | phase_map = {0: 'train', 1: 'test'} 151 | filtered_layer_names = set() 152 | filtered_layers = [] 153 | for layer in layers: 154 | phase = self.phase 155 | if len(layer.include): 156 | phase = phase_map[layer.include[0].phase] 157 | if len(layer.exclude): 158 | phase = phase_map[1 - layer.include[0].phase] 159 | exclude = (phase != self.phase) 160 | # Dropout layers appear in a fair number of Caffe 161 | # test-time networks. These are just ignored. We'll 162 | # filter them out here. 163 | if (not exclude) and (phase == 'test'): 164 | exclude = (layer.type == LayerType.Dropout) 165 | if not exclude: 166 | filtered_layers.append(layer) 167 | # Guard against dupes. 168 | assert layer.name not in filtered_layer_names 169 | filtered_layer_names.add(layer.name) 170 | return filtered_layers 171 | 172 | def make_node(self, layer): 173 | '''Create a graph node for the given layer.''' 174 | kind = NodeKind.map_raw_kind(layer.type) 175 | if kind is None: 176 | raise KaffeError('Unknown layer type encountered: %s' % layer.type) 177 | # We want to use the layer's top names (the "output" names), rather than the 178 | # name attribute, which is more of readability thing than a functional one. 179 | # Other layers will refer to a node by its "top name". 180 | return Node(layer.name, kind, layer=layer) 181 | 182 | def make_input_nodes(self): 183 | ''' 184 | Create data input nodes. 185 | 186 | This method is for old-style inputs, where the input specification 187 | was not treated as a first-class layer in the prototext. 188 | Newer models use the "Input layer" type. 189 | ''' 190 | nodes = [Node(name, NodeKind.Data) for name in self.params.input] 191 | if len(nodes): 192 | input_dim = map(int, self.params.input_dim) 193 | if not input_dim: 194 | if len(self.params.input_shape) > 0: 195 | input_dim = map(int, self.params.input_shape[0].dim) 196 | else: 197 | raise KaffeError('Dimensions for input not specified.') 198 | for node in nodes: 199 | node.output_shape = tuple(input_dim) 200 | return nodes 201 | 202 | def build(self): 203 | ''' 204 | Builds the graph from the Caffe layer definitions. 205 | ''' 206 | # Get the layers 207 | layers = self.params.layers or self.params.layer 208 | # Filter out phase-excluded layers 209 | layers = self.filter_layers(layers) 210 | # Get any separately-specified input layers 211 | nodes = self.make_input_nodes() 212 | nodes += [self.make_node(layer) for layer in layers] 213 | # Initialize the graph 214 | graph = Graph(nodes=nodes, name=self.params.name) 215 | # Connect the nodes 216 | # 217 | # A note on layers and outputs: 218 | # In Caffe, each layer can produce multiple outputs ("tops") from a set of inputs 219 | # ("bottoms"). The bottoms refer to other layers' tops. The top can rewrite a bottom 220 | # (in case of in-place operations). Note that the layer's name is not used for establishing 221 | # any connectivity. It's only used for data association. By convention, a layer with a 222 | # single top will often use the same name (although this is not required). 223 | # 224 | # The current implementation only supports single-output nodes (note that a node can still 225 | # have multiple children, since multiple child nodes can refer to the single top's name). 226 | node_outputs = {} 227 | for layer in layers: 228 | node = graph.get_node(layer.name) 229 | for input_name in layer.bottom: 230 | assert input_name != layer.name 231 | parent_node = node_outputs.get(input_name) 232 | if (parent_node is None) or (parent_node == node): 233 | parent_node = graph.get_node(input_name) 234 | node.add_parent(parent_node) 235 | if len(layer.top)>1: 236 | raise KaffeError('Multiple top nodes are not supported.') 237 | for output_name in layer.top: 238 | if output_name == layer.name: 239 | # Output is named the same as the node. No further action required. 240 | continue 241 | # There are two possibilities here: 242 | # 243 | # Case 1: output_name refers to another node in the graph. 244 | # This is an "in-place operation" that overwrites an existing node. 245 | # This would create a cycle in the graph. We'll undo the in-placing 246 | # by substituting this node wherever the overwritten node is referenced. 247 | # 248 | # Case 2: output_name violates the convention layer.name == output_name. 249 | # Since we are working in the single-output regime, we will can rename it to 250 | # match the layer name. 251 | # 252 | # For both cases, future references to this top re-routes to this node. 253 | node_outputs[output_name] = node 254 | 255 | graph.compute_output_shapes() 256 | return graph 257 | 258 | 259 | class NodeMapper(NodeDispatch): 260 | 261 | def __init__(self, graph): 262 | self.graph = graph 263 | 264 | def map(self): 265 | nodes = self.graph.topologically_sorted() 266 | # Remove input nodes - we'll handle them separately. 267 | input_nodes = self.graph.get_input_nodes() 268 | nodes = [t for t in nodes if t not in input_nodes] 269 | # Decompose DAG into chains. 270 | chains = [] 271 | for node in nodes: 272 | attach_to_chain = None 273 | if len(node.parents) == 1: 274 | parent = node.get_only_parent() 275 | for chain in chains: 276 | if chain[-1] == parent: 277 | # Node is part of an existing chain. 278 | attach_to_chain = chain 279 | break 280 | if attach_to_chain is None: 281 | # Start a new chain for this node. 282 | attach_to_chain = [] 283 | chains.append(attach_to_chain) 284 | attach_to_chain.append(node) 285 | # Map each chain. 286 | mapped_chains = [] 287 | for chain in chains: 288 | mapped_chains.append(self.map_chain(chain)) 289 | return self.commit(mapped_chains) 290 | 291 | def map_chain(self, chain): 292 | return [self.map_node(node) for node in chain] 293 | 294 | def map_node(self, node): 295 | map_func = self.get_handler(node.kind, 'map') 296 | mapped_node = map_func(node) 297 | assert mapped_node is not None 298 | mapped_node.node = node 299 | return mapped_node 300 | 301 | def commit(self, mapped_chains): 302 | raise NotImplementedError('Must be implemented by subclass.') 303 | -------------------------------------------------------------------------------- /caffe-tensorflow/kaffe/layers.py: -------------------------------------------------------------------------------- 1 | import re 2 | import numbers 3 | from collections import namedtuple 4 | 5 | from .shapes import * 6 | 7 | LAYER_DESCRIPTORS = { 8 | 9 | # Caffe Types 10 | 'AbsVal': shape_identity, 11 | 'Accuracy': shape_scalar, 12 | 'ArgMax': shape_not_implemented, 13 | 'BatchNorm': shape_identity, 14 | 'BN': shape_identity, 15 | 'BNLL': shape_not_implemented, 16 | 'Concat': shape_concat, 17 | 'ContrastiveLoss': shape_scalar, 18 | 'Convolution': shape_convolution, 19 | 'Deconvolution': shape_not_implemented, 20 | 'Data': shape_data, 21 | 'Dropout': shape_identity, 22 | 'Interp': shape_identity, 23 | 'DummyData': shape_data, 24 | 'EuclideanLoss': shape_scalar, 25 | 'Eltwise': shape_identity, 26 | 'Exp': shape_identity, 27 | 'Flatten': shape_not_implemented, 28 | 'HDF5Datapython': shape_data, 29 | 'HDF5Output': shape_identity, 30 | 'HingeLoss': shape_scalar, 31 | 'Im2col': shape_not_implemented, 32 | 'ImageData': shape_data, 33 | 'InfogainLoss': shape_scalar, 34 | 'InnerProduct': shape_inner_product, 35 | 'Input': shape_data, 36 | 'LRN': shape_identity, 37 | 'MemoryData': shape_mem_data, 38 | 'MultinomialLogisticLoss': shape_scalar, 39 | 'MVN': shape_not_implemented, 40 | 'Pooling': shape_pool, 41 | 'Power': shape_identity, 42 | 'ReLU': shape_identity, 43 | 'Scale': shape_identity, 44 | 'Sigmoid': shape_identity, 45 | 'SigmoidCrossEntropyLoss': shape_scalar, 46 | 'Silence': shape_not_implemented, 47 | 'Softmax': shape_identity, 48 | 'SoftmaxWithLoss': shape_scalar, 49 | 'Split': shape_not_implemented, 50 | 'Slice': shape_not_implemented, 51 | 'TanH': shape_identity, 52 | 'WindowData': shape_not_implemented, 53 | 'Threshold': shape_identity, 54 | } 55 | 56 | LAYER_TYPES = LAYER_DESCRIPTORS.keys() 57 | 58 | LayerType = type('LayerType', (), {t: t for t in LAYER_TYPES}) 59 | 60 | class NodeKind(LayerType): 61 | 62 | @staticmethod 63 | def map_raw_kind(kind): 64 | if kind in LAYER_TYPES: 65 | return kind 66 | return None 67 | 68 | @staticmethod 69 | def compute_output_shape(node): 70 | try: 71 | val = LAYER_DESCRIPTORS[node.kind](node) 72 | return val 73 | except NotImplementedError: 74 | raise KaffeError('Output shape computation not implemented for type: %s' % node.kind) 75 | 76 | 77 | class NodeDispatchError(KaffeError): 78 | 79 | pass 80 | 81 | 82 | class NodeDispatch(object): 83 | 84 | @staticmethod 85 | def get_handler_name(node_kind): 86 | if len(node_kind) <= 4: 87 | # A catch-all for things like ReLU and tanh 88 | return node_kind.lower() 89 | # Convert from CamelCase to under_scored 90 | name = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', node_kind) 91 | return re.sub('([a-z0-9])([A-Z])', r'\1_\2', name).lower() 92 | 93 | def get_handler(self, node_kind, prefix): 94 | name = self.get_handler_name(node_kind) 95 | name = '_'.join((prefix, name)) 96 | try: 97 | return getattr(self, name) 98 | except AttributeError: 99 | raise NodeDispatchError('No handler found for node kind: %s (expected: %s)' % 100 | (node_kind, name)) 101 | 102 | 103 | class LayerAdapter(object): 104 | 105 | def __init__(self, layer, kind): 106 | self.layer = layer 107 | self.kind = kind 108 | 109 | @property 110 | def parameters(self): 111 | name = NodeDispatch.get_handler_name(self.kind) 112 | name = '_'.join((name, 'param')) 113 | try: 114 | return getattr(self.layer, name) 115 | except AttributeError: 116 | raise NodeDispatchError('Caffe parameters not found for layer kind: %s' % (self.kind)) 117 | 118 | @staticmethod 119 | def get_kernel_value(scalar, repeated, idx, default=None): 120 | if scalar: 121 | return scalar 122 | if repeated: 123 | if isinstance(repeated, numbers.Number): 124 | return repeated 125 | if len(repeated) == 1: 126 | # Same value applies to all spatial dimensions 127 | return int(repeated[0]) 128 | assert idx < len(repeated) 129 | # Extract the value for the given spatial dimension 130 | return repeated[idx] 131 | if default is None: 132 | raise ValueError('Unable to determine kernel parameter!') 133 | return default 134 | 135 | @property 136 | def kernel_parameters(self): 137 | assert self.kind in (NodeKind.Convolution, NodeKind.Pooling) 138 | params = self.parameters 139 | k_h = self.get_kernel_value(params.kernel_h, params.kernel_size, 0) 140 | k_w = self.get_kernel_value(params.kernel_w, params.kernel_size, 1) 141 | s_h = self.get_kernel_value(params.stride_h, params.stride, 0, default=1) 142 | s_w = self.get_kernel_value(params.stride_w, params.stride, 1, default=1) 143 | p_h = self.get_kernel_value(params.pad_h, params.pad, 0, default=0) 144 | p_w = self.get_kernel_value(params.pad_h, params.pad, 1, default=0) 145 | return KernelParameters(k_h, k_w, s_h, s_w, p_h, p_w) 146 | 147 | 148 | KernelParameters = namedtuple('KernelParameters', ['kernel_h', 'kernel_w', 'stride_h', 'stride_w', 149 | 'pad_h', 'pad_w']) 150 | -------------------------------------------------------------------------------- /caffe-tensorflow/kaffe/shapes.py: -------------------------------------------------------------------------------- 1 | import math 2 | from collections import namedtuple 3 | 4 | from .errors import KaffeError 5 | 6 | TensorShape = namedtuple('TensorShape', ['batch_size', 'channels', 'height', 'width']) 7 | 8 | 9 | def get_filter_output_shape(i_h, i_w, params, round_func): 10 | o_h = (i_h + 2 * params.pad_h - params.kernel_h) / float(params.stride_h) + 1 11 | o_w = (i_w + 2 * params.pad_w - params.kernel_w) / float(params.stride_w) + 1 12 | return (int(round_func(o_h)), int(round_func(o_w))) 13 | 14 | 15 | def get_strided_kernel_output_shape(node, round_func): 16 | assert node.layer is not None 17 | input_shape = node.get_only_parent().output_shape 18 | o_h, o_w = get_filter_output_shape(input_shape.height, input_shape.width, 19 | node.layer.kernel_parameters, round_func) 20 | params = node.layer.parameters 21 | has_c_o = hasattr(params, 'num_output') 22 | c = params.num_output if has_c_o else input_shape.channels 23 | return TensorShape(input_shape.batch_size, c, o_h, o_w) 24 | 25 | 26 | def shape_not_implemented(node): 27 | raise NotImplementedError 28 | 29 | 30 | def shape_identity(node): 31 | assert len(node.parents) > 0 32 | return node.parents[0].output_shape 33 | 34 | 35 | def shape_scalar(node): 36 | return TensorShape(1, 1, 1, 1) 37 | 38 | 39 | def shape_data(node): 40 | if node.output_shape: 41 | # Old-style input specification 42 | return node.output_shape 43 | try: 44 | # New-style input specification 45 | return map(int, node.parameters.shape[0].dim) 46 | except: 47 | # We most likely have a data layer on our hands. The problem is, 48 | # Caffe infers the dimensions of the data from the source (eg: LMDB). 49 | # We want to avoid reading datasets here. Fail for now. 50 | # This can be temporarily fixed by transforming the data layer to 51 | # Caffe's "input" layer (as is usually used in the "deploy" version). 52 | # TODO: Find a better solution for this. 53 | raise KaffeError('Cannot determine dimensions of data layer.\n' 54 | 'See comments in function shape_data for more info.') 55 | 56 | 57 | def shape_mem_data(node): 58 | params = node.parameters 59 | return TensorShape(params.batch_size, params.channels, params.height, params.width) 60 | 61 | 62 | def shape_concat(node): 63 | axis = node.layer.parameters.axis 64 | output_shape = None 65 | for parent in node.parents: 66 | if output_shape is None: 67 | output_shape = list(parent.output_shape) 68 | else: 69 | output_shape[axis] += parent.output_shape[axis] 70 | return tuple(output_shape) 71 | 72 | 73 | def shape_convolution(node): 74 | return get_strided_kernel_output_shape(node, math.floor) 75 | 76 | 77 | def shape_pool(node): 78 | return get_strided_kernel_output_shape(node, math.ceil) 79 | 80 | 81 | def shape_inner_product(node): 82 | input_shape = node.get_only_parent().output_shape 83 | return TensorShape(input_shape.batch_size, node.layer.parameters.num_output, 1, 1) 84 | -------------------------------------------------------------------------------- /caffe-tensorflow/kaffe/tensorflow/__init__.py: -------------------------------------------------------------------------------- 1 | from .transformer import TensorFlowTransformer 2 | from .network import Network 3 | -------------------------------------------------------------------------------- /caffe-tensorflow/kaffe/tensorflow/network.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | 4 | DEFAULT_PADDING = 'SAME' 5 | 6 | 7 | def layer(op): 8 | '''Decorator for composable network layers.''' 9 | 10 | def layer_decorated(self, *args, **kwargs): 11 | # Automatically set a name if not provided. 12 | name = kwargs.setdefault('name', self.get_unique_name(op.__name__)) 13 | # Figure out the layer inputs. 14 | if len(self.terminals) == 0: 15 | raise RuntimeError('No input variables found for layer %s.' % name) 16 | elif len(self.terminals) == 1: 17 | layer_input = self.terminals[0] 18 | else: 19 | layer_input = list(self.terminals) 20 | # Perform the operation and get the output. 21 | layer_output = op(self, layer_input, *args, **kwargs) 22 | # Add to layer LUT. 23 | self.layers[name] = layer_output 24 | # This output is now the input for the next layer. 25 | self.feed(layer_output) 26 | # Return self for chained calls. 27 | return self 28 | 29 | return layer_decorated 30 | 31 | 32 | class Network(object): 33 | 34 | def __init__(self, inputs, trainable=True): 35 | # The input nodes for this network 36 | self.inputs = inputs 37 | # The current list of terminal nodes 38 | self.terminals = [] 39 | # Mapping from layer names to layers 40 | self.layers = dict(inputs) 41 | # If true, the resulting variables are set as trainable 42 | self.trainable = trainable 43 | # Switch variable for dropout 44 | self.use_dropout = tf.placeholder_with_default(tf.constant(1.0), 45 | shape=[], 46 | name='use_dropout') 47 | self.setup() 48 | 49 | def setup(self): 50 | '''Construct the network. ''' 51 | raise NotImplementedError('Must be implemented by the subclass.') 52 | 53 | def load(self, data_path, session, ignore_missing=False): 54 | '''Load network weights. 55 | data_path: The path to the numpy-serialized network weights 56 | session: The current TensorFlow session 57 | ignore_missing: If true, serialized weights for missing layers are ignored. 58 | ''' 59 | data_dict = np.load(data_path).item() 60 | for op_name in data_dict: 61 | with tf.variable_scope(op_name, reuse=True): 62 | for param_name, data in data_dict[op_name].iteritems(): 63 | try: 64 | var = tf.get_variable(param_name) 65 | session.run(var.assign(data)) 66 | except ValueError: 67 | if not ignore_missing: 68 | raise 69 | 70 | def feed(self, *args): 71 | '''Set the input(s) for the next operation by replacing the terminal nodes. 72 | The arguments can be either layer names or the actual layers. 73 | ''' 74 | assert len(args) != 0 75 | self.terminals = [] 76 | for fed_layer in args: 77 | if isinstance(fed_layer, basestring): 78 | try: 79 | fed_layer = self.layers[fed_layer] 80 | except KeyError: 81 | raise KeyError('Unknown layer name fed: %s' % fed_layer) 82 | self.terminals.append(fed_layer) 83 | return self 84 | 85 | def get_output(self): 86 | '''Returns the current network output.''' 87 | return self.terminals[-1] 88 | 89 | def get_unique_name(self, prefix): 90 | '''Returns an index-suffixed unique name for the given prefix. 91 | This is used for auto-generating layer names based on the type-prefix. 92 | ''' 93 | ident = sum(t.startswith(prefix) for t, _ in self.layers.items()) + 1 94 | return '%s_%d' % (prefix, ident) 95 | 96 | def make_var(self, name, shape): 97 | '''Creates a new TensorFlow variable.''' 98 | return tf.get_variable(name, shape, trainable=self.trainable) 99 | 100 | def validate_padding(self, padding): 101 | '''Verifies that the padding is one of the supported ones.''' 102 | assert padding in ('SAME', 'VALID') 103 | 104 | @layer 105 | def conv(self, 106 | input, 107 | k_h, 108 | k_w, 109 | c_o, 110 | s_h, 111 | s_w, 112 | name, 113 | relu=True, 114 | padding=DEFAULT_PADDING, 115 | group=1, 116 | biased=True): 117 | # Verify that the padding is acceptable 118 | self.validate_padding(padding) 119 | # Get the number of channels in the input 120 | c_i = input.get_shape()[-1] 121 | # Verify that the grouping parameter is valid 122 | assert c_i % group == 0 123 | assert c_o % group == 0 124 | # Convolution for a given input and kernel 125 | convolve = lambda i, k: tf.nn.conv2d(i, k, [1, s_h, s_w, 1], padding=padding) 126 | with tf.variable_scope(name) as scope: 127 | kernel = self.make_var('weights', shape=[k_h, k_w, c_i / group, c_o]) 128 | if group == 1: 129 | # This is the common-case. Convolve the input without any further complications. 130 | output = convolve(input, kernel) 131 | else: 132 | # Split the input into groups and then convolve each of them independently 133 | input_groups = tf.split(3, group, input) 134 | kernel_groups = tf.split(3, group, kernel) 135 | output_groups = [convolve(i, k) for i, k in zip(input_groups, kernel_groups)] 136 | # Concatenate the groups 137 | output = tf.concat(3, output_groups) 138 | # Add the biases 139 | if biased: 140 | biases = self.make_var('biases', [c_o]) 141 | output = tf.nn.bias_add(output, biases) 142 | if relu: 143 | # ReLU non-linearity 144 | output = tf.nn.relu(output, name=scope.name) 145 | return output 146 | 147 | @layer 148 | def relu(self, input, name): 149 | return tf.nn.relu(input, name=name) 150 | 151 | @layer 152 | def max_pool(self, input, k_h, k_w, s_h, s_w, name, padding=DEFAULT_PADDING): 153 | self.validate_padding(padding) 154 | return tf.nn.max_pool(input, 155 | ksize=[1, k_h, k_w, 1], 156 | strides=[1, s_h, s_w, 1], 157 | padding=padding, 158 | name=name) 159 | 160 | @layer 161 | def avg_pool(self, input, k_h, k_w, s_h, s_w, name, padding=DEFAULT_PADDING): 162 | self.validate_padding(padding) 163 | return tf.nn.avg_pool(input, 164 | ksize=[1, k_h, k_w, 1], 165 | strides=[1, s_h, s_w, 1], 166 | padding=padding, 167 | name=name) 168 | 169 | @layer 170 | def lrn(self, input, radius, alpha, beta, name, bias=1.0): 171 | return tf.nn.local_response_normalization(input, 172 | depth_radius=radius, 173 | alpha=alpha, 174 | beta=beta, 175 | bias=bias, 176 | name=name) 177 | 178 | @layer 179 | def concat(self, inputs, axis, name): 180 | return tf.concat(concat_dim=axis, values=inputs, name=name) 181 | 182 | @layer 183 | def add(self, inputs, name): 184 | return tf.add_n(inputs, name=name) 185 | 186 | @layer 187 | def fc(self, input, num_out, name, relu=True): 188 | with tf.variable_scope(name) as scope: 189 | input_shape = input.get_shape() 190 | if input_shape.ndims == 4: 191 | # The input is spatial. Vectorize it first. 192 | dim = 1 193 | for d in input_shape[1:].as_list(): 194 | dim *= d 195 | feed_in = tf.reshape(input, [-1, dim]) 196 | else: 197 | feed_in, dim = (input, input_shape[-1].value) 198 | weights = self.make_var('weights', shape=[dim, num_out]) 199 | biases = self.make_var('biases', [num_out]) 200 | op = tf.nn.relu_layer if relu else tf.nn.xw_plus_b 201 | fc = op(feed_in, weights, biases, name=scope.name) 202 | return fc 203 | 204 | @layer 205 | def softmax(self, input, name): 206 | input_shape = map(lambda v: v.value, input.get_shape()) 207 | if len(input_shape) > 2: 208 | # For certain models (like NiN), the singleton spatial dimensions 209 | # need to be explicitly squeezed, since they're not broadcast-able 210 | # in TensorFlow's NHWC ordering (unlike Caffe's NCHW). 211 | if input_shape[1] == 1 and input_shape[2] == 1: 212 | input = tf.squeeze(input, squeeze_dims=[1, 2]) 213 | else: 214 | raise ValueError('Rank 2 tensor input expected for softmax!') 215 | return tf.nn.softmax(input, name=name) 216 | 217 | @layer 218 | def batch_normalization(self, input, name, scale_offset=True, relu=False): 219 | # NOTE: Currently, only inference is supported 220 | with tf.variable_scope(name) as scope: 221 | shape = [input.get_shape()[-1]] 222 | if scale_offset: 223 | scale = self.make_var('scale', shape=shape) 224 | offset = self.make_var('offset', shape=shape) 225 | else: 226 | scale, offset = (None, None) 227 | output = tf.nn.batch_normalization( 228 | input, 229 | mean=self.make_var('mean', shape=shape), 230 | variance=self.make_var('variance', shape=shape), 231 | offset=offset, 232 | scale=scale, 233 | # TODO: This is the default Caffe batch norm eps 234 | # Get the actual eps from parameters 235 | variance_epsilon=1e-5, 236 | name=name) 237 | if relu: 238 | output = tf.nn.relu(output) 239 | return output 240 | 241 | @layer 242 | def dropout(self, input, keep_prob, name): 243 | keep = 1 - self.use_dropout + (self.use_dropout * keep_prob) 244 | return tf.nn.dropout(input, keep, name=name) 245 | -------------------------------------------------------------------------------- /caffe-tensorflow/kaffe/tensorflow/transformer.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from ..errors import KaffeError, print_stderr 4 | from ..graph import GraphBuilder, NodeMapper 5 | from ..layers import NodeKind 6 | from ..transformers import (DataInjector, DataReshaper, NodeRenamer, ReLUFuser, 7 | BatchNormScaleBiasFuser, BatchNormPreprocessor, ParameterNamer) 8 | 9 | from . import network 10 | 11 | 12 | def get_padding_type(kernel_params, input_shape, output_shape): 13 | '''Translates Caffe's numeric padding to one of ('SAME', 'VALID'). 14 | Caffe supports arbitrary padding values, while TensorFlow only 15 | supports 'SAME' and 'VALID' modes. So, not all Caffe paddings 16 | can be translated to TensorFlow. There are some subtleties to 17 | how the padding edge-cases are handled. These are described here: 18 | https://github.com/Yangqing/caffe2/blob/master/caffe2/proto/caffe2_legacy.proto 19 | ''' 20 | k_h, k_w, s_h, s_w, p_h, p_w = kernel_params 21 | s_o_h = np.ceil(input_shape.height / float(s_h)) 22 | s_o_w = np.ceil(input_shape.width / float(s_w)) 23 | if (output_shape.height == s_o_h) and (output_shape.width == s_o_w): 24 | return 'SAME' 25 | v_o_h = np.ceil((input_shape.height - k_h + 1.0) / float(s_h)) 26 | v_o_w = np.ceil((input_shape.width - k_w + 1.0) / float(s_w)) 27 | if (output_shape.height == v_o_h) and (output_shape.width == v_o_w): 28 | return 'VALID' 29 | return None 30 | 31 | 32 | class TensorFlowNode(object): 33 | '''An intermediate representation for TensorFlow operations.''' 34 | 35 | def __init__(self, op, *args, **kwargs): 36 | # A string corresponding to the TensorFlow operation 37 | self.op = op 38 | # Positional arguments for the operation 39 | self.args = args 40 | # Keyword arguments for the operation 41 | self.kwargs = list(kwargs.items()) 42 | # The source Caffe node 43 | self.node = None 44 | 45 | def format(self, arg): 46 | '''Returns a string representation for the given value.''' 47 | return "'%s'" % arg if isinstance(arg, basestring) else str(arg) 48 | 49 | def pair(self, key, value): 50 | '''Returns key=formatted(value).''' 51 | return '%s=%s' % (key, self.format(value)) 52 | 53 | def emit(self): 54 | '''Emits the Python source for this node.''' 55 | # Format positional arguments 56 | args = map(self.format, self.args) 57 | # Format any keyword arguments 58 | if self.kwargs: 59 | args += [self.pair(k, v) for k, v in self.kwargs] 60 | # Set the node name 61 | args.append(self.pair('name', self.node.name)) 62 | args = ', '.join(args) 63 | return '%s(%s)' % (self.op, args) 64 | 65 | 66 | class MaybeActivated(object): 67 | 68 | def __init__(self, node, default=True): 69 | self.inject_kwargs = {} 70 | if node.metadata.get('relu', False) != default: 71 | self.inject_kwargs['relu'] = not default 72 | 73 | def __call__(self, *args, **kwargs): 74 | kwargs.update(self.inject_kwargs) 75 | return TensorFlowNode(*args, **kwargs) 76 | 77 | 78 | class TensorFlowMapper(NodeMapper): 79 | 80 | def get_kernel_params(self, node): 81 | kernel_params = node.layer.kernel_parameters 82 | input_shape = node.get_only_parent().output_shape 83 | padding = get_padding_type(kernel_params, input_shape, node.output_shape) 84 | # Only emit the padding if it's not the default value. 85 | padding = {'padding': padding} if padding != network.DEFAULT_PADDING else {} 86 | return (kernel_params, padding) 87 | 88 | def map_convolution(self, node): 89 | (kernel_params, kwargs) = self.get_kernel_params(node) 90 | h = kernel_params.kernel_h 91 | w = kernel_params.kernel_w 92 | c_o = node.output_shape[1] 93 | c_i = node.parents[0].output_shape[1] 94 | group = node.parameters.group 95 | if group != 1: 96 | kwargs['group'] = group 97 | if not node.parameters.bias_term: 98 | kwargs['biased'] = False 99 | assert kernel_params.kernel_h == h 100 | assert kernel_params.kernel_w == w 101 | return MaybeActivated(node)('conv', kernel_params.kernel_h, kernel_params.kernel_w, c_o, 102 | kernel_params.stride_h, kernel_params.stride_w, **kwargs) 103 | 104 | def map_relu(self, node): 105 | return TensorFlowNode('relu') 106 | 107 | def map_interp(self, node): 108 | return TensorFlowNode('relu') 109 | 110 | def map_pooling(self, node): 111 | pool_type = node.parameters.pool 112 | if pool_type == 0: 113 | pool_op = 'max_pool' 114 | elif pool_type == 1: 115 | pool_op = 'avg_pool' 116 | else: 117 | # Stochastic pooling, for instance. 118 | raise KaffeError('Unsupported pooling type.') 119 | (kernel_params, padding) = self.get_kernel_params(node) 120 | return TensorFlowNode(pool_op, kernel_params.kernel_h, kernel_params.kernel_w, 121 | kernel_params.stride_h, kernel_params.stride_w, **padding) 122 | 123 | def map_inner_product(self, node): 124 | #TODO: Axis 125 | assert node.parameters.axis == 1 126 | #TODO: Unbiased 127 | assert node.parameters.bias_term == True 128 | return MaybeActivated(node)('fc', node.parameters.num_output) 129 | 130 | def map_softmax(self, node): 131 | return TensorFlowNode('softmax') 132 | 133 | def map_lrn(self, node): 134 | params = node.parameters 135 | # The window size must be an odd value. For a window 136 | # size of (2*n+1), TensorFlow defines depth_radius = n. 137 | assert params.local_size % 2 == 1 138 | # Caffe scales by (alpha/(2*n+1)), whereas TensorFlow 139 | # just scales by alpha (as does Krizhevsky's paper). 140 | # We'll account for that here. 141 | alpha = params.alpha / float(params.local_size) 142 | return TensorFlowNode('lrn', int(params.local_size / 2), alpha, params.beta) 143 | 144 | def map_concat(self, node): 145 | axis = (2, 3, 1, 0)[node.parameters.axis] 146 | return TensorFlowNode('concat', axis) 147 | 148 | def map_dropout(self, node): 149 | return TensorFlowNode('dropout', node.parameters.dropout_ratio) 150 | 151 | def map_batch_norm(self, node): 152 | scale_offset = len(node.data) == 4 153 | kwargs = {} if scale_offset else {'scale_offset': False} 154 | return MaybeActivated(node, default=False)('batch_normalization', **kwargs) 155 | 156 | def map_bn(self, node): 157 | scale_offset = len(node.data) == 4 158 | print node 159 | kwargs = {} if scale_offset else {'scale_offset': False} 160 | return MaybeActivated(node, default=False)('batch_normalization', **kwargs) 161 | 162 | def map_eltwise(self, node): 163 | operations = {0: 'multiply', 1: 'add', 2: 'max'} 164 | op_code = node.parameters.operation 165 | try: 166 | return TensorFlowNode(operations[op_code]) 167 | except KeyError: 168 | raise KaffeError('Unknown elementwise operation: {}'.format(op_code)) 169 | 170 | def commit(self, chains): 171 | return chains 172 | 173 | 174 | class TensorFlowEmitter(object): 175 | 176 | def __init__(self, tab=None): 177 | self.tab = tab or ' ' * 4 178 | self.prefix = '' 179 | 180 | def indent(self): 181 | self.prefix += self.tab 182 | 183 | def outdent(self): 184 | self.prefix = self.prefix[:-len(self.tab)] 185 | 186 | def statement(self, s): 187 | return self.prefix + s + '\n' 188 | 189 | def emit_imports(self): 190 | return self.statement('from kaffe.tensorflow import Network\n') 191 | 192 | def emit_class_def(self, name): 193 | return self.statement('class %s(Network):' % (name)) 194 | 195 | def emit_setup_def(self): 196 | return self.statement('def setup(self):') 197 | 198 | def emit_parents(self, chain): 199 | assert len(chain) 200 | s = '(self.feed(' 201 | sep = ', \n' + self.prefix + (' ' * len(s)) 202 | s += sep.join(["'%s'" % parent.name for parent in chain[0].node.parents]) 203 | return self.statement(s + ')') 204 | 205 | def emit_node(self, node): 206 | return self.statement(' ' * 5 + '.' + node.emit()) 207 | 208 | def emit(self, name, chains): 209 | s = self.emit_imports() 210 | s += self.emit_class_def(name) 211 | self.indent() 212 | s += self.emit_setup_def() 213 | self.indent() 214 | blocks = [] 215 | for chain in chains: 216 | b = '' 217 | b += self.emit_parents(chain) 218 | for node in chain: 219 | b += self.emit_node(node) 220 | blocks.append(b[:-1] + ')') 221 | s = s + '\n\n'.join(blocks) 222 | return s 223 | 224 | 225 | class TensorFlowTransformer(object): 226 | 227 | def __init__(self, def_path, data_path, verbose=True, phase='test'): 228 | self.verbose = verbose 229 | self.phase = phase 230 | self.load(def_path, data_path, phase) 231 | self.params = None 232 | self.source = None 233 | 234 | def load(self, def_path, data_path, phase): 235 | # Build the graph 236 | graph = GraphBuilder(def_path, phase).build() 237 | 238 | if data_path is not None: 239 | # Load and associate learned parameters 240 | graph = DataInjector(def_path, data_path)(graph) 241 | 242 | # Transform the graph 243 | transformers = [ 244 | # Fuse split batch normalization layers 245 | BatchNormScaleBiasFuser(), 246 | 247 | # Fuse ReLUs 248 | # TODO: Move non-linearity application to layer wrapper, allowing 249 | # any arbitrary operation to be optionally activated. 250 | ReLUFuser(allowed_parent_types=[NodeKind.Convolution, NodeKind.InnerProduct, 251 | NodeKind.BatchNorm]), 252 | 253 | # Rename nodes 254 | # Slashes are used for scoping in TensorFlow. Replace slashes 255 | # in node names with underscores. 256 | # (Caffe's GoogLeNet implementation uses slashes) 257 | NodeRenamer(lambda node: node.name.replace('/', '_')) 258 | ] 259 | self.graph = graph.transformed(transformers) 260 | 261 | # Display the graph 262 | if self.verbose: 263 | print_stderr(self.graph) 264 | 265 | def transform_data(self): 266 | if self.params is None: 267 | transformers = [ 268 | 269 | # Reshape the parameters to TensorFlow's ordering 270 | DataReshaper({ 271 | # (c_o, c_i, h, w) -> (h, w, c_i, c_o) 272 | NodeKind.Convolution: (2, 3, 1, 0), 273 | 274 | # (c_o, c_i) -> (c_i, c_o) 275 | NodeKind.InnerProduct: (1, 0) 276 | }), 277 | 278 | # Pre-process batch normalization data 279 | BatchNormPreprocessor(), 280 | 281 | # Convert parameters to dictionaries 282 | ParameterNamer(), 283 | ] 284 | self.graph = self.graph.transformed(transformers) 285 | self.params = {node.name: node.data for node in self.graph.nodes if node.data} 286 | return self.params 287 | 288 | def transform_source(self): 289 | if self.source is None: 290 | mapper = TensorFlowMapper(self.graph) 291 | chains = mapper.map() 292 | emitter = TensorFlowEmitter() 293 | self.source = emitter.emit(self.graph.name, chains) 294 | return self.source 295 | -------------------------------------------------------------------------------- /caffe-tensorflow/kaffe/transformers.py: -------------------------------------------------------------------------------- 1 | ''' 2 | A collection of graph transforms. 3 | 4 | A transformer is a callable that accepts a graph and returns a transformed version. 5 | ''' 6 | 7 | import numpy as np 8 | 9 | from .caffe import get_caffe_resolver, has_pycaffe 10 | from .errors import KaffeError, print_stderr 11 | from .layers import NodeKind 12 | 13 | 14 | class DataInjector(object): 15 | ''' 16 | Associates parameters loaded from a .caffemodel file with their corresponding nodes. 17 | ''' 18 | 19 | def __init__(self, def_path, data_path): 20 | # The .prototxt file defining the graph 21 | self.def_path = def_path 22 | # The .caffemodel file containing the learned parameters 23 | self.data_path = data_path 24 | # Set to true if the fallback protocol-buffer based backend was used 25 | self.did_use_pb = False 26 | # A list containing (layer name, parameters) tuples 27 | self.params = None 28 | # Load the parameters 29 | self.load() 30 | 31 | def load(self): 32 | if has_pycaffe(): 33 | self.load_using_caffe() 34 | else: 35 | self.load_using_pb() 36 | 37 | def load_using_caffe(self): 38 | caffe = get_caffe_resolver().caffe 39 | net = caffe.Net(self.def_path, self.data_path, caffe.TEST) 40 | data = lambda blob: blob.data 41 | self.params = [(k, map(data, v)) for k, v in net.params.items()] 42 | 43 | def load_using_pb(self): 44 | data = get_caffe_resolver().NetParameter() 45 | data.MergeFromString(open(self.data_path, 'rb').read()) 46 | pair = lambda layer: (layer.name, self.normalize_pb_data(layer)) 47 | layers = data.layers or data.layer 48 | self.params = [pair(layer) for layer in layers if layer.blobs] 49 | self.did_use_pb = True 50 | 51 | def normalize_pb_data(self, layer): 52 | transformed = [] 53 | for blob in layer.blobs: 54 | if len(blob.shape.dim): 55 | dims = blob.shape.dim 56 | c_o, c_i, h, w = map(int, [1] * (4 - len(dims)) + list(dims)) 57 | else: 58 | c_o = blob.num 59 | c_i = blob.channels 60 | h = blob.height 61 | w = blob.width 62 | data = np.array(blob.data, dtype=np.float32).reshape(c_o, c_i, h, w) 63 | transformed.append(data) 64 | return transformed 65 | 66 | def adjust_parameters(self, node, data): 67 | if not self.did_use_pb: 68 | return data 69 | # When using the protobuf-backend, each parameter initially has four dimensions. 70 | # In certain cases (like FC layers), we want to eliminate the singleton dimensions. 71 | # This implementation takes care of the common cases. However, it does leave the 72 | # potential for future issues. 73 | # The Caffe-backend does not suffer from this problem. 74 | data = list(data) 75 | squeeze_indices = [1] # Squeeze biases. 76 | if node.kind == NodeKind.InnerProduct: 77 | squeeze_indices.append(0) # Squeeze FC. 78 | for idx in squeeze_indices: 79 | data[idx] = np.squeeze(data[idx]) 80 | return data 81 | 82 | def __call__(self, graph): 83 | for layer_name, data in self.params: 84 | if layer_name in graph: 85 | node = graph.get_node(layer_name) 86 | node.data = self.adjust_parameters(node, data) 87 | else: 88 | print_stderr('Ignoring parameters for non-existent layer: %s' % layer_name) 89 | return graph 90 | 91 | 92 | class DataReshaper(object): 93 | 94 | def __init__(self, mapping, replace=True): 95 | # A dictionary mapping NodeKind to the transposed order. 96 | self.mapping = mapping 97 | # The node kinds eligible for reshaping 98 | self.reshaped_node_types = self.mapping.keys() 99 | # If true, the reshaped data will replace the old one. 100 | # Otherwise, it's set to the reshaped_data attribute. 101 | self.replace = replace 102 | 103 | def has_spatial_parent(self, node): 104 | try: 105 | parent = node.get_only_parent() 106 | s = parent.output_shape 107 | return s.height > 1 or s.width > 1 108 | except KaffeError: 109 | return False 110 | 111 | def map(self, node_kind): 112 | try: 113 | return self.mapping[node_kind] 114 | except KeyError: 115 | raise KaffeError('Ordering not found for node kind: {}'.format(node_kind)) 116 | 117 | def __call__(self, graph): 118 | for node in graph.nodes: 119 | if node.data is None: 120 | continue 121 | if node.kind not in self.reshaped_node_types: 122 | 123 | # Check for 2+ dimensional data 124 | if any(len(tensor.shape) > 1 for tensor in node.data): 125 | print_stderr('Warning: parmaters not reshaped for node: {}'.format(node)) 126 | continue 127 | transpose_order = self.map(node.kind) 128 | weights = node.data[0] 129 | if (node.kind == NodeKind.InnerProduct) and self.has_spatial_parent(node): 130 | # The FC layer connected to the spatial layer needs to be 131 | # re-wired to match the new spatial ordering. 132 | in_shape = node.get_only_parent().output_shape 133 | fc_shape = weights.shape 134 | output_channels = fc_shape[0] 135 | weights = weights.reshape((output_channels, in_shape.channels, in_shape.height, 136 | in_shape.width)) 137 | weights = weights.transpose(self.map(NodeKind.Convolution)) 138 | node.reshaped_data = weights.reshape(fc_shape[transpose_order[0]], 139 | fc_shape[transpose_order[1]]) 140 | else: 141 | node.reshaped_data = weights.transpose(transpose_order) 142 | 143 | if self.replace: 144 | for node in graph.nodes: 145 | if hasattr(node, 'reshaped_data'): 146 | # Set the weights 147 | node.data[0] = node.reshaped_data 148 | del node.reshaped_data 149 | return graph 150 | 151 | 152 | class SubNodeFuser(object): 153 | ''' 154 | An abstract helper for merging a single-child with its single-parent. 155 | ''' 156 | 157 | def __call__(self, graph): 158 | nodes = graph.nodes 159 | print graph.name 160 | fused_nodes = [] 161 | for node in nodes: 162 | if len(node.parents) != 1: 163 | # We're only fusing nodes with single parents 164 | continue 165 | parent = node.get_only_parent() 166 | if len(parent.children) != 1: 167 | # We can only fuse a node if its parent's 168 | # value isn't used by any other node. 169 | continue 170 | if not self.is_eligible_pair(parent, node): 171 | continue 172 | # Rewrite the fused node's children to its parent. 173 | for child in node.children: 174 | child.parents.remove(node) 175 | parent.add_child(child) 176 | # Disconnect the fused node from the graph. 177 | parent.children.remove(node) 178 | fused_nodes.append(node) 179 | # Let the sub-class merge the fused node in any arbitrary way. 180 | self.merge(parent, node) 181 | transformed_nodes = [node for node in nodes if node not in fused_nodes] 182 | return graph.replaced(transformed_nodes) 183 | 184 | def is_eligible_pair(self, parent, child): 185 | '''Returns true if this parent/child pair is eligible for fusion.''' 186 | raise NotImplementedError('Must be implemented by subclass.') 187 | 188 | def merge(self, parent, child): 189 | '''Merge the child node into the parent.''' 190 | raise NotImplementedError('Must be implemented by subclass') 191 | 192 | 193 | class ReLUFuser(SubNodeFuser): 194 | ''' 195 | Fuses rectified linear units with their parent nodes. 196 | ''' 197 | 198 | def __init__(self, allowed_parent_types=None): 199 | # Fuse ReLUs when the parent node is one of the given types. 200 | # If None, all node types are eligible. 201 | self.allowed_parent_types = allowed_parent_types 202 | 203 | def is_eligible_pair(self, parent, child): 204 | return ((self.allowed_parent_types is None or parent.kind in self.allowed_parent_types) and 205 | child.kind == NodeKind.ReLU) 206 | 207 | def merge(self, parent, _): 208 | parent.metadata['relu'] = True 209 | 210 | 211 | class BatchNormScaleBiasFuser(SubNodeFuser): 212 | ''' 213 | The original batch normalization paper includes two learned 214 | parameters: a scaling factor \gamma and a bias \beta. 215 | Caffe's implementation does not include these two. However, it is commonly 216 | replicated by adding a scaling+bias layer immidiately after the batch norm. 217 | 218 | This fuser merges the scaling+bias layer with the batch norm. 219 | ''' 220 | 221 | def is_eligible_pair_(self, parent, child): 222 | return (parent.kind == NodeKind.BatchNorm and child.kind == NodeKind.Scale and 223 | child.parameters.axis == 1 and child.parameters.bias_term == True) 224 | 225 | ''' 226 | Made for the purpose of 227 | ''' 228 | def is_eligible_pair(self, parent, child): 229 | if parent.kind == NodeKind.BatchNorm: 230 | print 'kaffe/transformers.py line 227: use function above' 231 | return (parent.kind == NodeKind.BN and child.kind == NodeKind.Scale and 232 | child.parameters.axis == 1 and child.parameters.bias_term == True) 233 | 234 | def merge(self, parent, child): 235 | 236 | parent.scale_bias_node = child 237 | 238 | 239 | class BatchNormPreprocessor(object): 240 | ''' 241 | Prescale batch normalization parameters. 242 | Concatenate gamma (scale) and beta (bias) terms if set. 243 | ''' 244 | 245 | def __call__(self, graph): 246 | for node in graph.nodes: 247 | if node.kind not in (NodeKind.BatchNorm, NodeKind.BN): 248 | continue 249 | else: 250 | continue 251 | assert node.data is not None 252 | print len(node.data) 253 | print node 254 | assert len(node.data) == 3 255 | 256 | mean, variance, scale = node.data 257 | # Prescale the stats 258 | scaling_factor = 1.0 / scale if scale != 0 else 0 259 | mean *= scaling_factor 260 | variance *= scaling_factor 261 | # Replace with the updated values 262 | node.data = [mean, variance] 263 | if hasattr(node, 'scale_bias_node'): 264 | # Include the scale and bias terms 265 | gamma, beta = node.scale_bias_node.data 266 | node.data += [gamma, beta] 267 | return graph 268 | 269 | 270 | class NodeRenamer(object): 271 | ''' 272 | Renames nodes in the graph using a given unary function that 273 | accepts a node and returns its new name. 274 | ''' 275 | 276 | def __init__(self, renamer): 277 | self.renamer = renamer 278 | 279 | def __call__(self, graph): 280 | for node in graph.nodes: 281 | node.name = self.renamer(node) 282 | return graph 283 | 284 | 285 | class ParameterNamer(object): 286 | ''' 287 | Convert layer data arrays to a dictionary mapping parameter names to their values. 288 | ''' 289 | 290 | def __call__(self, graph): 291 | for node in graph.nodes: 292 | if node.data is None: 293 | continue 294 | if node.kind in (NodeKind.Convolution, NodeKind.InnerProduct): 295 | names = ('weights',) 296 | if node.parameters.bias_term: 297 | names += ('biases',) 298 | elif node.kind in (NodeKind.BatchNorm, NodeKind.BN): 299 | names = ('mean', 'variance') 300 | if len(node.data) == 4: 301 | 302 | names += ('scale', 'offset') 303 | else: 304 | print_stderr('WARNING: Unhandled parameters: {}'.format(node.kind)) 305 | continue 306 | assert len(names) == len(node.data) 307 | node.data = dict(zip(names, node.data)) 308 | return graph 309 | -------------------------------------------------------------------------------- /cityscapes_labels.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # 3 | # Cityscapes labels 4 | # 5 | 6 | from collections import namedtuple 7 | 8 | 9 | # -------------------------------------------------------------------------------- 10 | # Definitions 11 | # -------------------------------------------------------------------------------- 12 | 13 | # a label and all meta information 14 | Label = namedtuple('Label', [ 15 | 16 | 'name', # The identifier of this label, e.g. 'car', 'person', ... . 17 | # We use them to uniquely name a class 18 | 19 | 'id', # An integer ID that is associated with this label. 20 | # The IDs are used to represent the label in ground truth images 21 | # An ID of -1 means that this label does not have an ID and thus 22 | # is ignored when creating ground truth images (e.g. license plate). 23 | # Do not modify these IDs, since exactly these IDs are expected by the 24 | # evaluation server. 25 | 26 | 'trainId', # Feel free to modify these IDs as suitable for your method. Then create 27 | # ground truth images with train IDs, using the tools provided in the 28 | # 'preparation' folder. However, make sure to validate or submit results 29 | # to our evaluation server using the regular IDs above! 30 | # For trainIds, multiple labels might have the same ID. Then, these labels 31 | # are mapped to the same class in the ground truth images. For the inverse 32 | # mapping, we use the label that is defined first in the list below. 33 | # For example, mapping all void-type classes to the same ID in training, 34 | # might make sense for some approaches. 35 | # Max value is 255! 36 | 37 | 'category', # The name of the category that this label belongs to 38 | 39 | 'categoryId', # The ID of this category. Used to create ground truth images 40 | # on category level. 41 | 42 | 'hasInstances', # Whether this label distinguishes between single instances or not 43 | 44 | 'ignoreInEval', # Whether pixels having this class as ground truth label are ignored 45 | # during evaluations or not 46 | 47 | 'color', # The color of this label 48 | ]) 49 | 50 | 51 | # -------------------------------------------------------------------------------- 52 | # A list of all labels 53 | # -------------------------------------------------------------------------------- 54 | 55 | # Please adapt the train IDs as appropriate for you approach. 56 | # Note that you might want to ignore labels with ID 255 during training. 57 | # Further note that the current train IDs are only a suggestion. You can use whatever you like. 58 | # Make sure to provide your results using the original IDs and not the training IDs. 59 | # Note that many IDs are ignored in evaluation and thus you never need to predict these! 60 | 61 | labels = [ 62 | # name id trainId category catId hasInstances ignoreInEval color 63 | Label('unlabeled', 0, 255, 'void', 0, False, True, (0, 0, 0)), 64 | Label('ego vehicle', 1, 255, 'void', 0, False, True, (0, 0, 0)), 65 | Label('rectification border', 2, 255, 66 | 'void', 0, False, True, (0, 0, 0)), 67 | Label('out of roi', 3, 255, 'void', 0, False, True, (0, 0, 0)), 68 | Label('static', 4, 255, 'void', 0, False, True, (0, 0, 0)), 69 | Label('dynamic', 5, 255, 'void', 0, False, True, (111, 74, 0)), 70 | Label('ground', 6, 255, 'void', 0, False, True, (81, 0, 81)), 71 | Label('road', 7, 0, 'flat', 1, False, False, (128, 64, 128)), 72 | Label('sidewalk', 8, 1, 'flat', 1, False, False, (244, 35, 232)), 73 | Label('parking', 9, 255, 'flat', 1, False, True, (250, 170, 160)), 74 | Label('rail track', 10, 255, 'flat', 1, False, True, (230, 150, 140)), 75 | Label('building', 11, 2, 'construction', 76 | 2, False, False, (70, 70, 70)), 77 | Label('wall', 12, 3, 'construction', 78 | 2, False, False, (102, 102, 156)), 79 | Label('fence', 13, 4, 'construction', 80 | 2, False, False, (190, 153, 153)), 81 | Label('guard rail', 14, 255, 'construction', 82 | 2, False, True, (180, 165, 180)), 83 | Label('bridge', 15, 255, 'construction', 84 | 2, False, True, (150, 100, 100)), 85 | Label('tunnel', 16, 255, 'construction', 86 | 2, False, True, (150, 120, 90)), 87 | Label('pole', 17, 5, 'object', 3, False, False, (153, 153, 153)), 88 | Label('polegroup', 18, 255, 'object', 89 | 3, False, True, (153, 153, 153)), 90 | Label('traffic light', 19, 6, 'object', 91 | 3, False, False, (250, 170, 30)), 92 | Label('traffic sign', 20, 7, 'object', 93 | 3, False, False, (220, 220, 0)), 94 | Label('vegetation', 21, 8, 'nature', 95 | 4, False, False, (107, 142, 35)), 96 | Label('terrain', 22, 9, 'nature', 4, False, False, (152, 251, 152)), 97 | Label('sky', 23, 10, 'sky', 5, False, False, (70, 130, 180)), 98 | Label('person', 24, 11, 'human', 6, True, False, (220, 20, 60)), 99 | Label('rider', 25, 12, 'human', 6, True, False, (255, 0, 0)), 100 | Label('car', 26, 13, 'vehicle', 7, True, False, (0, 0, 142)), 101 | Label('truck', 27, 14, 'vehicle', 7, True, False, (0, 0, 70)), 102 | Label('bus', 28, 15, 'vehicle', 7, True, False, (0, 60, 100)), 103 | Label('caravan', 29, 255, 'vehicle', 7, True, True, (0, 0, 90)), 104 | Label('trailer', 30, 255, 'vehicle', 7, True, True, (0, 0, 110)), 105 | Label('train', 31, 16, 'vehicle', 7, True, False, (0, 80, 100)), 106 | Label('motorcycle', 32, 17, 'vehicle', 7, True, False, (0, 0, 230)), 107 | Label('bicycle', 33, 18, 'vehicle', 7, True, False, (119, 11, 32)), 108 | Label('license plate', -1, -1, 109 | 'vehicle', 7, False, True, (0, 0, 142)), 110 | ] 111 | 112 | 113 | # -------------------------------------------------------------------------------- 114 | # Create dictionaries for a fast lookup 115 | # -------------------------------------------------------------------------------- 116 | 117 | # Please refer to the main method below for example usages! 118 | 119 | # name to label object 120 | name2label = {label.name: label for label in labels} 121 | # id to label object 122 | id2label = {label.id: label for label in labels} 123 | # trainId to label object 124 | trainId2label = {label.trainId: label for label in reversed(labels)} 125 | # category to list of label objects 126 | category2labels = {} 127 | for label in labels: 128 | category = label.category 129 | if category in category2labels: 130 | category2labels[category].append(label) 131 | else: 132 | category2labels[category] = [label] 133 | 134 | # -------------------------------------------------------------------------------- 135 | # Assure single instance name 136 | # -------------------------------------------------------------------------------- 137 | 138 | # returns the label name that describes a single instance (if possible) 139 | # e.g. input | output 140 | # ---------------------- 141 | # car | car 142 | # cargroup | car 143 | # foo | None 144 | # foogroup | None 145 | # skygroup | None 146 | 147 | 148 | def assureSingleInstanceName(name): 149 | # if the name is known, it is not a group 150 | if name in name2label: 151 | return name 152 | # test if the name actually denotes a group 153 | if not name.endswith("group"): 154 | return None 155 | # remove group 156 | name = name[:-len("group")] 157 | # test if the new name exists 158 | if name not in name2label: 159 | return None 160 | # test if the new name denotes a label that actually has instances 161 | if not name2label[name].hasInstances: 162 | return None 163 | # all good then 164 | return name 165 | 166 | # -------------------------------------------------------------------------------- 167 | # Main for testing 168 | # -------------------------------------------------------------------------------- 169 | 170 | 171 | # just a dummy main 172 | if __name__ == "__main__": 173 | # Print all the labels 174 | print("List of cityscapes labels:") 175 | print("") 176 | print(" {:>21} | {:>3} | {:>7} | {:>14} | {:>10} | {:>12} | {:>12}".format( 177 | 'name', 'id', 'trainId', 'category', 'categoryId', 'hasInstances', 'ignoreInEval')) 178 | print(" " + ('-' * 98)) 179 | for label in labels: 180 | print(" {:>21} | {:>3} | {:>7} | {:>14} | {:>10} | {:>12} | {:>12}".format( 181 | label.name, label.id, label.trainId, label.category, label.categoryId, label.hasInstances, label.ignoreInEval)) 182 | print("") 183 | 184 | print("Example usages:") 185 | 186 | # Map from name to label 187 | name = 'car' 188 | id = name2label[name].id 189 | print("ID of label '{name}': {id}".format(name=name, id=id)) 190 | 191 | # Map from ID to label 192 | category = id2label[id].category 193 | print("Category of label with ID '{id}': {category}".format( 194 | id=id, category=category)) 195 | 196 | # Map from trainID to label 197 | trainId = 0 198 | name = trainId2label[trainId].name 199 | print("Name of label with trainID '{id}': {name}".format( 200 | id=trainId, name=name)) 201 | -------------------------------------------------------------------------------- /drawImage/__init__.py: -------------------------------------------------------------------------------- 1 | from drawModule import * -------------------------------------------------------------------------------- /drawImage/__init__.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/drawImage/__init__.pyc -------------------------------------------------------------------------------- /drawImage/drawModule.py: -------------------------------------------------------------------------------- 1 | from PIL import Image, ImageDraw 2 | import scipy.ndimage 3 | import scipy.io 4 | import numpy as np 5 | import time 6 | import copy 7 | 8 | 9 | class BaseDraw: 10 | def __init__(self, color150, objectNames, img, pred_size, predicted_classes): 11 | self.class_colors = scipy.io.loadmat(color150) 12 | self.class_names = scipy.io.loadmat(objectNames, struct_as_record=False) 13 | self.im = img 14 | self.pred_size = pred_size 15 | self.predicted_classes = copy.deepcopy(predicted_classes) 16 | self.original_W = self.im.size[0] 17 | self.original_H = self.im.size[1] 18 | 19 | self.output_W = self.original_W 20 | self.output_H = self.original_H 21 | 22 | 23 | def dumpArray(self, array, i): 24 | test = array*100 25 | test = Image.fromarray(test.astype('uint8')) 26 | test = test.convert("RGB") 27 | test.save('/home/vlad/oS_AI/'+str(i)+'t.jpg', "JPEG") 28 | 29 | def calculateResize(self): 30 | W_coef = float(self.original_W)/float(self.output_W) 31 | H_coef = float(self.original_H)/float(self.output_H) 32 | horiz_pad = 0 33 | vert_pad = 0 34 | if W_coef > H_coef: 35 | coef = W_coef 36 | horiz_pad = int((self.output_H - self.original_H/coef)/2) 37 | return [coef, horiz_pad, vert_pad] 38 | else: 39 | coef = H_coef 40 | vert_pad = int((self.output_W - self.original_W/coef)/2) 41 | return [coef, horiz_pad, vert_pad] 42 | 43 | 44 | def resizeToOutput(self, image, coef, h_pad, w_pad): 45 | image = image.resize((int(self.original_W/coef), int(self.original_H/coef)), resample=Image.BILINEAR) 46 | outputImage = Image.new("RGB",(self.output_W,self.output_H),(0,0,0)) 47 | outputImage.paste(image,(w_pad,h_pad)) 48 | return outputImage 49 | 50 | 51 | 52 | def drawSimpleSegment(self): 53 | 54 | #Drawing module 55 | im_Width, im_Height = self.pred_size 56 | prediction_image = Image.new("RGB", (im_Width, im_Height) ,(0,0,0)) 57 | prediction_imageDraw = ImageDraw.Draw(prediction_image) 58 | 59 | #BASE all image segmentation 60 | for i in range(im_Width): 61 | for j in range(im_Height): 62 | #get matrix element class(0-149) 63 | px_Class = self.predicted_classes[j][i] 64 | #assign color from .mat list 65 | put_Px_Color = tuple(self.class_colors['colors'][px_Class]) 66 | 67 | #drawing 68 | prediction_imageDraw.point((i,j), fill=put_Px_Color) 69 | 70 | #Resize to original size and save 71 | self.coef, self.h_pad, self.w_pad = self.calculateResize() 72 | FullHdOutImage = self.resizeToOutput(prediction_image, self.coef, self.h_pad, self.w_pad) 73 | FullHdOutImage = Image.blend(FullHdOutImage, self.im, 0.5) 74 | 75 | return FullHdOutImage 76 | -------------------------------------------------------------------------------- /drawImage/drawModule.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/drawImage/drawModule.pyc -------------------------------------------------------------------------------- /example_images/ade20k.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/example_images/ade20k.jpg -------------------------------------------------------------------------------- /example_images/pascal_voc.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/example_images/pascal_voc.jpg -------------------------------------------------------------------------------- /example_images/test_pycaffe.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/example_images/test_pycaffe.jpg -------------------------------------------------------------------------------- /example_results/ade20k_probs.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/example_results/ade20k_probs.jpg -------------------------------------------------------------------------------- /example_results/ade20k_seg.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/example_results/ade20k_seg.jpg -------------------------------------------------------------------------------- /example_results/ade20k_seg_blended.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/example_results/ade20k_seg_blended.jpg -------------------------------------------------------------------------------- /example_results/ade20k_seg_pycaffe.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/example_results/ade20k_seg_pycaffe.jpg -------------------------------------------------------------------------------- /example_results/cityscapes_probs.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/example_results/cityscapes_probs.jpg -------------------------------------------------------------------------------- /example_results/cityscapes_seg.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/example_results/cityscapes_seg.jpg -------------------------------------------------------------------------------- /example_results/cityscapes_seg_blended.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/example_results/cityscapes_seg_blended.jpg -------------------------------------------------------------------------------- /example_results/pascal_voc_probs.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/example_results/pascal_voc_probs.jpg -------------------------------------------------------------------------------- /example_results/pascal_voc_seg.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/example_results/pascal_voc_seg.jpg -------------------------------------------------------------------------------- /example_results/pascal_voc_seg_blended.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/example_results/pascal_voc_seg_blended.jpg -------------------------------------------------------------------------------- /layers_builder.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | from math import ceil 3 | from keras.layers import Conv2D, MaxPooling2D, AveragePooling2D 4 | from keras.layers import BatchNormalization, Activation, Input, Dropout, ZeroPadding2D, Lambda 5 | from keras.layers import Concatenate, Add 6 | from keras.models import Model 7 | from keras.optimizers import SGD 8 | 9 | learning_rate = 1e-3 # Layer specific learning rate 10 | # Weight decay not implemented 11 | 12 | 13 | def BN(name=""): 14 | return BatchNormalization(momentum=0.95, name=name, epsilon=1e-5) 15 | 16 | 17 | def Interp(x, shape): 18 | from keras.backend import tf as ktf 19 | new_height, new_width = shape 20 | resized = ktf.image.resize_images(x, [new_height, new_width], 21 | align_corners=True) 22 | return resized 23 | 24 | 25 | def residual_conv(prev, level, pad=1, lvl=1, sub_lvl=1, modify_stride=False): 26 | lvl = str(lvl) 27 | sub_lvl = str(sub_lvl) 28 | names = ["conv"+lvl+"_" + sub_lvl + "_1x1_reduce", 29 | "conv"+lvl+"_" + sub_lvl + "_1x1_reduce_bn", 30 | "conv"+lvl+"_" + sub_lvl + "_3x3", 31 | "conv"+lvl+"_" + sub_lvl + "_3x3_bn", 32 | "conv"+lvl+"_" + sub_lvl + "_1x1_increase", 33 | "conv"+lvl+"_" + sub_lvl + "_1x1_increase_bn"] 34 | if modify_stride is False: 35 | prev = Conv2D(64 * level, (1, 1), strides=(1, 1), name=names[0], 36 | use_bias=False)(prev) 37 | elif modify_stride is True: 38 | prev = Conv2D(64 * level, (1, 1), strides=(2, 2), name=names[0], 39 | use_bias=False)(prev) 40 | 41 | prev = BN(name=names[1])(prev) 42 | prev = Activation('relu')(prev) 43 | 44 | prev = ZeroPadding2D(padding=(pad, pad))(prev) 45 | prev = Conv2D(64 * level, (3, 3), strides=(1, 1), dilation_rate=pad, 46 | name=names[2], use_bias=False)(prev) 47 | 48 | prev = BN(name=names[3])(prev) 49 | prev = Activation('relu')(prev) 50 | prev = Conv2D(256 * level, (1, 1), strides=(1, 1), name=names[4], 51 | use_bias=False)(prev) 52 | prev = BN(name=names[5])(prev) 53 | return prev 54 | 55 | 56 | def short_convolution_branch(prev, level, lvl=1, sub_lvl=1, modify_stride=False): 57 | lvl = str(lvl) 58 | sub_lvl = str(sub_lvl) 59 | names = ["conv" + lvl+"_" + sub_lvl + "_1x1_proj", 60 | "conv" + lvl+"_" + sub_lvl + "_1x1_proj_bn"] 61 | 62 | if modify_stride is False: 63 | prev = Conv2D(256 * level, (1, 1), strides=(1, 1), name=names[0], 64 | use_bias=False)(prev) 65 | elif modify_stride is True: 66 | prev = Conv2D(256 * level, (1, 1), strides=(2, 2), name=names[0], 67 | use_bias=False)(prev) 68 | 69 | prev = BN(name=names[1])(prev) 70 | return prev 71 | 72 | 73 | def empty_branch(prev): 74 | return prev 75 | 76 | 77 | def residual_short(prev_layer, level, pad=1, lvl=1, sub_lvl=1, modify_stride=False): 78 | prev_layer = Activation('relu')(prev_layer) 79 | block_1 = residual_conv(prev_layer, level, 80 | pad=pad, lvl=lvl, sub_lvl=sub_lvl, 81 | modify_stride=modify_stride) 82 | 83 | block_2 = short_convolution_branch(prev_layer, level, 84 | lvl=lvl, sub_lvl=sub_lvl, 85 | modify_stride=modify_stride) 86 | added = Add()([block_1, block_2]) 87 | return added 88 | 89 | 90 | def residual_empty(prev_layer, level, pad=1, lvl=1, sub_lvl=1): 91 | prev_layer = Activation('relu')(prev_layer) 92 | 93 | block_1 = residual_conv(prev_layer, level, pad=pad, 94 | lvl=lvl, sub_lvl=sub_lvl) 95 | block_2 = empty_branch(prev_layer) 96 | added = Add()([block_1, block_2]) 97 | return added 98 | 99 | 100 | def ResNet(inp, layers): 101 | # Names for the first couple layers of model 102 | names = ["conv1_1_3x3_s2", 103 | "conv1_1_3x3_s2_bn", 104 | "conv1_2_3x3", 105 | "conv1_2_3x3_bn", 106 | "conv1_3_3x3", 107 | "conv1_3_3x3_bn"] 108 | 109 | # Short branch(only start of network) 110 | 111 | cnv1 = Conv2D(64, (3, 3), strides=(2, 2), padding='same', name=names[0], 112 | use_bias=False)(inp) # "conv1_1_3x3_s2" 113 | bn1 = BN(name=names[1])(cnv1) # "conv1_1_3x3_s2/bn" 114 | relu1 = Activation('relu')(bn1) # "conv1_1_3x3_s2/relu" 115 | 116 | cnv1 = Conv2D(64, (3, 3), strides=(1, 1), padding='same', name=names[2], 117 | use_bias=False)(relu1) # "conv1_2_3x3" 118 | bn1 = BN(name=names[3])(cnv1) # "conv1_2_3x3/bn" 119 | relu1 = Activation('relu')(bn1) # "conv1_2_3x3/relu" 120 | 121 | cnv1 = Conv2D(128, (3, 3), strides=(1, 1), padding='same', name=names[4], 122 | use_bias=False)(relu1) # "conv1_3_3x3" 123 | bn1 = BN(name=names[5])(cnv1) # "conv1_3_3x3/bn" 124 | relu1 = Activation('relu')(bn1) # "conv1_3_3x3/relu" 125 | 126 | res = MaxPooling2D(pool_size=(3, 3), padding='same', 127 | strides=(2, 2))(relu1) # "pool1_3x3_s2" 128 | 129 | # ---Residual layers(body of network) 130 | 131 | """ 132 | Modify_stride --Used only once in first 3_1 convolutions block. 133 | changes stride of first convolution from 1 -> 2 134 | """ 135 | 136 | # 2_1- 2_3 137 | res = residual_short(res, 1, pad=1, lvl=2, sub_lvl=1) 138 | for i in range(2): 139 | res = residual_empty(res, 1, pad=1, lvl=2, sub_lvl=i+2) 140 | 141 | # 3_1 - 3_3 142 | res = residual_short(res, 2, pad=1, lvl=3, sub_lvl=1, modify_stride=True) 143 | for i in range(3): 144 | res = residual_empty(res, 2, pad=1, lvl=3, sub_lvl=i+2) 145 | if layers is 50: 146 | # 4_1 - 4_6 147 | res = residual_short(res, 4, pad=2, lvl=4, sub_lvl=1) 148 | for i in range(5): 149 | res = residual_empty(res, 4, pad=2, lvl=4, sub_lvl=i+2) 150 | elif layers is 101: 151 | # 4_1 - 4_23 152 | res = residual_short(res, 4, pad=2, lvl=4, sub_lvl=1) 153 | for i in range(22): 154 | res = residual_empty(res, 4, pad=2, lvl=4, sub_lvl=i+2) 155 | else: 156 | print("This ResNet is not implemented") 157 | 158 | # 5_1 - 5_3 159 | res = residual_short(res, 8, pad=4, lvl=5, sub_lvl=1) 160 | for i in range(2): 161 | res = residual_empty(res, 8, pad=4, lvl=5, sub_lvl=i+2) 162 | 163 | res = Activation('relu')(res) 164 | return res 165 | 166 | 167 | def interp_block(prev_layer, level, feature_map_shape, str_lvl=1, ): 168 | str_lvl = str(str_lvl) 169 | 170 | names = [ 171 | "conv5_3_pool"+str_lvl+"_conv", 172 | "conv5_3_pool"+str_lvl+"_conv_bn" 173 | ] 174 | 175 | kernel = (10*level, 10*level) 176 | strides = (10*level, 10*level) 177 | prev_layer = AveragePooling2D(kernel, strides=strides)(prev_layer) 178 | prev_layer = Conv2D(512, (1, 1), strides=(1, 1), name=names[0], 179 | use_bias=False)(prev_layer) 180 | prev_layer = BN(name=names[1])(prev_layer) 181 | prev_layer = Activation('relu')(prev_layer) 182 | prev_layer = Lambda(Interp, arguments={'shape': feature_map_shape})(prev_layer) 183 | return prev_layer 184 | 185 | 186 | def build_pyramid_pooling_module(res, input_shape): 187 | """Build the Pyramid Pooling Module.""" 188 | # ---PSPNet concat layers with Interpolation 189 | feature_map_size = tuple(int(ceil(input_dim / 8.0)) for input_dim in input_shape) 190 | print("PSP module will interpolate to a final feature map size of %s" % (feature_map_size, )) 191 | 192 | interp_block1 = interp_block(res, 6, feature_map_size, str_lvl=1) 193 | interp_block2 = interp_block(res, 3, feature_map_size, str_lvl=2) 194 | interp_block3 = interp_block(res, 2, feature_map_size, str_lvl=3) 195 | interp_block6 = interp_block(res, 1, feature_map_size, str_lvl=6) 196 | 197 | # concat all these layers. resulted shape=(1,feature_map_size_x,feature_map_size_y,4096) 198 | res = Concatenate()([res, 199 | interp_block6, 200 | interp_block3, 201 | interp_block2, 202 | interp_block1]) 203 | return res 204 | 205 | 206 | def build_pspnet(nb_classes, resnet_layers, input_shape, activation='softmax'): 207 | """Build PSPNet.""" 208 | print("Building a PSPNet based on ResNet %i expecting inputs of shape %s predicting %i classes" % (resnet_layers, input_shape, nb_classes)) 209 | 210 | inp = Input((input_shape[0], input_shape[1], 3)) 211 | res = ResNet(inp, layers=resnet_layers) 212 | psp = build_pyramid_pooling_module(res, input_shape) 213 | 214 | x = Conv2D(512, (3, 3), strides=(1, 1), padding="same", name="conv5_4", 215 | use_bias=False)(psp) 216 | x = BN(name="conv5_4_bn")(x) 217 | x = Activation('relu')(x) 218 | x = Dropout(0.1)(x) 219 | 220 | x = Conv2D(nb_classes, (1, 1), strides=(1, 1), name="conv6")(x) 221 | x = Lambda(Interp, arguments={'shape': (input_shape[0], input_shape[1])})(x) 222 | x = Activation('softmax')(x) 223 | 224 | model = Model(inputs=inp, outputs=x) 225 | 226 | # Solver 227 | sgd = SGD(lr=learning_rate, momentum=0.9, nesterov=True) 228 | model.compile(optimizer=sgd, 229 | loss='categorical_crossentropy', 230 | metrics=['accuracy']) 231 | return model 232 | -------------------------------------------------------------------------------- /pascal_voc_labels.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # 3 | # Pascal VOC labels 4 | # 5 | 6 | from collections import namedtuple 7 | import numpy as np 8 | 9 | Label = namedtuple('Label', [ 10 | 11 | 'name', 12 | 'id', 13 | 'color' 14 | ]) 15 | 16 | labels = [Label('background', 0, (0, 0, 0)), 17 | Label('aeroplane', 1, (128, 0, 0)), 18 | Label('bicycle', 2, (0, 128, 0)), 19 | Label('bird', 3, (128, 128, 0)), 20 | Label('boat', 4, (0, 0, 128)), 21 | Label('bottle', 5, (128, 0, 128)), 22 | Label('bus', 6, (0, 128, 128)), 23 | Label('car', 7, (128, 128, 128)), 24 | Label('cat', 8, (64, 0, 0)), 25 | Label('chair', 9, (192, 0, 0)), 26 | Label('cow', 10, (64, 128, 0)), 27 | Label('diningtable', 11, (192, 128, 0)), 28 | Label('dog', 12, (64, 0, 128)), 29 | Label('horse', 13, (192, 0, 128)), 30 | Label('motorbike', 14, (64, 128, 128)), 31 | Label('person', 15, (192, 128, 128)), 32 | Label('pottedplant', 16, (0, 64, 0)), 33 | Label('sheep', 17, (128, 64, 0)), 34 | Label('sofa', 18, (0, 192, 0)), 35 | Label('train', 19, (128, 192, 0)), 36 | Label('tvmonitor', 20, (0, 64, 128)), 37 | Label('void', 21, (128, 64, 12))] 38 | 39 | voc_id2label = {label.id: label for label in labels} 40 | 41 | 42 | def generate_color_map(N=256, normalized=False): 43 | """from https://gist.github.com/wllhf/a4533e0adebe57e3ed06d4b50c8419ae .""" 44 | def bitget(byteval, idx): 45 | return ((byteval & (1 << idx)) != 0) 46 | 47 | dtype = 'float32' if normalized else 'uint8' 48 | cmap = np.zeros((N, 3), dtype=dtype) 49 | for i in range(N): 50 | r = g = b = 0 51 | c = i 52 | for j in range(8): 53 | r = r | (bitget(c, 0) << 7 - j) 54 | g = g | (bitget(c, 1) << 7 - j) 55 | b = b | (bitget(c, 2) << 7 - j) 56 | c = c >> 3 57 | 58 | cmap[i] = np.array([r, g, b]) 59 | 60 | cmap = cmap / 255 if normalized else cmap 61 | return cmap 62 | 63 | 64 | def generate_voc_labels(): 65 | labels = ['background', 'aeroplane', 'bicycle', 'bird', 'boat', 'bottle', 'bus', 'car', 'cat', 'chair', 'cow', 66 | 'diningtable', 'dog', 'horse', 'motorbike', 'person', 'pottedplant', 'sheep', 'sofa', 'train', 'tvmonitor', 'void'] 67 | color_map = generate_color_map() 68 | for id, name in enumerate(labels): 69 | color = color_map[id] 70 | print("Label(\'%s\', %i, (%i, %i, %i))," % 71 | (name, id, color[0], color[1], color[2])) 72 | -------------------------------------------------------------------------------- /pspnet.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ 3 | This module is a Keras/Tensorflow based implementation of Pyramid Scene Parsing Networks. 4 | 5 | Original paper & code published by Hengshuang Zhao et al. (2017) 6 | """ 7 | from __future__ import print_function 8 | from __future__ import division 9 | from os.path import splitext, join, isfile 10 | from os import environ 11 | from math import ceil 12 | import argparse 13 | import numpy as np 14 | from scipy import misc, ndimage 15 | from keras import backend as K 16 | from keras.models import model_from_json 17 | import tensorflow as tf 18 | import layers_builder as layers 19 | import utils 20 | import matplotlib.pyplot as plt 21 | import cv2 22 | from tensorflow.python.client import device_lib 23 | 24 | __author__ = "Vlad Kryvoruchko, Chaoyue Wang, Jeffrey Hu & Julian Tatsch" 25 | 26 | 27 | # These are the means for the ImageNet pretrained ResNet 28 | DATA_MEAN = np.array([[[123.68, 116.779, 103.939]]]) # RGB order 29 | EVALUATION_SCALES = [1.0] # must be all floats! 30 | 31 | 32 | def getGPUname(): 33 | local_device_protos = device_lib.list_local_devices() 34 | l = [x.physical_device_desc for x in local_device_protos if x.device_type == 'GPU'] 35 | s='' 36 | for t in l: 37 | s+=t[t.find("name: ")+len("name: "):t.find(", pci")] + " " 38 | return s 39 | 40 | 41 | GPU_NAME = getGPUname() 42 | 43 | 44 | class PSPNet(object): 45 | """Pyramid Scene Parsing Network by Hengshuang Zhao et al 2017.""" 46 | 47 | def __init__(self, nb_classes, resnet_layers, input_shape, weights): 48 | """Instanciate a PSPNet.""" 49 | self.input_shape = input_shape 50 | json_path = join("weights", "keras", weights + ".json") 51 | h5_path = join("weights", "keras", weights + ".h5") 52 | if isfile(json_path) and isfile(h5_path): 53 | print("Keras model & weights found, loading...") 54 | with open(json_path, 'r') as file_handle: 55 | self.model = model_from_json(file_handle.read()) 56 | self.model.load_weights(h5_path) 57 | else: 58 | print("No Keras model & weights found, import from npy weights.") 59 | self.model = layers.build_pspnet(nb_classes=nb_classes, 60 | resnet_layers=resnet_layers, 61 | input_shape=self.input_shape) 62 | self.set_npy_weights(weights) 63 | 64 | def predict(self, img, flip_evaluation): 65 | """ 66 | Predict segementation for an image. 67 | 68 | Arguments: 69 | img: must be rowsxcolsx3 70 | """ 71 | h_ori, w_ori = img.shape[:2] 72 | if img.shape[0:2] != self.input_shape: 73 | print("Input %s not fitting for network size %s, resizing. You may want to try sliding prediction for better results." % (img.shape[0:2], self.input_shape)) 74 | img = misc.imresize(img, self.input_shape) 75 | input_data = self.preprocess_image(img) 76 | # utils.debug(self.model, input_data) 77 | 78 | regular_prediction = self.model.predict(input_data)[0] 79 | if flip_evaluation: 80 | print("Predict flipped") 81 | flipped_prediction = np.fliplr(self.model.predict(np.flip(input_data, axis=2))[0]) 82 | prediction = (regular_prediction + flipped_prediction) / 2.0 83 | else: 84 | prediction = regular_prediction 85 | 86 | if img.shape[0:1] != self.input_shape: # upscale prediction if necessary 87 | h, w = prediction.shape[:2] 88 | # prediction = ndimage.zoom(prediction, (1.*h_ori/h, 1.*w_ori/w, 1.), 89 | # order=1, prefilter=False) 90 | prediction = cv2.resize(prediction,(w_ori,h_ori)) 91 | return prediction 92 | 93 | def preprocess_image(self, img): 94 | """Preprocess an image as input.""" 95 | float_img = img.astype('float16') 96 | centered_image = float_img - DATA_MEAN 97 | bgr_image = centered_image[:, :, ::-1] # RGB => BGR 98 | input_data = bgr_image[np.newaxis, :, :, :] # Append sample dimension for keras 99 | return input_data 100 | 101 | def set_npy_weights(self, weights_path): 102 | """Set weights from the intermediary npy file.""" 103 | npy_weights_path = join("weights", "npy", weights_path + ".npy") 104 | json_path = join("weights", "keras", weights_path + ".json") 105 | h5_path = join("weights", "keras", weights_path + ".h5") 106 | 107 | print("Importing weights from %s" % npy_weights_path) 108 | weights = np.load(npy_weights_path, encoding="latin1").item() 109 | 110 | whitelist = ["InputLayer", "Activation", "ZeroPadding2D", "Add", "MaxPooling2D", "AveragePooling2D", "Lambda", "Concatenate", "Dropout"] 111 | 112 | weights_set = 0 113 | for layer in self.model.layers: 114 | print("Processing %s" % layer.name) 115 | if layer.name[:4] == 'conv' and layer.name[-2:] == 'bn': 116 | mean = weights[layer.name]['mean'].reshape(-1) 117 | variance = weights[layer.name]['variance'].reshape(-1) 118 | scale = weights[layer.name]['scale'].reshape(-1) 119 | offset = weights[layer.name]['offset'].reshape(-1) 120 | 121 | self.model.get_layer(layer.name).set_weights([mean, variance, 122 | scale, offset]) 123 | weights_set += 1 124 | elif layer.name[:4] == 'conv' and not layer.name[-4:] == 'relu': 125 | try: 126 | weight = weights[layer.name]['weights'] 127 | self.model.get_layer(layer.name).set_weights([weight]) 128 | except Exception: 129 | biases = weights[layer.name]['biases'] 130 | self.model.get_layer(layer.name).set_weights([weight, 131 | biases]) 132 | weights_set += 1 133 | elif layer.__class__.__name__ in whitelist: 134 | # print("Nothing to set in %s" % layer.__class__.__name__) 135 | pass 136 | else: 137 | print("Warning: Did not find weights for keras layer %s in numpy weights" % layer) 138 | 139 | print("Set a total of %i weights" % weights_set) 140 | 141 | print('Finished importing weights.') 142 | 143 | print("Writing keras model & weights") 144 | json_string = self.model.to_json() 145 | with open(json_path, 'w') as file_handle: 146 | file_handle.write(json_string) 147 | self.model.save_weights(h5_path) 148 | print("Finished writing Keras model & weights") 149 | 150 | 151 | class PSPNet50(PSPNet): 152 | """Build a PSPNet based on a 50-Layer ResNet.""" 153 | 154 | def __init__(self, nb_classes, weights, input_shape): 155 | """Instanciate a PSPNet50.""" 156 | PSPNet.__init__(self, nb_classes=nb_classes, resnet_layers=50, 157 | input_shape=input_shape, weights=weights) 158 | 159 | 160 | class PSPNet101(PSPNet): 161 | """Build a PSPNet based on a 101-Layer ResNet.""" 162 | 163 | def __init__(self, nb_classes, weights, input_shape): 164 | """Instanciate a PSPNet101.""" 165 | PSPNet.__init__(self, nb_classes=nb_classes, resnet_layers=101, 166 | input_shape=input_shape, weights=weights) 167 | 168 | 169 | def pad_image(img, target_size): 170 | """Pad an image up to the target size.""" 171 | rows_missing = target_size[0] - img.shape[0] 172 | cols_missing = target_size[1] - img.shape[1] 173 | padded_img = np.pad(img, ((0, rows_missing), (0, cols_missing), (0, 0)), 'constant') 174 | return padded_img 175 | 176 | 177 | def visualize_prediction(prediction): 178 | """Visualize prediction.""" 179 | cm = np.argmax(prediction, axis=2) + 1 180 | color_cm = utils.add_color(cm) 181 | plt.imshow(color_cm) 182 | plt.show() 183 | 184 | 185 | def predict_sliding(full_image, net, flip_evaluation): 186 | """Predict on tiles of exactly the network input shape so nothing gets squeezed.""" 187 | tile_size = net.input_shape 188 | classes = net.model.outputs[0].shape[3] 189 | overlap = 1/3 190 | 191 | stride = ceil(tile_size[0] * (1 - overlap)) 192 | tile_rows = int(ceil((full_image.shape[0] - tile_size[0]) / stride) + 1) # strided convolution formula 193 | tile_cols = int(ceil((full_image.shape[1] - tile_size[1]) / stride) + 1) 194 | print("Need %i x %i prediction tiles @ stride %i px" % (tile_cols, tile_rows, stride)) 195 | full_probs = np.zeros((full_image.shape[0], full_image.shape[1], classes)) 196 | count_predictions = np.zeros((full_image.shape[0], full_image.shape[1], classes)) 197 | tile_counter = 0 198 | for row in range(tile_rows): 199 | for col in range(tile_cols): 200 | x1 = int(col * stride) 201 | y1 = int(row * stride) 202 | x2 = min(x1 + tile_size[1], full_image.shape[1]) 203 | y2 = min(y1 + tile_size[0], full_image.shape[0]) 204 | x1 = max(int(x2 - tile_size[1]), 0) # for portrait images the x1 underflows sometimes 205 | y1 = max(int(y2 - tile_size[0]), 0) # for very few rows y1 underflows 206 | 207 | img = full_image[y1:y2, x1:x2] 208 | padded_img = pad_image(img, tile_size) 209 | # plt.imshow(padded_img) 210 | # plt.show() 211 | tile_counter += 1 212 | print("Predicting tile %i" % tile_counter) 213 | padded_prediction = net.predict(padded_img, flip_evaluation) 214 | prediction = padded_prediction[0:img.shape[0], 0:img.shape[1], :] 215 | count_predictions[y1:y2, x1:x2] += 1 216 | full_probs[y1:y2, x1:x2] += prediction # accumulate the predictions also in the overlapping regions 217 | 218 | # average the predictions in the overlapping regions 219 | full_probs /= count_predictions 220 | # visualize normalization Weights 221 | # plt.imshow(np.mean(count_predictions, axis=2)) 222 | # plt.show() 223 | return full_probs 224 | 225 | 226 | def predict_multi_scale(full_image, net, scales, sliding_evaluation, flip_evaluation): 227 | """Predict an image by looking at it with different scales.""" 228 | classes = net.model.outputs[0].shape[3] 229 | full_probs = np.zeros((full_image.shape[0], full_image.shape[1], classes)) 230 | h_ori, w_ori = full_image.shape[:2] 231 | for scale in scales: 232 | print("Predicting image scaled by %f" % scale) 233 | scaled_img = misc.imresize(full_image, size=scale, interp="bilinear") 234 | if sliding_evaluation: 235 | scaled_probs = predict_sliding(scaled_img, net, flip_evaluation) 236 | else: 237 | scaled_probs = net.predict(scaled_img, flip_evaluation) 238 | # scale probs up to full size 239 | h, w = scaled_probs.shape[:2] 240 | probs = ndimage.zoom(scaled_probs, (1.*h_ori/h, 1.*w_ori/w, 1.), 241 | order=1, prefilter=False) 242 | # visualize_prediction(probs) 243 | # integrate probs over all scales 244 | full_probs += probs 245 | full_probs /= len(scales) 246 | return full_probs 247 | 248 | 249 | def _predict(full_image, net, flip_evaluation): 250 | classes = net.model.outputs[0].shape[3] 251 | full_probs = np.zeros((full_image.shape[0], full_image.shape[1], classes)) 252 | h_ori, w_ori = full_image.shape[:2] 253 | scaled_probs = net.predict(full_image, flip_evaluation) 254 | # scale probs up to full size 255 | h, w = scaled_probs.shape[:2] 256 | probs = cv2.resize(scaled_probs,(w_ori,h_ori)) 257 | # visualize_prediction(probs) 258 | # integrate probs over all scales 259 | full_probs += probs 260 | return full_probs 261 | 262 | 263 | if __name__ == "__main__": 264 | parser = argparse.ArgumentParser() 265 | parser.add_argument('-m', '--model', type=str, default='pspnet50_ade20k', 266 | help='Model/Weights to use', 267 | choices=['pspnet50_ade20k', 268 | 'pspnet101_cityscapes', 269 | 'pspnet101_voc2012']) 270 | parser.add_argument('-i', '--input_path', type=str, default='example_images/ade20k.jpg', 271 | help='Path the input image') 272 | parser.add_argument('-o', '--output_path', type=str, default='example_results/ade20k.jpg', 273 | help='Path to output') 274 | parser.add_argument('--id', default="0") 275 | parser.add_argument('-s', '--sliding', action='store_true', 276 | help="Whether the network should be slided over the original image for prediction.") 277 | parser.add_argument('-f', '--flip', action='store_true', 278 | help="Whether the network should predict on both image and flipped image.") 279 | parser.add_argument('-ms', '--multi_scale', action='store_true', 280 | help="Whether the network should predict on multiple scales.") 281 | args = parser.parse_args() 282 | 283 | environ["CUDA_VISIBLE_DEVICES"] = args.id 284 | 285 | sess = tf.Session() 286 | K.set_session(sess) 287 | 288 | with sess.as_default(): 289 | img = cv2.imread(args.input_path) 290 | print(args.input_path) 291 | 292 | if "pspnet50" in args.model: 293 | pspnet = PSPNet50(nb_classes=150, input_shape=(473, 473), 294 | weights=args.model) 295 | elif "pspnet101" in args.model: 296 | if "cityscapes" in args.model: 297 | pspnet = PSPNet101(nb_classes=19, input_shape=(713, 713), 298 | weights=args.model) 299 | if "voc2012" in args.model: 300 | pspnet = PSPNet101(nb_classes=21, input_shape=(473, 473), 301 | weights=args.model) 302 | 303 | else: 304 | print("Network architecture not implemented.") 305 | 306 | if args.multi_scale: 307 | EVALUATION_SCALES = [0.5, 0.75, 1.0, 1.25, 1.5, 1.75] # must be all floats! 308 | EVALUATION_SCALES = [0.15, 0.25, 0.5] # must be all floats! 309 | 310 | img = cv2.cvtColor(img,cv2.COLOR_BGR2RGB) 311 | 312 | # class_scores = predict_multi_scale(img, pspnet, EVALUATION_SCALES, args.sliding, args.flip) 313 | class_scores = _predict(img, pspnet, args.flip) 314 | 315 | print("Writing results...") 316 | 317 | class_image = np.argmax(class_scores, axis=2) 318 | pm = np.max(class_scores, axis=2) 319 | colored_class_image = utils.color_class_image(class_image, args.model) 320 | 321 | alpha_blended = 0.5 * colored_class_image + 0.5 * img 322 | filename, ext = splitext(args.output_path) 323 | misc.imsave(filename + "_seg" + ext, colored_class_image) 324 | misc.imsave(filename + "_probs" + ext, pm) 325 | misc.imsave(filename + "_seg_blended" + ext, alpha_blended) 326 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | numpy 2 | scipy 3 | tensorflow 4 | tensorflow-gpu 5 | keras 6 | -------------------------------------------------------------------------------- /utils.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | import colorsys 3 | import numpy as np 4 | from keras.models import Model 5 | from cityscapes_labels import trainId2label 6 | from ade20k_labels import ade20k_id2label 7 | from pascal_voc_labels import voc_id2label 8 | 9 | 10 | def class_image_to_image(class_id_image, class_id_to_rgb_map): 11 | """Map the class image to a rgb-color image.""" 12 | colored_image = np.zeros((class_id_image.shape[0], class_id_image.shape[1], 3), np.uint8) 13 | for i in range(-1,256): 14 | try: 15 | cl = class_id_to_rgb_map[i] 16 | colored_image[class_id_image[:,:]==i] = cl.color 17 | except KeyError as key_error: 18 | pass 19 | return colored_image 20 | 21 | 22 | def class_image_to_image_slow(class_id_image, class_id_to_rgb_map): 23 | """Map the class image to a rgb-color image.""" 24 | colored_image = np.zeros((class_id_image.shape[0], class_id_image.shape[1], 3), np.uint8) 25 | for row in range(class_id_image.shape[0]): 26 | for col in range(class_id_image.shape[1]): 27 | try: 28 | colored_image[row, col, :] = class_id_to_rgb_map[int(class_id_image[row, col])].color 29 | except KeyError as key_error: 30 | print("Warning: could not resolve classid %s" % key_error) 31 | return colored_image 32 | 33 | 34 | def color_class_image(class_image, model_name): 35 | """Color classed depending on the model used.""" 36 | if 'cityscapes' in model_name: 37 | colored_image = class_image_to_image(class_image, trainId2label) 38 | elif 'voc' in model_name: 39 | colored_image = class_image_to_image(class_image, voc_id2label) 40 | elif 'ade20k' in model_name: 41 | colored_image = class_image_to_image(class_image, ade20k_id2label) 42 | else: 43 | colored_image = add_color(class_image) 44 | return colored_image 45 | 46 | def color_class_image_slow(class_image, model_name): 47 | """Color classed depending on the model used.""" 48 | if 'cityscapes' in model_name: 49 | colored_image = class_image_to_image_slow(class_image, trainId2label) 50 | elif 'voc' in model_name: 51 | colored_image = class_image_to_image_slow(class_image, voc_id2label) 52 | elif 'ade20k' in model_name: 53 | colored_image = class_image_to_image_slow(class_image, ade20k_id2label) 54 | else: 55 | colored_image = add_color(class_image) 56 | return colored_image 57 | 58 | 59 | def add_color(img): 60 | """Color classes a good distance away from each other.""" 61 | h, w = img.shape 62 | img_color = np.zeros((h, w, 3)) 63 | for i in xrange(1, 151): 64 | img_color[img == i] = to_color(i) 65 | return img_color * 255 # is [0.0-1.0] should be [0-255] 66 | 67 | 68 | def to_color(category): 69 | """Map each category color a good distance away from each other on the HSV color space.""" 70 | v = (category-1)*(137.5/360) 71 | return colorsys.hsv_to_rgb(v, 1, 1) 72 | 73 | 74 | def debug(model, data): 75 | """Debug model by printing the activations in each layer.""" 76 | names = [layer.name for layer in model.layers] 77 | for name in names[:]: 78 | print_activation(model, name, data) 79 | 80 | 81 | def print_activation(model, layer_name, data): 82 | """Print the activations in each layer.""" 83 | intermediate_layer_model = Model(inputs=model.input, 84 | outputs=model.get_layer(layer_name).output) 85 | io = intermediate_layer_model.predict(data) 86 | print(layer_name, array_to_str(io)) 87 | 88 | 89 | def array_to_str(a): 90 | return "{} {} {} {} {}".format(a.dtype, a.shape, np.min(a), 91 | np.max(a), np.mean(a)) 92 | -------------------------------------------------------------------------------- /utils/color150/airplane.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/airplane.jpg -------------------------------------------------------------------------------- /utils/color150/animal.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/animal.jpg -------------------------------------------------------------------------------- /utils/color150/apparel.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/apparel.jpg -------------------------------------------------------------------------------- /utils/color150/arcade machine.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/arcade machine.jpg -------------------------------------------------------------------------------- /utils/color150/armchair.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/armchair.jpg -------------------------------------------------------------------------------- /utils/color150/ashcan.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/ashcan.jpg -------------------------------------------------------------------------------- /utils/color150/awning.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/awning.jpg -------------------------------------------------------------------------------- /utils/color150/bag.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/bag.jpg -------------------------------------------------------------------------------- /utils/color150/ball.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/ball.jpg -------------------------------------------------------------------------------- /utils/color150/bannister.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/bannister.jpg -------------------------------------------------------------------------------- /utils/color150/bar.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/bar.jpg -------------------------------------------------------------------------------- /utils/color150/barrel.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/barrel.jpg -------------------------------------------------------------------------------- /utils/color150/base.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/base.jpg -------------------------------------------------------------------------------- /utils/color150/basket.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/basket.jpg -------------------------------------------------------------------------------- /utils/color150/bathtub.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/bathtub.jpg -------------------------------------------------------------------------------- /utils/color150/bed.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/bed.jpg -------------------------------------------------------------------------------- /utils/color150/bench.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/bench.jpg -------------------------------------------------------------------------------- /utils/color150/bicycle.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/bicycle.jpg -------------------------------------------------------------------------------- /utils/color150/bike.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/bike.jpg -------------------------------------------------------------------------------- /utils/color150/blanket.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/blanket.jpg -------------------------------------------------------------------------------- /utils/color150/blind.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/blind.jpg -------------------------------------------------------------------------------- /utils/color150/boat.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/boat.jpg -------------------------------------------------------------------------------- /utils/color150/book.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/book.jpg -------------------------------------------------------------------------------- /utils/color150/bookcase.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/bookcase.jpg -------------------------------------------------------------------------------- /utils/color150/booth.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/booth.jpg -------------------------------------------------------------------------------- /utils/color150/bottle.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/bottle.jpg -------------------------------------------------------------------------------- /utils/color150/box.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/box.jpg -------------------------------------------------------------------------------- /utils/color150/bridge.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/bridge.jpg -------------------------------------------------------------------------------- /utils/color150/buffet.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/buffet.jpg -------------------------------------------------------------------------------- /utils/color150/building.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/building.jpg -------------------------------------------------------------------------------- /utils/color150/bulletin board.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/bulletin board.jpg -------------------------------------------------------------------------------- /utils/color150/bus.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/bus.jpg -------------------------------------------------------------------------------- /utils/color150/cabinet.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/cabinet.jpg -------------------------------------------------------------------------------- /utils/color150/canopy.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/canopy.jpg -------------------------------------------------------------------------------- /utils/color150/car.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/car.jpg -------------------------------------------------------------------------------- /utils/color150/case.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/case.jpg -------------------------------------------------------------------------------- /utils/color150/ceiling.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/ceiling.jpg -------------------------------------------------------------------------------- /utils/color150/chair.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/chair.jpg -------------------------------------------------------------------------------- /utils/color150/chandelier.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/chandelier.jpg -------------------------------------------------------------------------------- /utils/color150/chest of drawers.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/chest of drawers.jpg -------------------------------------------------------------------------------- /utils/color150/clock.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/clock.jpg -------------------------------------------------------------------------------- /utils/color150/coffee table.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/coffee table.jpg -------------------------------------------------------------------------------- /utils/color150/column.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/column.jpg -------------------------------------------------------------------------------- /utils/color150/computer.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/computer.jpg -------------------------------------------------------------------------------- /utils/color150/conveyer belt.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/conveyer belt.jpg -------------------------------------------------------------------------------- /utils/color150/counter.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/counter.jpg -------------------------------------------------------------------------------- /utils/color150/countertop.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/countertop.jpg -------------------------------------------------------------------------------- /utils/color150/cradle.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/cradle.jpg -------------------------------------------------------------------------------- /utils/color150/crt screen.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/crt screen.jpg -------------------------------------------------------------------------------- /utils/color150/curtain.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/curtain.jpg -------------------------------------------------------------------------------- /utils/color150/cushion.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/cushion.jpg -------------------------------------------------------------------------------- /utils/color150/desk.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/desk.jpg -------------------------------------------------------------------------------- /utils/color150/dirt track.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/dirt track.jpg -------------------------------------------------------------------------------- /utils/color150/dishwasher.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/dishwasher.jpg -------------------------------------------------------------------------------- /utils/color150/door.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/door.jpg -------------------------------------------------------------------------------- /utils/color150/earth.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/earth.jpg -------------------------------------------------------------------------------- /utils/color150/escalator.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/escalator.jpg -------------------------------------------------------------------------------- /utils/color150/fan.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/fan.jpg -------------------------------------------------------------------------------- /utils/color150/fence.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/fence.jpg -------------------------------------------------------------------------------- /utils/color150/field.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/field.jpg -------------------------------------------------------------------------------- /utils/color150/fireplace.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/fireplace.jpg -------------------------------------------------------------------------------- /utils/color150/flag.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/flag.jpg -------------------------------------------------------------------------------- /utils/color150/floor.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/floor.jpg -------------------------------------------------------------------------------- /utils/color150/flower.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/flower.jpg -------------------------------------------------------------------------------- /utils/color150/food.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/food.jpg -------------------------------------------------------------------------------- /utils/color150/fountain.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/fountain.jpg -------------------------------------------------------------------------------- /utils/color150/furniture.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/furniture.jpg -------------------------------------------------------------------------------- /utils/color150/glass.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/glass.jpg -------------------------------------------------------------------------------- /utils/color150/grandstand.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/grandstand.jpg -------------------------------------------------------------------------------- /utils/color150/grass.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/grass.jpg -------------------------------------------------------------------------------- /utils/color150/hill.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/hill.jpg -------------------------------------------------------------------------------- /utils/color150/hood.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/hood.jpg -------------------------------------------------------------------------------- /utils/color150/house.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/house.jpg -------------------------------------------------------------------------------- /utils/color150/houseware.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/houseware.jpg -------------------------------------------------------------------------------- /utils/color150/hovel.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/hovel.jpg -------------------------------------------------------------------------------- /utils/color150/kitchen island.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/kitchen island.jpg -------------------------------------------------------------------------------- /utils/color150/lake.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/lake.jpg -------------------------------------------------------------------------------- /utils/color150/lamp.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/lamp.jpg -------------------------------------------------------------------------------- /utils/color150/land.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/land.jpg -------------------------------------------------------------------------------- /utils/color150/light.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/light.jpg -------------------------------------------------------------------------------- /utils/color150/microwave.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/microwave.jpg -------------------------------------------------------------------------------- /utils/color150/minibike.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/minibike.jpg -------------------------------------------------------------------------------- /utils/color150/mirror.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/mirror.jpg -------------------------------------------------------------------------------- /utils/color150/monitor.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/monitor.jpg -------------------------------------------------------------------------------- /utils/color150/mountain.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/mountain.jpg -------------------------------------------------------------------------------- /utils/color150/ottoman.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/ottoman.jpg -------------------------------------------------------------------------------- /utils/color150/oven.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/oven.jpg -------------------------------------------------------------------------------- /utils/color150/painting.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/painting.jpg -------------------------------------------------------------------------------- /utils/color150/palm.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/palm.jpg -------------------------------------------------------------------------------- /utils/color150/path.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/path.jpg -------------------------------------------------------------------------------- /utils/color150/person.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/person.jpg -------------------------------------------------------------------------------- /utils/color150/pier.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/pier.jpg -------------------------------------------------------------------------------- /utils/color150/pillow.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/pillow.jpg -------------------------------------------------------------------------------- /utils/color150/plant.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/plant.jpg -------------------------------------------------------------------------------- /utils/color150/plate.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/plate.jpg -------------------------------------------------------------------------------- /utils/color150/plaything.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/plaything.jpg -------------------------------------------------------------------------------- /utils/color150/pole.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/pole.jpg -------------------------------------------------------------------------------- /utils/color150/pool table.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/pool table.jpg -------------------------------------------------------------------------------- /utils/color150/poster.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/poster.jpg -------------------------------------------------------------------------------- /utils/color150/pot.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/pot.jpg -------------------------------------------------------------------------------- /utils/color150/radiator.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/radiator.jpg -------------------------------------------------------------------------------- /utils/color150/railing.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/railing.jpg -------------------------------------------------------------------------------- /utils/color150/refrigerator.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/refrigerator.jpg -------------------------------------------------------------------------------- /utils/color150/river.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/river.jpg -------------------------------------------------------------------------------- /utils/color150/road.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/road.jpg -------------------------------------------------------------------------------- /utils/color150/rock.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/rock.jpg -------------------------------------------------------------------------------- /utils/color150/rug.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/rug.jpg -------------------------------------------------------------------------------- /utils/color150/runway.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/runway.jpg -------------------------------------------------------------------------------- /utils/color150/sand.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/sand.jpg -------------------------------------------------------------------------------- /utils/color150/sconce.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/sconce.jpg -------------------------------------------------------------------------------- /utils/color150/screen door.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/screen door.jpg -------------------------------------------------------------------------------- /utils/color150/screen.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/screen.jpg -------------------------------------------------------------------------------- /utils/color150/sculpture.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/sculpture.jpg -------------------------------------------------------------------------------- /utils/color150/sea.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/sea.jpg -------------------------------------------------------------------------------- /utils/color150/seat.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/seat.jpg -------------------------------------------------------------------------------- /utils/color150/shelf.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/shelf.jpg -------------------------------------------------------------------------------- /utils/color150/ship.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/ship.jpg -------------------------------------------------------------------------------- /utils/color150/shower.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/shower.jpg -------------------------------------------------------------------------------- /utils/color150/sidewalk.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/sidewalk.jpg -------------------------------------------------------------------------------- /utils/color150/signboard.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/signboard.jpg -------------------------------------------------------------------------------- /utils/color150/sink.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/sink.jpg -------------------------------------------------------------------------------- /utils/color150/sky.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/sky.jpg -------------------------------------------------------------------------------- /utils/color150/skyscraper.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/skyscraper.jpg -------------------------------------------------------------------------------- /utils/color150/sofa.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/sofa.jpg -------------------------------------------------------------------------------- /utils/color150/stage.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/stage.jpg -------------------------------------------------------------------------------- /utils/color150/stairs.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/stairs.jpg -------------------------------------------------------------------------------- /utils/color150/stairway.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/stairway.jpg -------------------------------------------------------------------------------- /utils/color150/step.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/step.jpg -------------------------------------------------------------------------------- /utils/color150/stool.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/stool.jpg -------------------------------------------------------------------------------- /utils/color150/stove.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/stove.jpg -------------------------------------------------------------------------------- /utils/color150/streetlight.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/streetlight.jpg -------------------------------------------------------------------------------- /utils/color150/swimming pool.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/swimming pool.jpg -------------------------------------------------------------------------------- /utils/color150/swivel chair.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/swivel chair.jpg -------------------------------------------------------------------------------- /utils/color150/table.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/table.jpg -------------------------------------------------------------------------------- /utils/color150/tank.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/tank.jpg -------------------------------------------------------------------------------- /utils/color150/television receiver.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/television receiver.jpg -------------------------------------------------------------------------------- /utils/color150/tent.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/tent.jpg -------------------------------------------------------------------------------- /utils/color150/toilet.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/toilet.jpg -------------------------------------------------------------------------------- /utils/color150/towel.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/towel.jpg -------------------------------------------------------------------------------- /utils/color150/tower.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/tower.jpg -------------------------------------------------------------------------------- /utils/color150/trade name.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/trade name.jpg -------------------------------------------------------------------------------- /utils/color150/traffic light.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/traffic light.jpg -------------------------------------------------------------------------------- /utils/color150/tray.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/tray.jpg -------------------------------------------------------------------------------- /utils/color150/tree.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/tree.jpg -------------------------------------------------------------------------------- /utils/color150/truck.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/truck.jpg -------------------------------------------------------------------------------- /utils/color150/van.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/van.jpg -------------------------------------------------------------------------------- /utils/color150/vase.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/vase.jpg -------------------------------------------------------------------------------- /utils/color150/wall.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/wall.jpg -------------------------------------------------------------------------------- /utils/color150/wardrobe.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/wardrobe.jpg -------------------------------------------------------------------------------- /utils/color150/washer.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/washer.jpg -------------------------------------------------------------------------------- /utils/color150/water.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/water.jpg -------------------------------------------------------------------------------- /utils/color150/waterfall.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/waterfall.jpg -------------------------------------------------------------------------------- /utils/color150/windowpane.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/color150/windowpane.jpg -------------------------------------------------------------------------------- /utils/colorization/color150.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/colorization/color150.mat -------------------------------------------------------------------------------- /utils/colorization/objectName150.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/utils/colorization/objectName150.mat -------------------------------------------------------------------------------- /weight_converter.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from __future__ import print_function 4 | 5 | import sys 6 | from os.path import splitext 7 | import numpy as np 8 | 9 | import caffe 10 | 11 | # Not needed because Tensorflow and Caffe do convolution the same way 12 | # Needed for conversion to Theano 13 | 14 | 15 | def rot90(W): 16 | for i in range(W.shape[0]): 17 | for j in range(W.shape[1]): 18 | W[i, j] = np.rot90(W[i, j], 2) 19 | return W 20 | 21 | 22 | weights = {} 23 | assert "prototxt" in splitext(sys.argv[1])[1], "First argument must be caffe prototxt %s" % sys.argv[1] 24 | assert "caffemodel" in splitext(sys.argv[2])[1], "Second argument must be caffe weights %s" % sys.argv[2] 25 | net = caffe.Net(sys.argv[1], sys.argv[2], caffe.TEST) 26 | for k, v in net.params.items(): 27 | print ("Layer %s, has %d params." % (k, len(v))) 28 | if len(v) == 1: 29 | W = v[0].data[...] 30 | W = np.transpose(W, (2, 3, 1, 0)) 31 | weights[k] = {"weights": W} 32 | elif len(v) == 2: 33 | W = v[0].data[...] 34 | W = np.transpose(W, (2, 3, 1, 0)) 35 | b = v[1].data[...] 36 | weights[k] = {"weights": W, "biases": b} 37 | elif len(v) == 4: # Batchnorm layer 38 | k = k.replace('/', '_') 39 | mean = v[0].data[...] 40 | variance = v[1].data[...] 41 | scale = v[2].data[...] 42 | offset = v[3].data[...] 43 | weights[k] = {"mean": mean, "variance": variance, "scale": scale, "offset": offset} 44 | else: 45 | print("Undefined layer") 46 | exit() 47 | 48 | arr = np.asarray(weights) 49 | weights_name = splitext(sys.argv[2])[0]+".npy" 50 | np.save(weights_name.lower(), arr) 51 | -------------------------------------------------------------------------------- /weights/caffe/Put caffemodels & prototxts here: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/weights/caffe/Put caffemodels & prototxts here -------------------------------------------------------------------------------- /weights/keras/Put keras weights here: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/weights/keras/Put keras weights here -------------------------------------------------------------------------------- /weights/npy/Put the npy weights here: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karolmajek/PSPNet-Keras-tensorflow/e3d696b0ee94228fdc35db58bbb2ed2b2243fcb5/weights/npy/Put the npy weights here --------------------------------------------------------------------------------