├── _config.yml ├── includes ├── flask │ ├── doit.sh │ ├── static │ │ ├── css │ │ │ └── index.css │ │ └── js │ │ │ └── index.js │ ├── templates │ │ └── index.html │ └── app.py ├── Makefile.config ├── processAll.py ├── JooJanta200.ipynb └── TVG_CRFRNN_new_deploy.prototxt.dg ├── Dockerfile.joojanta200.bulkProcess ├── Dockerfile.joojanta200.flask ├── Dockerfile.joojanta200.notebook └── README.md /_config.yml: -------------------------------------------------------------------------------- 1 | theme: jekyll-theme-tactile -------------------------------------------------------------------------------- /includes/flask/doit.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | cd /work/ 3 | python processAll.py dummy 4 | -------------------------------------------------------------------------------- /includes/flask/static/css/index.css: -------------------------------------------------------------------------------- 1 | 2 | dy { 3 | padding-top: 40px; 4 | padding-bottom: 40px; 5 | } 6 | 7 | .form-signin { 8 | max-width: 330px; 9 | padding: 15px; 10 | margin: 0 auto; 11 | } 12 | .form-signin .form-signin-heading, 13 | .form-signin .checkbox { 14 | margin-bottom: 10px; 15 | } 16 | .form-signin .checkbox { 17 | font-weight: normal; 18 | } 19 | .form-signin .form-control { 20 | position: relative; 21 | height: auto; 22 | -webkit-box-sizing: border-box; 23 | -moz-box-sizing: border-box; 24 | box-sizing: border-box; 25 | padding: 10px; 26 | font-size: 16px; 27 | } 28 | .form-signin .form-control:focus { 29 | z-index: 2; 30 | } 31 | .form-signin input[type="url"] { 32 | margin-bottom: -1px; 33 | border-bottom-right-radius: 0; 34 | border-bottom-left-radius: 0; 35 | } 36 | -------------------------------------------------------------------------------- /Dockerfile.joojanta200.bulkProcess: -------------------------------------------------------------------------------- 1 | from bvlc/caffe:cpu 2 | maintainer dgrossman@iqt.org 3 | RUN apt-get update 4 | RUN apt-get install -y vim 5 | run pip install jupyter 6 | RUN mkdir /work 7 | WORKDIR /work 8 | 9 | ADD ./includes /includes 10 | 11 | RUN mkdir model 12 | RUN git clone https://github.com/torrvision/crfasrnn.git 13 | WORKDIR /work/crfasrnn 14 | RUN git checkout 3b9c6b61e5416d2d38b5c6cdf6f8dcc02fafcc43 15 | 16 | RUN git clone https://github.com/torrvision/caffe.git 17 | WORKDIR /work/crfasrnn/caffe 18 | RUN git checkout e7f25fa2bdbb291a067930d60f337d305c128583 19 | 20 | RUN cp /includes/Makefile.config . 21 | RUN make 22 | RUN make pycaffe 23 | RUN make distribute 24 | WORKDIR /work/crfasrnn/python-scripts 25 | RUN cp *par /work 26 | RUN cp /includes/TVG_CRFRNN_new_deploy.prototxt.dg /work/crfasrnn/python-scripts 27 | RUN cp /includes/JooJanta200.ipynb /work 28 | RUN cp /includes/*py /work 29 | WORKDIR /work 30 | 31 | RUN mkdir /work/inputImages 32 | RUN mkdir /work/outputImages 33 | 34 | EXPOSE 8888 35 | #CMD ["/usr/local/bin/jupyter","notebook","--allow-root","--ip=0.0.0.0"]] 36 | CMD ["/usr/bin/python","./processAll.py"] 37 | 38 | -------------------------------------------------------------------------------- /Dockerfile.joojanta200.flask: -------------------------------------------------------------------------------- 1 | from bvlc/caffe:cpu 2 | maintainer dgrossman@iqt.org 3 | RUN apt-get update 4 | RUN apt-get install -y vim 5 | run pip install jupyter 6 | RUN mkdir /work 7 | WORKDIR /work 8 | 9 | ADD ./includes /includes 10 | 11 | RUN mkdir model 12 | RUN git clone https://github.com/torrvision/crfasrnn.git 13 | WORKDIR /work/crfasrnn 14 | RUN git checkout 3b9c6b61e5416d2d38b5c6cdf6f8dcc02fafcc43 15 | 16 | RUN git clone https://github.com/torrvision/caffe.git 17 | WORKDIR /work/crfasrnn/caffe 18 | RUN git checkout e7f25fa2bdbb291a067930d60f337d305c128583 19 | 20 | RUN cp /includes/Makefile.config . 21 | RUN make 22 | RUN make pycaffe 23 | RUN make distribute 24 | WORKDIR /work/crfasrnn/python-scripts 25 | RUN cp *par /work 26 | RUN cp /includes/TVG_CRFRNN_new_deploy.prototxt.dg /work/crfasrnn/python-scripts 27 | RUN cp /includes/JooJanta200.ipynb /work 28 | RUN cp /includes/*py /work 29 | WORKDIR /work 30 | 31 | RUN mkdir /work/inputImages 32 | RUN mkdir /work/outputImages 33 | RUN mv /includes/flask /work/flask 34 | WORKDIR /work/flask 35 | RUN pip install flask 36 | 37 | EXPOSE 5000 38 | CMD ["/usr/bin/python","./app.py"] 39 | 40 | -------------------------------------------------------------------------------- /Dockerfile.joojanta200.notebook: -------------------------------------------------------------------------------- 1 | from bvlc/caffe:cpu 2 | maintainer dgrossman@iqt.org 3 | RUN apt-get update 4 | RUN apt-get install -y vim 5 | run pip install jupyter 6 | RUN mkdir /work 7 | WORKDIR /work 8 | 9 | ADD ./includes /includes 10 | 11 | RUN mkdir model 12 | RUN git clone https://github.com/torrvision/crfasrnn.git 13 | WORKDIR /work/crfasrnn 14 | RUN git checkout 3b9c6b61e5416d2d38b5c6cdf6f8dcc02fafcc43 15 | 16 | RUN git clone https://github.com/torrvision/caffe.git 17 | WORKDIR /work/crfasrnn/caffe 18 | RUN git checkout e7f25fa2bdbb291a067930d60f337d305c128583 19 | 20 | RUN cp /includes/Makefile.config . 21 | RUN make 22 | RUN make pycaffe 23 | RUN make distribute 24 | WORKDIR /work/crfasrnn/python-scripts 25 | RUN cp *par /work 26 | RUN cp /includes/TVG_CRFRNN_new_deploy.prototxt.dg /work/crfasrnn/python-scripts 27 | RUN cp /includes/JooJanta200.ipynb /work 28 | RUN cp /includes/*py /work 29 | WORKDIR /work 30 | 31 | RUN mkdir /work/inputImages 32 | RUN mkdir /work/outputImages 33 | 34 | EXPOSE 8888 35 | CMD ["/usr/local/bin/jupyter","notebook","--allow-root","--ip=0.0.0.0"]] 36 | #CMD ["/usr/bin/python","./processAll.py"] 37 | 38 | -------------------------------------------------------------------------------- /includes/flask/templates/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Joo Janta 200 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 |
17 |
18 | 22 |
23 | 24 |
25 |

Joo Janta 200 app

26 | 33 |
34 | 42 |
43 | 44 | 45 | -------------------------------------------------------------------------------- /includes/flask/app.py: -------------------------------------------------------------------------------- 1 | from flask import Flask 2 | from flask import render_template 3 | from flask import request 4 | import urllib 5 | from subprocess import call 6 | 7 | import json 8 | 9 | app = Flask(__name__) 10 | 11 | 12 | @app.route("/") 13 | def main(): 14 | return render_template('index.html') 15 | 16 | 17 | @app.route('/', methods=['POST']) 18 | def work_it(): 19 | uri = request.form['uri'] 20 | 21 | print(request.form) 22 | if uri: 23 | print('downloading uri {0}'.format(uri)) 24 | try: 25 | urllib.urlretrieve(uri, '/work/flask/static/top.jpg') 26 | except: 27 | return json.dumps({'ret': 'False', 28 | 'msg': 'could not download {0}'.format(uri)}) 29 | 30 | try: 31 | call(['bash', '/work/flask/doit.sh']) 32 | return json.dumps({'ret': 'True', 33 | 'topImage': '/static/top.jpg', 34 | 'bottomImage': '/static/bottom.bmp', 35 | 'msg': 'success'}) 36 | except: 37 | return json.dumps({'ret': 'False', 38 | 'msg': 'failure running back end process'}) 39 | else: 40 | return json.dumps( 41 | {'ret': 'False', 'msg': 'include a url for download'}) 42 | 43 | 44 | @app.after_request 45 | def add_header(r): 46 | """ 47 | Add headers to both force latest IE rendering engine or Chrome Frame, 48 | and also to cache the rendered page for 10 minutes. 49 | """ 50 | r.headers[ 51 | "Cache-Control"] = "no-cache, no-store, must-revalidate, public, max-age=0" 52 | r.headers["Pragma"] = "no-cache" 53 | r.headers["Expires"] = "0" 54 | return r 55 | 56 | if __name__ == "__main__": 57 | app.run(host='0.0.0.0') 58 | -------------------------------------------------------------------------------- /includes/flask/static/js/index.js: -------------------------------------------------------------------------------- 1 | $(function(){ 2 | $('#btnSignUp').click(function(){ 3 | var btn = $(this); 4 | 5 | btn.prop('disabled',true); 6 | 7 | $.ajax({ 8 | url: '/', 9 | data: $('form').serialize(), 10 | type: 'POST', 11 | success: function(response){ 12 | console.log(response); 13 | 14 | var json = JSON.parse(response); 15 | var _ret = json['ret']; 16 | var _msg = json['msg']; 17 | 18 | console.log('response'); 19 | console.log(_ret); 20 | console.log(_msg); 21 | if (_ret === 'True'){ 22 | 23 | $("#topImage").html("\n function rollover(my_image) {\n my_image.src=\"/static/bottom.bmp\";\n }\n \n"+"\n function mouseaway(my_image) {\n my_image.src=\"/static/top.jpg\";\n }\n \n"+ ""); 24 | console.log('top'); 25 | $("#bottomImage").html(""); 26 | console.log('bottom'); 27 | btn.prop('disabled',false); 28 | 29 | } else { 30 | 31 | $("#topImage").html("WARNING: " + _msg); 32 | console.log('error'); 33 | btn.prop('disabled',false); 34 | } 35 | }, 36 | error: function(error){ 37 | console.log(error); 38 | btn.prop('disabled',false); 39 | } 40 | }); 41 | 42 | }); 43 | }); 44 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # joojanta200 2 | 3 | Named after the glasses worn by [Zaphod Beeblebrox in Hitch hikers guide to the galaxy](http://hitchhikers.wikia.com/wiki/Joo_Janta_200_Super-Chromatic_Peril_Sensitive_Sunglasses). 4 | 5 | The project utilizes [CRF as RNN model](http://www.robots.ox.ac.uk/~szheng/papers/CRFasRNN.pdf) to assign labels to pixels. The demo preserves pixels assigned to people while blacking out pixels associated with other labels. (Think of it as automating background removal around people) 6 | 7 | ## Three different docker files, Three different purposes. 8 | 9 | all the dockers need the model. The docker containers appear to be memory pigs during inference. I havent tested the lower bounds of their memory lust, but >= 4Gig ram makes things run smoothly. 10 | 11 | download TVG_CRFRNN_COCO_VOC.caffemodel model somewhere and pass directory holding it to the container. 12 | 13 | ``` 14 | wget -O TVG_CRFRNN_COCO_VOC.caffemodel http://goo.gl/j7PrPZ 15 | ``` 16 | 17 | ### Dockerfile.joojanta200.bulkProcess 18 | 19 | for people that want to remove backgrounds in bulk. 20 | 21 | Build: 22 | ``` 23 | docker build -t jj.bulk -f Dockerfile.joojanta200.bulkProcess . 24 | ``` 25 | Run: 26 | 1. **dirOfInputImages** - directory of images to process 27 | 1. **dirToWriteImages** - directory to write processed images 28 | 1. **dirTo500megModel** - directory containing TVG_CRFRNN_COCO_VOC.caffemodel 29 | 1. **containerid** - containerid from the build 30 | 31 | ``` 32 | docker run -v /dirOfInputImages:/work/inputImages \ 33 | -v /dirToWriteImages:/work/outputImages \ 34 | -v /dirTo500megModel:/work/model jj.bulk 35 | ``` 36 | 37 | ### Dockerfile.joojanta200.notebook 38 | 39 | jupyter notebook for looking under the covers 40 | 41 | The container will spew forth a webpage to go to. replace the 8888 in the webpage with **somePort** 42 | 43 | Build: 44 | 45 | ``` 46 | docker build -t jj.notebook -f Dockerfile.joojanta200.notebook . 47 | ``` 48 | Run: 49 | 1. **somePort** - local port to listen on 50 | 1. **dirTo500megModel** - directory containing TVG_CRFRNN_COCO_VOC.caffemodel 51 | 1. **containerid** - containerid from the build 52 | 53 | ``` 54 | docker run -v /dirTo500megModel:/work/model \ 55 | -p :8888 jj.notebook 56 | ``` 57 | ### Dockerfile.joojanta200.flask 58 | 59 | Simple webapp, user pastes image uri, container downloads image, processes and displays original image over processed image. 60 | 61 | Build: 62 | ``` 63 | docker build -t jj.flask -f Dockerfile.joojanta200.flask . 64 | ``` 65 | Run: 66 | 1. **somePort** - local port to listen on 67 | 1. **dirTo500megModel** - directory containing TVG_CRFRNN_COCO_VOC.caffemodel 68 | 1. **containerid** - containerid from the build 69 | 70 | ``` 71 | docker run -v /dirTo500megModel:/work/model \ 72 | -p :5000 jj.flask 73 | ``` 74 | 75 | ### how not to screw things up in the future.. 76 | things work with the following versions 77 | 1. https://github.com/BVLC/caffe/tree/master/docker commit 4db619aec9cd384b11a1c55fac257d14b704bb15 78 | 1. https://github.com/torrvision/crfasrnn commit 3b9c6b61e5416d2d38b5c6cdf6f8dcc02fafcc43 [done] 79 | 1. https://github.com/torrvision/caffe commit e7f25fa2bdbb291a067930d60f337d305c128583 [done] 80 | 1. TVG_CRFRNN_COCO_VOC.caffemodel bc4926ad00ecc9a1c627db82377ecf56 (md5sum) 81 | -------------------------------------------------------------------------------- /includes/Makefile.config: -------------------------------------------------------------------------------- 1 | ## Refer to http://caffe.berkeleyvision.org/installation.html 2 | # Contributions simplifying and improving our build system are welcome! 3 | 4 | # cuDNN acceleration switch (uncomment to build with cuDNN). 5 | # USE_CUDNN := 1 6 | 7 | # CPU-only switch (uncomment to build without GPU support). 8 | CPU_ONLY := 1 9 | 10 | # To customize your choice of compiler, uncomment and set the following. 11 | # N.B. the default for Linux is g++ and the default for OSX is clang++ 12 | # CUSTOM_CXX := g++ 13 | 14 | # CUDA directory contains bin/ and lib/ directories that we need. 15 | CUDA_DIR := /usr/local/cuda 16 | # On Ubuntu 14.04, if cuda tools are installed via 17 | # "sudo apt-get install nvidia-cuda-toolkit" then use this instead: 18 | # CUDA_DIR := /usr 19 | 20 | # CUDA architecture setting: going with all of them. 21 | # For CUDA < 6.0, comment the *_50 lines for compatibility. 22 | CUDA_ARCH := -gencode arch=compute_20,code=sm_20 \ 23 | -gencode arch=compute_20,code=sm_21 \ 24 | -gencode arch=compute_30,code=sm_30 \ 25 | -gencode arch=compute_35,code=sm_35 \ 26 | -gencode arch=compute_50,code=sm_50 \ 27 | -gencode arch=compute_50,code=compute_50 28 | 29 | # BLAS choice: 30 | # atlas for ATLAS (default) 31 | # mkl for MKL 32 | # open for OpenBlas 33 | BLAS := atlas 34 | # Custom (MKL/ATLAS/OpenBLAS) include and lib directories. 35 | # Leave commented to accept the defaults for your choice of BLAS 36 | # (which should work)! 37 | # BLAS_INCLUDE := /path/to/your/blas 38 | # BLAS_LIB := /path/to/your/blas 39 | 40 | # Homebrew puts openblas in a directory that is not on the standard search path 41 | # BLAS_INCLUDE := $(shell brew --prefix openblas)/include 42 | # BLAS_LIB := $(shell brew --prefix openblas)/lib 43 | 44 | # This is required only if you will compile the matlab interface. 45 | # MATLAB directory should contain the mex binary in /bin. 46 | # MATLAB_DIR := /usr/local 47 | # MATLAB_DIR := /Applications/MATLAB_R2012b.app 48 | 49 | # NOTE: this is required only if you will compile the python interface. 50 | # We need to be able to find Python.h and numpy/arrayobject.h. 51 | PYTHON_INCLUDE := /usr/include/python2.7 \ 52 | /usr/lib/python2.7/dist-packages/numpy/core/include 53 | # Anaconda Python distribution is quite popular. Include path: 54 | # Verify anaconda location, sometimes it's in root. 55 | # ANACONDA_HOME := $(HOME)/anaconda 56 | # PYTHON_INCLUDE := $(ANACONDA_HOME)/include \ 57 | # $(ANACONDA_HOME)/include/python2.7 \ 58 | # $(ANACONDA_HOME)/lib/python2.7/site-packages/numpy/core/include \ 59 | 60 | # We need to be able to find libpythonX.X.so or .dylib. 61 | PYTHON_LIB := /usr/lib 62 | # PYTHON_LIB := $(ANACONDA_HOME)/lib 63 | 64 | # Homebrew installs numpy in a non standard path (keg only) 65 | # PYTHON_INCLUDE += $(dir $(shell python -c 'import numpy.core; print(numpy.core.__file__)'))/include 66 | # PYTHON_LIB += $(shell brew --prefix numpy)/lib 67 | 68 | # Uncomment to support layers written in Python (will link against Python libs) 69 | # WITH_PYTHON_LAYER := 1 70 | 71 | # Whatever else you find you need goes here. 72 | INCLUDE_DIRS := $(PYTHON_INCLUDE) /usr/local/include /usr/include/hdf5/serial 73 | LIBRARY_DIRS := $(PYTHON_LIB) /usr/local/lib /usr/lib /usr/lib/x86_64-linux-gnu/hdf5/serial 74 | 75 | # If Homebrew is installed at a non standard location (for example your home directory) and you use it for general dependencies 76 | # INCLUDE_DIRS += $(shell brew --prefix)/include 77 | # LIBRARY_DIRS += $(shell brew --prefix)/lib 78 | 79 | # Uncomment to use `pkg-config` to specify OpenCV library paths. 80 | # (Usually not necessary -- OpenCV libraries are normally installed in one of the above $LIBRARY_DIRS.) 81 | # USE_PKG_CONFIG := 1 82 | 83 | BUILD_DIR := build 84 | DISTRIBUTE_DIR := distribute 85 | 86 | # Uncomment for debugging. Does not work on OSX due to https://github.com/BVLC/caffe/issues/171 87 | # DEBUG := 1 88 | 89 | # The ID of the GPU that 'make runtest' will use to run unit tests. 90 | TEST_GPUID := 0 91 | 92 | # enable pretty build (comment to see full commands) 93 | Q ?= @ 94 | -------------------------------------------------------------------------------- /includes/processAll.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf-8 3 | 4 | """ 5 | This package contains code for the "CRF-RNN" semantic image segmentation method, published in the 6 | ICCV 2015 paper Conditional Random Fields as Recurrent Neural Networks. Our software is built on 7 | top of the Caffe deep learning library. 8 | 9 | Contact: 10 | Shuai Zheng (szheng@robots.ox.ac.uk), Sadeep Jayasumana (sadeep@robots.ox.ac.uk), Bernardino Romera-Paredes (bernard@robots.ox.ac.uk) 11 | 12 | Supervisor: 13 | Philip Torr (philip.torr@eng.ox.ac.uk) 14 | 15 | For more information about CRF-RNN, please vist the project website http://crfasrnn.torr.vision. 16 | """ 17 | 18 | import sys 19 | import time 20 | import getopt 21 | import os 22 | import numpy as np 23 | from PIL import Image as PILImage 24 | 25 | # Path of the Caffe installation. 26 | _CAFFE_ROOT = "/work/crfasrnn/caffe" 27 | 28 | # Model definition and model file paths 29 | # Contains the network definition 30 | _MODEL_DEF_FILE = "/work/crfasrnn/python-scripts/TVG_CRFRNN_new_deploy.prototxt.dg" 31 | # Contains the trained weights. Download from http://goo.gl/j7PrPZ 32 | _MODEL_FILE = "/work/model/TVG_CRFRNN_COCO_VOC.caffemodel" 33 | 34 | sys.path.insert(0, _CAFFE_ROOT + "/python") 35 | sys.path.insert(0, _CAFFE_ROOT + "/python/caffe") 36 | import os 37 | 38 | os.environ['PYCAFFE_ROOT'] = _CAFFE_ROOT 39 | os.environ['CAFFE_ROOT'] = _CAFFE_ROOT 40 | os.environ['PYTHONPATH'] = _CAFFE_ROOT + "/python" 41 | os.environ['CAFFE_ROOT'] = _CAFFE_ROOT 42 | os.environ['PATH'] = '/work/crfasrnn/caffe/build/tools:/work/crfasrnn/caffe/python:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' 43 | 44 | import caffe 45 | 46 | _MAX_DIM = 500 47 | 48 | 49 | def apply_mask(img, mask, color): 50 | len_x = mask.shape[0] 51 | len_y = mask.shape[1] 52 | for x in range(len_x): 53 | for y in range(len_y): 54 | if mask[x, y] != color: 55 | img[x, y][0] = 0 56 | img[x, y][1] = 0 57 | img[x, y][2] = 0 58 | return img 59 | 60 | 61 | def get_palette(num_cls): 62 | """ Returns the color map for visualizing the segmentation mask. 63 | 64 | Args: 65 | num_cls: Number of classes 66 | 67 | Returns: 68 | The color map 69 | """ 70 | 71 | n = num_cls 72 | palette = [0] * (n * 3) 73 | for j in xrange(0, n): 74 | lab = j 75 | palette[j * 3 + 0] = 0 76 | palette[j * 3 + 1] = 0 77 | palette[j * 3 + 2] = 0 78 | i = 0 79 | while lab: 80 | palette[j * 3 + 0] |= (((lab >> 0) & 1) << (7 - i)) 81 | palette[j * 3 + 1] |= (((lab >> 1) & 1) << (7 - i)) 82 | palette[j * 3 + 2] |= (((lab >> 2) & 1) << (7 - i)) 83 | i += 1 84 | lab >>= 3 85 | return palette 86 | 87 | 88 | def crfrnn_segmenter(model_def_file, model_file, gpu_device, inputs): 89 | """ Returns the segmentation of the given image. 90 | 91 | Args: 92 | model_def_file: File path of the Caffe model definition prototxt file 93 | model_file: File path of the trained model file (contains trained weights) 94 | gpu_device: ID of the GPU device. If using the CPU, set this to -1 95 | inputs: List of images to be segmented 96 | 97 | Returns: 98 | The segmented image 99 | """ 100 | 101 | assert os.path.isfile( 102 | model_def_file), "File {} is missing".format(model_def_file) 103 | assert os.path.isfile(model_file), ("File {} is missing. Please download it using " 104 | "./download_trained_model.sh").format(model_file) 105 | 106 | if gpu_device >= 0: 107 | caffe.set_device(gpu_device) 108 | caffe.set_mode_gpu() 109 | else: 110 | caffe.set_mode_cpu() 111 | 112 | net = caffe.Net(model_def_file, model_file, caffe.TEST) 113 | 114 | num_images = len(inputs) 115 | num_channels = inputs[0].shape[2] 116 | assert num_channels == 3, "Unexpected channel count. A 3-channel RGB image is exptected." 117 | 118 | caffe_in = np.zeros( 119 | (num_images, 120 | num_channels, 121 | _MAX_DIM, 122 | _MAX_DIM), 123 | dtype=np.float32) 124 | for ix, in_ in enumerate(inputs): 125 | caffe_in[ix] = in_.transpose((2, 0, 1)) 126 | 127 | start_time = time.time() 128 | out = net.forward_all(**{net.inputs[0]: caffe_in}) 129 | end_time = time.time() 130 | 131 | print( 132 | "Time taken to run the network: {:.4f} seconds".format( 133 | end_time - 134 | start_time)) 135 | predictions = out[net.outputs[0]] 136 | 137 | return predictions[0].argmax(axis=0).astype(np.uint8) 138 | 139 | 140 | def run_crfrnn(input_file, output_file, gpu_device): 141 | """ Runs the CRF-RNN segmentation on the given RGB image and saves the segmentation mask. 142 | 143 | Args: 144 | input_file: Input RGB image file (e.g. in JPEG format) 145 | output_file: Path to save the resulting segmentation in PNG format 146 | gpu_device: ID of the GPU device. If using the CPU, set this to -1 147 | """ 148 | 149 | input_image = 255 * caffe.io.load_image(input_file) 150 | input_image = resize_image(input_image) 151 | 152 | image = PILImage.fromarray(np.uint8(input_image)) 153 | image = np.array(image) 154 | 155 | palette = get_palette(256) 156 | # PIL reads image in the form of RGB, while cv2 reads image in the form of 157 | # BGR, mean_vec = [R,G,B] 158 | mean_vec = np.array([123.68, 116.779, 103.939], dtype=np.float32) 159 | mean_vec = mean_vec.reshape(1, 1, 3) 160 | 161 | # Rearrange channels to form BGR 162 | im = image[:, :, ::-1] 163 | # Subtract mean 164 | im = im - mean_vec 165 | 166 | # Pad as necessary 167 | cur_h, cur_w, cur_c = im.shape 168 | pad_h = _MAX_DIM - cur_h 169 | pad_w = _MAX_DIM - cur_w 170 | im = np.pad(im, pad_width=((0, pad_h), (0, pad_w), (0, 0)), 171 | mode='constant', constant_values=0) 172 | 173 | # Get predictions 174 | segmentation = crfrnn_segmenter( 175 | _MODEL_DEF_FILE, _MODEL_FILE, gpu_device, [im]) 176 | segmentation = segmentation[0:cur_h, 0:cur_w] 177 | 178 | output_im = PILImage.fromarray(segmentation) 179 | 180 | # compute and mask everything but humans 181 | output_im.putpalette(palette) 182 | 183 | real_output = apply_mask(image, segmentation, 15) 184 | output_im = PILImage.fromarray(real_output) 185 | output_im.save(output_file) 186 | 187 | # return (image,segmentation) 188 | 189 | 190 | def resize_image(image): 191 | """ Resizes the image so that the largest dimension is not larger than 500 pixels. 192 | If the image's largest dimension is already less than 500, no changes are made. 193 | 194 | Args: 195 | Input image 196 | 197 | Returns: 198 | Resized image where the largest dimension is less than 500 pixels 199 | """ 200 | 201 | width, height = image.shape[0], image.shape[1] 202 | max_dim = max(width, height) 203 | 204 | if max_dim > _MAX_DIM: 205 | if height > width: 206 | ratio = float(_MAX_DIM) / height 207 | else: 208 | ratio = float(_MAX_DIM) / width 209 | image = PILImage.fromarray(np.uint8(image)) 210 | image = image.resize( 211 | (int(height * ratio), int(width * ratio)), resample=PILImage.BILINEAR) 212 | image = np.array(image) 213 | 214 | return image 215 | 216 | 217 | def run_ppl_only(in_filename, out_filename): 218 | run_crfrnn(in_filename, out_filename, -1) 219 | 220 | 221 | def find_images(topdir): 222 | retval = [] 223 | exten = ['jpg', 'bmp', 'png'] 224 | 225 | for dirpath, dirnames, files in os.walk(topdir): 226 | for name in files: 227 | if name.lower().split('.')[-1] in exten: 228 | retval.append(name) 229 | #retval.append(os.path.join(dirpath, name)) 230 | return retval 231 | 232 | 233 | def main(): 234 | print 'Starting' 235 | print sys.argv, len(sys.argv) 236 | 237 | if len(sys.argv) != 1: 238 | run_crfrnn('/work/flask/static/top.jpg', 239 | '/work/flask/static/bottom.bmp', -1) 240 | return 0 241 | 242 | work_in = '/work/inputImages' 243 | work_out = '/work/outputImages' 244 | work_items = find_images(work_in) 245 | 246 | for work_item in work_items: 247 | in_filename = os.path.join(work_in, work_item) 248 | out_work_item = work_item.split('.')[0] 249 | out_work_item += '.png' 250 | print(work_item, out_work_item) 251 | out_filename = os.path.join(work_out, out_work_item) 252 | print(in_filename, out_filename) 253 | run_ppl_only(in_filename, out_filename) 254 | 255 | 256 | if __name__ == '__main__': 257 | main() 258 | -------------------------------------------------------------------------------- /includes/JooJanta200.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": { 7 | "collapsed": true 8 | }, 9 | "outputs": [], 10 | "source": [ 11 | "# -*- coding: utf-8 -*-\n", 12 | "\"\"\"\n", 13 | "This package contains code for the \"CRF-RNN\" semantic image segmentation method, published in the\n", 14 | "ICCV 2015 paper Conditional Random Fields as Recurrent Neural Networks. Our software is built on\n", 15 | "top of the Caffe deep learning library.\n", 16 | "\n", 17 | "Contact:\n", 18 | "Shuai Zheng (szheng@robots.ox.ac.uk), Sadeep Jayasumana (sadeep@robots.ox.ac.uk), Bernardino Romera-Paredes (bernard@robots.ox.ac.uk)\n", 19 | "\n", 20 | "Supervisor:\n", 21 | "Philip Torr (philip.torr@eng.ox.ac.uk)\n", 22 | "\n", 23 | "For more information about CRF-RNN, please vist the project website http://crfasrnn.torr.vision.\n", 24 | "\"\"\"\n", 25 | "\n", 26 | "import sys\n", 27 | "import time\n", 28 | "import getopt\n", 29 | "import os\n", 30 | "import numpy as np\n", 31 | "from PIL import Image as PILImage\n", 32 | "\n", 33 | "# Path of the Caffe installation.\n", 34 | "_CAFFE_ROOT = \"/work/crfasrnn/caffe\"\n", 35 | "\n", 36 | "# Model definition and model file paths\n", 37 | "_MODEL_DEF_FILE = \"/work/crfasrnn/python-scripts/TVG_CRFRNN_new_deploy.prototxt.dg\" # Contains the network definition\n", 38 | "_MODEL_FILE = \"/work/model/TVG_CRFRNN_COCO_VOC.caffemodel\" # Contains the trained weights. Download from http://goo.gl/j7PrPZ\n", 39 | "\n", 40 | "sys.path.insert(0, _CAFFE_ROOT + \"/python\")\n", 41 | "sys.path.insert(0, _CAFFE_ROOT + \"/python/caffe\")\n", 42 | "import os\n", 43 | "\n", 44 | "os.environ['PYCAFFE_ROOT'] = _CAFFE_ROOT\n", 45 | "os.environ['CAFFE_ROOT'] = _CAFFE_ROOT\n", 46 | "os.environ['PYTHONPATH'] = _CAFFE_ROOT + \"/python\"\n", 47 | "os.environ['CAFFE_ROOT'] = _CAFFE_ROOT\n", 48 | "os.environ['PATH'] = '/work/crfasrnn/caffe/build/tools:/work/crfasrnn/caffe/python:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'\n", 49 | "\n", 50 | "import caffe\n", 51 | "\n", 52 | "_MAX_DIM = 500\n", 53 | "\n", 54 | "def apply_mask(img,mask,color):\n", 55 | " len_x = mask.shape[0]\n", 56 | " len_y = mask.shape[1]\n", 57 | " for x in range(len_x):\n", 58 | " for y in range(len_y):\n", 59 | " if mask[x,y] != color:\n", 60 | " img[x,y][0] = 0\n", 61 | " img[x,y][1] = 0\n", 62 | " img[x,y][2] = 0\n", 63 | " return img\n", 64 | "\n", 65 | "\n", 66 | "def get_palette(num_cls):\n", 67 | " \"\"\" Returns the color map for visualizing the segmentation mask.\n", 68 | "\n", 69 | " Args:\n", 70 | " num_cls: Number of classes\n", 71 | "\n", 72 | " Returns:\n", 73 | " The color map\n", 74 | " \"\"\"\n", 75 | "\n", 76 | " n = num_cls\n", 77 | " palette = [0] * (n * 3)\n", 78 | " for j in xrange(0, n):\n", 79 | " lab = j\n", 80 | " palette[j * 3 + 0] = 0\n", 81 | " palette[j * 3 + 1] = 0\n", 82 | " palette[j * 3 + 2] = 0\n", 83 | " i = 0\n", 84 | " while lab:\n", 85 | " palette[j * 3 + 0] |= (((lab >> 0) & 1) << (7 - i))\n", 86 | " palette[j * 3 + 1] |= (((lab >> 1) & 1) << (7 - i))\n", 87 | " palette[j * 3 + 2] |= (((lab >> 2) & 1) << (7 - i))\n", 88 | " i += 1\n", 89 | " lab >>= 3\n", 90 | " return palette\n", 91 | "\n", 92 | "\n", 93 | "def crfrnn_segmenter(model_def_file, model_file, gpu_device, inputs):\n", 94 | " \"\"\" Returns the segmentation of the given image.\n", 95 | "\n", 96 | " Args:\n", 97 | " model_def_file: File path of the Caffe model definition prototxt file\n", 98 | " model_file: File path of the trained model file (contains trained weights)\n", 99 | " gpu_device: ID of the GPU device. If using the CPU, set this to -1\n", 100 | " inputs: List of images to be segmented \n", 101 | "\n", 102 | " Returns:\n", 103 | " The segmented image\n", 104 | " \"\"\"\n", 105 | " \n", 106 | " assert os.path.isfile(model_def_file), \"File {} is missing\".format(model_def_file)\n", 107 | " assert os.path.isfile(model_file), (\"File {} is missing. Please download it using \"\n", 108 | " \"./download_trained_model.sh\").format(model_file)\n", 109 | "\n", 110 | " if gpu_device >= 0:\n", 111 | " caffe.set_device(gpu_device)\n", 112 | " caffe.set_mode_gpu()\n", 113 | " else:\n", 114 | " caffe.set_mode_cpu()\n", 115 | "\n", 116 | " net = caffe.Net(model_def_file, model_file, caffe.TEST)\n", 117 | "\n", 118 | " num_images = len(inputs)\n", 119 | " num_channels = inputs[0].shape[2]\n", 120 | " assert num_channels == 3, \"Unexpected channel count. A 3-channel RGB image is exptected.\"\n", 121 | " \n", 122 | " caffe_in = np.zeros((num_images, num_channels, _MAX_DIM, _MAX_DIM), dtype=np.float32)\n", 123 | " for ix, in_ in enumerate(inputs):\n", 124 | " caffe_in[ix] = in_.transpose((2, 0, 1))\n", 125 | "\n", 126 | " start_time = time.time()\n", 127 | " out = net.forward_all(**{net.inputs[0]: caffe_in})\n", 128 | " end_time = time.time()\n", 129 | "\n", 130 | " print(\"Time taken to run the network: {:.4f} seconds\".format(end_time - start_time))\n", 131 | " predictions = out[net.outputs[0]]\n", 132 | "\n", 133 | " return predictions[0].argmax(axis=0).astype(np.uint8)\n", 134 | "\n", 135 | "\n", 136 | "def run_crfrnn(input_file, output_file, gpu_device):\n", 137 | " \"\"\" Runs the CRF-RNN segmentation on the given RGB image and saves the segmentation mask.\n", 138 | "\n", 139 | " Args:\n", 140 | " input_file: Input RGB image file (e.g. in JPEG format)\n", 141 | " output_file: Path to save the resulting segmentation in PNG format\n", 142 | " gpu_device: ID of the GPU device. If using the CPU, set this to -1\n", 143 | " \"\"\"\n", 144 | "\n", 145 | " input_image = 255 * caffe.io.load_image(input_file)\n", 146 | " input_image = resize_image(input_image)\n", 147 | "\n", 148 | " image = PILImage.fromarray(np.uint8(input_image))\n", 149 | " image = np.array(image)\n", 150 | "\n", 151 | " palette = get_palette(256)\n", 152 | " #PIL reads image in the form of RGB, while cv2 reads image in the form of BGR, mean_vec = [R,G,B] \n", 153 | " mean_vec = np.array([123.68, 116.779, 103.939], dtype=np.float32)\n", 154 | " mean_vec = mean_vec.reshape(1, 1, 3)\n", 155 | "\n", 156 | " # Rearrange channels to form BGR\n", 157 | " im = image[:, :, ::-1]\n", 158 | " # Subtract mean\n", 159 | " im = im - mean_vec\n", 160 | "\n", 161 | " # Pad as necessary\n", 162 | " cur_h, cur_w, cur_c = im.shape\n", 163 | " pad_h = _MAX_DIM - cur_h\n", 164 | " pad_w = _MAX_DIM - cur_w\n", 165 | " im = np.pad(im, pad_width=((0, pad_h), (0, pad_w), (0, 0)), mode='constant', constant_values=0)\n", 166 | "\n", 167 | " # Get predictions\n", 168 | " segmentation = crfrnn_segmenter(_MODEL_DEF_FILE, _MODEL_FILE, gpu_device, [im])\n", 169 | " segmentation = segmentation[0:cur_h, 0:cur_w]\n", 170 | "\n", 171 | " output_im = PILImage.fromarray(segmentation)\n", 172 | "\n", 173 | " #compute and mask everything but humans\n", 174 | " output_im.putpalette(palette)\n", 175 | " \n", 176 | "\n", 177 | " real_output = apply_mask(image,segmentation,15)\n", 178 | " output_im = PILImage.fromarray(real_output)\n", 179 | " output_im.save(output_file)\n", 180 | "\n", 181 | " \n", 182 | " #return (image,segmentation)\n", 183 | "\n", 184 | "def resize_image(image):\n", 185 | " \"\"\" Resizes the image so that the largest dimension is not larger than 500 pixels.\n", 186 | " If the image's largest dimension is already less than 500, no changes are made.\n", 187 | "\n", 188 | " Args:\n", 189 | " Input image\n", 190 | "\n", 191 | " Returns:\n", 192 | " Resized image where the largest dimension is less than 500 pixels\n", 193 | " \"\"\"\n", 194 | "\n", 195 | " width, height = image.shape[0], image.shape[1]\n", 196 | " max_dim = max(width, height)\n", 197 | "\n", 198 | " if max_dim > _MAX_DIM:\n", 199 | " if height > width:\n", 200 | " ratio = float(_MAX_DIM) / height\n", 201 | " else:\n", 202 | " ratio = float(_MAX_DIM) / width\n", 203 | " image = PILImage.fromarray(np.uint8(image))\n", 204 | " image = image.resize((int(height * ratio), int(width * ratio)), resample=PILImage.BILINEAR)\n", 205 | " image = np.array(image)\n", 206 | "\n", 207 | " return image" 208 | ] 209 | }, 210 | { 211 | "cell_type": "code", 212 | "execution_count": null, 213 | "metadata": { 214 | "collapsed": true 215 | }, 216 | "outputs": [], 217 | "source": [ 218 | "def run_ppl_only(in_filename, out_filename):\n", 219 | " run_crfrnn(in_filename,out_filename,-1)\n", 220 | " " 221 | ] 222 | }, 223 | { 224 | "cell_type": "code", 225 | "execution_count": null, 226 | "metadata": { 227 | "collapsed": true 228 | }, 229 | "outputs": [], 230 | "source": [ 231 | "def find_images(topdir):\n", 232 | " retval = []\n", 233 | " exten = ['jpg', 'bmp', 'png']\n", 234 | "\n", 235 | " for dirpath, dirnames, files in os.walk(topdir):\n", 236 | " for name in files:\n", 237 | " if name.lower().split('.')[-1] in exten:\n", 238 | " retval.append(name)\n", 239 | " #retval.append(os.path.join(dirpath, name))\n", 240 | " return retval\n", 241 | "\n", 242 | "def main():\n", 243 | " print 'here'\n", 244 | " work_in = '/work/inputImages'\n", 245 | " work_out = '/work/outputImages'\n", 246 | " work_items = find_images(work_in)\n", 247 | " \n", 248 | " for work_item in work_items:\n", 249 | " in_filename = os.path.join(work_in,work_item)\n", 250 | " out_work_item = work_item.split('.')[0]\n", 251 | " out_work_item += '.png'\n", 252 | " print(work_item,out_work_item)\n", 253 | " out_filename = os.path.join(work_out,out_work_item)\n", 254 | " print(in_filename,out_filename)\n", 255 | " run_ppl_only(in_filename,out_filename)\n" 256 | ] 257 | }, 258 | { 259 | "cell_type": "code", 260 | "execution_count": null, 261 | "metadata": {}, 262 | "outputs": [], 263 | "source": [ 264 | "#x = run_crfrnn('./crfasrnn/python-scripts/input.jpg','./output.png',-1)" 265 | ] 266 | }, 267 | { 268 | "cell_type": "code", 269 | "execution_count": null, 270 | "metadata": {}, 271 | "outputs": [], 272 | "source": [ 273 | "#from IPython.display import Image\n", 274 | "#image = Image('./output.png')\n", 275 | "#image\n" 276 | ] 277 | }, 278 | { 279 | "cell_type": "code", 280 | "execution_count": null, 281 | "metadata": { 282 | "collapsed": true 283 | }, 284 | "outputs": [], 285 | "source": [] 286 | } 287 | ], 288 | "metadata": { 289 | "kernelspec": { 290 | "display_name": "Python 2", 291 | "language": "python", 292 | "name": "python2" 293 | }, 294 | "language_info": { 295 | "codemirror_mode": { 296 | "name": "ipython", 297 | "version": 2 298 | }, 299 | "file_extension": ".py", 300 | "mimetype": "text/x-python", 301 | "name": "python", 302 | "nbconvert_exporter": "python", 303 | "pygments_lexer": "ipython2", 304 | "version": "2.7.12" 305 | } 306 | }, 307 | "nbformat": 4, 308 | "nbformat_minor": 2 309 | } 310 | 311 | -------------------------------------------------------------------------------- /includes/TVG_CRFRNN_new_deploy.prototxt.dg: -------------------------------------------------------------------------------- 1 | name: "CRFRNN-VOC" 2 | input: 'data' 3 | input_dim: 1 4 | input_dim: 3 5 | input_dim: 500 6 | input_dim: 500 7 | force_backward: true 8 | 9 | layer { 10 | name: "conv1_1" 11 | type: "Convolution" 12 | bottom: "data" 13 | top: "conv1_1" 14 | param { 15 | lr_mult: 1 16 | decay_mult: 1 17 | } 18 | param { 19 | lr_mult: 2 20 | decay_mult: 0 21 | } 22 | convolution_param { 23 | num_output: 64 24 | pad: 100 25 | kernel_size: 3 26 | engine: CAFFE 27 | } 28 | } 29 | layer { 30 | name: "relu1_1" 31 | type: "ReLU" 32 | bottom: "conv1_1" 33 | top: "conv1_1" 34 | } 35 | layer { 36 | name: "conv1_2" 37 | type: "Convolution" 38 | bottom: "conv1_1" 39 | top: "conv1_2" 40 | param { 41 | lr_mult: 1 42 | decay_mult: 1 43 | } 44 | param { 45 | lr_mult: 2 46 | decay_mult: 0 47 | } 48 | convolution_param { 49 | num_output: 64 50 | pad: 1 51 | kernel_size: 3 52 | engine: CAFFE 53 | } 54 | } 55 | layer { 56 | name: "relu1_2" 57 | type: "ReLU" 58 | bottom: "conv1_2" 59 | top: "conv1_2" 60 | } 61 | layer { 62 | name: "pool1" 63 | type: "Pooling" 64 | bottom: "conv1_2" 65 | top: "pool1" 66 | pooling_param { 67 | pool: MAX 68 | kernel_size: 2 69 | stride: 2 70 | } 71 | } 72 | layer { 73 | name: "conv2_1" 74 | type: "Convolution" 75 | bottom: "pool1" 76 | top: "conv2_1" 77 | param { 78 | lr_mult: 1 79 | decay_mult: 1 80 | } 81 | param { 82 | lr_mult: 2 83 | decay_mult: 0 84 | } 85 | convolution_param { 86 | num_output: 128 87 | pad: 1 88 | kernel_size: 3 89 | engine: CAFFE 90 | } 91 | } 92 | layer { 93 | name: "relu2_1" 94 | type: "ReLU" 95 | bottom: "conv2_1" 96 | top: "conv2_1" 97 | } 98 | layer { 99 | name: "conv2_2" 100 | type: "Convolution" 101 | bottom: "conv2_1" 102 | top: "conv2_2" 103 | param { 104 | lr_mult: 1 105 | decay_mult: 1 106 | } 107 | param { 108 | lr_mult: 2 109 | decay_mult: 0 110 | } 111 | convolution_param { 112 | num_output: 128 113 | pad: 1 114 | kernel_size: 3 115 | engine: CAFFE 116 | } 117 | } 118 | layer { 119 | name: "relu2_2" 120 | type: "ReLU" 121 | bottom: "conv2_2" 122 | top: "conv2_2" 123 | } 124 | layer { 125 | name: "pool2" 126 | type: "Pooling" 127 | bottom: "conv2_2" 128 | top: "pool2" 129 | pooling_param { 130 | pool: MAX 131 | kernel_size: 2 132 | stride: 2 133 | } 134 | } 135 | layer { 136 | name: "conv3_1" 137 | type: "Convolution" 138 | bottom: "pool2" 139 | top: "conv3_1" 140 | param { 141 | lr_mult: 1 142 | decay_mult: 1 143 | } 144 | param { 145 | lr_mult: 2 146 | decay_mult: 0 147 | } 148 | convolution_param { 149 | num_output: 256 150 | pad: 1 151 | kernel_size: 3 152 | engine: CAFFE 153 | } 154 | } 155 | layer { 156 | name: "relu3_1" 157 | type: "ReLU" 158 | bottom: "conv3_1" 159 | top: "conv3_1" 160 | } 161 | layer { 162 | name: "conv3_2" 163 | type: "Convolution" 164 | bottom: "conv3_1" 165 | top: "conv3_2" 166 | param { 167 | lr_mult: 1 168 | decay_mult: 1 169 | } 170 | param { 171 | lr_mult: 2 172 | decay_mult: 0 173 | } 174 | convolution_param { 175 | num_output: 256 176 | pad: 1 177 | kernel_size: 3 178 | engine: CAFFE 179 | } 180 | } 181 | layer { 182 | name: "relu3_2" 183 | type: "ReLU" 184 | bottom: "conv3_2" 185 | top: "conv3_2" 186 | } 187 | layer { 188 | name: "conv3_3" 189 | type: "Convolution" 190 | bottom: "conv3_2" 191 | top: "conv3_3" 192 | param { 193 | lr_mult: 1 194 | decay_mult: 1 195 | } 196 | param { 197 | lr_mult: 2 198 | decay_mult: 0 199 | } 200 | convolution_param { 201 | num_output: 256 202 | pad: 1 203 | kernel_size: 3 204 | engine: CAFFE 205 | } 206 | } 207 | layer { 208 | name: "relu3_3" 209 | type: "ReLU" 210 | bottom: "conv3_3" 211 | top: "conv3_3" 212 | } 213 | layer { 214 | name: "pool3" 215 | type: "Pooling" 216 | bottom: "conv3_3" 217 | top: "pool3" 218 | pooling_param { 219 | pool: MAX 220 | kernel_size: 2 221 | stride: 2 222 | } 223 | } 224 | layer { 225 | name: "conv4_1" 226 | type: "Convolution" 227 | bottom: "pool3" 228 | top: "conv4_1" 229 | param { 230 | lr_mult: 1 231 | decay_mult: 1 232 | } 233 | param { 234 | lr_mult: 2 235 | decay_mult: 0 236 | } 237 | convolution_param { 238 | num_output: 512 239 | pad: 1 240 | kernel_size: 3 241 | engine: CAFFE 242 | } 243 | } 244 | layer { 245 | name: "relu4_1" 246 | type: "ReLU" 247 | bottom: "conv4_1" 248 | top: "conv4_1" 249 | } 250 | layer { 251 | name: "conv4_2" 252 | type: "Convolution" 253 | bottom: "conv4_1" 254 | top: "conv4_2" 255 | param { 256 | lr_mult: 1 257 | decay_mult: 1 258 | } 259 | param { 260 | lr_mult: 2 261 | decay_mult: 0 262 | } 263 | convolution_param { 264 | num_output: 512 265 | pad: 1 266 | kernel_size: 3 267 | engine: CAFFE 268 | } 269 | } 270 | layer { 271 | name: "relu4_2" 272 | type: "ReLU" 273 | bottom: "conv4_2" 274 | top: "conv4_2" 275 | } 276 | layer { 277 | name: "conv4_3" 278 | type: "Convolution" 279 | bottom: "conv4_2" 280 | top: "conv4_3" 281 | param { 282 | lr_mult: 1 283 | decay_mult: 1 284 | } 285 | param { 286 | lr_mult: 2 287 | decay_mult: 0 288 | } 289 | convolution_param { 290 | num_output: 512 291 | pad: 1 292 | kernel_size: 3 293 | engine: CAFFE 294 | } 295 | } 296 | layer { 297 | name: "relu4_3" 298 | type: "ReLU" 299 | bottom: "conv4_3" 300 | top: "conv4_3" 301 | } 302 | layer { 303 | name: "pool4" 304 | type: "Pooling" 305 | bottom: "conv4_3" 306 | top: "pool4" 307 | pooling_param { 308 | pool: MAX 309 | kernel_size: 2 310 | stride: 2 311 | } 312 | } 313 | layer { 314 | name: "conv5_1" 315 | type: "Convolution" 316 | bottom: "pool4" 317 | top: "conv5_1" 318 | param { 319 | lr_mult: 1 320 | decay_mult: 1 321 | } 322 | param { 323 | lr_mult: 2 324 | decay_mult: 0 325 | } 326 | convolution_param { 327 | num_output: 512 328 | pad: 1 329 | kernel_size: 3 330 | engine: CAFFE 331 | } 332 | } 333 | layer { 334 | name: "relu5_1" 335 | type: "ReLU" 336 | bottom: "conv5_1" 337 | top: "conv5_1" 338 | } 339 | layer { 340 | name: "conv5_2" 341 | type: "Convolution" 342 | bottom: "conv5_1" 343 | top: "conv5_2" 344 | param { 345 | lr_mult: 1 346 | decay_mult: 1 347 | } 348 | param { 349 | lr_mult: 2 350 | decay_mult: 0 351 | } 352 | convolution_param { 353 | num_output: 512 354 | pad: 1 355 | kernel_size: 3 356 | engine: CAFFE 357 | } 358 | } 359 | layer { 360 | name: "relu5_2" 361 | type: "ReLU" 362 | bottom: "conv5_2" 363 | top: "conv5_2" 364 | } 365 | layer { 366 | name: "conv5_3" 367 | type: "Convolution" 368 | bottom: "conv5_2" 369 | top: "conv5_3" 370 | param { 371 | lr_mult: 1 372 | decay_mult: 1 373 | } 374 | param { 375 | lr_mult: 2 376 | decay_mult: 0 377 | } 378 | convolution_param { 379 | num_output: 512 380 | pad: 1 381 | kernel_size: 3 382 | engine: CAFFE 383 | } 384 | } 385 | layer { 386 | name: "relu5_3" 387 | type: "ReLU" 388 | bottom: "conv5_3" 389 | top: "conv5_3" 390 | } 391 | layer { 392 | name: "pool5" 393 | type: "Pooling" 394 | bottom: "conv5_3" 395 | top: "pool5" 396 | pooling_param { 397 | pool: MAX 398 | kernel_size: 2 399 | stride: 2 400 | } 401 | } 402 | layer { 403 | name: "fc6" 404 | type: "Convolution" 405 | bottom: "pool5" 406 | top: "fc6" 407 | param { 408 | lr_mult: 1 409 | decay_mult: 1 410 | } 411 | param { 412 | lr_mult: 2 413 | decay_mult: 0 414 | } 415 | convolution_param { 416 | num_output: 4096 417 | kernel_size: 7 418 | engine: CAFFE 419 | } 420 | } 421 | layer { 422 | name: "relu6" 423 | type: "ReLU" 424 | bottom: "fc6" 425 | top: "fc6" 426 | } 427 | layer { 428 | name: "drop6" 429 | type: "Dropout" 430 | bottom: "fc6" 431 | top: "fc6" 432 | dropout_param { 433 | dropout_ratio: 0.5 434 | } 435 | } 436 | layer { 437 | name: "fc7" 438 | type: "Convolution" 439 | bottom: "fc6" 440 | top: "fc7" 441 | param { 442 | lr_mult: 1 443 | decay_mult: 1 444 | } 445 | param { 446 | lr_mult: 2 447 | decay_mult: 0 448 | } 449 | convolution_param { 450 | num_output: 4096 451 | kernel_size: 1 452 | engine: CAFFE 453 | } 454 | } 455 | layer { 456 | name: "relu7" 457 | type: "ReLU" 458 | bottom: "fc7" 459 | top: "fc7" 460 | } 461 | layer { 462 | name: "drop7" 463 | type: "Dropout" 464 | bottom: "fc7" 465 | top: "fc7" 466 | dropout_param { 467 | dropout_ratio: 0.5 468 | } 469 | } 470 | layer { 471 | name: "score-fr" 472 | type: "Convolution" 473 | bottom: "fc7" 474 | top: "score" 475 | param { 476 | lr_mult: 1 477 | decay_mult: 1 478 | } 479 | param { 480 | lr_mult: 2 481 | decay_mult: 0 482 | } 483 | convolution_param { 484 | num_output: 21 485 | kernel_size: 1 486 | engine: CAFFE 487 | } 488 | } 489 | layer { 490 | name: "score2" 491 | type: "Deconvolution" 492 | bottom: "score" 493 | top: "score2" 494 | param { 495 | lr_mult: 1 496 | } 497 | convolution_param { 498 | num_output: 21 499 | kernel_size: 4 500 | stride: 2 501 | weight_filler: { type: "bilinear" } 502 | } 503 | } 504 | layer { 505 | name: "score-pool4" 506 | type: "Convolution" 507 | bottom: "pool4" 508 | top: "score-pool4" 509 | param { 510 | lr_mult: 1 511 | decay_mult: 1 512 | } 513 | param { 514 | lr_mult: 2 515 | decay_mult: 0 516 | } 517 | convolution_param { 518 | num_output: 21 519 | kernel_size: 1 520 | engine: CAFFE 521 | } 522 | } 523 | layer { type: 'Crop' name: 'crop' bottom: 'score-pool4' bottom: 'score2' 524 | top: 'score-pool4c' } 525 | layer { 526 | name: "fuse" 527 | type: "Eltwise" 528 | bottom: "score2" 529 | bottom: "score-pool4c" 530 | top: "score-fused" 531 | eltwise_param { 532 | operation: SUM 533 | } 534 | } 535 | layer { 536 | name: "score4" 537 | type: "Deconvolution" 538 | bottom: "score-fused" 539 | top: "score4" 540 | param { 541 | lr_mult: 1 542 | decay_mult: 1 543 | } 544 | convolution_param { 545 | num_output: 21 546 | bias_term: false 547 | kernel_size: 4 548 | stride: 2 549 | } 550 | } 551 | layer { 552 | name: "score-pool3" 553 | type: "Convolution" 554 | bottom: "pool3" 555 | top: "score-pool3" 556 | param { 557 | lr_mult: 1 558 | decay_mult: 1 559 | } 560 | param { 561 | lr_mult: 2 562 | decay_mult: 0 563 | } 564 | convolution_param { 565 | num_output: 21 566 | kernel_size: 1 567 | engine: CAFFE 568 | } 569 | } 570 | layer { type: 'Crop' name: 'crop' bottom: 'score-pool3' bottom: 'score4' 571 | top: 'score-pool3c' } 572 | layer { 573 | name: "fuse" 574 | type: "Eltwise" 575 | bottom: "score4" 576 | bottom: "score-pool3c" 577 | top: "score-final" 578 | eltwise_param { 579 | operation: SUM 580 | } 581 | } 582 | layer { 583 | name: "upsample" 584 | type: "Deconvolution" 585 | bottom: "score-final" 586 | top: "bigscore" 587 | param { 588 | lr_mult: 0 589 | } 590 | convolution_param { 591 | num_output: 21 592 | bias_term: false 593 | kernel_size: 16 594 | stride: 8 595 | } 596 | } 597 | layer { type: 'Crop' name: 'crop' bottom: 'bigscore' bottom: 'data' top: 'coarse' } 598 | 599 | layer { type: 'Split' name: 'splitting' 600 | bottom: 'coarse' top: 'unary' top: 'Q0' 601 | } 602 | 603 | layer { 604 | name: "inference1"#if you set name "inference1", code will load parameters from caffemodel. 605 | type: "MultiStageMeanfield" 606 | bottom: "unary" 607 | bottom: "Q0" 608 | bottom: "data" 609 | top: "pred" 610 | param { 611 | lr_mult: 10000#learning rate for W_G 612 | } 613 | param { 614 | lr_mult: 10000#learning rate for W_B 615 | } 616 | param { 617 | lr_mult: 1000 #learning rate for compatiblity transform matrix 618 | } 619 | multi_stage_meanfield_param { 620 | num_iterations: 10 621 | compatibility_mode: POTTS#Initialize the compatilibity transform matrix with a matrix whose diagonal is -1. 622 | threshold: 2 623 | theta_alpha: 160 624 | theta_beta: 3 625 | theta_gamma: 3 626 | spatial_filter_weight: 3 627 | bilateral_filter_weight: 5 628 | } 629 | } 630 | --------------------------------------------------------------------------------