├── doc ├── ss_sample.png ├── hierarchy_example.gif ├── showcandidates_scr.png ├── segmentation_example.png ├── showcandidates_scr_more.png └── hierarchy_example_composited.png ├── .gitignore ├── requirements.txt ├── CMakeLists.txt ├── LICENSE.txt ├── color_space.py ├── demo_showhierarchy.py ├── segment_py.cpp ├── test_color_space.py ├── selective_search.py ├── demo_showcandidates.py ├── features.py ├── README.md ├── test_features.py └── test_selective_search.py /doc/ss_sample.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/belltailjp/selective_search_py/HEAD/doc/ss_sample.png -------------------------------------------------------------------------------- /doc/hierarchy_example.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/belltailjp/selective_search_py/HEAD/doc/hierarchy_example.gif -------------------------------------------------------------------------------- /doc/showcandidates_scr.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/belltailjp/selective_search_py/HEAD/doc/showcandidates_scr.png -------------------------------------------------------------------------------- /doc/segmentation_example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/belltailjp/selective_search_py/HEAD/doc/segmentation_example.png -------------------------------------------------------------------------------- /doc/showcandidates_scr_more.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/belltailjp/selective_search_py/HEAD/doc/showcandidates_scr_more.png -------------------------------------------------------------------------------- /doc/hierarchy_example_composited.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/belltailjp/selective_search_py/HEAD/doc/hierarchy_example_composited.png -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__/ 2 | *.py[cod] 3 | *.so 4 | *.png 5 | *.jpg 6 | *.jpeg 7 | CMakeCache.txt 8 | CMakeFiles 9 | Makefile 10 | cmake_install.cmake 11 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | Cython==0.22 2 | joblib==0.8.4 3 | matplotlib==1.4.3 4 | numpy==1.9.2 5 | Pillow==2.8.1 6 | PySide==1.2.2 7 | PyYAML==3.11 8 | scikit-image==0.11.3 9 | scikit-learn==0.16.0 10 | scipy==0.15.1 11 | pytest==2.7.0 12 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 2.8) 2 | 3 | project(selective_search) 4 | 5 | execute_process(COMMAND python3.4-config --cflags OUTPUT_VARIABLE PYTHON3_CFLAGS OUTPUT_STRIP_TRAILING_WHITESPACE) 6 | execute_process(COMMAND python3.4-config --ldflags OUTPUT_VARIABLE PYTHON3_LDFLAGS OUTPUT_STRIP_TRAILING_WHITESPACE) 7 | set(BOOST_LIBS "-lboost_system -lboost_python3 -lboost_numpy") 8 | 9 | set(SUPPRESS_WARNING "-Wall -Wno-unused-function -Wno-unused-variable -Wno-unused-local-typedefs") 10 | 11 | include_directories("segment") 12 | set(CMAKE_CXX_FLAGS "-fPIC -O3 -std=c++11 ${PYTHON3_CFLAGS} ${SUPPRESS_WARNING}") 13 | set(CMAKE_SHARED_LINKER_FLAGS "${PYTHON3_LDFLAGS} -shared -fPIC") 14 | 15 | add_library(segment SHARED segment_py.cpp) 16 | target_link_libraries(segment ${BOOST_LIBS} segment) 17 | set_target_properties(segment PROPERTIES PREFIX "") 18 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2015 Daichi SUZUO 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all 11 | copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | SOFTWARE. 20 | -------------------------------------------------------------------------------- /color_space.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import numpy 5 | import skimage.io 6 | import skimage.color 7 | 8 | def convert_color(I, name): 9 | if len(I.shape) != 3: 10 | I = skimage.color.gray2rgb(I) 11 | 12 | converters = {'rgb' : lambda I: I, 13 | 'lab' : to_Lab, 14 | 'rgi' : to_rgI, 15 | 'hsv' : to_HSV, 16 | 'nrgb' : to_nRGB, 17 | 'hue' : to_Hue} 18 | 19 | return converters[name](I) 20 | 21 | def to_grey(I): 22 | grey_img = (255 * skimage.color.rgb2grey(I)).astype(numpy.uint8) 23 | return numpy.dstack([grey_img, grey_img, grey_img]) 24 | 25 | def to_Lab(I): 26 | lab = skimage.color.rgb2lab(I) 27 | l = 255 * lab[:, :, 0] / 100 # L component ranges from 0 to 100 28 | a = 127 + lab[:, :, 1] # a component ranges from -127 to 127 29 | b = 127 + lab[:, :, 2] # b component ranges from -127 to 127 30 | return numpy.dstack([l, a, b]).astype(numpy.uint8) 31 | 32 | def to_rgI(I): 33 | rgi = I.copy() 34 | rgi[:, :, 2] = to_grey(I)[:, :, 0] 35 | return rgi 36 | 37 | def to_HSV(I): 38 | return (255 * skimage.color.rgb2hsv(I)).astype(numpy.uint8) 39 | 40 | def to_nRGB(I): 41 | _I = I / 255.0 42 | norm_I = numpy.sqrt(_I[:, :, 0] ** 2 + _I[:, :, 1] ** 2 + _I[:, :, 2] ** 2) 43 | norm_r = (_I[:, :, 0] / norm_I * 255).astype(numpy.uint8) 44 | norm_g = (_I[:, :, 1] / norm_I * 255).astype(numpy.uint8) 45 | norm_b = (_I[:, :, 2] / norm_I * 255).astype(numpy.uint8) 46 | return numpy.dstack([norm_r, norm_g, norm_b]) 47 | 48 | def to_Hue(I): 49 | I_h = to_HSV(I)[:, :, 0] 50 | return numpy.dstack([I_h, I_h, I_h]) 51 | 52 | -------------------------------------------------------------------------------- /demo_showhierarchy.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import sys 5 | import argparse 6 | import warnings 7 | import numpy 8 | import skimage.io 9 | import features 10 | import color_space 11 | import selective_search 12 | 13 | def generate_color_table(R): 14 | # generate initial color 15 | colors = numpy.random.randint(0, 255, (len(R), 3)) 16 | 17 | # merged-regions are colored same as larger parent 18 | for region, parent in R.items(): 19 | if not len(parent) == 0: 20 | colors[region] = colors[parent[0]] 21 | 22 | return colors 23 | 24 | if __name__=="__main__": 25 | parser = argparse.ArgumentParser() 26 | parser.add_argument('image', type=str, help='filename of the image') 27 | parser.add_argument('-k', '--k', type=int, default=100, help='threshold k for initial segmentation') 28 | parser.add_argument('-c', '--color', nargs=1, default='rgb', choices=['rgb', 'lab', 'rgi', 'hsv', 'nrgb', 'hue'], help='color space') 29 | parser.add_argument('-f', '--feature', nargs="+", default=['texture', 'fill'], choices=['size', 'color', 'texture', 'fill'], help='feature for similarity calculation') 30 | parser.add_argument('-o', '--output', type=str, default='result', help='prefix of resulting images') 31 | parser.add_argument('-a', '--alpha', type=float, default=1.0, help='alpha value for compositing result image with input image') 32 | args = parser.parse_args() 33 | 34 | img = skimage.io.imread(args.image) 35 | if len(img.shape) == 2: 36 | img = skimage.color.gray2rgb(img) 37 | 38 | print('k:', args.k) 39 | print('color:', args.color) 40 | print('feature:', ' '.join(args.feature)) 41 | 42 | mask = features.SimilarityMask('size' in args.feature, 'color' in args.feature, 'texture' in args.feature, 'fill' in args.feature) 43 | (R, F, L) = selective_search.hierarchical_segmentation(img, args.k, mask) 44 | print('result filename: %s_[0000-%04d].png' % (args.output, len(F) - 1)) 45 | 46 | # suppress warning when saving result images 47 | warnings.filterwarnings("ignore", category = UserWarning) 48 | 49 | colors = generate_color_table(R) 50 | for depth, label in enumerate(F): 51 | result = colors[label] 52 | result = (result * args.alpha + img * (1. - args.alpha)).astype(numpy.uint8) 53 | fn = "%s_%04d.png" % (args.output, depth) 54 | skimage.io.imsave(fn, result) 55 | print('.', end="") 56 | sys.stdout.flush() 57 | 58 | print() 59 | 60 | -------------------------------------------------------------------------------- /segment_py.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include "segment/segment-image.h" 6 | 7 | static int operator<(const rgb& x, const rgb& y) 8 | { 9 | return (x.r << 16 | x.g << 8 | x.b) < (y.r << 16 | y.g << 8 | y.b); 10 | } 11 | 12 | static void check_image_format(const boost::numpy::ndarray& input_image) 13 | { 14 | const int nd = input_image.get_nd(); 15 | if(nd != 3) 16 | throw std::runtime_error("input_image must be 3-dimensional"); 17 | 18 | const int depth = input_image.shape(2); 19 | 20 | if(depth != 3) 21 | throw std::runtime_error("input_image must have rgb channel"); 22 | 23 | if(input_image.get_dtype() != boost::numpy::dtype::get_builtin()) 24 | throw std::runtime_error("dtype of input_image must be uint8"); 25 | 26 | if(!input_image.get_flags() & boost::numpy::ndarray::C_CONTIGUOUS) 27 | throw std::runtime_error("input_image must be C-style contiguous"); 28 | } 29 | 30 | boost::python::tuple segment(const boost::numpy::ndarray& input_image, float sigma, float c, int min_size) 31 | { 32 | check_image_format(input_image); 33 | 34 | const int h = input_image.shape(0); 35 | const int w = input_image.shape(1); 36 | 37 | // Convert to internal format 38 | image seg_input_img(w, h); 39 | rgb* p = reinterpret_cast(input_image.get_data()); 40 | std::copy(p, p + w * h, seg_input_img.data); 41 | 42 | int num_css; 43 | image *seg_result_img = segment_image(&seg_input_img, sigma, c, min_size, &num_css); 44 | 45 | // Convert from internal format 46 | boost::numpy::ndarray result_image = boost::numpy::empty(input_image.get_nd(), input_image.get_shape(), input_image.get_dtype()); 47 | std::copy(seg_result_img->data, seg_result_img->data + w * h, reinterpret_cast(result_image.get_data())); 48 | 49 | delete seg_result_img; 50 | return boost::python::make_tuple(result_image, num_css); 51 | } 52 | 53 | boost::python::tuple segment_label(const boost::numpy::ndarray& input_image, float sigma, float c, int min_size) 54 | { 55 | check_image_format(input_image); 56 | 57 | const int h = input_image.shape(0); 58 | const int w = input_image.shape(1); 59 | 60 | // Convert to internal format 61 | image seg_input_img(w, h); 62 | rgb* p = reinterpret_cast(input_image.get_data()); 63 | std::copy(p, p + w * h, seg_input_img.data); 64 | 65 | // Execute segmentation 66 | int num_css; 67 | image *seg_result_img = segment_image(&seg_input_img, sigma, c, min_size, &num_css); 68 | 69 | // Convert per-region-color to label 70 | boost::numpy::ndarray result_label = boost::numpy::empty(2, input_image.get_shape(), boost::numpy::dtype::get_builtin()); 71 | rgb* in_p = seg_result_img->data; 72 | int* out_p = reinterpret_cast(result_label.get_data()); 73 | 74 | std::map color_label_map; 75 | int current_label = 0; 76 | for(int i = 0; i < w * h; ++i, ++in_p, ++out_p) 77 | { 78 | auto label = color_label_map.find(*in_p); 79 | if(label != color_label_map.end()) 80 | *out_p = label->second; 81 | else 82 | color_label_map[*in_p] = (*out_p = current_label++); 83 | } 84 | 85 | delete seg_result_img; 86 | return boost::python::make_tuple(result_label, num_css); 87 | } 88 | 89 | BOOST_PYTHON_MODULE(segment) 90 | { 91 | boost::numpy::initialize(); 92 | boost::python::def("segment", segment); 93 | boost::python::def("segment_label", segment_label); 94 | } 95 | 96 | -------------------------------------------------------------------------------- /test_color_space.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import pytest 5 | import numpy 6 | from color_space import * 7 | 8 | class TestColorSpace: 9 | def _assert_range(self, img): 10 | assert img.dtype == numpy.uint8 11 | assert img.shape == (10, 10, 3) 12 | assert 0 <= numpy.min(img) 13 | assert 1 < numpy.max(img) <= 255 14 | 15 | def setup_method(self, method): 16 | self.I = numpy.ndarray((10, 10, 3), dtype=numpy.uint8) 17 | self.I[:, :, 0] = 50 18 | self.I[:, :, 1] = 100 19 | self.I[:, :, 2] = 150 20 | self.Irand = numpy.random.randint(0, 256, (10, 10, 3)).astype(numpy.uint8) 21 | 22 | 23 | def test_to_grey_range(self): 24 | self._assert_range(to_grey(self.Irand)) 25 | 26 | def test_to_grey_value(self): 27 | img = to_grey(self.I) 28 | grey_value = int(0.2125 * 50 + 0.7154 * 100 + 0.0721 * 150) 29 | assert ((img == grey_value).all()) 30 | 31 | 32 | def test_to_Lab_range(self): 33 | self._assert_range(to_Lab(self.Irand)) 34 | 35 | def test_to_Lab_value(self): 36 | img = to_Lab(self.I) 37 | 38 | 39 | def test_to_rgI_range(self): 40 | self._assert_range(to_rgI(self.Irand)) 41 | 42 | def test_to_rgI_value(self): 43 | img = to_rgI(self.I) 44 | grey_value = int(0.2125 * 50 + 0.7154 * 100 + 0.0721 * 150) 45 | assert ((img[:, :, 0] == 50).all()) 46 | assert ((img[:, :, 1] == 100).all()) 47 | assert ((img[:, :, 2] == grey_value).all()) 48 | 49 | 50 | def test_to_HSV_range(self): 51 | self._assert_range(to_HSV(self.Irand)) 52 | 53 | def test_to_HSV_value(self): 54 | img = to_HSV(self.I) 55 | h, s, v = 148, 170, 150 56 | assert ((img[:, :, 0] == h).all()) 57 | assert ((img[:, :, 1] == s).all()) 58 | assert ((img[:, :, 2] == v).all()) 59 | 60 | 61 | def test_to_nRGB_range(self): 62 | self._assert_range(to_nRGB(self.Irand)) 63 | 64 | def test_to_nRGB_value(self): 65 | img = to_nRGB(self.I) 66 | denom = numpy.sqrt(50 ** 2 + 100 ** 2 + 150 ** 2) / 255.0 67 | r, g, b = 50 / denom, 100 / denom, 150 / denom 68 | assert ((img[:, :, 0] == int(r)).all()) 69 | assert ((img[:, :, 1] == int(g)).all()) 70 | assert ((img[:, :, 2] == int(b)).all()) 71 | 72 | 73 | def test_to_Hue_range(self): 74 | self._assert_range(to_Hue(self.Irand)) 75 | 76 | def test_to_Hue_value(self): 77 | img = to_Hue(self.I) 78 | expected_h = 148 79 | assert ((img[:, :, 0] == expected_h).all()) 80 | assert ((img[:, :, 1] == expected_h).all()) 81 | assert ((img[:, :, 2] == expected_h).all()) 82 | 83 | 84 | def test_convert_color_nonexisting_color(self): 85 | with pytest.raises(KeyError): 86 | convert_color(self.Irand, 'nonexisting-colorspace') 87 | 88 | def test_convert_color_give_singlechannel_image(self): 89 | I = numpy.random.randint(0, 255, (10, 10)).astype(numpy.uint8) 90 | assert numpy.array_equal(convert_color(I, 'rgb')[:, :, 0], I) 91 | 92 | def test_convert_color_value(self): 93 | assert numpy.array_equal(convert_color(self.Irand, 'rgb'), self.Irand) 94 | assert numpy.array_equal(convert_color(self.Irand, 'lab'), to_Lab(self.Irand)) 95 | assert numpy.array_equal(convert_color(self.Irand, 'rgi'), to_rgI(self.Irand)) 96 | assert numpy.array_equal(convert_color(self.Irand, 'hsv'), to_HSV(self.Irand)) 97 | assert numpy.array_equal(convert_color(self.Irand, 'nrgb'), to_nRGB(self.Irand)) 98 | assert numpy.array_equal(convert_color(self.Irand, 'hue'), to_Hue(self.Irand)) 99 | 100 | -------------------------------------------------------------------------------- /selective_search.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import datetime 5 | import itertools 6 | import copy 7 | import joblib 8 | import numpy 9 | import scipy.sparse 10 | import segment 11 | import collections 12 | import skimage.io 13 | import features 14 | import color_space 15 | 16 | def _calc_adjacency_matrix(label_img, n_region): 17 | r = numpy.vstack([label_img[:, :-1].ravel(), label_img[:, 1:].ravel()]) 18 | b = numpy.vstack([label_img[:-1, :].ravel(), label_img[1:, :].ravel()]) 19 | t = numpy.hstack([r, b]) 20 | A = scipy.sparse.coo_matrix((numpy.ones(t.shape[1]), (t[0], t[1])), shape=(n_region, n_region), dtype=bool).todense().getA() 21 | A = A | A.transpose() 22 | 23 | for i in range(n_region): 24 | A[i, i] = True 25 | 26 | dic = {i : {i} ^ set(numpy.flatnonzero(A[i])) for i in range(n_region)} 27 | 28 | Adjacency = collections.namedtuple('Adjacency', ['matrix', 'dictionary']) 29 | return Adjacency(matrix = A, dictionary = dic) 30 | 31 | def _new_adjacency_dict(A, i, j, t): 32 | Ak = copy.deepcopy(A) 33 | Ak[t] = (Ak[i] | Ak[j]) - {i, j} 34 | del Ak[i], Ak[j] 35 | for (p, Q) in Ak.items(): 36 | if i in Q or j in Q: 37 | Q -= {i, j} 38 | Q.add(t) 39 | 40 | return Ak 41 | 42 | def _new_label_image(F, i, j, t): 43 | Fk = numpy.copy(F) 44 | Fk[Fk == i] = Fk[Fk == j] = t 45 | return Fk 46 | 47 | def _build_initial_similarity_set(A0, feature_extractor): 48 | S = list() 49 | for (i, J) in A0.items(): 50 | S += [(feature_extractor.similarity(i, j), (i, j)) for j in J if i < j] 51 | 52 | return sorted(S) 53 | 54 | def _merge_similarity_set(feature_extractor, Ak, S, i, j, t): 55 | # remove entries which have i or j 56 | S = list(filter(lambda x: not(i in x[1] or j in x[1]), S)) 57 | 58 | # calculate similarity between region t and its adjacencies 59 | St = [(feature_extractor.similarity(t, x), (t, x)) for x in Ak[t] if t < x] +\ 60 | [(feature_extractor.similarity(x, t), (x, t)) for x in Ak[t] if x < t] 61 | 62 | return sorted(S + St) 63 | 64 | def hierarchical_segmentation(I, k = 100, feature_mask = features.SimilarityMask(1, 1, 1, 1)): 65 | F0, n_region = segment.segment_label(I, 0.8, k, 100) 66 | adj_mat, A0 = _calc_adjacency_matrix(F0, n_region) 67 | feature_extractor = features.Features(I, F0, n_region) 68 | 69 | # stores list of regions sorted by their similarity 70 | S = _build_initial_similarity_set(A0, feature_extractor) 71 | 72 | # stores region label and its parent (empty if initial). 73 | R = {i : () for i in range(n_region)} 74 | 75 | A = [A0] # stores adjacency relation for each step 76 | F = [F0] # stores label image for each step 77 | 78 | # greedy hierarchical grouping loop 79 | while len(S): 80 | (s, (i, j)) = S.pop() 81 | t = feature_extractor.merge(i, j) 82 | 83 | # record merged region (larger region should come first) 84 | R[t] = (i, j) if feature_extractor.size[j] < feature_extractor.size[i] else (j, i) 85 | 86 | Ak = _new_adjacency_dict(A[-1], i, j, t) 87 | A.append(Ak) 88 | 89 | S = _merge_similarity_set(feature_extractor, Ak, S, i, j, t) 90 | 91 | F.append(_new_label_image(F[-1], i, j, t)) 92 | 93 | # bounding boxes for each hierarchy 94 | L = feature_extractor.bbox 95 | 96 | return (R, F, L) 97 | 98 | def _generate_regions(R, L): 99 | n_ini = sum(not parent for parent in R.values()) 100 | n_all = len(R) 101 | 102 | regions = list() 103 | for label in R.keys(): 104 | i = min(n_all - n_ini + 1, n_all - label) 105 | vi = numpy.random.rand() * i 106 | regions.append((vi, L[i])) 107 | 108 | return sorted(regions) 109 | 110 | def _selective_search_one(I, color, k, mask): 111 | I_color = color_space.convert_color(I, color) 112 | (R, F, L) = hierarchical_segmentation(I_color, k, mask) 113 | return _generate_regions(R, L) 114 | 115 | def selective_search(I, color_spaces = ['rgb'], ks = [100], feature_masks = [features.SimilarityMask(1, 1, 1, 1)], n_jobs = -1): 116 | parameters = itertools.product(color_spaces, ks, feature_masks) 117 | region_set = joblib.Parallel(n_jobs = n_jobs)(joblib.delayed(_selective_search_one)(I, color, k, mask) for (color, k, mask) in parameters) 118 | 119 | #flatten list of list of tuple to list of tuple 120 | regions = sum(region_set, []) 121 | return sorted(regions) 122 | 123 | -------------------------------------------------------------------------------- /demo_showcandidates.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import os 5 | import sys 6 | import argparse 7 | import itertools 8 | import numpy 9 | import skimage.io 10 | import color_space 11 | import features 12 | import selective_search 13 | 14 | from PySide.QtCore import * 15 | from PySide.QtGui import * 16 | 17 | #color_choises = ["RGB", "Lab", "rgI", "HSV", "nRGB", "Hue"] 18 | color_choises = ["RGB", "rgI", "HSV", "nRGB", "Hue"] 19 | k_choises = ["50", "100", "150", "300"] 20 | similarity_choises = ["C", "T", "S", "F",\ 21 | "C+T", "C+S", "C+F", "T+S", "T+F", "S+F",\ 22 | "C+T+S", "C+T+F", "C+S+F", "T+S+F",\ 23 | "C+T+S+F"] 24 | 25 | class Demo(QWidget): 26 | chosen_colors = {"RGB"} 27 | chosen_ks = {"100"} 28 | chosen_similarities = {"C+T+S+F", "T+S+F", "F", "S"} 29 | regions = list() 30 | 31 | def __init__(self, ndimg): 32 | super().__init__() 33 | self.ndimg = ndimg 34 | h, w = ndimg.shape[:2] 35 | self.qimg = QImage(ndimg.flatten(), w, h, QImage.Format_RGB888) 36 | 37 | self.layout = QGridLayout() 38 | self.setLayout(self.layout) 39 | self.__init_parameter_choises() 40 | self.__init_runbutton() 41 | self.__init_imagearea() 42 | self.__init_slider() 43 | self.__draw() 44 | 45 | def keyPressEvent(self, e): 46 | if e.key() == Qt.Key_Escape or e.key() == Qt.Key_Q: 47 | self.close() 48 | 49 | def __init_imagearea(self): 50 | self.label = QLabel() 51 | self.layout.addWidget(self.label, 0, 2) 52 | 53 | def __init_runbutton(self): 54 | button = QPushButton("Run") 55 | button.clicked.connect(self.runButtonClicked) 56 | self.layout.addWidget(button, 2, 1) 57 | 58 | def runButtonClicked(self): 59 | self.__parameter_changed() 60 | 61 | def __init_parameter_choises(self): 62 | color_checkbox = self.__init_choises('Color space', color_choises, self.chosen_colors, self.color_selected) 63 | k_checkbox = self.__init_choises('k', k_choises, self.chosen_ks, self.k_selected) 64 | sim_checkbox = self.__init_choises('Similarity measure', similarity_choises, self.chosen_similarities, self.similarity_selected) 65 | 66 | color_k_vbox = QVBoxLayout() 67 | color_k_vbox.addWidget(color_checkbox) 68 | color_k_vbox.addWidget(k_checkbox) 69 | spacer = QSpacerItem(0, 0, QSizePolicy.Minimum, QSizePolicy.Expanding) 70 | color_k_vbox.addSpacerItem(spacer) 71 | self.layout.addLayout(color_k_vbox, 0, 0) 72 | 73 | self.layout.addWidget(sim_checkbox, 0, 1) 74 | 75 | def __init_choises(self, title, choises, default_choises, handler): 76 | group = QGroupBox(title) 77 | group.setFlat(False) 78 | 79 | vbox = QVBoxLayout() 80 | for choise in choises: 81 | checkbox = QCheckBox(choise) 82 | if choise in default_choises: 83 | checkbox.setCheckState(Qt.Checked) 84 | checkbox.stateChanged.connect(handler) 85 | vbox.addWidget(checkbox) 86 | 87 | group.setLayout(vbox) 88 | return group 89 | 90 | def __init_slider(self): 91 | hbox = QHBoxLayout() 92 | 93 | label = QLabel() 94 | label.setText('count:') 95 | hbox.addWidget(label) 96 | 97 | self.slider = QSlider(Qt.Horizontal) 98 | self.slider.setMinimum(0) 99 | self.slider.valueChanged.connect(self.count_changed) 100 | hbox.addWidget(self.slider) 101 | 102 | self.count_label = QLabel() 103 | hbox.addWidget(self.count_label) 104 | 105 | self.layout.addLayout(hbox, 1, 2) 106 | 107 | 108 | def count_changed(self, value): 109 | self.__draw() 110 | self.count_label.setText(str(value)) 111 | 112 | def color_selected(self, value): 113 | color = self.sender().text() 114 | if value: 115 | self.chosen_colors.add(color) 116 | else: 117 | self.chosen_colors.remove(color) 118 | if len(self.chosen_colors) == 0: 119 | self.sender().setCheckState(Qt.Checked) 120 | 121 | def k_selected(self, value): 122 | k = self.sender().text() 123 | if value: 124 | self.chosen_ks.add(k) 125 | else: 126 | self.chosen_ks.remove(k) 127 | if len(self.chosen_ks) == 0: 128 | self.sender().setCheckState(Qt.Checked) 129 | 130 | def similarity_selected(self, value): 131 | similarity = self.sender().text() 132 | if value: 133 | self.chosen_similarities.add(similarity) 134 | else: 135 | self.chosen_similarities.remove(similarity) 136 | if len(self.chosen_similarities) == 0: 137 | self.sender().setCheckState(Qt.Checked) 138 | 139 | def __parameter_changed(self): 140 | # obtain parameters 141 | color_spaces = [color.lower() for color in self.chosen_colors] 142 | ks = [float(k) for k in self.chosen_ks] 143 | similarity_masks = [features.SimilarityMask('S' in mask, 'C' in mask, 'T' in mask, 'F' in mask) for mask in self.chosen_similarities] 144 | 145 | self.regions = selective_search.selective_search(self.ndimg, color_spaces, ks, similarity_masks) 146 | self.slider.setMaximum(len(self.regions)) 147 | self.slider.setValue(int(len(self.regions) / 4)) 148 | self.__draw() 149 | 150 | def __draw(self): 151 | self.pixmap = QPixmap(self.qimg) 152 | painter = QPainter(self.pixmap) 153 | painter.setPen(QColor(0, 255, 0)) 154 | for v, (y0, x0, y1, x1) in self.regions[:int(self.slider.value())]: 155 | painter.drawRect(x0, y0, x1, y1) 156 | 157 | self.label.setPixmap(self.pixmap) 158 | 159 | 160 | if __name__=="__main__": 161 | parser = argparse.ArgumentParser() 162 | parser.add_argument('-i', '--image', type=str, required=True, help='filename of the image') 163 | args = parser.parse_args() 164 | 165 | img = skimage.io.imread(args.image) 166 | if len(img.shape) == 2: 167 | img = skimage.color.gray2rgb(img) 168 | 169 | app = QApplication(sys.argv) 170 | wnd = Demo(img) 171 | wnd.show() 172 | app.exec_() 173 | 174 | -------------------------------------------------------------------------------- /features.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import collections 5 | import math 6 | import numpy 7 | import skimage 8 | import skimage.filters 9 | import scipy.ndimage.filters 10 | 11 | SimilarityMask = collections.namedtuple("SimilarityMask", ["size", "color", "texture", "fill"]) 12 | 13 | class Features: 14 | def __init__(self, image, label, n_region, similarity_weight = SimilarityMask(1, 1, 1, 1)): 15 | self.image = image 16 | self.label = label 17 | self.w = similarity_weight 18 | 19 | self.imsize = float(label.shape[0] * label.shape[1]) 20 | self.size = self.__init_size(n_region) 21 | self.color = self.__init_color(n_region) 22 | self.bbox = self.__init_bounding_box(n_region) 23 | self.texture = self.__init_texture(n_region) 24 | 25 | def __init_size(self, n_region): 26 | bincnt = numpy.bincount(self.label.ravel(), minlength = n_region) 27 | return {i : bincnt[i] for i in range(n_region)} 28 | 29 | def __init_color(self, n_region): 30 | n_bin = 25 31 | bin_width = int(math.ceil(255.0 / n_bin)) 32 | 33 | bins_color = [i * bin_width for i in range(n_bin + 1)] 34 | bins_label = range(n_region + 1) 35 | bins = [bins_label, bins_color] 36 | 37 | r_hist = numpy.histogram2d(self.label.ravel(), self.image[:, :, 0].ravel(), bins=bins)[0] #shape=(n_region, n_bin) 38 | g_hist = numpy.histogram2d(self.label.ravel(), self.image[:, :, 1].ravel(), bins=bins)[0] 39 | b_hist = numpy.histogram2d(self.label.ravel(), self.image[:, :, 2].ravel(), bins=bins)[0] 40 | hist = numpy.hstack([r_hist, g_hist, b_hist]) 41 | l1_norm = numpy.sum(hist, axis = 1).reshape((n_region, 1)) 42 | 43 | hist = numpy.nan_to_num(hist / l1_norm) 44 | return {i : hist[i] for i in range(n_region)} 45 | 46 | def __init_bounding_box(self, n_region): 47 | bbox = dict() 48 | for region in range(n_region): 49 | I, J = numpy.where(self.label == region) 50 | bbox[region] = (min(I), min(J), max(I), max(J)) 51 | return bbox 52 | 53 | def __init_texture(self, n_region): 54 | ar = numpy.ndarray((n_region, 240)) 55 | return {i : ar[i] for i in range(n_region)} 56 | 57 | def __calc_gradient_histogram(self, label, gaussian, n_region, nbins_orientation = 8, nbins_inten = 10): 58 | op = numpy.array([[-1, 0, 1]], dtype=numpy.float32) 59 | h = scipy.ndimage.filters.convolve(gaussian, op) 60 | v = scipy.ndimage.filters.convolve(gaussian, op.transpose()) 61 | g = numpy.arctan2(v, h) 62 | 63 | # define each axis for texture histogram 64 | bin_width = 2 * math.pi / 8 65 | bins_label = range(n_region + 1) 66 | bins_angle = numpy.linspace(-math.pi, math.pi, nbins_orientation + 1) 67 | bins_inten = numpy.linspace(.0, 1., nbins_inten + 1) 68 | bins = [bins_label, bins_angle, bins_inten] 69 | 70 | # calculate 3 dimensional histogram 71 | ar = numpy.vstack([label.ravel(), g.ravel(), gaussian.ravel()]).transpose() 72 | hist = numpy.histogramdd(ar, bins = bins)[0] 73 | 74 | # orientation_wise intensity histograms are serialized for each region 75 | return numpy.reshape(hist, (n_region, nbins_orientation * nbins_inten)) 76 | 77 | def __init_texture(self, n_region): 78 | gaussian = skimage.filters.gaussian_filter(self.image, sigma = 1.0, multichannel = True).astype(numpy.float32) 79 | r_hist = self.__calc_gradient_histogram(self.label, gaussian[:, :, 0], n_region) 80 | g_hist = self.__calc_gradient_histogram(self.label, gaussian[:, :, 1], n_region) 81 | b_hist = self.__calc_gradient_histogram(self.label, gaussian[:, :, 2], n_region) 82 | 83 | hist = numpy.hstack([r_hist, g_hist, b_hist]) 84 | l1_norm = numpy.sum(hist, axis = 1).reshape((n_region, 1)) 85 | 86 | hist = numpy.nan_to_num(hist / l1_norm) 87 | return {i : hist[i] for i in range(n_region)} 88 | 89 | 90 | def __sim_size(self, i, j): 91 | return 1. - (self.size[i] + self.size[j]) / self.imsize 92 | 93 | def __calc_histogram_intersection(self, vec1, vec2): 94 | return numpy.sum(numpy.minimum(vec1, vec2)) 95 | 96 | def __sim_texture(self, i, j): 97 | return self.__calc_histogram_intersection(self.texture[i], self.texture[j]) 98 | 99 | def __sim_color(self, i, j): 100 | return self.__calc_histogram_intersection(self.color[i], self.color[j]) 101 | 102 | def __sim_fill(self, i, j): 103 | (bi0, bi1, bi2, bi3), (bj0, bj1, bj2, bj3) = self.bbox[i], self.bbox[j] 104 | (bij0, bij1, bij2, bij3) = min(bi0, bj0), min(bi1, bj1), max(bi2, bj2), max(bi3, bj3) 105 | bij_size = (bij2 - bij0) * (bij3 - bij1) 106 | return 1. - (bij_size - self.size[i] - self.size[j]) / self.imsize 107 | 108 | def similarity(self, i, j): 109 | return self.w.size * self.__sim_size(i, j) + \ 110 | self.w.texture * self.__sim_texture(i, j) + \ 111 | self.w.color * self.__sim_color(i, j) + \ 112 | self.w.fill * self.__sim_fill(i, j) 113 | 114 | 115 | def __merge_size(self, i, j, new_region_id): 116 | self.size[new_region_id] = self.size[i] + self.size[j] 117 | 118 | def __histogram_merge(self, vec1, vec2, w1, w2): 119 | return (w1 * vec1 + w2 * vec2) / (w1 + w2) 120 | 121 | def __merge_color(self, i, j, new_region_id): 122 | self.color[new_region_id] = self.__histogram_merge(self.color[i], self.color[j], self.size[i], self.size[j]) 123 | 124 | def __merge_texture(self, i, j, new_region_id): 125 | self.texture[new_region_id] = self.__histogram_merge(self.texture[i], self.texture[j], self.size[i], self.size[j]) 126 | 127 | def __merge_bbox(self, i, j, new_region_id): 128 | (bi0, bi1, bi2, bi3), (bj0, bj1, bj2, bj3) = self.bbox[i], self.bbox[j] 129 | self.bbox[new_region_id] = (min(bi0, bj0), min(bi1, bj1), max(bi2, bj2), max(bi3, bj3)) 130 | 131 | def merge(self, i, j): 132 | new_region_id = len(self.size) 133 | self.__merge_size(i, j, new_region_id) 134 | self.__merge_color(i, j, new_region_id) 135 | self.__merge_texture(i, j, new_region_id) 136 | self.__merge_bbox(i, j, new_region_id) 137 | return new_region_id 138 | 139 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Overview 2 | 3 | This is a python implementation of the Selective Search [[1]](#selective_search_ijcv)[[2]](#selective_search_iccv). 4 | 5 | The Selective Search is used as a preprocess of object detection/recognition pipeline.
6 | It finds regions likely to contain any objects from an input image regardless of its scale and location, 7 | that allows detectors to concentrate only for such 'prospective' regions.
8 | Therefore you can configure more computationally efficient detector, 9 | or use more rich feature representation and classification method [[3]](#deeplearning) 10 | compared to the conventional exhaustive search scheme. 11 | 12 | For more details about the method, please refer the original paper. 13 | 14 | This implementation is based on the journal edition of the original paper, and giving similar parameter variations. 15 | 16 | ![segmentation example](doc/segmentation_example.png) 17 | ![selective search example](doc/ss_sample.png) 18 | 19 | 20 | # Requirements 21 | 22 | * CMake (>= 2.8) 23 | * GCC (>= 4.8.2) 24 | * Python (>= 3.4.3) 25 | * For required packages, see `requirements.txt` 26 | * Boost (>= 1.58.0) built with python support 27 | * [Boost.NumPy](https://github.com/ndarray/Boost.NumPy) 28 | * If you got an error to build, see [belltailjp/Boost.NumPy](https://github.com/belltailjp/Boost.NumPy)) 29 | 30 | In addition, this is only tested on x64 Linux environment. 31 | 32 | 33 | # Preparation 34 | 35 | This implementation contains a few C++ code which wraps the Efficient Graph-Based Image Segmentation [[4]](#segmentation) used for generating an initial value. 36 | It works as a python module, so build it first. 37 | 38 | ```sh 39 | % git clone https://github.com/belltailjp/selective_search_py.git 40 | % cd selective_search_py 41 | % wget http://cs.brown.edu/~pff/segment/segment.zip; unzip segment.zip; rm segment.zip 42 | % cmake . 43 | % make 44 | ``` 45 | 46 | Then you will see a shared object `segment.so` in the directory. 47 | Keep it on the same directory of main Python script, or referrable location described in `LD_LIBRARY_PATH`. 48 | 49 | 50 | # Demo 51 | 52 | ## Interactively show regions likely to contain objects 53 | 54 | *showcandidate* demo allows you to interactively see the result of selective search. 55 | 56 | ```sh 57 | % ./demo_showcandidates.py image.jpg 58 | ``` 59 | 60 | ![showcandidate GUI example](doc/showcandidates_scr.png) 61 | 62 | You can choose any combination of parameters on the left side of the screen. 63 | Then click the "Run" button and wait for a while. You will see the generated regions on the right side. 64 | 65 | By changing the slider on the bottom, you can increase/decrease number of region candidates. 66 | The more slider goes to left, the more confident regions are shown like this: 67 | 68 | ![showcandidate GUI example more region](doc/showcandidates_scr_more.png) 69 | 70 | 71 | ## Show image segmentation hierarchy 72 | 73 | *showhierarchy* demo visualizes colored region images for each step in iteration. 74 | 75 | ```sh 76 | % ./demo_showhierarchy.py image.jpg --k 500 --feature color texture --color rgb 77 | ``` 78 | 79 | ![image segmentation hierarchy visualization](doc/hierarchy_example.gif) 80 | 81 | If you want to see labels composited with the input image, give a particular alpha-value. 82 | 83 | ```sh 84 | % ./demo_showhierarchy.py image.jpg --k 500 --feature color texture --color rgb --alpha 0.6 85 | ``` 86 | 87 | ![image heerarchy with original image](doc/hierarchy_example_composited.png) 88 | 89 | 90 | # Implementation 91 | 92 | ## Overview 93 | 94 | Algorithm of the method is described in Journal edition of the original paper in detail ([[1]](#selective_search_ijcv)). 95 | For diversification strategy, this implementation supports to vary the following parameter as the original paper proposed. 96 | 97 | * Color space 98 | * *RGB, Lab, rgI, HSV, normalized RGB* and *Hue* 99 | * *C* of Color invariance [[5]](#color_invariance) is currently not supported. 100 | * Similarity measure 101 | * Texture, Color, Fill and Size 102 | * Initial segmentation parameter *k* 103 | * As the initial (fine-grained) segmentation, this implementation uses [[4]](#segmentation). *k* is one of the parameters of the method. 104 | 105 | You can give any combinations for each strategy. 106 | 107 | 108 | ## How to integrate to your code 109 | 110 | If you just want to use this implementation as a black box, only the `selective_search` module is necessary to import. 111 | 112 | ```python 113 | from selective_search import * 114 | 115 | img = skimage.io.imread('image.png') 116 | regions = selective_search(img) 117 | for v, (i0, j0, i1, j1) in regions: 118 | ... 119 | ``` 120 | 121 | Then you can get a list regions sorted by score in ascending order. 122 | Regions with larger score (latter elements of the list) are considered as 'non-prospective' regions, so they can be filtered out as you need. 123 | 124 | To change parameters, just give a list of values for each diversification strategy. Note that they must be given as a list. 125 | `selective_search` returns a single list of generated regions which contains every combination of selective search result. 126 | This result is also sorted. 127 | 128 | ```python 129 | regions = selective_search(img, \ 130 | color_spaces = ['rgb', 'hsv'],\ #color space. should be lower case. 131 | ks = [50, 150, 300],\ #k. 132 | feature_masks = [(0, 0, 1, 1)]) #indicates whether S/C/T/F similarity is used, respectively. 133 | ``` 134 | 135 | 136 | ## Test 137 | 138 | This implementation contains automated unit tests using PyTest. 139 | 140 | To execute full test, type: 141 | 142 | ```sh 143 | % py.test 144 | ``` 145 | 146 | 147 | # License 148 | 149 | This implementation is publicly available under the MIT license. See LICENSE.txt for more details. 150 | 151 | However regarding the selective search method itself, authors of the original paper have not mention anything so far. 152 | Please ask the original authors if you have any concens. 153 | 154 | 155 | # References 156 | 157 | \[1\] [J. R. R. Uijlings et al., Selective Search for Object Recognition, IJCV, 2013](https://ivi.fnwi.uva.nl/isis/publications/bibtexbrowser.php?key=UijlingsIJCV2013&bib=all.bib)
158 | \[2\] [Koen van de Sande et al., Segmentation As Selective Search for Object Recognition, ICCV, 2011](https://ivi.fnwi.uva.nl/isis/publications/bibtexbrowser.php?key=UijlingsIJCV2013&bib=all.bib)
159 | \[3\] [R. Girshick et al., Rich Feature Hierarchies for Accurate Object Detection and Semantic Segmentation, CVPR, 2014](http://www.cs.berkeley.edu/~rbg/papers/r-cnn-cvpr.pdf)
160 | \[4\] [P. Felzenszwalb et al., Efficient Graph-Based Image Segmentation, IJCV, 2004](http://cs.brown.edu/~pff/segment/)
161 | \[5\] J. M. Geusebroek et al., Color invariance, TPAMI, 2001 162 | -------------------------------------------------------------------------------- /test_features.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import numpy 5 | import features 6 | 7 | class TestFeaturesColorHistogram: 8 | def setup_method(self, method = None, w = 10, h = 10): 9 | self.h, self.w = h, w 10 | image = numpy.zeros((self.h, self.w, 3), dtype=numpy.uint8) 11 | label = numpy.zeros((self.h, self.w), dtype=int) 12 | self.f = features.Features(image, label, 1) 13 | 14 | def test_1region_1color(self): 15 | hist = self.f._Features__init_color(1) 16 | assert len(hist) == 1 17 | assert hist[0].shape == (75,) 18 | r_expected = [0.333333333] + [0] * 24 19 | g_expected = [0.333333333] + [0] * 24 20 | b_expected = [0.333333333] + [0] * 24 21 | numpy.testing.assert_array_almost_equal(hist[0].ravel(), r_expected + g_expected + b_expected) 22 | 23 | def test_1region_255color(self): 24 | self.setup_method(self, w = 1, h = 256) 25 | for y in range(self.h): 26 | self.f.image[y, :, :] = y 27 | 28 | hist = self.f._Features__init_color(1) 29 | assert len(hist) == 1 30 | assert hist[0].shape == (75,) 31 | r_expected = [11] * 23 + [3, 0] # because bin width equals 11 32 | g_expected = [11] * 23 + [3, 0] 33 | b_expected = [11] * 23 + [3, 0] 34 | expected = numpy.array(r_expected + g_expected + b_expected) 35 | numpy.testing.assert_array_almost_equal(hist[0].ravel(), expected / numpy.sum(expected)) 36 | 37 | def test_2region_1color(self): 38 | self.setup_method(self, w = 1, h = 2) 39 | for y in range(self.h): 40 | self.f.label[y, :] = y 41 | 42 | hist = self.f._Features__init_color(2) 43 | assert len(hist) == 2 44 | assert hist[0].shape == (75,) 45 | r1_expected = ([1.0/3] + [0] * 24) + ([1.0/3] + [0] * 24) + ([1.0/3] + [0] * 24) 46 | r2_expected = ([1.0/3] + [0] * 24) + ([1.0/3] + [0] * 24) + ([1.0/3] + [0] * 24) 47 | numpy.testing.assert_array_almost_equal(hist[0].ravel(), r1_expected) 48 | numpy.testing.assert_array_almost_equal(hist[1].ravel(), r2_expected) 49 | 50 | 51 | class TestFeaturesSize: 52 | def setup_method(self, method): 53 | image = numpy.zeros((10, 10, 3), dtype=numpy.uint8) 54 | label = numpy.zeros((10, 10), dtype=int) 55 | self.f = features.Features(image, label, 1) 56 | 57 | def test_1region(self): 58 | sizes = self.f._Features__init_size(1) 59 | assert len(sizes) == 1 60 | assert sizes[0] == 100 61 | 62 | def test_2region(self): 63 | self.f.label[:5, :] = 1 64 | sizes = self.f._Features__init_size(2) 65 | assert len(sizes) == 2 66 | assert sizes[0] == 50 67 | assert sizes[1] == 50 68 | 69 | class TestFeaturesBoundingBox: 70 | def setup_method(self, method): 71 | image = numpy.zeros((10, 10, 3), dtype=numpy.uint8) 72 | label = numpy.zeros((10, 10), dtype=int) 73 | self.f = features.Features(image, label, 1) 74 | 75 | def test_1region(self): 76 | bb = self.f._Features__init_bounding_box(1) 77 | assert len(bb) == 1 78 | assert bb[0] == (0, 0, 9, 9) 79 | 80 | def test_4region(self): 81 | self.f.label[:5, :5] = 0 82 | self.f.label[:5, 5:] = 1 83 | self.f.label[5:, :5] = 2 84 | self.f.label[5:, 5:] = 3 85 | bb = self.f._Features__init_bounding_box(4) 86 | assert len(bb) == 4 87 | assert bb[0] == (0, 0, 4, 4) 88 | assert bb[1] == (0, 5, 4, 9) 89 | assert bb[2] == (5, 0, 9, 4) 90 | assert bb[3] == (5, 5, 9, 9) 91 | 92 | 93 | class TestSimilarity: 94 | def setup_method(self, method): 95 | self.dummy_image = numpy.zeros((10, 10, 3), dtype=numpy.uint8) 96 | self.dummy_label = numpy.zeros((10, 10), dtype=int) 97 | self.f = features.Features(self.dummy_image, self.dummy_label, 1) 98 | 99 | def test_similarity_size(self): 100 | self.f.size = {0 : 10, 1 : 20} 101 | 102 | s = self.f._Features__sim_size(0, 1) 103 | assert s == 0.7 104 | 105 | def test_similarity_color_simple(self): 106 | self.f.color[0] = numpy.array([1] * 75) 107 | self.f.color[1] = numpy.array([2] * 75) 108 | s = self.f._Features__sim_color(0, 1) 109 | assert s == 75 110 | 111 | def test_similarity_color_complex(self): 112 | # build 75-dimensional arrays as color histogram 113 | self.f.color[0] = numpy.array([1, 2, 1, 2, 1] * 15) 114 | self.f.color[1] = numpy.array([2, 1, 2, 1, 2] * 15) 115 | s = self.f._Features__sim_color(0, 1) 116 | assert s == 75 117 | 118 | def test_similarity_texture(self): 119 | # build 240-dimensional arrays as texture histogram 120 | self.f.texture[0] = numpy.array([1, 2, 1, 2, 1, 2] * 40) 121 | self.f.texture[1] = numpy.array([2, 1, 2, 1, 2, 1] * 40) 122 | s = self.f._Features__sim_texture(0, 1) 123 | assert s == 240 124 | 125 | def test_similarity_fill(self): 126 | self.f.bbox[0] = numpy.array([10, 10, 20, 20]) 127 | self.f.size[0] = 100 128 | self.f.bbox[1] = numpy.array([20, 20, 30, 30]) 129 | self.f.size[1] = 100 130 | s = self.f._Features__sim_fill(0, 1) 131 | assert s == 1. - float(400 - 200) / 100 132 | 133 | def test_similarity_user_all(self, monkeypatch): 134 | monkeypatch.setattr(features.Features, '_Features__sim_size', lambda self, i, j: 1) 135 | monkeypatch.setattr(features.Features, '_Features__sim_texture',lambda self, i, j: 1) 136 | monkeypatch.setattr(features.Features, '_Features__sim_color', lambda self, i, j: 1) 137 | monkeypatch.setattr(features.Features, '_Features__sim_fill', lambda self, i, j: 1) 138 | w = features.SimilarityMask(1, 1, 1, 1) 139 | f = features.Features(self.dummy_image, self.dummy_label, 1, w) 140 | assert f.similarity(0, 1) == 4 141 | 142 | 143 | class TestMerge: 144 | def setup_method(self, method): 145 | dummy_image = numpy.zeros((10, 10, 3), dtype=numpy.uint8) 146 | dummy_label = numpy.zeros((10, 10), dtype=int) 147 | self.f = features.Features(dummy_image, dummy_label, 1) 148 | 149 | def test_merge_size(self): 150 | self.f.size = {0: 10, 1: 20} 151 | self.f._Features__merge_size(0, 1, 2) 152 | assert self.f.size[2] == 30 153 | 154 | def test_merge_color(self): 155 | self.f.color[0] = numpy.array([1.] * 75) 156 | self.f.size[0] = 100 157 | self.f.color[1] = numpy.array([2.] * 75) 158 | self.f.size[1] = 50 159 | self.f._Features__merge_color(0, 1, 2) 160 | 161 | expected = (100 * 1. + 50 * 2.) / (100 + 50) 162 | assert numpy.array_equal(self.f.color[2], [expected] * 75) 163 | 164 | def test_merge_texture(self): 165 | self.f.texture[0] = numpy.array([1.] * 240) 166 | self.f.size[0] = 100 167 | self.f.texture[1] = numpy.array([2.] * 240) 168 | self.f.size[1] = 50 169 | self.f._Features__merge_texture(0, 1, 2) 170 | 171 | expected = (100 * 1. + 50 * 2.) / (100 + 50) 172 | assert numpy.array_equal(self.f.texture[2], [expected] * 240) 173 | 174 | def test_merge_bbox(self): 175 | self.f.bbox[0] = numpy.array([10, 10, 20, 20]) 176 | self.f.size[0] = 100 177 | self.f.bbox[1] = numpy.array([20, 20, 30, 30]) 178 | self.f.size[1] = 50 179 | self.f.imsize = 1000 180 | self.f._Features__merge_bbox(0, 1, 2) 181 | 182 | assert numpy.array_equal(self.f.bbox[2], [10, 10, 30, 30]) 183 | 184 | def test_merge(self): 185 | self.f.imsize = 1000 186 | self.f.size = {0: 10, 1: 20} 187 | self.f.color = {0: numpy.array([1.] * 75), 1: numpy.array([2.] * 75)} 188 | self.f.texture = {0: numpy.array([1.] * 240), 1: numpy.array([2.] * 240)} 189 | self.f.bbox = {0: numpy.array([10, 10, 20, 20]), 1: numpy.array([20, 20, 30, 30])} 190 | assert self.f.merge(0, 1) == 2 191 | assert len(self.f.size) == 3 192 | assert len(self.f.color) == 3 193 | assert len(self.f.texture) == 3 194 | assert len(self.f.bbox) == 3 195 | 196 | -------------------------------------------------------------------------------- /test_selective_search.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import nose 5 | import nose.tools 6 | 7 | import numpy 8 | import selective_search 9 | 10 | class TestCalcAdjecencyMatrix: 11 | def setup_method(self, method): 12 | self.label = numpy.zeros((4, 4), dtype=int) 13 | 14 | def test_only_1_segment(self): 15 | # 0, 0, 0, 0 16 | # 0, 0, 0, 0 17 | # 0, 0, 0, 0 18 | # 0, 0, 0, 0 19 | (adj_mat, adj_dic) = selective_search._calc_adjacency_matrix(self.label, 1) 20 | assert type(adj_mat) == numpy.ndarray 21 | assert adj_mat.shape == (1, 1) and adj_mat.dtype == bool 22 | assert adj_mat[0, 0] == True 23 | assert adj_dic[0] == set() 24 | 25 | def test_fully_adjacent(self): 26 | # 1, 1, 0, 0 27 | # 1, 1, 0, 0 28 | # 1, 1, 0, 0 29 | # 1, 1, 0, 0 30 | self.label[:2, :] = 1 31 | expected_mat = numpy.array([[True, True],\ 32 | [True, True]]) 33 | 34 | (adj_mat, adj_dic) = selective_search._calc_adjacency_matrix(self.label, 2) 35 | assert adj_mat.shape == (2, 2) and adj_mat.dtype == bool 36 | assert numpy.array_equal(adj_mat, expected_mat) 37 | assert adj_dic[0] == {1} 38 | assert adj_dic[1] == {0} 39 | 40 | def test_partially_adjacent(self): 41 | # 0, 0, 1, 1 42 | # 0, 0, 1, 1 43 | # 2, 2, 3, 3 44 | # 2, 2, 3, 3 45 | self.label[:2, :2] = 0 46 | self.label[:2, 2:] = 1 47 | self.label[2:, :2] = 2 48 | self.label[2:, 2:] = 3 49 | expected_mat = numpy.array([[True, True, True, False],\ 50 | [True, True, False, True],\ 51 | [True, False, True, True],\ 52 | [False, True, True, True]]) 53 | 54 | (adj_mat, adj_dic) = selective_search._calc_adjacency_matrix(self.label, 4) 55 | assert numpy.diag(adj_mat).all() 56 | assert numpy.array_equal(adj_mat.transpose(), adj_mat) 57 | assert numpy.array_equal(adj_mat, expected_mat) 58 | 59 | assert adj_dic[0] == {1, 2} 60 | assert adj_dic[1] == {0, 3} 61 | assert adj_dic[2] == {0, 3} 62 | assert adj_dic[3] == {1, 2} 63 | 64 | def test_edge_case_vertical(self): 65 | # 0, 0, 0, 1 66 | # 0, 0, 0, 1 67 | # 0, 0, 0, 2 68 | # 0, 0, 0, 2 69 | self.label[:2, -1:] = 1 70 | self.label[2:, -1:] = 2 71 | expected_mat = numpy.array([[True, True, True],\ 72 | [True, True, True],\ 73 | [True, True, True]]) 74 | 75 | (adj_mat, adj_dic) = selective_search._calc_adjacency_matrix(self.label, 3) 76 | assert numpy.array_equal(expected_mat, adj_mat) 77 | assert adj_dic[0] == {1, 2} 78 | assert adj_dic[1] == {0, 2} 79 | assert adj_dic[2] == {0, 1} 80 | 81 | def test_edge_case_horizontal(self): 82 | # 0, 0, 0, 0 83 | # 0, 0, 0, 0 84 | # 0, 0, 0, 0 85 | # 1, 1, 2, 2 86 | self.label[-1:, :2] = 1 87 | self.label[-1:, 2:] = 2 88 | expected_mat = numpy.array([[True, True, True],\ 89 | [True, True, True],\ 90 | [True, True, True]]) 91 | 92 | (adj_mat, adj_dic) = selective_search._calc_adjacency_matrix(self.label, 3) 93 | assert numpy.array_equal(expected_mat, adj_mat) 94 | assert adj_dic[0] == {1, 2} 95 | assert adj_dic[1] == {0, 2} 96 | assert adj_dic[2] == {0, 1} 97 | 98 | def test_extreme_example(self): 99 | # 0, 1, 2, 3 100 | # 4, 5, 6, 7 101 | # 8, 9,10,11 102 | #12,13,14,15 103 | self.label = numpy.array(range(16)).reshape((4,4)) 104 | (adj_mat, adj_dic) = selective_search._calc_adjacency_matrix(self.label, 16) 105 | assert numpy.array_equal(adj_mat.transpose(), adj_mat) 106 | assert set(numpy.flatnonzero(adj_mat[ 0])) == { 0, 1, 4} 107 | assert set(numpy.flatnonzero(adj_mat[ 1])) == { 0, 1, 2, 5} 108 | assert set(numpy.flatnonzero(adj_mat[ 2])) == { 1, 2, 3, 6} 109 | assert set(numpy.flatnonzero(adj_mat[ 3])) == { 2, 3, 7} 110 | assert set(numpy.flatnonzero(adj_mat[ 4])) == { 0, 4, 5, 8} 111 | assert set(numpy.flatnonzero(adj_mat[ 5])) == { 1, 4, 5, 6, 9} 112 | assert set(numpy.flatnonzero(adj_mat[ 6])) == { 2, 5, 6, 7, 10} 113 | assert set(numpy.flatnonzero(adj_mat[ 7])) == { 3, 6, 7, 11} 114 | assert set(numpy.flatnonzero(adj_mat[ 8])) == { 4, 8, 9, 12} 115 | assert set(numpy.flatnonzero(adj_mat[ 9])) == { 5, 8, 9, 10, 13} 116 | assert set(numpy.flatnonzero(adj_mat[10])) == { 6, 9, 10, 11, 14} 117 | assert set(numpy.flatnonzero(adj_mat[11])) == { 7, 10, 11, 15} 118 | assert set(numpy.flatnonzero(adj_mat[12])) == { 8, 12, 13} 119 | assert set(numpy.flatnonzero(adj_mat[13])) == { 9, 12, 13, 14} 120 | assert set(numpy.flatnonzero(adj_mat[14])) == {10, 13, 14, 15} 121 | assert set(numpy.flatnonzero(adj_mat[15])) == {11, 14, 15} 122 | 123 | for (i, adj_labels) in adj_dic.items(): 124 | assert set(numpy.flatnonzero(adj_mat[i])) - {i} == adj_labels 125 | 126 | 127 | class TestNewAdjacencyDict: 128 | def setup_method(self, method): 129 | # from: 130 | # 000000 131 | # 122334 132 | # 122334 133 | # 555555 134 | # to: 135 | # 000000 136 | # 166664 137 | # 166664 138 | # 555555 139 | self.A = {0: {1, 2, 3, 4},\ 140 | 1: {0, 2, 5},\ 141 | 2: {0, 1, 3, 5},\ 142 | 3: {0, 2, 4, 5},\ 143 | 4: {0, 3, 5},\ 144 | 5: {1, 2, 3, 4}} 145 | 146 | def test_exclusiveness(self): 147 | """ 148 | It should never violate source dictionary A 149 | """ 150 | assert self.A[0] == {1, 2, 3, 4} 151 | assert self.A[1] == {0, 2, 5} 152 | assert self.A[2] == {0, 1, 3, 5} 153 | assert self.A[3] == {0, 2, 4, 5} 154 | assert self.A[4] == {0, 3, 5} 155 | assert self.A[5] == {1, 2, 3, 4} 156 | assert 6 not in self.A 157 | 158 | def test_label(self): 159 | Ak = selective_search._new_adjacency_dict(self.A, 2, 3, 6) 160 | assert 2 not in Ak 161 | assert 3 not in Ak 162 | assert Ak[0] == {1, 4, 6} 163 | assert Ak[1] == {0, 5, 6} 164 | assert Ak[4] == {0, 5, 6} 165 | assert Ak[5] == {1, 4, 6} 166 | assert Ak[6] == {0, 1, 4, 5} 167 | 168 | 169 | class TestNewLabel: 170 | def setup_method(self, method): 171 | self.L = numpy.array([[0, 0, 0, 0, 0, 0],\ 172 | [1, 2, 2, 3, 3, 4],\ 173 | [1, 2, 2, 3, 3, 4],\ 174 | [5, 5, 5, 5, 5, 5]]) 175 | self.Lk = numpy.array([[0, 0, 0, 0, 0, 0],\ 176 | [1, 6, 6, 6, 6, 4],\ 177 | [1, 6, 6, 6, 6, 4],\ 178 | [5, 5, 5, 5, 5, 5]]) 179 | 180 | def test_exclusiveness(self): 181 | selective_search._new_label_image(self.L, 2, 3, 6) 182 | assert len(self.L[self.L == 2]) == 4 183 | assert len(self.L[self.L == 3]) == 4 184 | 185 | def test_new_label(self): 186 | Lk_actual = selective_search._new_label_image(self.L, 2, 3, 6) 187 | assert numpy.array_equal(self.Lk, Lk_actual) 188 | 189 | 190 | class TestBuildInitialSimilaritySet: 191 | def setup_method(self, method): 192 | class stub_feature_extractor: 193 | def similarity(self, i, j): 194 | return i + j # Dummy similarity. 195 | self.feature_extractor = stub_feature_extractor() 196 | 197 | # 0011 198 | # 2233 199 | self.A0 = {0: {1, 2}, 1: {0, 3}, 2: {0, 3}, 3: {1, 2}} 200 | 201 | def test_valie(self): 202 | # each line: sim, i, j (where sim=i+j in this test) 203 | # commented out lines: i should be smaller than j 204 | expected = [(1, (0, 1)),\ 205 | #(1, (1, 0)),\ 206 | (2, (0, 2)),\ 207 | #(2, (2, 0)),\ 208 | (4, (1, 3)),\ 209 | #(4, (3, 1)),\ 210 | (5, (2, 3)),\ 211 | #(5, (3, 2)) 212 | ] 213 | 214 | S = selective_search._build_initial_similarity_set(self.A0, self.feature_extractor) 215 | assert S == expected 216 | 217 | class TestMergeSimilaritySet: 218 | def setup_method(self, method): 219 | # 0011 => 0011 220 | # 2233 => 4444 221 | # (i, j, t) = (2, 3, 4) 222 | 223 | # target similarity set (similarity values are all dummy) 224 | self.S = [(1, (0, 1)),\ 225 | (2, (0, 2)),\ 226 | (3, (1, 3)),\ 227 | (5, (2, 3))] 228 | 229 | # assumption: adjacency dict is already updated 230 | self.Ak = {0: {1, 4},\ 231 | 1: {0, 4},\ 232 | 4: {0, 1}} 233 | 234 | self.extractor = type('Feature', (), {'similarity' : (lambda i, j: i + j)}) 235 | 236 | def test_value(self): 237 | S_ = selective_search._merge_similarity_set(self.extractor, self.Ak, self.S, 2, 3, 4) 238 | expect_S = [(1, (0, 1)),\ 239 | (4, (0, 4)),\ 240 | (5, (1, 4))] 241 | 242 | assert S_ == expect_S 243 | 244 | --------------------------------------------------------------------------------