├── .gitattributes ├── .github └── workflows │ └── c-cpp.yml ├── .gitignore ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── deer.jpg ├── example.jpg ├── selective_search.hpp └── test.cpp /.gitattributes: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # Set default behavior to automatically normalize line endings. 3 | ############################################################################### 4 | * text=auto 5 | 6 | ############################################################################### 7 | # Set default behavior for command prompt diff. 8 | # 9 | # This is need for earlier builds of msysgit that does not have it on by 10 | # default for csharp files. 11 | # Note: This is only used by command line 12 | ############################################################################### 13 | #*.cs diff=csharp 14 | 15 | ############################################################################### 16 | # Set the merge driver for project and solution files 17 | # 18 | # Merging from the command prompt will add diff markers to the files if there 19 | # are conflicts (Merging from VS is not affected by the settings below, in VS 20 | # the diff markers are never inserted). Diff markers may cause the following 21 | # file extensions to fail to load in VS. An alternative would be to treat 22 | # these files as binary and thus will always conflict and require user 23 | # intervention with every merge. To do so, just uncomment the entries below 24 | ############################################################################### 25 | #*.sln merge=binary 26 | #*.csproj merge=binary 27 | #*.vbproj merge=binary 28 | #*.vcxproj merge=binary 29 | #*.vcproj merge=binary 30 | #*.dbproj merge=binary 31 | #*.fsproj merge=binary 32 | #*.lsproj merge=binary 33 | #*.wixproj merge=binary 34 | #*.modelproj merge=binary 35 | #*.sqlproj merge=binary 36 | #*.wwaproj merge=binary 37 | 38 | ############################################################################### 39 | # behavior for image files 40 | # 41 | # image files are treated as binary by default. 42 | ############################################################################### 43 | #*.jpg binary 44 | #*.png binary 45 | #*.gif binary 46 | 47 | ############################################################################### 48 | # diff behavior for common document formats 49 | # 50 | # Convert binary document formats to text before diffing them. This feature 51 | # is only available from the command line. Turn it on by uncommenting the 52 | # entries below. 53 | ############################################################################### 54 | #*.doc diff=astextplain 55 | #*.DOC diff=astextplain 56 | #*.docx diff=astextplain 57 | #*.DOCX diff=astextplain 58 | #*.dot diff=astextplain 59 | #*.DOT diff=astextplain 60 | #*.pdf diff=astextplain 61 | #*.PDF diff=astextplain 62 | #*.rtf diff=astextplain 63 | #*.RTF diff=astextplain 64 | -------------------------------------------------------------------------------- /.github/workflows/c-cpp.yml: -------------------------------------------------------------------------------- 1 | name: C/C++ CI 2 | 3 | on: 4 | push: 5 | branches: [ master ] 6 | pull_request: 7 | branches: [ master ] 8 | 9 | jobs: 10 | build: 11 | runs-on: ubuntu-20.04 12 | steps: 13 | - name: Checkout 14 | uses: actions/checkout@v2 15 | - name: Install OpenCV 16 | run: sudo apt-get install -y libopencv-dev 17 | - name: Build 18 | run: make 19 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ## Ignore Visual Studio temporary files, build results, and 2 | ## files generated by popular Visual Studio add-ons. 3 | 4 | # User-specific files 5 | *.suo 6 | *.user 7 | *.userosscache 8 | *.sln.docstates 9 | 10 | # User-specific files (MonoDevelop/Xamarin Studio) 11 | *.userprefs 12 | 13 | # Build results 14 | [Dd]ebug/ 15 | [Dd]ebugPublic/ 16 | [Rr]elease/ 17 | [Rr]eleases/ 18 | x64/ 19 | x86/ 20 | build/ 21 | bld/ 22 | [Bb]in/ 23 | [Oo]bj/ 24 | 25 | # Visual Studio 2015 cache/options directory 26 | .vs/ 27 | 28 | # MSTest test Results 29 | [Tt]est[Rr]esult*/ 30 | [Bb]uild[Ll]og.* 31 | 32 | # NUNIT 33 | *.VisualState.xml 34 | TestResult.xml 35 | 36 | # Build Results of an ATL Project 37 | [Dd]ebugPS/ 38 | [Rr]eleasePS/ 39 | dlldata.c 40 | 41 | # DNX 42 | project.lock.json 43 | artifacts/ 44 | 45 | *_i.c 46 | *_p.c 47 | *_i.h 48 | *.ilk 49 | *.meta 50 | *.obj 51 | *.pch 52 | *.pdb 53 | *.pgc 54 | *.pgd 55 | *.rsp 56 | *.sbr 57 | *.tlb 58 | *.tli 59 | *.tlh 60 | *.tmp 61 | *.tmp_proj 62 | *.log 63 | *.vspscc 64 | *.vssscc 65 | .builds 66 | *.pidb 67 | *.svclog 68 | *.scc 69 | 70 | # Chutzpah Test files 71 | _Chutzpah* 72 | 73 | # Visual C++ cache files 74 | ipch/ 75 | *.aps 76 | *.ncb 77 | *.opensdf 78 | *.sdf 79 | *.cachefile 80 | 81 | # Visual Studio profiler 82 | *.psess 83 | *.vsp 84 | *.vspx 85 | 86 | # TFS 2012 Local Workspace 87 | $tf/ 88 | 89 | # Guidance Automation Toolkit 90 | *.gpState 91 | 92 | # ReSharper is a .NET coding add-in 93 | _ReSharper*/ 94 | *.[Rr]e[Ss]harper 95 | *.DotSettings.user 96 | 97 | # JustCode is a .NET coding add-in 98 | .JustCode 99 | 100 | # TeamCity is a build add-in 101 | _TeamCity* 102 | 103 | # DotCover is a Code Coverage Tool 104 | *.dotCover 105 | 106 | # NCrunch 107 | _NCrunch_* 108 | .*crunch*.local.xml 109 | 110 | # MightyMoose 111 | *.mm.* 112 | AutoTest.Net/ 113 | 114 | # Web workbench (sass) 115 | .sass-cache/ 116 | 117 | # Installshield output folder 118 | [Ee]xpress/ 119 | 120 | # DocProject is a documentation generator add-in 121 | DocProject/buildhelp/ 122 | DocProject/Help/*.HxT 123 | DocProject/Help/*.HxC 124 | DocProject/Help/*.hhc 125 | DocProject/Help/*.hhk 126 | DocProject/Help/*.hhp 127 | DocProject/Help/Html2 128 | DocProject/Help/html 129 | 130 | # Click-Once directory 131 | publish/ 132 | 133 | # Publish Web Output 134 | *.[Pp]ublish.xml 135 | *.azurePubxml 136 | ## TODO: Comment the next line if you want to checkin your 137 | ## web deploy settings but do note that will include unencrypted 138 | ## passwords 139 | #*.pubxml 140 | 141 | *.publishproj 142 | 143 | # NuGet Packages 144 | *.nupkg 145 | # The packages folder can be ignored because of Package Restore 146 | **/packages/* 147 | # except build/, which is used as an MSBuild target. 148 | !**/packages/build/ 149 | # Uncomment if necessary however generally it will be regenerated when needed 150 | #!**/packages/repositories.config 151 | 152 | # Windows Azure Build Output 153 | csx/ 154 | *.build.csdef 155 | 156 | # Windows Store app package directory 157 | AppPackages/ 158 | 159 | # Visual Studio cache files 160 | # files ending in .cache can be ignored 161 | *.[Cc]ache 162 | # but keep track of directories ending in .cache 163 | !*.[Cc]ache/ 164 | 165 | # Others 166 | ClientBin/ 167 | [Ss]tyle[Cc]op.* 168 | ~$* 169 | *~ 170 | *.dbmdl 171 | *.dbproj.schemaview 172 | *.pfx 173 | *.publishsettings 174 | node_modules/ 175 | orleans.codegen.cs 176 | 177 | # RIA/Silverlight projects 178 | Generated_Code/ 179 | 180 | # Backup & report files from converting an old project file 181 | # to a newer Visual Studio version. Backup files are not needed, 182 | # because we have git ;-) 183 | _UpgradeReport_Files/ 184 | Backup*/ 185 | UpgradeLog*.XML 186 | UpgradeLog*.htm 187 | 188 | # SQL Server files 189 | *.mdf 190 | *.ldf 191 | 192 | # Business Intelligence projects 193 | *.rdl.data 194 | *.bim.layout 195 | *.bim_*.settings 196 | 197 | # Microsoft Fakes 198 | FakesAssemblies/ 199 | 200 | # Node.js Tools for Visual Studio 201 | .ntvs_analysis.dat 202 | 203 | # Visual Studio 6 build log 204 | *.plg 205 | 206 | # Visual Studio 6 workspace options file 207 | *.opt 208 | 209 | # LightSwitch generated files 210 | GeneratedArtifacts/ 211 | _Pvt_Extensions/ 212 | ModelManifest.xml 213 | 214 | # bug of VS 215 | *.VC.opendb 216 | *.db 217 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:20.04 2 | 3 | ENV DEBIAN_FRONTEND=noninteractive 4 | 5 | RUN apt-get update -y \ 6 | && apt-get install -y build-essential \ 7 | && apt-get install -y libopencv-dev \ 8 | && apt-get clean \ 9 | && rm -rf /var/lib/apt/lists/* 10 | 11 | CMD ["/bin/bash"] 12 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2016 watanika 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all 11 | copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | SOFTWARE. -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | CXX = g++ 2 | CXXFLAGS = -std=c++11 -O2 -Wall `pkg-config --cflags opencv4` 3 | LDFLAGS = `pkg-config --libs opencv4` 4 | OBJS = test 5 | 6 | all: $(OBJS) 7 | 8 | clean: 9 | $(RM) $(OBJS) 10 | 11 | .PHONY: all clean 12 | 13 | .SUFFIXES: .cpp .o 14 | 15 | .cpp.o: 16 | $(CXX) $(CXXFLAGS) -o $@ -c $^ 17 | .cpp: 18 | $(CXX) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # C++ Selective Search 2 | ![C/C++ CI](https://github.com/ukiyoyo/selective-search-cpp/workflows/C/C++%20CI/badge.svg) 3 | 4 | This is a simple, small C++ implementation of Selective Search [1, 2] that can be easily integrated into your projects. 5 | 6 | For initial segmentation, this implementation uses Efficient Graph-Based Image Segmentation [3]. 7 | 8 | ![example](example.jpg) 9 | 10 | ## Dependencies 11 | 12 | - C++11 features 13 | - OpenCV (tested version: 4.0) 14 | 15 | ## Usage 16 | 17 | You can test this implementation as: 18 | 19 | ```sh 20 | % make 21 | % ./test 22 | ``` 23 | 24 | To include it in your project, you just need to: 25 | 26 | ```cpp 27 | #include "selective_search.hpp" 28 | 29 | ... 30 | 31 | // Get object proposals 32 | auto proposals = ss::selectiveSearch( img, scale, sigma, minSize, smallest, largest, distorted ); 33 | 34 | ``` 35 | 36 | ## License 37 | 38 | MIT 39 | 40 | ## References 41 | 42 | [1] J. R. R. Uijlings et al., "Selective Search for Object Recognition", IJCV, 2013 43 | 44 | [2] K. E. A. van de Sande et al., "Segmentation As Selective Search for Object Recognition", ICCV, 2011 45 | 46 | [3] P. Felzenszwalb et al., "Efficient Graph-Based Image Segmentation", IJCV, 2004 47 | -------------------------------------------------------------------------------- /deer.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tttanikawa/selective-search-cpp/4fa9445f14efee6f552a6e3fb70bc77f78083a16/deer.jpg -------------------------------------------------------------------------------- /example.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tttanikawa/selective-search-cpp/4fa9445f14efee6f552a6e3fb70bc77f78083a16/example.jpg -------------------------------------------------------------------------------- /selective_search.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | 22 | namespace std { 23 | template <> 24 | class hash> { 25 | public: 26 | std::size_t operator()(const std::pair& x) const { 27 | return hash()(x.first) ^ hash()(x.second); 28 | } 29 | }; 30 | } // namespace std 31 | 32 | namespace ss { 33 | 34 | inline double square(double a) { 35 | return a * a; 36 | } 37 | 38 | inline double diff(const cv::Mat& img, int x1, int y1, int x2, int y2) { 39 | return sqrt(square(img.at(y1, x1)[0] - img.at(y2, x2)[0]) + 40 | square(img.at(y1, x1)[1] - img.at(y2, x2)[1]) + 41 | square(img.at(y1, x1)[2] - img.at(y2, x2)[2])); 42 | } 43 | 44 | struct UniverseElement { 45 | int rank; 46 | int p; 47 | int size; 48 | 49 | UniverseElement() : rank(0), size(1), p(0) {} 50 | UniverseElement(int rank, int size, int p) : rank(rank), size(size), p(p) {} 51 | 52 | bool operator==(const UniverseElement& other) const { 53 | return rank == other.rank && p == other.p && size == other.size; 54 | } 55 | }; 56 | 57 | class Universe { 58 | private: 59 | std::vector elements; 60 | int num; 61 | 62 | public: 63 | Universe(int num) : num(num) { 64 | elements.reserve(num); 65 | 66 | for (int i = 0; i < num; i++) { 67 | elements.emplace_back(0, 1, i); 68 | } 69 | } 70 | 71 | ~Universe() {} 72 | 73 | int findFast(int x) { 74 | return elements[x].p; 75 | } 76 | 77 | int find(int x) { 78 | int y = x; 79 | while (y != elements[y].p) { 80 | y = elements[y].p; 81 | } 82 | elements[x].p = y; 83 | 84 | return y; 85 | } 86 | 87 | void join(int x, int y) { 88 | if (elements[x].rank > elements[y].rank) { 89 | elements[y].p = x; 90 | elements[x].size += elements[y].size; 91 | } else { 92 | elements[x].p = y; 93 | elements[y].size += elements[x].size; 94 | if (elements[x].rank == elements[y].rank) { 95 | elements[y].rank++; 96 | } 97 | } 98 | num--; 99 | } 100 | 101 | int size(int x) const { return elements[x].size; } 102 | int numSets() const { return num; } 103 | }; 104 | 105 | struct edge { 106 | int a; 107 | int b; 108 | double w; 109 | }; 110 | 111 | bool operator<(const edge& a, const edge& b) { 112 | return a.w < b.w; 113 | } 114 | 115 | inline double calThreshold(int size, double scale) { 116 | return scale / size; 117 | } 118 | 119 | std::shared_ptr segmentGraph(int numVertices, int numEdges, std::vector& edges, double scale) { 120 | std::sort(edges.begin(), edges.end()); 121 | auto universe = std::make_shared(numVertices); 122 | std::vector threshold(numVertices, scale); 123 | 124 | for (auto& pedge : edges) { 125 | int a = universe->find(pedge.a); 126 | int b = universe->find(pedge.b); 127 | if (a != b) { 128 | if ((pedge.w <= threshold[a]) && (pedge.w <= threshold[b])) { 129 | universe->join(a, b); 130 | a = universe->find(a); 131 | threshold[a] = pedge.w + calThreshold(universe->size(a), scale); 132 | } 133 | } 134 | } 135 | 136 | return universe; 137 | } 138 | 139 | // image segmentation using "Efficient Graph-Based Image Segmentation" 140 | std::shared_ptr segmentation(const cv::Mat& img, double scale, double sigma, int minSize) { 141 | const int width = img.cols; 142 | const int height = img.rows; 143 | 144 | cv::Mat imgF; 145 | img.convertTo(imgF, CV_32FC3); 146 | 147 | cv::Mat blurred; 148 | cv::GaussianBlur(imgF, blurred, cv::Size(5, 5), sigma); 149 | 150 | std::vector edges(width * height * 4); 151 | int num = 0; 152 | for (int y = 0; y < height; y++) { 153 | for (int x = 0; x < width; x++) { 154 | if (x < width - 1) { 155 | edges[num].a = y * width + x; 156 | edges[num].b = y * width + (x + 1); 157 | edges[num].w = diff(blurred, x, y, x + 1, y); 158 | num++; 159 | } 160 | 161 | if (y < height - 1) { 162 | edges[num].a = y * width + x; 163 | edges[num].b = (y + 1) * width + x; 164 | edges[num].w = diff(blurred, x, y, x, y + 1); 165 | num++; 166 | } 167 | 168 | if ((x < width - 1) && (y < height - 1)) { 169 | edges[num].a = y * width + x; 170 | edges[num].b = (y + 1) * width + (x + 1); 171 | edges[num].w = diff(blurred, x, y, x + 1, y + 1); 172 | num++; 173 | } 174 | 175 | if ((x < width - 1) && (y > 0)) { 176 | edges[num].a = y * width + x; 177 | edges[num].b = (y - 1) * width + (x + 1); 178 | edges[num].w = diff(blurred, x, y, x + 1, y - 1); 179 | num++; 180 | } 181 | } 182 | } 183 | 184 | edges.erase(edges.begin() + num, edges.end()); 185 | auto universe = segmentGraph(width * height, num, edges, scale); 186 | for (int i = 0; i < num; i++) { 187 | int a = universe->find(edges[i].a); 188 | int b = universe->find(edges[i].b); 189 | if ((a != b) && ((universe->size(a) < minSize) || (universe->size(b) < minSize))) { 190 | universe->join(a, b); 191 | } 192 | } 193 | return universe; 194 | } 195 | 196 | void visualize(const cv::Mat& img, std::shared_ptr universe) { 197 | const int height = img.rows; 198 | const int width = img.cols; 199 | std::vector colors; 200 | cv::Mat segmentated(height, width, CV_8UC3); 201 | std::random_device rnd; 202 | std::mt19937 mt(rnd()); 203 | std::uniform_int_distribution<> rand256(0, 255); 204 | for (int i = 0; i < height * width; i++) { 205 | cv::Vec3b color(rand256(mt), rand256(mt), rand256(mt)); 206 | colors.push_back(color); 207 | } 208 | 209 | for (int y = 0; y < height; y++) { 210 | for (int x = 0; x < width; x++) { 211 | segmentated.at(y, x) = colors[universe->findFast(y * width + x)]; 212 | } 213 | } 214 | cv::imshow("Initial Segmentation Result", segmentated); 215 | cv::waitKey(1); 216 | } 217 | 218 | struct Region { 219 | int size; 220 | cv::Rect rect; 221 | std::vector labels; 222 | std::vector colourHist; 223 | std::vector textureHist; 224 | std::vector points; 225 | 226 | Region() {} 227 | 228 | Region(const cv::Rect& rect, int label) : rect(rect) { 229 | labels.push_back(label); 230 | } 231 | 232 | Region( 233 | const cv::Rect& rect, int size, const std::vector&& colourHist, const std::vector&& textureHist, const std::vector&& labels) 234 | : rect(rect), size(size), colourHist(std::move(colourHist)), textureHist(std::move(textureHist)), labels(std::move(labels)) {} 235 | 236 | Region& operator=(const Region& region) = default; 237 | 238 | Region& operator=(Region&& region) noexcept { 239 | if (this != ®ion) { 240 | this->size = region.size; 241 | this->rect = region.rect; 242 | this->labels = std::move(region.labels); 243 | this->colourHist = std::move(region.colourHist); 244 | this->textureHist = std::move(region.textureHist); 245 | } 246 | 247 | return *this; 248 | } 249 | 250 | Region(Region&& region) noexcept { 251 | *this = std::move(region); 252 | } 253 | }; 254 | 255 | std::shared_ptr generateSegments(const cv::Mat& img, double scale, double sigma, int minSize) { 256 | auto universe = segmentation(img, scale, sigma, minSize); 257 | // visualize(img, universe); 258 | return universe; 259 | } 260 | 261 | double calcSimOfColour(const Region& r1, const Region& r2) { 262 | assert(r1.colourHist.size() == r2.colourHist.size()); 263 | float sum = 0.0; 264 | for (auto i1 = r1.colourHist.cbegin(), i2 = r2.colourHist.cbegin(); i1 != r1.colourHist.cend(); i1++, i2++) { 265 | sum += std::min(*i1, *i2); 266 | } 267 | return sum; 268 | } 269 | 270 | double calcSimOfTexture(const Region& r1, const Region& r2) { 271 | assert(r1.colourHist.size() == r2.colourHist.size()); 272 | double sum = 0.0; 273 | for (auto i1 = r1.textureHist.cbegin(), i2 = r2.textureHist.cbegin(); i1 != r1.textureHist.cend(); i1++, i2++) { 274 | sum += std::min(*i1, *i2); 275 | } 276 | 277 | return sum; 278 | } 279 | 280 | inline double calcSimOfSize(const Region& r1, const Region& r2, int imSize) { 281 | return (1.0 - (double)(r1.size + r2.size) / imSize); 282 | } 283 | 284 | inline double calcSimOfRect(const Region& r1, const Region& r2, int imSize) { 285 | return (1.0 - (double)((r1.rect | r2.rect).area() - r1.size - r2.size) / imSize); 286 | } 287 | 288 | inline double calcSimilarity(const Region& r1, const Region& r2, int imSize) { 289 | return (calcSimOfColour(r1, r2) + calcSimOfTexture(r1, r2) + calcSimOfSize(r1, r2, imSize) + calcSimOfRect(r1, r2, imSize)); 290 | } 291 | 292 | void calcColourHist(const cv::Mat& img, std::shared_ptr universe, int label, Region& region) { 293 | std::array, 3> hsv; 294 | for (auto& e : hsv) { 295 | e.reserve(region.points.size()); 296 | } 297 | 298 | for (cv::Vec2i point : region.points) { 299 | for (int channel = 0; channel < 3; channel++) { 300 | hsv[channel].push_back(img.at(point[0], point[1])[channel]); 301 | } 302 | } 303 | 304 | int channels[] = {0}; 305 | const int bins = 25; 306 | int histSize[] = {bins}; 307 | float range[] = {0, 256}; 308 | const float* ranges[] = {range}; 309 | for (int channel = 0; channel < 3; channel++) { 310 | cv::Mat hist; 311 | cv::Mat input(hsv[channel]); 312 | cv::calcHist(&input, 1, channels, cv::Mat(), hist, 1, histSize, ranges, true, false); 313 | cv::normalize(hist, hist, 1.0, 0.0, cv::NORM_L1); 314 | 315 | std::vector histogram; 316 | hist.copyTo(histogram); 317 | if (region.colourHist.empty()) { 318 | region.colourHist = std::move(histogram); 319 | } else { 320 | std::copy(histogram.begin(), histogram.end(), std::back_inserter(region.colourHist)); 321 | } 322 | } 323 | } 324 | 325 | cv::Mat calcTextureGradient(const cv::Mat& img) { 326 | cv::Mat sobelX, sobelY; 327 | cv::Sobel(img, sobelX, CV_32F, 1, 0); 328 | cv::Sobel(img, sobelY, CV_32F, 0, 1); 329 | cv::Mat magnitude, angle; 330 | cv::cartToPolar(sobelX, sobelY, magnitude, angle, true); 331 | return angle; 332 | } 333 | 334 | void calcTextureHist(const cv::Mat& img, const cv::Mat& gradient, std::shared_ptr universe, int label, Region& region) { 335 | const int orientations = 8; 336 | std::array, orientations>, 3> intensity; 337 | for (auto& e : intensity) { 338 | for (auto& ee : e) { 339 | ee.reserve(region.points.size()); 340 | } 341 | } 342 | for (cv::Vec2i point : region.points) { 343 | for (int channel = 0; channel < 3; channel++) { 344 | int angle = (int)(gradient.at(point[0], point[1])[channel] / 22.5) % orientations; 345 | intensity[channel][angle].push_back(img.at(point[0], point[1])[channel]); 346 | } 347 | } 348 | 349 | int channels[] = {0}; 350 | const int bins = 10; 351 | int histSize[] = {bins}; 352 | float range[] = {0, 256}; 353 | const float* ranges[] = {range}; 354 | for (int channel = 0; channel < 3; channel++) { 355 | for (int angle = 0; angle < orientations; angle++) { 356 | cv::Mat hist; 357 | cv::Mat input(intensity[channel][angle]); 358 | cv::calcHist(&input, 1, channels, cv::Mat(), hist, 1, histSize, ranges, true, false); 359 | cv::normalize(hist, hist, 1.0, 0.0, cv::NORM_L1); 360 | std::vector histogram; 361 | hist.copyTo(histogram); 362 | if (region.textureHist.empty()) { 363 | region.textureHist = std::move(histogram); 364 | } else { 365 | std::copy(histogram.begin(), histogram.end(), std::back_inserter(region.textureHist)); 366 | } 367 | } 368 | } 369 | } 370 | 371 | std::map extractRegions(const cv::Mat& img, std::shared_ptr universe) { 372 | std::map R; 373 | for (int y = 0; y < img.rows; y++) { 374 | for (int x = 0; x < img.cols; x++) { 375 | int label = universe->findFast(y * img.cols + x); 376 | if (R.find(label) == R.end()) { 377 | R[label] = Region(cv::Rect(100000, 100000, 0, 0), label); 378 | } 379 | 380 | Region& region = R[label]; 381 | if (region.rect.x > x) { 382 | region.rect.x = x; 383 | } 384 | if (region.rect.y > y) { 385 | region.rect.y = y; 386 | } 387 | if (region.rect.br().x < x) { 388 | region.rect.width = x - region.rect.x + 1; 389 | } 390 | if (region.rect.br().y < y) { 391 | region.rect.height = y - region.rect.y + 1; 392 | } 393 | region.points.push_back(cv::Vec2i(y, x)); 394 | } 395 | } 396 | 397 | cv::Mat gradient = calcTextureGradient(img); 398 | cv::Mat hsv; 399 | cv::cvtColor(img, hsv, cv::COLOR_BGR2HSV); 400 | for (auto& labelRegion : R) { 401 | labelRegion.second.size = labelRegion.second.points.size(); 402 | calcColourHist(hsv, universe, labelRegion.first, labelRegion.second); 403 | calcTextureHist(img, gradient, universe, labelRegion.first, labelRegion.second); 404 | } 405 | return R; 406 | } 407 | 408 | inline bool isIntersecting(const Region& a, const Region& b) { 409 | return ((a.rect & b.rect).area() != 0); 410 | } 411 | 412 | using LabelRegion = std::pair; 413 | using Neighbour = std::pair; 414 | std::vector extractNeighbours(const std::map& R) { 415 | std::vector neighbours; 416 | neighbours.reserve(R.size() * (R.size() - 1) / 2); 417 | for (auto a = R.cbegin(); a != R.cend(); a++) { 418 | auto tmp = a; 419 | tmp++; 420 | for (auto b = tmp; b != R.cend(); b++) { 421 | if (isIntersecting(a->second, b->second)) { 422 | neighbours.push_back(std::make_pair(std::min(a->first, b->first), std::max(a->first, b->first))); 423 | } 424 | } 425 | } 426 | return neighbours; 427 | } 428 | 429 | std::vector merge(const std::vector& a, const std::vector& b, int asize, int bsize) { 430 | std::vector newVector; 431 | newVector.reserve(a.size()); 432 | for (auto ai = a.begin(), bi = b.begin(); ai != a.end(); ai++, bi++) { 433 | newVector.push_back(((*ai) * asize + (*bi) * bsize) / (asize + bsize)); 434 | } 435 | return newVector; 436 | }; 437 | 438 | Region mergeRegions(const Region& r1, const Region& r2) { 439 | assert(r1.colourHist.size() == r2.colourHist.size()); 440 | assert(r1.textureHist.size() == r2.textureHist.size()); 441 | int newSize = r1.size + r2.size; 442 | std::vector newLabels(r1.labels); 443 | std::copy(r2.labels.begin(), r2.labels.end(), std::back_inserter(newLabels)); 444 | return Region(r1.rect | r2.rect, 445 | newSize, 446 | std::move(merge(r1.colourHist, r2.colourHist, r1.size, r2.size)), 447 | std::move(merge(r1.textureHist, r2.textureHist, r1.size, r2.size)), 448 | std::move(newLabels)); 449 | } 450 | 451 | std::vector selectiveSearch(const cv::Mat& img, double scale = 1.0, double sigma = 0.8, int minSize = 50, int smallest = 1000, int largest = 270000, double distorted = 5.0) { 452 | assert(img.channels() == 3); 453 | auto universe = generateSegments(img, scale, sigma, minSize); 454 | int imgSize = img.total(); 455 | auto R = extractRegions(img, universe); 456 | auto neighbours = extractNeighbours(R); 457 | std::unordered_map, double> S; 458 | for (auto& n : neighbours) { 459 | S[n] = calcSimilarity(R[n.first], R[n.second], imgSize); 460 | } 461 | 462 | using NeighbourSim = std::pair, double>; 463 | while (!S.empty()) { 464 | auto cmp = [](const NeighbourSim& a, const NeighbourSim& b) { return a.second < b.second; }; 465 | auto m = std::max_element(S.begin(), S.end(), cmp); 466 | int i = m->first.first; 467 | int j = m->first.second; 468 | auto ij = std::make_pair(i, j); 469 | int t = R.rbegin()->first + 1; 470 | R[t] = mergeRegions(R[i], R[j]); 471 | 472 | std::vector> keyToDelete; 473 | for (auto& s : S) { 474 | auto key = s.first; 475 | if ((i == key.first) || (i == key.second) || (j == key.first) || (j == key.second)) { 476 | keyToDelete.push_back(key); 477 | } 478 | } 479 | 480 | for (auto& key : keyToDelete) { 481 | S.erase(key); 482 | if (key == ij) { 483 | continue; 484 | } 485 | int n = (key.first == i || key.first == j) ? key.second : key.first; 486 | S[std::make_pair(n, t)] = calcSimilarity(R[n], R[t], imgSize); 487 | } 488 | } 489 | 490 | std::vector proposals; 491 | proposals.reserve(R.size()); 492 | for (auto& r : R) { 493 | // exclude same rectangle (with different segments) 494 | if (std::find(proposals.begin(), proposals.end(), r.second.rect) != proposals.end()) { 495 | continue; 496 | } 497 | // exclude regions that is smaller/larger than assigned size 498 | if (r.second.size < smallest || r.second.size > largest) { 499 | continue; 500 | } 501 | double w = r.second.rect.width; 502 | double h = r.second.rect.height; 503 | 504 | // exclude distorted rects 505 | if ((w / h > distorted) || (h / w > distorted)) { 506 | continue; 507 | } 508 | proposals.push_back(r.second.rect); 509 | } 510 | return proposals; 511 | } 512 | } // namespace ss 513 | -------------------------------------------------------------------------------- /test.cpp: -------------------------------------------------------------------------------- 1 | #include "selective_search.hpp" 2 | #include 3 | 4 | int main(int argc, char** argv) { 5 | std::string fileName = "./deer.jpg"; 6 | cv::Mat img = cv::imread(fileName, cv::IMREAD_COLOR); 7 | 8 | // selective search 9 | auto proposals = ss::selectiveSearch(img, 500, 0.8, 50, 20000, 100000, 2.5); 10 | // do something... 11 | 12 | for (auto&& rect : proposals) { 13 | cv::rectangle(img, rect, cv::Scalar(0, 255, 0), 3, 8); 14 | } 15 | cv::imwrite("./result.jpg", img); 16 | cv::imshow("result", img); 17 | cv::waitKey(0); 18 | return 0; 19 | } --------------------------------------------------------------------------------