├── pic1.png ├── pic2.png ├── pic3.png ├── pic4.png ├── pic5.png ├── pic6.png ├── README.md └── squares.cpp /pic1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/warrenlp/OpenCV_Rectangle_Detection/HEAD/pic1.png -------------------------------------------------------------------------------- /pic2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/warrenlp/OpenCV_Rectangle_Detection/HEAD/pic2.png -------------------------------------------------------------------------------- /pic3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/warrenlp/OpenCV_Rectangle_Detection/HEAD/pic3.png -------------------------------------------------------------------------------- /pic4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/warrenlp/OpenCV_Rectangle_Detection/HEAD/pic4.png -------------------------------------------------------------------------------- /pic5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/warrenlp/OpenCV_Rectangle_Detection/HEAD/pic5.png -------------------------------------------------------------------------------- /pic6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/warrenlp/OpenCV_Rectangle_Detection/HEAD/pic6.png -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | OpenCV_Rectangle_Detection 2 | ========================== 3 | 4 | Alters the default rectangle detection algorithm in OpenCV to filter out found rectangles. 5 | 6 | ### Installing opencv on Ubuntu: 7 | Simple way: 8 | sudo apt-get install libopencv-dev 9 | 10 | ### Compile on Ubuntu in project folder: 11 | g++ squares.cpp -o squares `pkg-config --cflags --libs opencv` 12 | 13 | ### Help Example: 14 | A program using pyramid scaling, Canny, contours, contour simpification and 15 | memory storage (it's got it all folks) to find 16 | squares in a list of images pic1-6.png 17 | Returns sequence of squares detected on the image. 18 | the sequence is stored in the specified memory storage 19 | Call: 20 | ./squares 21 | Using OpenCV version 3.2.0 22 | 23 | -------------------------------------------------------------------------------- /squares.cpp: -------------------------------------------------------------------------------- 1 | // The "Square Detector" program. 2 | // It loads several images sequentially and tries to find squares in 3 | // each image 4 | 5 | #include "opencv2/core/core.hpp" 6 | #include "opencv2/imgproc/imgproc.hpp" 7 | #include "opencv2/highgui/highgui.hpp" 8 | 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | 15 | using namespace cv; 16 | using namespace std; 17 | 18 | 19 | template 20 | std::string string_format(const std::string& format, Args ... args) 21 | { 22 | size_t size = snprintf(nullptr, 0, format.c_str(), args ... ) + 1; 23 | if (size <= 0) { throw std::runtime_error("Error during formatting."); } 24 | std::unique_ptr buf(new char[size]); 25 | snprintf(buf.get(), size, format.c_str(), args ...); 26 | return std::string(buf.get(), buf.get() + size - 1); 27 | } 28 | 29 | static void help() 30 | { 31 | std::string help_msg_format( 32 | "\nA program using pyramid scaling, Canny, contours, contour simpification and\n" 33 | "memory storage (it's got it all folks) to find\n" 34 | "squares in a list of images pic1-6.png\n" 35 | "Returns sequence of squares detected on the image.\n" 36 | "the sequence is stored in the specified memory storage\n" 37 | "Call:\n" 38 | "./squares\n" 39 | "Using OpenCV version %s\n"); 40 | 41 | cout << string_format(help_msg_format, CV_VERSION) << endl; 42 | } 43 | 44 | int thresh = 50, N = 11; 45 | const char *wndname = "Square Detection Demo"; 46 | 47 | struct Center 48 | { 49 | Point2f location; 50 | float radius; 51 | // double confidence; 52 | }; 53 | 54 | // helper function: 55 | // finds a cosine of angle between vectors 56 | // from pt0->pt1 and from pt0->pt2 57 | static double angle(Point pt1, Point pt2, Point pt0) 58 | { 59 | double dx1 = pt1.x - pt0.x; 60 | double dy1 = pt1.y - pt0.y; 61 | double dx2 = pt2.x - pt0.x; 62 | double dy2 = pt2.y - pt0.y; 63 | return (dx1 * dx2 + dy1 * dy2) / sqrt((dx1 * dx1 + dy1 * dy1) * (dx2 * dx2 + dy2 * dy2) + 1e-10); 64 | } 65 | 66 | //***************************************************************************** 67 | /*! 68 | * \brief FilterByMaxSize 69 | * 70 | * \version 71 | * - W Parsons 04/15/2020 72 | * Updated to std::copy_if 73 | * 74 | *****************************************************************************/ 75 | static void FilterByMaxSize(const vector> colorSquares[], vector> filteredBySizeSquares[], const int maxsize=100000) 76 | { 77 | const int numOfColorPlanes = 3; 78 | // Eliminate squares that are greater than maximum allowable size (min size has already been filtered out) 79 | for (int c = 0; c < numOfColorPlanes; ++c) 80 | { 81 | // Eliminate squares that are greater than maximum allowable size 82 | copy_if(colorSquares[c].cbegin(), colorSquares[c].cend(), std::back_inserter(filteredBySizeSquares[c]), 83 | [=](auto x){return fabs(contourArea(Mat(x))) < maxsize;} ); 84 | } 85 | } 86 | 87 | //***************************************************************************** 88 | /*! 89 | * \brief SortByCenters 90 | * 91 | * \param colorSquares, sortedSquares, minDist 92 | * 93 | * \version 94 | * - W Parsons 01/12/2014 95 | * Initial Version 96 | * 97 | *****************************************************************************/ 98 | static void SortByCenters(const vector> &colorSquares, vector>> &sortedSquares, 99 | const int minDist=50) 100 | { 101 | for (size_t i = 0; i < colorSquares.size(); ++i) 102 | { 103 | bool isNew = true; 104 | Center curCenter; 105 | minEnclosingCircle(Mat(colorSquares[i]), curCenter.location, curCenter.radius); 106 | 107 | for (size_t j = 0; j < sortedSquares.size() && isNew; ++j) 108 | { 109 | Center sortedCenter; 110 | 111 | // For this algorithm we pick the sorted square in the middle of the array to check against since they are sorted by radius size 112 | minEnclosingCircle(Mat(sortedSquares[j][sortedSquares[j].size() / 2]), sortedCenter.location, sortedCenter.radius); 113 | 114 | double dist = norm(sortedCenter.location - curCenter.location); 115 | isNew = dist >= minDist && dist >= sortedCenter.radius && dist >= curCenter.radius; 116 | if (!isNew) 117 | { 118 | // Determine where this radius fits in the group 119 | size_t k = sortedSquares[j].size() - 1; 120 | minEnclosingCircle(Mat(sortedSquares[j][k]), sortedCenter.location, sortedCenter.radius); 121 | while (k > 0 && curCenter.radius < sortedCenter.radius) 122 | { 123 | k--; 124 | minEnclosingCircle(Mat(sortedSquares[j][k]), sortedCenter.location, sortedCenter.radius); 125 | } 126 | if (curCenter.radius > sortedCenter.radius) 127 | { 128 | ++k; 129 | } 130 | sortedSquares[j].insert(sortedSquares[j].begin() + k, colorSquares[i]); 131 | } 132 | } 133 | if (isNew) 134 | { 135 | // Start a new group of squares 136 | sortedSquares.push_back(vector>(1, colorSquares[i])); 137 | } 138 | } 139 | } 140 | 141 | //***************************************************************************** 142 | /*! 143 | * \brief SortByCenters 144 | * 145 | * \version 146 | * - W Parsons 01/12/2014 147 | * Initial Version 148 | * 149 | *****************************************************************************/ 150 | static void SortByCenters(const vector> colorSquares[], vector>> sortededByCentersSquares[], 151 | const int minDist = 50) 152 | { 153 | const int numOfColorPlanes = 3; 154 | // Eliminate squares that are greater than maximum allowable size (min size has already been filtered out) 155 | for (int c = 0; c < numOfColorPlanes; ++c) 156 | { 157 | SortByCenters(colorSquares[c], sortededByCentersSquares[c], minDist); 158 | } 159 | } 160 | 161 | //***************************************************************************** 162 | /*! 163 | * \brief DumpSortedCenterData 164 | * 165 | * \version 166 | * - W Parsons 04/16/2020 167 | * Updated to C++11-style string join using accumulator 168 | * 169 | *****************************************************************************/ 170 | static void DumpSortedCenterData(const vector>> &sortedByCenterSquares) 171 | { 172 | auto centerInfoStr = [](const vector& center) { 173 | Center curCenter; 174 | minEnclosingCircle(Mat(center), curCenter.location, curCenter.radius); 175 | return (dynamic_cast( 176 | ostringstream() << "C: " << curCenter.location << " R: " << curCenter.radius 177 | )).str(); 178 | }; 179 | 180 | for_each(sortedByCenterSquares.cbegin(), sortedByCenterSquares.cend(), 181 | [¢erInfoStr](const vector>& sortedByCenterSquare) { 182 | vector centerInfos(sortedByCenterSquare.size()); 183 | transform(sortedByCenterSquare.cbegin(), sortedByCenterSquare.cend(), centerInfos.begin(), 184 | centerInfoStr); 185 | 186 | // Join strings with "," 187 | string centerStr = accumulate(centerInfos.cbegin(), centerInfos.cend(), std::string(), 188 | [](const std::string& a, const std::string& b) -> std::string { 189 | return a + (a.length() > 0 ? ", " : "") + b; 190 | }); 191 | 192 | cout << centerStr << endl; 193 | } 194 | ); 195 | 196 | cout << endl; 197 | } 198 | 199 | //***************************************************************************** 200 | /*! 201 | * \brief DumpSortedCenterData 202 | * 203 | * \version 204 | * - W Parsons 01/12/2014 205 | * Initial Version 206 | * 207 | *****************************************************************************/ 208 | static void DumpSortedCenterData(const vector>> sortedByCenterSquares[]) 209 | { 210 | const int numOfColorPlanes = 3; 211 | // Eliminate squares that are greater than maximum allowable size (min size has already been filtered out) 212 | for (int c = 0; c < numOfColorPlanes; ++c) 213 | { 214 | cout << "Dumping Data from layer \"" << c << "\" sortedByCenterSquares:" << endl; 215 | DumpSortedCenterData(sortedByCenterSquares[c]); 216 | } 217 | } 218 | 219 | //***************************************************************************** 220 | /*! 221 | * \brief ConsolidateSquares 222 | * 223 | * \version 224 | * - W Parsons 01/13/2014 225 | * Initial Version 226 | * 227 | *****************************************************************************/ 228 | static void ConsolidateSquares(const vector>> sortedSquares[], vector> &consolidatedSquares) 229 | { 230 | // Make sure we don't have any residual data in this vector or we could end up with repeats 231 | consolidatedSquares.clear(); 232 | 233 | const int numOfColorPlanes = 3; 234 | vector>> sortedByCenterSquares; 235 | const int minDist = 50; 236 | 237 | // For this algorithm we simply select the square located in the middle of the vector since they are sorted by radius size 238 | 239 | // Pull center from each plane into one vector 240 | for (int c = 0; c < numOfColorPlanes; ++c) 241 | { 242 | vector>>::const_iterator cSortedIter = sortedSquares[c].begin(); 243 | for (; cSortedIter != sortedSquares[c].end(); ++cSortedIter) 244 | { 245 | size_t middle = cSortedIter->size() / 2; 246 | vector middleSquare = cSortedIter->at(middle); 247 | consolidatedSquares.push_back(middleSquare); 248 | } 249 | } 250 | 251 | // Reduce to one unique square 252 | // Color information is gone by this point 253 | SortByCenters(consolidatedSquares, sortedByCenterSquares, minDist); 254 | 255 | DumpSortedCenterData(sortedByCenterSquares); 256 | 257 | consolidatedSquares.clear(); 258 | 259 | vector>>::const_iterator cSortedIter = sortedByCenterSquares.begin(); 260 | for (; cSortedIter != sortedByCenterSquares.end(); ++cSortedIter) 261 | { 262 | size_t middle = cSortedIter->size() / 2; 263 | vector middleSquare = cSortedIter->at(middle); 264 | consolidatedSquares.push_back(middleSquare); 265 | } 266 | } 267 | 268 | //***************************************************************************** 269 | /*! 270 | * \brief FilterByBGR 271 | * 272 | * \version 273 | * - W Parsons 01/13/2014 274 | * Initial Version 275 | * 276 | *****************************************************************************/ 277 | static void FilterByBGR(const Mat &image, const vector> &sortedSquares, 278 | vector> &sortedByRGBSquares, 279 | const vector &colorRange) 280 | { 281 | // Make sure that we don't have any residual data in the returned vector 282 | sortedByRGBSquares.clear(); 283 | 284 | Mat upperMat; 285 | Mat lowerMat; 286 | Mat resultMat; 287 | 288 | bool isInRange(true); 289 | 290 | vector>::const_iterator cSortedSquaresIter = sortedSquares.begin(); 291 | for (; cSortedSquaresIter != sortedSquares.end(); ++cSortedSquaresIter) 292 | { 293 | Rect rect = boundingRect(*cSortedSquaresIter); 294 | 295 | // Reduce rect to 25% (50% per side) with center as anchor point 296 | Point2i topLeft(rect.tl()); 297 | Point2i botRight(rect.br()); 298 | Point2i delta = botRight - topLeft; 299 | topLeft += (delta * 0.25); 300 | botRight -= (delta * 0.25); 301 | 302 | rect = Rect(topLeft, botRight); 303 | 304 | // Define the Region of Interest (ROI) 305 | Mat imageRect = image(rect); 306 | 307 | lowerMat = Mat(imageRect.size(), CV_8UC3, colorRange[0]); 308 | upperMat = Mat(imageRect.size(), CV_8UC3, colorRange[1]); 309 | resultMat = Mat::zeros(imageRect.size(), CV_8U); 310 | 311 | inRange(imageRect, lowerMat, upperMat, resultMat); 312 | 313 | vector planes; 314 | split(imageRect, planes); 315 | for (int c = 0; c < 3; ++c) 316 | { 317 | resultMat.copyTo(planes[c]); 318 | } 319 | merge(planes, imageRect); 320 | 321 | // For this algorithm, we make sure that we have 100% containment within the color range 322 | // For this we use a pixel-wise logical-AND operation 323 | int nr = resultMat.rows; 324 | int nc = resultMat.cols; // * resultMat.channels(); // Should be only one channel, but just in case... 325 | for (int i = 0; i < nr && isInRange; ++i) 326 | { 327 | uchar *data = resultMat.ptr(i); 328 | for (int j = 0; j < nc && isInRange; ++j) 329 | { 330 | if (data[j] != 255) 331 | { 332 | isInRange = false; 333 | } 334 | } 335 | } 336 | 337 | if (isInRange) 338 | { 339 | sortedByRGBSquares.push_back(*cSortedSquaresIter); 340 | } 341 | else 342 | { 343 | isInRange = true; 344 | } 345 | } 346 | } 347 | 348 | // filters out squares found based on color, position, and size 349 | static void FilterSquares(const Mat &image, vector> colorSquares[], vector> &squares) 350 | { 351 | squares.clear(); 352 | 353 | const int maxsize = 100000; 354 | const int minDstBtnCtrs = 50; 355 | vector> filteredBySizeSquares[3]; 356 | vector>> sortedByCenterSquares[3]; 357 | vector> consolidatedSquares; 358 | vector> sortedByRGBSquares; 359 | 360 | // Eliminate squares that are greater than maximum allowable size (min size has already been filtered out) 361 | FilterByMaxSize(colorSquares, filteredBySizeSquares, maxsize); 362 | 363 | // Sorts Squares into groups which share the same approximate center 364 | // This is calculated based on the containing circle of the contours as well as the distance between centers 365 | SortByCenters(filteredBySizeSquares, sortedByCenterSquares, minDstBtnCtrs); 366 | 367 | DumpSortedCenterData(sortedByCenterSquares); 368 | 369 | // Reduces the squares to a single, unique square across all color planes 370 | ConsolidateSquares(sortedByCenterSquares, consolidatedSquares); 371 | 372 | // This is a very permissive range being the whole upper-half of the gray-scale spectrum 373 | // Later on the algorithm will reject any image with any pixels out of this range, so we need it to be permissive 374 | vector colorRange; 375 | colorRange.push_back(Scalar(125, 125, 125)); 376 | colorRange.push_back(Scalar(255, 255, 255)); 377 | 378 | // Removes squares that do not fit within the prescribed color range 379 | FilterByBGR(image, consolidatedSquares, sortedByRGBSquares, colorRange); 380 | 381 | //TODO: Add FilterByHSL()??? // This might come in useful as resistance to different lighting conditions 382 | 383 | for (size_t c = 0; c < 3; ++c) 384 | { 385 | // Make sure we don't need the old, unfiltered squares data here any more 386 | colorSquares[c].clear(); 387 | std::copy(filteredBySizeSquares[c].begin(), filteredBySizeSquares[c].end(), std::back_inserter(colorSquares[c])); 388 | } 389 | 390 | std::copy(sortedByRGBSquares.begin(), sortedByRGBSquares.end(), std::back_inserter(squares)); 391 | } 392 | 393 | // Returns sequence of squares detected on the image. 394 | // The sequence is stored in the specified memory storage. 395 | static void FindSquares(const Mat &image, vector> &squares, vector> colorSquares[]) 396 | { 397 | squares.clear(); 398 | 399 | Mat pyr, timg, gray0(image.size(), CV_8U), gray; 400 | 401 | // down-scale and upscale the image to filter out the noise 402 | pyrDown(image, pyr, Size(image.cols / 2, image.rows / 2)); 403 | pyrUp(pyr, timg, image.size()); 404 | vector> contours; 405 | 406 | // find squares in every color plane of the image 407 | for (int c = 0; c < 3; ++c) 408 | { 409 | colorSquares[c].clear(); 410 | 411 | int ch[] = {c, 0}; 412 | mixChannels(&timg, 1, &gray0, 1, ch, 1); 413 | 414 | // try several threshold levels 415 | for (int l = 0; l < N; l++) 416 | { 417 | // hack: use Canny instead of zero threshold level. 418 | // Canny helps to catch squares with gradient shading 419 | if (l == 0) 420 | { 421 | // apply Canny. Take the upper threshold from slider 422 | // and set the lower to 0 (which forces edges merging) 423 | Canny(gray0, gray, 0, thresh, 5); 424 | // dilate canny output to remove potential 425 | // holes between edge segments 426 | dilate(gray, gray, Mat(), Point(-1, -1)); 427 | } 428 | else 429 | { 430 | // apply threshold if l!=0: 431 | // tgray(x,y) = gray(x,y) < (l+1)*255/N ? 255 : 0 432 | gray = gray0 >= (l + 1) * 255 / N; 433 | } 434 | 435 | // find contours and store them all as a list 436 | findContours(gray, contours, CV_RETR_LIST, CV_CHAIN_APPROX_SIMPLE); 437 | 438 | vector approx; 439 | 440 | // test each contour 441 | for (size_t i = 0; i < contours.size(); ++i) 442 | { 443 | // approximate contour with accuracy proportional 444 | // to the contour perimeter 445 | approxPolyDP(Mat(contours[i]), approx, arcLength(Mat(contours[i]), true) * 0.02, true); 446 | 447 | // square contours should have 4 vertices after approximation 448 | // relatively large area (to filter out noisy contours) 449 | // and be convex. 450 | // Note: absolute value of an area is used because 451 | // area may be positive or negative - in accordance with the 452 | // contour orientation 453 | if (approx.size() == 4 && 454 | fabs(contourArea(Mat(approx))) > 1000 && 455 | isContourConvex(Mat(approx))) 456 | { 457 | double maxCosine = 0; 458 | 459 | for (int j = 2; j < 5; j++) 460 | { 461 | // find the maximum cosine of the angle between joint edges 462 | double cosine = fabs(angle(approx[j % 4], approx[j - 2], approx[j - 1])); 463 | maxCosine = MAX(maxCosine, cosine); 464 | } 465 | 466 | // if cosines of all angles are small 467 | // (all angles are ~90 degree) then write quandrange 468 | // vertices to resultant sequence 469 | if (maxCosine < 0.3) 470 | colorSquares[c].push_back(approx); 471 | } 472 | } 473 | } 474 | } 475 | 476 | FilterSquares(image, colorSquares, squares); 477 | } 478 | 479 | // the function draws all the squares in the image 480 | static void DrawSquares(Mat &image, const vector> &squares) 481 | { 482 | for (size_t i = 0; i < squares.size(); i++) 483 | { 484 | const Point *p = &squares[i][0]; 485 | int n = (int)squares[i].size(); 486 | polylines(image, &p, &n, 1, true, Scalar(0, 255, 0), 3, CV_AA); 487 | } 488 | 489 | imshow(wndname, image); 490 | } 491 | 492 | // the function draws all the squares in the image on their respective color planes and colors 493 | static void DrawSquares(Mat &image, const vector> squares[]) 494 | { 495 | const int numOfColorPlanes = 3; 496 | Scalar color; 497 | const Scalar CV_BLUE(255, 0, 0); 498 | const Scalar CV_GREEN(0, 255, 0); 499 | const Scalar CV_RED(0, 0, 255); 500 | 501 | for (size_t c = 0; c < numOfColorPlanes; ++c) 502 | { 503 | switch (c) 504 | { 505 | case 0: 506 | color = CV_BLUE; 507 | break; 508 | case 1: 509 | color = CV_GREEN; 510 | break; 511 | case 2: 512 | color = CV_RED; 513 | break; 514 | default: 515 | break; 516 | } 517 | 518 | for (size_t i = 0; i < squares[c].size(); i++) 519 | { 520 | const Point *p = &squares[c][i][0]; 521 | int n = (int)squares[c][i].size(); 522 | polylines(image, &p, &n, 1, true, color, 3, CV_AA); 523 | } 524 | } 525 | 526 | imshow(wndname, image); 527 | } 528 | 529 | int main(int /*argc*/, char ** /*argv*/) 530 | { 531 | vector x(6); 532 | // Generate vector of ints from 1 to 6 533 | iota(begin(x), end(x), 1); 534 | 535 | vector names(6); 536 | transform(x.cbegin(), x.cend(), names.begin(), [](const int in) { return "pic" + to_string(in) + ".png"; }); 537 | 538 | help(); 539 | namedWindow(wndname, 1); 540 | vector> squares; 541 | vector> colorSquares[3]; 542 | 543 | for (auto name : names) 544 | { 545 | cout << "Name: " << name << endl; 546 | Mat image = imread(name, 1); 547 | if (image.empty()) 548 | { 549 | cout << "Couldn't load " << name << endl; 550 | continue; 551 | } 552 | 553 | FindSquares(image, squares, colorSquares); 554 | DrawSquares(image, squares); 555 | 556 | int c = waitKey(); 557 | if ((char)c == 27) 558 | break; 559 | } 560 | 561 | return 0; 562 | } 563 | --------------------------------------------------------------------------------