├── .gitignore ├── LICENSE ├── README.md ├── disparity_SGBM_norm.png ├── disparity_map.py ├── epilines.png ├── keypoint_matches.png ├── left_img.png ├── original_images.png ├── rectified_1.png ├── rectified_2.png ├── rectified_images.png ├── right_img.png └── sift_keypoints.png /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Andreas Jakl 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Depth Maps with Python and OpenCV 2 | Calculate and visualize depth maps (disparity maps) using OpenCV for Python. 3 | 4 | Normalized disparity map generated by this script: 5 | 6 | ![](https://raw.githubusercontent.com/andijakl/python-depthmaps/main/disparity_SGBM_norm.png) 7 | 8 | Source image (left camera image): 9 | 10 | ![](https://raw.githubusercontent.com/andijakl/python-depthmaps/main/left_img.png) 11 | 12 | ## How can I learn more? 13 | 14 | The tutorial and background info is available in the blog post series [Easily Create a Depth Map with Smartphone AR](https://www.andreasjakl.com/easily-create-depth-maps-with-smartphone-ar-part-1/). 15 | 16 | 17 | ## Credits 18 | 19 | Released under the MIT License - see the LICENSE file for details. 20 | 21 | Developed by Andreas Jakl, Professor at the St. Pölten University of Applied Sciences, Austria. 22 | 23 | * 24 | * 25 | * 26 | -------------------------------------------------------------------------------- /disparity_SGBM_norm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andijakl/python-depthmaps/ef4e395ca48b2c47f8cd0cc73afb43f62cf4c933/disparity_SGBM_norm.png -------------------------------------------------------------------------------- /disparity_map.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import cv2 as cv 3 | import matplotlib.pyplot as plt 4 | 5 | # Read both images and convert to grayscale 6 | img1 = cv.imread('left_img.png', cv.IMREAD_GRAYSCALE) 7 | img2 = cv.imread('right_img.png', cv.IMREAD_GRAYSCALE) 8 | 9 | # ------------------------------------------------------------ 10 | # PREPROCESSING 11 | 12 | # Compare unprocessed images 13 | fig, axes = plt.subplots(1, 2, figsize=(15, 10)) 14 | axes[0].imshow(img1, cmap="gray") 15 | axes[1].imshow(img2, cmap="gray") 16 | axes[0].axhline(250) 17 | axes[1].axhline(250) 18 | axes[0].axhline(450) 19 | axes[1].axhline(450) 20 | plt.suptitle("Original images") 21 | plt.savefig("original_images.png") 22 | plt.show() 23 | 24 | # 1. Detect keypoints and their descriptors 25 | # Based on: https://docs.opencv.org/master/dc/dc3/tutorial_py_matcher.html 26 | 27 | # Initiate SIFT detector 28 | sift = cv.SIFT_create() 29 | # find the keypoints and descriptors with SIFT 30 | kp1, des1 = sift.detectAndCompute(img1, None) 31 | kp2, des2 = sift.detectAndCompute(img2, None) 32 | 33 | # Visualize keypoints 34 | imgSift = cv.drawKeypoints( 35 | img1, kp1, None, flags=cv.DRAW_MATCHES_FLAGS_DRAW_RICH_KEYPOINTS) 36 | cv.imshow("SIFT Keypoints", imgSift) 37 | cv.imwrite("sift_keypoints.png", imgSift) 38 | 39 | # Match keypoints in both images 40 | # Based on: https://docs.opencv.org/master/dc/dc3/tutorial_py_matcher.html 41 | FLANN_INDEX_KDTREE = 1 42 | index_params = dict(algorithm=FLANN_INDEX_KDTREE, trees=5) 43 | search_params = dict(checks=50) # or pass empty dictionary 44 | flann = cv.FlannBasedMatcher(index_params, search_params) 45 | matches = flann.knnMatch(des1, des2, k=2) 46 | 47 | # Keep good matches: calculate distinctive image features 48 | # Lowe, D.G. Distinctive Image Features from Scale-Invariant Keypoints. International Journal of Computer Vision 60, 91–110 (2004). https://doi.org/10.1023/B:VISI.0000029664.99615.94 49 | # https://www.cs.ubc.ca/~lowe/papers/ijcv04.pdf 50 | matchesMask = [[0, 0] for i in range(len(matches))] 51 | good = [] 52 | pts1 = [] 53 | pts2 = [] 54 | 55 | for i, (m, n) in enumerate(matches): 56 | if m.distance < 0.7*n.distance: 57 | # Keep this keypoint pair 58 | matchesMask[i] = [1, 0] 59 | good.append(m) 60 | pts2.append(kp2[m.trainIdx].pt) 61 | pts1.append(kp1[m.queryIdx].pt) 62 | 63 | # Draw the keypoint matches between both pictures 64 | # Still based on: https://docs.opencv.org/master/dc/dc3/tutorial_py_matcher.html 65 | draw_params = dict(matchColor=(0, 255, 0), 66 | singlePointColor=(255, 0, 0), 67 | matchesMask=matchesMask[300:500], 68 | flags=cv.DrawMatchesFlags_DEFAULT) 69 | 70 | keypoint_matches = cv.drawMatchesKnn( 71 | img1, kp1, img2, kp2, matches[300:500], None, **draw_params) 72 | cv.imshow("Keypoint matches", keypoint_matches) 73 | cv.imwrite("keypoint_matches.png", keypoint_matches) 74 | 75 | 76 | # ------------------------------------------------------------ 77 | # STEREO RECTIFICATION 78 | 79 | # Calculate the fundamental matrix for the cameras 80 | # https://docs.opencv.org/master/da/de9/tutorial_py_epipolar_geometry.html 81 | pts1 = np.int32(pts1) 82 | pts2 = np.int32(pts2) 83 | fundamental_matrix, inliers = cv.findFundamentalMat(pts1, pts2, cv.FM_RANSAC) 84 | 85 | # We select only inlier points 86 | pts1 = pts1[inliers.ravel() == 1] 87 | pts2 = pts2[inliers.ravel() == 1] 88 | 89 | # Visualize epilines 90 | # Adapted from: https://docs.opencv.org/master/da/de9/tutorial_py_epipolar_geometry.html 91 | 92 | 93 | def drawlines(img1src, img2src, lines, pts1src, pts2src): 94 | ''' img1 - image on which we draw the epilines for the points in img2 95 | lines - corresponding epilines ''' 96 | r, c = img1src.shape 97 | img1color = cv.cvtColor(img1src, cv.COLOR_GRAY2BGR) 98 | img2color = cv.cvtColor(img2src, cv.COLOR_GRAY2BGR) 99 | # Edit: use the same random seed so that two images are comparable! 100 | np.random.seed(0) 101 | for r, pt1, pt2 in zip(lines, pts1src, pts2src): 102 | color = tuple(np.random.randint(0, 255, 3).tolist()) 103 | x0, y0 = map(int, [0, -r[2]/r[1]]) 104 | x1, y1 = map(int, [c, -(r[2]+r[0]*c)/r[1]]) 105 | img1color = cv.line(img1color, (x0, y0), (x1, y1), color, 1) 106 | img1color = cv.circle(img1color, tuple(pt1), 5, color, -1) 107 | img2color = cv.circle(img2color, tuple(pt2), 5, color, -1) 108 | return img1color, img2color 109 | 110 | 111 | # Find epilines corresponding to points in right image (second image) and 112 | # drawing its lines on left image 113 | lines1 = cv.computeCorrespondEpilines( 114 | pts2.reshape(-1, 1, 2), 2, fundamental_matrix) 115 | lines1 = lines1.reshape(-1, 3) 116 | img5, img6 = drawlines(img1, img2, lines1, pts1, pts2) 117 | 118 | # Find epilines corresponding to points in left image (first image) and 119 | # drawing its lines on right image 120 | lines2 = cv.computeCorrespondEpilines( 121 | pts1.reshape(-1, 1, 2), 1, fundamental_matrix) 122 | lines2 = lines2.reshape(-1, 3) 123 | img3, img4 = drawlines(img2, img1, lines2, pts2, pts1) 124 | 125 | plt.subplot(121), plt.imshow(img5) 126 | plt.subplot(122), plt.imshow(img3) 127 | plt.suptitle("Epilines in both images") 128 | plt.savefig("epilines.png") 129 | plt.show() 130 | 131 | 132 | # Stereo rectification (uncalibrated variant) 133 | # Adapted from: https://stackoverflow.com/a/62607343 134 | h1, w1 = img1.shape 135 | h2, w2 = img2.shape 136 | _, H1, H2 = cv.stereoRectifyUncalibrated( 137 | np.float32(pts1), np.float32(pts2), fundamental_matrix, imgSize=(w1, h1) 138 | ) 139 | 140 | # Rectify (undistort) the images and save them 141 | # Adapted from: https://stackoverflow.com/a/62607343 142 | img1_rectified = cv.warpPerspective(img1, H1, (w1, h1)) 143 | img2_rectified = cv.warpPerspective(img2, H2, (w2, h2)) 144 | cv.imwrite("rectified_1.png", img1_rectified) 145 | cv.imwrite("rectified_2.png", img2_rectified) 146 | 147 | # Draw the rectified images 148 | fig, axes = plt.subplots(1, 2, figsize=(15, 10)) 149 | axes[0].imshow(img1_rectified, cmap="gray") 150 | axes[1].imshow(img2_rectified, cmap="gray") 151 | axes[0].axhline(250) 152 | axes[1].axhline(250) 153 | axes[0].axhline(450) 154 | axes[1].axhline(450) 155 | plt.suptitle("Rectified images") 156 | plt.savefig("rectified_images.png") 157 | plt.show() 158 | 159 | # ------------------------------------------------------------ 160 | # CALCULATE DISPARITY (DEPTH MAP) 161 | # Adapted from: https://github.com/opencv/opencv/blob/master/samples/python/stereo_match.py 162 | # and: https://docs.opencv.org/master/dd/d53/tutorial_py_depthmap.html 163 | 164 | # StereoSGBM Parameter explanations: 165 | # https://docs.opencv.org/4.5.0/d2/d85/classcv_1_1StereoSGBM.html 166 | 167 | # Matched block size. It must be an odd number >=1 . Normally, it should be somewhere in the 3..11 range. 168 | block_size = 11 169 | min_disp = -128 170 | max_disp = 128 171 | # Maximum disparity minus minimum disparity. The value is always greater than zero. 172 | # In the current implementation, this parameter must be divisible by 16. 173 | num_disp = max_disp - min_disp 174 | # Margin in percentage by which the best (minimum) computed cost function value should "win" the second best value to consider the found match correct. 175 | # Normally, a value within the 5-15 range is good enough 176 | uniquenessRatio = 5 177 | # Maximum size of smooth disparity regions to consider their noise speckles and invalidate. 178 | # Set it to 0 to disable speckle filtering. Otherwise, set it somewhere in the 50-200 range. 179 | speckleWindowSize = 200 180 | # Maximum disparity variation within each connected component. 181 | # If you do speckle filtering, set the parameter to a positive value, it will be implicitly multiplied by 16. 182 | # Normally, 1 or 2 is good enough. 183 | speckleRange = 2 184 | disp12MaxDiff = 0 185 | 186 | stereo = cv.StereoSGBM_create( 187 | minDisparity=min_disp, 188 | numDisparities=num_disp, 189 | blockSize=block_size, 190 | uniquenessRatio=uniquenessRatio, 191 | speckleWindowSize=speckleWindowSize, 192 | speckleRange=speckleRange, 193 | disp12MaxDiff=disp12MaxDiff, 194 | P1=8 * 1 * block_size * block_size, 195 | P2=32 * 1 * block_size * block_size, 196 | ) 197 | disparity_SGBM = stereo.compute(img1_rectified, img2_rectified) 198 | 199 | plt.imshow(disparity_SGBM, cmap='plasma') 200 | plt.colorbar() 201 | plt.show() 202 | 203 | # Normalize the values to a range from 0..255 for a grayscale image 204 | disparity_SGBM = cv.normalize(disparity_SGBM, disparity_SGBM, alpha=255, 205 | beta=0, norm_type=cv.NORM_MINMAX) 206 | disparity_SGBM = np.uint8(disparity_SGBM) 207 | cv.imshow("Disparity", disparity_SGBM) 208 | cv.imwrite("disparity_SGBM_norm.png", disparity_SGBM) 209 | 210 | 211 | cv.waitKey() 212 | cv.destroyAllWindows() 213 | # --------------------------------------------------------------- 214 | -------------------------------------------------------------------------------- /epilines.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andijakl/python-depthmaps/ef4e395ca48b2c47f8cd0cc73afb43f62cf4c933/epilines.png -------------------------------------------------------------------------------- /keypoint_matches.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andijakl/python-depthmaps/ef4e395ca48b2c47f8cd0cc73afb43f62cf4c933/keypoint_matches.png -------------------------------------------------------------------------------- /left_img.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andijakl/python-depthmaps/ef4e395ca48b2c47f8cd0cc73afb43f62cf4c933/left_img.png -------------------------------------------------------------------------------- /original_images.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andijakl/python-depthmaps/ef4e395ca48b2c47f8cd0cc73afb43f62cf4c933/original_images.png -------------------------------------------------------------------------------- /rectified_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andijakl/python-depthmaps/ef4e395ca48b2c47f8cd0cc73afb43f62cf4c933/rectified_1.png -------------------------------------------------------------------------------- /rectified_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andijakl/python-depthmaps/ef4e395ca48b2c47f8cd0cc73afb43f62cf4c933/rectified_2.png -------------------------------------------------------------------------------- /rectified_images.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andijakl/python-depthmaps/ef4e395ca48b2c47f8cd0cc73afb43f62cf4c933/rectified_images.png -------------------------------------------------------------------------------- /right_img.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andijakl/python-depthmaps/ef4e395ca48b2c47f8cd0cc73afb43f62cf4c933/right_img.png -------------------------------------------------------------------------------- /sift_keypoints.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andijakl/python-depthmaps/ef4e395ca48b2c47f8cd0cc73afb43f62cf4c933/sift_keypoints.png --------------------------------------------------------------------------------