├── .gitignore ├── Disparity_Window_5_simulation.pdf ├── Disparity_Window_7_simulation.pdf ├── Img ├── FlowChart.png ├── Tsukuba_L.bmp ├── Tsukuba_L.png ├── Tsukuba_R.bmp ├── Tsukuba_R.png ├── Tsukuba_R_t.png ├── VerilogSimulationTime.png ├── maskL.bmp ├── maskL.png ├── maskR.bmp ├── maskR.png ├── test.png ├── toyL.bmp ├── toyL.png ├── toyR.bmp └── toyR.png ├── Python_test_implementation ├── Disparity_Python_implementation_scratch.ipynb ├── Disparity_TSukuba_5.jpg ├── Disparity_Tsukuba.jpg ├── Disparity__colorMap_Toy_5_python.jpg ├── Disparity__colorMap_Tsukuba.jpg ├── Disparity__colorMap_Tsukuba_5_python.jpg ├── Disparity_toy_5.jpg ├── img2hex.ipynb ├── imgtohex.ipynb ├── number.txt └── python.py ├── README.md ├── Tsukuba_L.hex ├── Tsukuba_R.hex ├── Tsukuba_output_3.bmp ├── Tsukuba_output_5.bmp ├── Tsukuba_output_7.bmp ├── image.hex ├── image_read.v ├── image_write.v ├── imgtohex.ipynb ├── output.bmp ├── output.png ├── parameter.v ├── simulation.pdf └── tb_simulation.v /.gitignore: -------------------------------------------------------------------------------- 1 | .ipynb_checkpoints/ 2 | _xmsgs/ 3 | iseconfig/ 4 | isim/ 5 | 6 | *.log 7 | *.xmsgs 8 | *.html 9 | *.htm 10 | *.cmd 11 | *.ini 12 | *.exe 13 | *.xise 14 | *.gise 15 | *.wdb 16 | *.prj 17 | *.bld 18 | *.cmd_log 19 | 20 | # intermediate build files 21 | *.bgn 22 | *.bit 23 | *.bld 24 | *.cmd_log 25 | *.drc 26 | *.ll 27 | *.lso 28 | *.msd 29 | *.msk 30 | *.ncd 31 | *.ngc 32 | *.ngd 33 | *.ngr 34 | *.pad 35 | *.par 36 | *.pcf 37 | *.prj 38 | *.ptwx 39 | *.rbb 40 | *.rbd 41 | *.stx 42 | *.syr 43 | *.twr 44 | *.twx 45 | *.unroutes 46 | *.ut 47 | *.xpi 48 | *.xst 49 | *_bitgen.xwbt 50 | *_envsettings.html 51 | *_map.map 52 | *_map.mrp 53 | *_map.ngm 54 | *_map.xrpt 55 | *_ngdbuild.xrpt 56 | *_pad.csv 57 | *_pad.txt 58 | *_par.xrpt 59 | *_summary.html 60 | *_summary.xml 61 | *_usage.xml 62 | *_xst.xrpt 63 | 64 | # iMPACT generated files 65 | _impactbatch.log 66 | impact.xsl 67 | impact_impact.xwbt 68 | ise_impact.cmd 69 | webtalk_impact.xml 70 | 71 | # Core Generator generated files 72 | xaw2verilog.log 73 | 74 | # project-wide generated files 75 | *.gise 76 | par_usage_statistics.html 77 | usage_statistics_webtalk.html 78 | webtalk.log 79 | webtalk_pn.xml 80 | 81 | # generated folders 82 | iseconfig/ 83 | xlnx_auto_0_xdb/ 84 | xst/ 85 | _ngo/ 86 | _xmsgs/ -------------------------------------------------------------------------------- /Disparity_Window_5_simulation.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Archfx/FPGA_depthMap/e718aa9433c02baa277ddcdabe67d3d3c6d78a10/Disparity_Window_5_simulation.pdf -------------------------------------------------------------------------------- /Disparity_Window_7_simulation.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Archfx/FPGA_depthMap/e718aa9433c02baa277ddcdabe67d3d3c6d78a10/Disparity_Window_7_simulation.pdf -------------------------------------------------------------------------------- /Img/FlowChart.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Archfx/FPGA_depthMap/e718aa9433c02baa277ddcdabe67d3d3c6d78a10/Img/FlowChart.png -------------------------------------------------------------------------------- /Img/Tsukuba_L.bmp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Archfx/FPGA_depthMap/e718aa9433c02baa277ddcdabe67d3d3c6d78a10/Img/Tsukuba_L.bmp -------------------------------------------------------------------------------- /Img/Tsukuba_L.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Archfx/FPGA_depthMap/e718aa9433c02baa277ddcdabe67d3d3c6d78a10/Img/Tsukuba_L.png -------------------------------------------------------------------------------- /Img/Tsukuba_R.bmp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Archfx/FPGA_depthMap/e718aa9433c02baa277ddcdabe67d3d3c6d78a10/Img/Tsukuba_R.bmp -------------------------------------------------------------------------------- /Img/Tsukuba_R.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Archfx/FPGA_depthMap/e718aa9433c02baa277ddcdabe67d3d3c6d78a10/Img/Tsukuba_R.png -------------------------------------------------------------------------------- /Img/Tsukuba_R_t.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Archfx/FPGA_depthMap/e718aa9433c02baa277ddcdabe67d3d3c6d78a10/Img/Tsukuba_R_t.png -------------------------------------------------------------------------------- /Img/VerilogSimulationTime.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Archfx/FPGA_depthMap/e718aa9433c02baa277ddcdabe67d3d3c6d78a10/Img/VerilogSimulationTime.png -------------------------------------------------------------------------------- /Img/maskL.bmp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Archfx/FPGA_depthMap/e718aa9433c02baa277ddcdabe67d3d3c6d78a10/Img/maskL.bmp -------------------------------------------------------------------------------- /Img/maskL.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Archfx/FPGA_depthMap/e718aa9433c02baa277ddcdabe67d3d3c6d78a10/Img/maskL.png -------------------------------------------------------------------------------- /Img/maskR.bmp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Archfx/FPGA_depthMap/e718aa9433c02baa277ddcdabe67d3d3c6d78a10/Img/maskR.bmp -------------------------------------------------------------------------------- /Img/maskR.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Archfx/FPGA_depthMap/e718aa9433c02baa277ddcdabe67d3d3c6d78a10/Img/maskR.png -------------------------------------------------------------------------------- /Img/test.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Archfx/FPGA_depthMap/e718aa9433c02baa277ddcdabe67d3d3c6d78a10/Img/test.png -------------------------------------------------------------------------------- /Img/toyL.bmp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Archfx/FPGA_depthMap/e718aa9433c02baa277ddcdabe67d3d3c6d78a10/Img/toyL.bmp -------------------------------------------------------------------------------- /Img/toyL.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Archfx/FPGA_depthMap/e718aa9433c02baa277ddcdabe67d3d3c6d78a10/Img/toyL.png -------------------------------------------------------------------------------- /Img/toyR.bmp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Archfx/FPGA_depthMap/e718aa9433c02baa277ddcdabe67d3d3c6d78a10/Img/toyR.bmp -------------------------------------------------------------------------------- /Img/toyR.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Archfx/FPGA_depthMap/e718aa9433c02baa277ddcdabe67d3d3c6d78a10/Img/toyR.png -------------------------------------------------------------------------------- /Python_test_implementation/Disparity_TSukuba_5.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Archfx/FPGA_depthMap/e718aa9433c02baa277ddcdabe67d3d3c6d78a10/Python_test_implementation/Disparity_TSukuba_5.jpg -------------------------------------------------------------------------------- /Python_test_implementation/Disparity_Tsukuba.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Archfx/FPGA_depthMap/e718aa9433c02baa277ddcdabe67d3d3c6d78a10/Python_test_implementation/Disparity_Tsukuba.jpg -------------------------------------------------------------------------------- /Python_test_implementation/Disparity__colorMap_Toy_5_python.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Archfx/FPGA_depthMap/e718aa9433c02baa277ddcdabe67d3d3c6d78a10/Python_test_implementation/Disparity__colorMap_Toy_5_python.jpg -------------------------------------------------------------------------------- /Python_test_implementation/Disparity__colorMap_Tsukuba.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Archfx/FPGA_depthMap/e718aa9433c02baa277ddcdabe67d3d3c6d78a10/Python_test_implementation/Disparity__colorMap_Tsukuba.jpg -------------------------------------------------------------------------------- /Python_test_implementation/Disparity__colorMap_Tsukuba_5_python.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Archfx/FPGA_depthMap/e718aa9433c02baa277ddcdabe67d3d3c6d78a10/Python_test_implementation/Disparity__colorMap_Tsukuba_5_python.jpg -------------------------------------------------------------------------------- /Python_test_implementation/Disparity_toy_5.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Archfx/FPGA_depthMap/e718aa9433c02baa277ddcdabe67d3d3c6d78a10/Python_test_implementation/Disparity_toy_5.jpg -------------------------------------------------------------------------------- /Python_test_implementation/img2hex.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 66, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import cv2\n", 10 | "import base64\n", 11 | "import numpy as np" 12 | ] 13 | }, 14 | { 15 | "cell_type": "code", 16 | "execution_count": 83, 17 | "metadata": {}, 18 | "outputs": [ 19 | { 20 | "name": "stdout", 21 | "output_type": "stream", 22 | "text": [ 23 | "77878\n", 24 | "76800\n" 25 | ] 26 | } 27 | ], 28 | "source": [ 29 | "image = cv2.imread('Img/image.jpg')\n", 30 | "gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)\n", 31 | "#buffer = cv2.imencode('.jpg', image)\n", 32 | "(tmp,buffer) = cv2.imencode('.bmp', gray)\n", 33 | "#imcode = base64.b64encode(buffer)\n", 34 | "#print(tmp)\n", 35 | "print (len(buffer))\n", 36 | "#230454\n", 37 | "buffer=buffer[1078:]\n", 38 | "print (len(buffer))" 39 | ] 40 | }, 41 | { 42 | "cell_type": "code", 43 | "execution_count": 88, 44 | "metadata": {}, 45 | "outputs": [ 46 | { 47 | "name": "stderr", 48 | "output_type": "stream", 49 | "text": [ 50 | "c:\\users\\aruna\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\\ipykernel_launcher.py:1: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n", 51 | " \"\"\"Entry point for launching an IPython kernel.\n" 52 | ] 53 | }, 54 | { 55 | "ename": "TypeError", 56 | "evalue": "bad argument type for built-in operation", 57 | "output_type": "error", 58 | "traceback": [ 59 | "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", 60 | "\u001b[1;31mTypeError\u001b[0m Traceback (most recent call last)", 61 | "\u001b[1;32m\u001b[0m in \u001b[0;36m\u001b[1;34m\u001b[0m\n\u001b[0;32m 1\u001b[0m \u001b[0mimage\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfromstring\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mbuffer\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0muint8\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[1;33m(\u001b[0m \u001b[1;36m320\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;36m240\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m1\u001b[0m \u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[0mcv2\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mimshow\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mimage\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;36m320\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m", 62 | "\u001b[1;31mTypeError\u001b[0m: bad argument type for built-in operation" 63 | ] 64 | } 65 | ], 66 | "source": [ 67 | "image = np.fromstring(buffer, np.uint8).reshape( 320,240, 1 )\n", 68 | "cv2.imshow(image,320)" 69 | ] 70 | }, 71 | { 72 | "cell_type": "code", 73 | "execution_count": 25, 74 | "metadata": {}, 75 | "outputs": [], 76 | "source": [ 77 | "file = open('image.hex', 'w')\n", 78 | "for i in buffer:\n", 79 | " #print (str(hex(i[0]))[2:])\n", 80 | " file.write(str(hex(i[0]))[2:])\n", 81 | " file.write('\\n')\n", 82 | " #print (i)\n", 83 | "\n", 84 | "file.close()" 85 | ] 86 | }, 87 | { 88 | "cell_type": "code", 89 | "execution_count": 64, 90 | "metadata": {}, 91 | "outputs": [], 92 | "source": [ 93 | "import struct\n", 94 | "\n", 95 | "bmp = open('Img/fl.bmp', 'rb')\n", 96 | "file1 = open('header.txt', 'w')\n", 97 | "for i in range(54):\n", 98 | " y=bmp.read(1)\n", 99 | " file1.write(str(int.from_bytes(y, \"little\")))\n", 100 | " file1.write('\\n')\n", 101 | "file1.close()\n", 102 | "# print('Type:', bmp.read(2).decode())\n", 103 | "# print(bmp.read(1))\n", 104 | "# print(bmp.read(1))\n", 105 | "# print(bmp.read(1))\n", 106 | "# print(bmp.read(1))\n", 107 | "# #print('Size: %s' % struct.unpack('I', bmp.read(4)))\n", 108 | "# print('Reserved 1: %s' % struct.unpack('H', bmp.read(2)))\n", 109 | "# print('Reserved 2: %s' % struct.unpack('H', bmp.read(2)))\n", 110 | "# print('Offset: %s' % struct.unpack('I', bmp.read(4)))\n", 111 | "\n", 112 | "# print('DIB Header Size: %s' % struct.unpack('I', bmp.read(4)))\n", 113 | "# print(bmp.read(1))\n", 114 | "# print(bmp.read(1))\n", 115 | "# print(bmp.read(1))\n", 116 | "# print(bmp.read(1))\n", 117 | "\n", 118 | "# print(str(bmp.read(1)))\n", 119 | "# print(bmp.read(1))\n", 120 | "# print(bmp.read(1))\n", 121 | "# print(bmp.read(1))\n", 122 | "# #print('Width: %s' % struct.unpack('I', bmp.read(4)))\n", 123 | "# #print('Height: %s' % struct.unpack('I', bmp.read(4)))\n", 124 | "# print('Colour Planes: %s' % struct.unpack('H', bmp.read(2)))\n", 125 | "# print('Bits per Pixel: %s' % struct.unpack('H', bmp.read(2)))\n", 126 | "# print('Compression Method: %s' % struct.unpack('I', bmp.read(4)))\n", 127 | "# print('Raw Image Size: %s' % struct.unpack('I', bmp.read(4)))\n", 128 | "# print('Horizontal Resolution: %s' % struct.unpack('I', bmp.read(4)))\n", 129 | "# print('Vertical Resolution: %s' % struct.unpack('I', bmp.read(4)))\n", 130 | "# print('Number of Colours: %s' % struct.unpack('I', bmp.read(4)))\n", 131 | "# print('Important Colours: %s' % struct.unpack('I', bmp.read(4)))" 132 | ] 133 | }, 134 | { 135 | "cell_type": "code", 136 | "execution_count": null, 137 | "metadata": {}, 138 | "outputs": [], 139 | "source": [] 140 | } 141 | ], 142 | "metadata": { 143 | "kernelspec": { 144 | "display_name": "Python 3", 145 | "language": "python", 146 | "name": "python3" 147 | }, 148 | "language_info": { 149 | "codemirror_mode": { 150 | "name": "ipython", 151 | "version": 3 152 | }, 153 | "file_extension": ".py", 154 | "mimetype": "text/x-python", 155 | "name": "python", 156 | "nbconvert_exporter": "python", 157 | "pygments_lexer": "ipython3", 158 | "version": "3.6.8" 159 | } 160 | }, 161 | "nbformat": 4, 162 | "nbformat_minor": 2 163 | } 164 | -------------------------------------------------------------------------------- /Python_test_implementation/imgtohex.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 48, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import cv2\n", 10 | "import base64\n", 11 | "import numpy as np" 12 | ] 13 | }, 14 | { 15 | "cell_type": "code", 16 | "execution_count": 53, 17 | "metadata": {}, 18 | "outputs": [ 19 | { 20 | "name": "stdout", 21 | "output_type": "stream", 22 | "text": [ 23 | "77878\n", 24 | "240\n", 25 | "320\n", 26 | "(240, 320)\n" 27 | ] 28 | } 29 | ], 30 | "source": [ 31 | "image = cv2.imread('Img/Tsukuba_R.bmp')\n", 32 | "width, height = image.shape[:2]\n", 33 | "gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)\n", 34 | "#buffer = cv2.imencode('.jpg', image)\n", 35 | "(tmp,buffer) = cv2.imencode('.bmp', gray)\n", 36 | "print (len(buffer))\n", 37 | "print (width)\n", 38 | "print (height)\n", 39 | "#230454\n", 40 | "print (gray.shape)" 41 | ] 42 | }, 43 | { 44 | "cell_type": "code", 45 | "execution_count": null, 46 | "metadata": {}, 47 | "outputs": [], 48 | "source": [] 49 | }, 50 | { 51 | "cell_type": "code", 52 | "execution_count": 54, 53 | "metadata": {}, 54 | "outputs": [ 55 | { 56 | "name": "stdout", 57 | "output_type": "stream", 58 | "text": [ 59 | "(240, 320)\n" 60 | ] 61 | } 62 | ], 63 | "source": [ 64 | "# for x in gray:\n", 65 | "# print (x)\n", 66 | "print (gray.shape)\n", 67 | "gray=gray.reshape(76800)" 68 | ] 69 | }, 70 | { 71 | "cell_type": "code", 72 | "execution_count": 51, 73 | "metadata": {}, 74 | "outputs": [], 75 | "source": [ 76 | "gray=gray.astype(int)" 77 | ] 78 | }, 79 | { 80 | "cell_type": "code", 81 | "execution_count": 55, 82 | "metadata": {}, 83 | "outputs": [], 84 | "source": [ 85 | "file = open('Tsukuba_R', 'w')\n", 86 | "for i in gray:\n", 87 | " #print (hex(i)[2:])#(str(hex(i[0]))[2:])\n", 88 | " file.write(str(hex(i))[2:])\n", 89 | " file.write('\\n')\n", 90 | " #print (i)\n", 91 | "\n", 92 | "file.close()" 93 | ] 94 | }, 95 | { 96 | "cell_type": "code", 97 | "execution_count": 10, 98 | "metadata": {}, 99 | "outputs": [], 100 | "source": [] 101 | }, 102 | { 103 | "cell_type": "code", 104 | "execution_count": null, 105 | "metadata": {}, 106 | "outputs": [], 107 | "source": [] 108 | }, 109 | { 110 | "cell_type": "code", 111 | "execution_count": null, 112 | "metadata": {}, 113 | "outputs": [], 114 | "source": [] 115 | }, 116 | { 117 | "cell_type": "code", 118 | "execution_count": null, 119 | "metadata": {}, 120 | "outputs": [], 121 | "source": [] 122 | }, 123 | { 124 | "cell_type": "code", 125 | "execution_count": null, 126 | "metadata": {}, 127 | "outputs": [], 128 | "source": [] 129 | } 130 | ], 131 | "metadata": { 132 | "kernelspec": { 133 | "display_name": "Python 3", 134 | "language": "python", 135 | "name": "python3" 136 | }, 137 | "language_info": { 138 | "codemirror_mode": { 139 | "name": "ipython", 140 | "version": 3 141 | }, 142 | "file_extension": ".py", 143 | "mimetype": "text/x-python", 144 | "name": "python", 145 | "nbconvert_exporter": "python", 146 | "pygments_lexer": "ipython3", 147 | "version": "3.6.8" 148 | } 149 | }, 150 | "nbformat": 4, 151 | "nbformat_minor": 2 152 | } 153 | -------------------------------------------------------------------------------- /Python_test_implementation/number.txt: -------------------------------------------------------------------------------- 1 | ssd <= ssd + std_logic_vector 2 | (to_unsigned( 3 | (to_integer(unsigned(org_L((to_integer(unsigned(row)) -2 ) * WIDTH + to_integer(unsigned(col)) -2 )))-to_integer(unsigned(org_R((to_integer(unsigned(row)) -2 ) * WIDTH + to_integer(unsigned(col)) -2 - to_integer(unsigned(offset))))))*(to_integer(unsigned(org_L((to_integer(unsigned(row)) -2 ) * WIDTH + to_integer(unsigned(col)) -2 )))-to_integer(unsigned(org_R((to_integer(unsigned(row)) -2 ) * WIDTH + to_integer(unsigned(col)) -2 -to_integer(unsigned(offset)))))) 4 | +(to_integer(unsigned(org_L((to_integer(unsigned(row)) -2 ) * WIDTH + to_integer(unsigned(col)) -1 )))-to_integer(unsigned(org_R((to_integer(unsigned(row)) -2 ) * WIDTH + to_integer(unsigned(col)) -1 - to_integer(unsigned(offset))))))*(to_integer(unsigned(org_L((to_integer(unsigned(row)) -2 ) * WIDTH + to_integer(unsigned(col)) -1 )))-to_integer(unsigned(org_R((to_integer(unsigned(row)) -2 ) * WIDTH + to_integer(unsigned(col)) -1 -to_integer(unsigned(offset)))))) 5 | +(to_integer(unsigned(org_L((to_integer(unsigned(row)) -2 ) * WIDTH + to_integer(unsigned(col)) + 0 )))-to_integer(unsigned(org_R((to_integer(unsigned(row)) -2 ) * WIDTH + to_integer(unsigned(col)) + 0 - to_integer(unsigned(offset))))))*(to_integer(unsigned(org_L((to_integer(unsigned(row)) -2 ) * WIDTH + to_integer(unsigned(col)) + 0 )))-to_integer(unsigned(org_R((to_integer(unsigned(row)) -2 ) * WIDTH + to_integer(unsigned(col)) + 0 -to_integer(unsigned(offset)))))) 6 | +(to_integer(unsigned(org_L((to_integer(unsigned(row)) -2 ) * WIDTH + to_integer(unsigned(col)) + 1 )))-to_integer(unsigned(org_R((to_integer(unsigned(row)) -2 ) * WIDTH + to_integer(unsigned(col)) + 1 - to_integer(unsigned(offset))))))*(to_integer(unsigned(org_L((to_integer(unsigned(row)) -2 ) * WIDTH + to_integer(unsigned(col)) + 1 )))-to_integer(unsigned(org_R((to_integer(unsigned(row)) -2 ) * WIDTH + to_integer(unsigned(col)) + 1 -to_integer(unsigned(offset)))))) 7 | +(to_integer(unsigned(org_L((to_integer(unsigned(row)) -2 ) * WIDTH + to_integer(unsigned(col)) + 2 )))-to_integer(unsigned(org_R((to_integer(unsigned(row)) -2 ) * WIDTH + to_integer(unsigned(col)) + 2 - to_integer(unsigned(offset))))))*(to_integer(unsigned(org_L((to_integer(unsigned(row)) -2 ) * WIDTH + to_integer(unsigned(col)) + 2 )))-to_integer(unsigned(org_R((to_integer(unsigned(row)) -2 ) * WIDTH + to_integer(unsigned(col)) + 2 -to_integer(unsigned(offset)))))) 8 | +(to_integer(unsigned(org_L((to_integer(unsigned(row)) -1 ) * WIDTH + to_integer(unsigned(col)) -2 )))-to_integer(unsigned(org_R((to_integer(unsigned(row)) -1 ) * WIDTH + to_integer(unsigned(col)) -2 - to_integer(unsigned(offset))))))*(to_integer(unsigned(org_L((to_integer(unsigned(row)) -1 ) * WIDTH + to_integer(unsigned(col)) -2 )))-to_integer(unsigned(org_R((to_integer(unsigned(row)) -1 ) * WIDTH + to_integer(unsigned(col)) -2 -to_integer(unsigned(offset)))))) 9 | +(to_integer(unsigned(org_L((to_integer(unsigned(row)) -1 ) * WIDTH + to_integer(unsigned(col)) -1 )))-to_integer(unsigned(org_R((to_integer(unsigned(row)) -1 ) * WIDTH + to_integer(unsigned(col)) -1 - to_integer(unsigned(offset))))))*(to_integer(unsigned(org_L((to_integer(unsigned(row)) -1 ) * WIDTH + to_integer(unsigned(col)) -1 )))-to_integer(unsigned(org_R((to_integer(unsigned(row)) -1 ) * WIDTH + to_integer(unsigned(col)) -1 -to_integer(unsigned(offset)))))) 10 | +(to_integer(unsigned(org_L((to_integer(unsigned(row)) -1 ) * WIDTH + to_integer(unsigned(col)) + 0 )))-to_integer(unsigned(org_R((to_integer(unsigned(row)) -1 ) * WIDTH + to_integer(unsigned(col)) + 0 - to_integer(unsigned(offset))))))*(to_integer(unsigned(org_L((to_integer(unsigned(row)) -1 ) * WIDTH + to_integer(unsigned(col)) + 0 )))-to_integer(unsigned(org_R((to_integer(unsigned(row)) -1 ) * WIDTH + to_integer(unsigned(col)) + 0 -to_integer(unsigned(offset)))))) 11 | +(to_integer(unsigned(org_L((to_integer(unsigned(row)) -1 ) * WIDTH + to_integer(unsigned(col)) + 1 )))-to_integer(unsigned(org_R((to_integer(unsigned(row)) -1 ) * WIDTH + to_integer(unsigned(col)) + 1 - to_integer(unsigned(offset))))))*(to_integer(unsigned(org_L((to_integer(unsigned(row)) -1 ) * WIDTH + to_integer(unsigned(col)) + 1 )))-to_integer(unsigned(org_R((to_integer(unsigned(row)) -1 ) * WIDTH + to_integer(unsigned(col)) + 1 -to_integer(unsigned(offset)))))) 12 | +(to_integer(unsigned(org_L((to_integer(unsigned(row)) -1 ) * WIDTH + to_integer(unsigned(col)) + 2 )))-to_integer(unsigned(org_R((to_integer(unsigned(row)) -1 ) * WIDTH + to_integer(unsigned(col)) + 2 - to_integer(unsigned(offset))))))*(to_integer(unsigned(org_L((to_integer(unsigned(row)) -1 ) * WIDTH + to_integer(unsigned(col)) + 2 )))-to_integer(unsigned(org_R((to_integer(unsigned(row)) -1 ) * WIDTH + to_integer(unsigned(col)) + 2 -to_integer(unsigned(offset)))))) 13 | +(to_integer(unsigned(org_L((to_integer(unsigned(row)) + 0 ) * WIDTH + to_integer(unsigned(col)) -2 )))-to_integer(unsigned(org_R((to_integer(unsigned(row)) + 0 ) * WIDTH + to_integer(unsigned(col)) -2 - to_integer(unsigned(offset))))))*(to_integer(unsigned(org_L((to_integer(unsigned(row)) + 0 ) * WIDTH + to_integer(unsigned(col)) -2 )))-to_integer(unsigned(org_R((to_integer(unsigned(row)) + 0 ) * WIDTH + to_integer(unsigned(col)) -2 -to_integer(unsigned(offset)))))) 14 | +(to_integer(unsigned(org_L((to_integer(unsigned(row)) + 0 ) * WIDTH + to_integer(unsigned(col)) -1 )))-to_integer(unsigned(org_R((to_integer(unsigned(row)) + 0 ) * WIDTH + to_integer(unsigned(col)) -1 - to_integer(unsigned(offset))))))*(to_integer(unsigned(org_L((to_integer(unsigned(row)) + 0 ) * WIDTH + to_integer(unsigned(col)) -1 )))-to_integer(unsigned(org_R((to_integer(unsigned(row)) + 0 ) * WIDTH + to_integer(unsigned(col)) -1 -to_integer(unsigned(offset)))))) 15 | +(to_integer(unsigned(org_L((to_integer(unsigned(row)) + 0 ) * WIDTH + to_integer(unsigned(col)) + 0 )))-to_integer(unsigned(org_R((to_integer(unsigned(row)) + 0 ) * WIDTH + to_integer(unsigned(col)) + 0 - to_integer(unsigned(offset))))))*(to_integer(unsigned(org_L((to_integer(unsigned(row)) + 0 ) * WIDTH + to_integer(unsigned(col)) + 0 )))-to_integer(unsigned(org_R((to_integer(unsigned(row)) + 0 ) * WIDTH + to_integer(unsigned(col)) + 0 -to_integer(unsigned(offset)))))) 16 | +(to_integer(unsigned(org_L((to_integer(unsigned(row)) + 0 ) * WIDTH + to_integer(unsigned(col)) + 1 )))-to_integer(unsigned(org_R((to_integer(unsigned(row)) + 0 ) * WIDTH + to_integer(unsigned(col)) + 1 - to_integer(unsigned(offset))))))*(to_integer(unsigned(org_L((to_integer(unsigned(row)) + 0 ) * WIDTH + to_integer(unsigned(col)) + 1 )))-to_integer(unsigned(org_R((to_integer(unsigned(row)) + 0 ) * WIDTH + to_integer(unsigned(col)) + 1 -to_integer(unsigned(offset)))))) 17 | +(to_integer(unsigned(org_L((to_integer(unsigned(row)) + 0 ) * WIDTH + to_integer(unsigned(col)) + 2 )))-to_integer(unsigned(org_R((to_integer(unsigned(row)) + 0 ) * WIDTH + to_integer(unsigned(col)) + 2 - to_integer(unsigned(offset))))))*(to_integer(unsigned(org_L((to_integer(unsigned(row)) + 0 ) * WIDTH + to_integer(unsigned(col)) + 2 )))-to_integer(unsigned(org_R((to_integer(unsigned(row)) + 0 ) * WIDTH + to_integer(unsigned(col)) + 2 -to_integer(unsigned(offset)))))) 18 | +(to_integer(unsigned(org_L((to_integer(unsigned(row)) + 1 ) * WIDTH + to_integer(unsigned(col)) -2 )))-to_integer(unsigned(org_R((to_integer(unsigned(row)) + 1 ) * WIDTH + to_integer(unsigned(col)) -2 - to_integer(unsigned(offset))))))*(to_integer(unsigned(org_L((to_integer(unsigned(row)) + 1 ) * WIDTH + to_integer(unsigned(col)) -2 )))-to_integer(unsigned(org_R((to_integer(unsigned(row)) + 1 ) * WIDTH + to_integer(unsigned(col)) -2 -to_integer(unsigned(offset)))))) 19 | +(to_integer(unsigned(org_L((to_integer(unsigned(row)) + 1 ) * WIDTH + to_integer(unsigned(col)) -1 )))-to_integer(unsigned(org_R((to_integer(unsigned(row)) + 1 ) * WIDTH + to_integer(unsigned(col)) -1 - to_integer(unsigned(offset))))))*(to_integer(unsigned(org_L((to_integer(unsigned(row)) + 1 ) * WIDTH + to_integer(unsigned(col)) -1 )))-to_integer(unsigned(org_R((to_integer(unsigned(row)) + 1 ) * WIDTH + to_integer(unsigned(col)) -1 -to_integer(unsigned(offset)))))) 20 | +(to_integer(unsigned(org_L((to_integer(unsigned(row)) + 1 ) * WIDTH + to_integer(unsigned(col)) + 0 )))-to_integer(unsigned(org_R((to_integer(unsigned(row)) + 1 ) * WIDTH + to_integer(unsigned(col)) + 0 - to_integer(unsigned(offset))))))*(to_integer(unsigned(org_L((to_integer(unsigned(row)) + 1 ) * WIDTH + to_integer(unsigned(col)) + 0 )))-to_integer(unsigned(org_R((to_integer(unsigned(row)) + 1 ) * WIDTH + to_integer(unsigned(col)) + 0 -to_integer(unsigned(offset)))))) 21 | +(to_integer(unsigned(org_L((to_integer(unsigned(row)) + 1 ) * WIDTH + to_integer(unsigned(col)) + 1 )))-to_integer(unsigned(org_R((to_integer(unsigned(row)) + 1 ) * WIDTH + to_integer(unsigned(col)) + 1 - to_integer(unsigned(offset))))))*(to_integer(unsigned(org_L((to_integer(unsigned(row)) + 1 ) * WIDTH + to_integer(unsigned(col)) + 1 )))-to_integer(unsigned(org_R((to_integer(unsigned(row)) + 1 ) * WIDTH + to_integer(unsigned(col)) + 1 -to_integer(unsigned(offset)))))) 22 | +(to_integer(unsigned(org_L((to_integer(unsigned(row)) + 1 ) * WIDTH + to_integer(unsigned(col)) + 2 )))-to_integer(unsigned(org_R((to_integer(unsigned(row)) + 1 ) * WIDTH + to_integer(unsigned(col)) + 2 - to_integer(unsigned(offset))))))*(to_integer(unsigned(org_L((to_integer(unsigned(row)) + 1 ) * WIDTH + to_integer(unsigned(col)) + 2 )))-to_integer(unsigned(org_R((to_integer(unsigned(row)) + 1 ) * WIDTH + to_integer(unsigned(col)) + 2 -to_integer(unsigned(offset)))))) 23 | +(to_integer(unsigned(org_L((to_integer(unsigned(row)) + 2 ) * WIDTH + to_integer(unsigned(col)) -2 )))-to_integer(unsigned(org_R((to_integer(unsigned(row)) + 2 ) * WIDTH + to_integer(unsigned(col)) -2 - to_integer(unsigned(offset))))))*(to_integer(unsigned(org_L((to_integer(unsigned(row)) + 2 ) * WIDTH + to_integer(unsigned(col)) -2 )))-to_integer(unsigned(org_R((to_integer(unsigned(row)) + 2 ) * WIDTH + to_integer(unsigned(col)) -2 -to_integer(unsigned(offset)))))) 24 | +(to_integer(unsigned(org_L((to_integer(unsigned(row)) + 2 ) * WIDTH + to_integer(unsigned(col)) -1 )))-to_integer(unsigned(org_R((to_integer(unsigned(row)) + 2 ) * WIDTH + to_integer(unsigned(col)) -1 - to_integer(unsigned(offset))))))*(to_integer(unsigned(org_L((to_integer(unsigned(row)) + 2 ) * WIDTH + to_integer(unsigned(col)) -1 )))-to_integer(unsigned(org_R((to_integer(unsigned(row)) + 2 ) * WIDTH + to_integer(unsigned(col)) -1 -to_integer(unsigned(offset)))))) 25 | +(to_integer(unsigned(org_L((to_integer(unsigned(row)) + 2 ) * WIDTH + to_integer(unsigned(col)) + 0 )))-to_integer(unsigned(org_R((to_integer(unsigned(row)) + 2 ) * WIDTH + to_integer(unsigned(col)) + 0 - to_integer(unsigned(offset))))))*(to_integer(unsigned(org_L((to_integer(unsigned(row)) + 2 ) * WIDTH + to_integer(unsigned(col)) + 0 )))-to_integer(unsigned(org_R((to_integer(unsigned(row)) + 2 ) * WIDTH + to_integer(unsigned(col)) + 0 -to_integer(unsigned(offset)))))) 26 | +(to_integer(unsigned(org_L((to_integer(unsigned(row)) + 2 ) * WIDTH + to_integer(unsigned(col)) + 1 )))-to_integer(unsigned(org_R((to_integer(unsigned(row)) + 2 ) * WIDTH + to_integer(unsigned(col)) + 1 - to_integer(unsigned(offset))))))*(to_integer(unsigned(org_L((to_integer(unsigned(row)) + 2 ) * WIDTH + to_integer(unsigned(col)) + 1 )))-to_integer(unsigned(org_R((to_integer(unsigned(row)) + 2 ) * WIDTH + to_integer(unsigned(col)) + 1 -to_integer(unsigned(offset)))))) 27 | +(to_integer(unsigned(org_L((to_integer(unsigned(row)) + 2 ) * WIDTH + to_integer(unsigned(col)) + 2 )))-to_integer(unsigned(org_R((to_integer(unsigned(row)) + 2 ) * WIDTH + to_integer(unsigned(col)) + 2 - to_integer(unsigned(offset))))))*(to_integer(unsigned(org_L((to_integer(unsigned(row)) + 2 ) * WIDTH + to_integer(unsigned(col)) + 2 )))-to_integer(unsigned(org_R((to_integer(unsigned(row)) + 2 ) * WIDTH + to_integer(unsigned(col)) + 2 -to_integer(unsigned(offset)))))) 28 | ,ssd'length)); -------------------------------------------------------------------------------- /Python_test_implementation/python.py: -------------------------------------------------------------------------------- 1 | for m in range(0,5): 2 | for n in range(0,5): 3 | x=m-2 4 | y=n-2 5 | print("+(to_integer(unsigned(org_L((to_integer(unsigned(row)) +",x,") * WIDTH + to_integer(unsigned(col)) +",y," ))-to_integer(unsigned(org_R((to_integer(unsigned(row)) +",x,") * WIDTH + to_integer(unsigned(col)) +",y,"- to_integer(unsigned(offset)))))*(to_integer(unsigned(org_L((to_integer(unsigned(row)) + ",x,") * WIDTH + to_integer(unsigned(col)) +",y," ))-to_integer(unsigned(org_R((to_integer(unsigned(row)) + ",x,") * WIDTH + to_integer(unsigned(col)) +",y,"-to_integer(unsigned(offset)))))") 6 | #print("+(org_L[(row +",x,") * WIDTH + col +",y," + 1 ]-org_R[(row +",x,") * WIDTH + col +",y," - offset + 1 ])*(org_L[(row + ",x,") * WIDTH + col +",y," + 1 ]-org_R[(row + ",x,") * WIDTH + col +",y," - offset + 1])") 7 | 8 | 9 | #ssd_1<=ssd_1+(org_L[(WIDTH+x) * row + col+y +1 ]-org_R[(WIDTH+x) * row + col+y-offset+1])*(org_L[(WIDTH+x) * row + col+y +1]-org_R[(WIDTH+x) * row + col+y-offset+1]); 10 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | # DepthMap generation on FPGA 3 | 4 | ## Cite this work 5 | 6 | Jayasena, A., 2021. Register Transfer Level Disparity generator with Stereo Vision. Journal of Open Research Software, 9(1), p.18. DOI: http://doi.org/10.5334/jors.339 7 | 8 | ``` 9 | Citation (bibtex) [Switch view]@article{Jayasena_2021, 10 | doi = {10.5334/jors.339}, 11 | url = {https://doi.org/10.5334%2Fjors.339}, 12 | year = 2021, 13 | publisher = {Ubiquity Press, Ltd.}, 14 | volume = {9}, 15 | author = {Aruna Jayasena}, 16 | title = {Register Transfer Level Disparity generator with Stereo Vision}, 17 | journal = {Journal of Open Research Software} 18 | } 19 | ``` 20 | ## About 21 | 22 | Most of the image processing projects in academia has been done on higher-end FPGA's with a considerable amount of resources. The main objective of this project is to implement a reliable embedded system on a lower end FPGA with limited resources. This project is based on Disparity calculation based on SAD (Sum of Absolute Difference) algorithm and creating a depth map. 23 | 24 | 25 | 26 | 27 | Hardware used for this project 28 | 29 | - Basys 3 FPGA board 30 | - 2x OV7670 image sensor modules 31 |
32 |
33 |
34 |
35 | 36 | This project has 3 major sections 37 | 38 | 1. [Functional verification of disparity generator based on Verilog](https://github.com/Archfx/FPGA_depthMap) 39 | 2. [Stereo camera implementation using OV7670 sensors based on VHDL](https://github.com/Archfx/FPGA-stereo-Camera-Basys3) 40 | 3. [Real time disparity generation on Basys3 FPGA](https://github.com/Archfx/FPGA-DepthMap-Basys3) 41 | 42 | 43 | ## Functional verification 44 | 45 | Hardware description languages(HDL) are not meant to be for rapid prototyping. Therefore, in this case, I have used python as the prototyping tool. The SAD algorithm was implemented on python from scratch without using any external library. I refrained from using 2D image arrays to store data because then the HDL implementation is straight forward. 46 | 47 | **SAD/SSD theory** 48 | 49 | Sum of Absolute difference and Sum of Squared Difference Disparity calculation theory is based on a simple geometric concept. Where they use the stereo vision to calculate the distance to the objects. For the implementation, two cameras should be on the same plane and they should not have any vertical offsets in their alignments. 50 | 51 | 52 | 53 | 54 | 55 | **Python implementation** 56 | 57 | The python implementation can be found [here](https://github.com/Archfx/FPGA_depthMap/blob/master/Python_test_implementation/Disparity_Python_implementation_scratch.ipynb) 58 | 59 | Test images used 60 | For the functional verification, I have used the most famous stereo image pair "Tsukuba" stereo pair 61 | 62 | 63 | 64 | 65 |

66 | Left image and Right Tsukuba images 67 |

68 | 69 |
70 |
71 |
72 |
73 |
74 |
75 |
76 |
77 |
78 | 79 |

80 | 81 |

82 | Python results 83 |

84 |

85 | 86 | For this generation, it took more than 4 seconds using an average laptop computer without any accelerating techniques. 87 | Based on the Python implementation Abstract flow chart is generated as follows. 88 | 89 |

90 | 91 |

92 | Disparity generation Flow chart 93 |

94 |

95 | 96 | Then this algorithm is directly ported to Verilog. The implementation was done using ISE design suite by Xilinx. The image files were converted to hex and imported to the simulation and the output is directly saved as a Bitmap image. 97 | 98 | 99 |

100 | 101 |

102 | Timing diagrams at 50MHz 103 |

104 |

105 | 106 |

107 | 108 |

109 | Simulation Output 110 |

111 |

112 | 113 | *** these modules are only for simulation purposes, Do not synthesize the code. 114 | 115 | 116 | ## Stereo Camera implementation 117 | 118 | The cameras that were used for this project is very inexpensive OV7670 modules. They are commonly available and the output can be configured to 8bit parallel. 119 | These cameras are using I2C interface to communicate with the master. We can configure the camera output by changing the internal registers of the cameras. 120 |

121 | 122 |

123 | Pmod connections with Cameras 124 |

125 |

126 |

127 | 128 |

129 | Pmod connector pinouts 130 |

131 |

132 |

133 | 134 |

135 | Basys3 Pmod pinout diagram 136 |

137 |

138 | This repo contains VHDL implementation for image read from two cameras and displaying the average of two images from the VGA output. 139 | 140 | OV7670 dual camera mount was designed using a cad tool and 3D printed to mount the cameras. STL files for camera mount can be found from [here](https://github.com/Archfx/FPGA-stereo-Camera-Basys3/tree/master/CamMountCAD). 141 |

142 | 143 |

144 | CAD Stereo camera mount 145 |

146 |

147 |

148 | 149 |

150 | Hardware connected together 151 |

152 |

153 | 154 | **Camera configuration** 155 | 156 | OV7670 camera module comes with I2C interface to configure it's internal registers. The problem here is we are using two cameras with the same type. By taking the advantage of paralel hardware implementation on FPGA two seperate I2C buses were used for the dual camera intergration. Fortunatly prioir work related to OV7670 Camera intergration to Zedboard FPGA has been done by the Engineer [Mike Field](https://github.com/hamsternz) at [here](http://hamsterworks.co.nz/mediawiki/index.php/OV7670_camera). 157 | This I2C driver was direcly ported to the Basys3 FPGA. Camera register configuration was done inorder to get required output from the Camera. 158 | 159 | ## Real-time depth map generation on FPGA 160 | 161 | When converting the functional verification module into synthesizable code due to limited functionalities in Verilog, VHDL was selected as the developing language. 162 | 163 | **Resource Utilization** 164 | 165 | Basys 3 is a entry level FPGA board. Hence it is not designed for image processing tasks. The Challange here was to run complex image processing algorithm on limited resources. Basys 3 Trainer FPGA board consists of following resources. 166 | 167 | 168 | 169 | 170 | 171 | 172 | 173 | 174 | 175 | 176 | 177 | 178 | 179 | 180 | 181 | 182 | 183 | 184 | 185 | 186 | 187 | 188 | 189 | 190 | 191 | 192 | 193 | 194 | 195 | 196 | 197 | 198 | 199 | 200 | 201 | 202 | 203 | 204 |
ResourceAvailable
LUT
20800
LUTRAM9600
FF41600
BRAM50
DSP90
IO106
BUFG32
MMCM5
205 | 206 | The main bottle necks were the Block memory and the LUTRAM. Basys 3 has 1,800Kbits of memory in 50 cells as 36,000bits in each cell. We are getting the camera output in YCbCr format. Here Y is the gray scale image of individual pixel size 4bits. 207 | 208 | Memory requirement calculation 209 | For left image : 320x240x4 bits 210 | For right image : 320x240x4 bits 211 | For Average image : 320x240x4 bits 212 | For Disparity image : 320x240x8 bits 213 | Total memory required : 1,536,000 bits/1,800 000 bits 214 | 215 | We cant process them images while it is in the BRAM because BRAM is FIFO (First In First Out) memory device. Therefor the Image sholud be loaded in a cache so that data can be accessed parallely. The cache can be created using the LUTRAM. The problem we have is the LUTRAM is insufficient to store two 320x240 images. Even the 160x120 size images cannot be stored to support SAD algorithm to compare. 216 | 217 |

218 | 219 |

220 | LUT bottleneck for 160x120 resolution 221 |

222 |

223 | 224 | Therefore a blockwise disparity calculation was used in order to utilize the full 320x240 resolution with available resources. The caching is done in block wise and then the caculated dispairty values are saved to the disparity_buffer circuit. 225 | 226 | ```diff 227 | - Although there are 9600 LUTRAMS are available we cannot ulilize 100% of that due to routing issues. 228 | 229 | ``` 230 |

231 | 232 |

233 | Routing failure 234 |

235 |

236 |

237 | 238 |

239 | Blockwise disparity calculation Utilization at 320x240 resolution 240 |

241 |

242 | 243 | **VGA Output** 244 | 245 | The system outputs the generated disparity map using the VGA output of the FPGA. 246 | Following are recorded output from the monitor using a camera. 247 | 248 |

249 | 250 |

251 | Demo -1 252 |

253 |

254 | 255 |

256 | 257 |

258 | Demo -2 259 |

260 |

261 | 262 | In both the demonstrations you may observe that camera exposure changes with the environement changes. Improvements are needed to fix this. It will reduce the noise in the output. 263 | Auto Exposure Correction (AEC) has been disabled from the cameras by editing the internal register modules. After disableing AEC, the result was much more clear and the noise was removed from the background. 264 | 265 | **Image Rectification and Camera Caliberation** 266 | 267 | The offsets of the two cameras are fixed using a image rectification module. Although the Automatic Exposure Caliberation is turned of one of the Cameras output is very darker while the other one is too bright. This should be corrected for the Disparity aldorithm to work correctly. 268 | 269 |

270 | 271 |

272 | Demo -3 ( Left : Disparity output | Right : average image of two cameras ) 273 |

274 |

275 | If we observe closely left camera brightness is too lower than the right hand side camera. 276 | 277 | After caliberation of exposure in cameras individually and improving the Image rectification module final output was in a good condition. There are noice and miss calculated points due to the inability of the cameras to identify the features correctly. But the output is smooth and clear. 278 | 279 |

280 | 281 |

282 | Demo -4 ( Left : Disparity output | Right : average image of two cameras ) 283 |

284 |

285 | 286 |

287 | 288 |

289 | Optimized system demo ( Left : Disparity output | Right : average image of two cameras ) 290 |

291 |

292 | 293 | 294 | 295 | -------------------------------------------------------------------------------- /Tsukuba_output_3.bmp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Archfx/FPGA_depthMap/e718aa9433c02baa277ddcdabe67d3d3c6d78a10/Tsukuba_output_3.bmp -------------------------------------------------------------------------------- /Tsukuba_output_5.bmp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Archfx/FPGA_depthMap/e718aa9433c02baa277ddcdabe67d3d3c6d78a10/Tsukuba_output_5.bmp -------------------------------------------------------------------------------- /Tsukuba_output_7.bmp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Archfx/FPGA_depthMap/e718aa9433c02baa277ddcdabe67d3d3c6d78a10/Tsukuba_output_7.bmp -------------------------------------------------------------------------------- /image_read.v: -------------------------------------------------------------------------------- 1 | /******************************************************************************/ 2 | /****************** Module for reading and processing image **************/ 3 | /******************************************************************************/ 4 | `include "parameter.v" // Include definition file 5 | module image_read 6 | #( 7 | parameter WIDTH = 320, // Image width 8 | HEIGHT = 240, // Image height 9 | INFILE_L = "Tsukuba_L.hex", // image file 10 | INFILE_R = "Tsukuba_R.hex", // image file 11 | START_UP_DELAY = 100, // Delay during start up time 12 | HSYNC_DELAY = 160, // Delay between HSYNC pulses 13 | VALUE= 100, // value for Brightness operation 14 | THRESHOLD= 90, // Threshold value for Threshold operation 15 | SIGN=0 // Sign value using for brightness operation 16 | // SIGN = 0: Brightness subtraction 17 | // SIGN = 1: Brightness addition 18 | ) 19 | ( 20 | input HCLK, // clock 21 | input HRESETn, // Reset (active low) 22 | output VSYNC, // Vertical synchronous pulse 23 | // This signal is often a way to indicate that one entire image is transmitted. 24 | // Just create and is not used, will be used once a video or many images are transmitted. 25 | output reg HSYNC, // Horizontal synchronous pulse 26 | // An HSYNC indicates that one line of the image is transmitted. 27 | // Used to be a horizontal synchronous signals for writing bmp file. 28 | output reg [7:0] DATA_0_L, // 8 bit Red data (even) 29 | output reg [7:0] DATA_1_L, // 8 bit Green data (even) 30 | output reg [7:0] DATA_0_R, // 8 bit Blue data (even) 31 | output reg [7:0] DATA_1_R, // 8 bit Red data (odd) 32 | // Process and transmit 2 pixels in parallel to make the process faster, you can modify to transmit 1 pixels or more if needed 33 | output ctrl_done // Done flag 34 | ); 35 | //------------------------------------------------- 36 | // Internal Signals 37 | //------------------------------------------------- 38 | 39 | parameter sizeOfWidth = 8; // data width 40 | parameter sizeOfLengthReal = 76800; // image data : 1179648 bytes: 512 * 768 *3 41 | // local parameters for FSM 42 | localparam ST_IDLE = 2'b00, // idle state 43 | ST_VSYNC = 2'b01, // state for creating vsync 44 | ST_HSYNC = 2'b10, // state for creating hsync 45 | ST_DATA = 2'b11; // state for data processing 46 | reg [1:0] cstate, // current state 47 | nstate; // next state 48 | reg start; // start signal: trigger Finite state machine beginning to operate 49 | reg HRESETn_d; // delayed reset signal: use to create start signal 50 | reg ctrl_vsync_run; // control signal for vsync counter 51 | reg [8:0] ctrl_vsync_cnt; // counter for vsync 52 | reg ctrl_hsync_run; // control signal for hsync counter 53 | reg [8:0] ctrl_hsync_cnt; // counter for hsync 54 | reg ctrl_data_run; // control signal for data processing 55 | reg [31 : 0] in_memory [0 : sizeOfLengthReal/4]; // memory to store 32-bit data image 56 | reg [7 : 0] total_memory_L [0 : sizeOfLengthReal-1]; // memory to store 8-bit data image 57 | reg [7 : 0] total_memory_R [0 : sizeOfLengthReal-1]; // memory to store 8-bit data image 58 | // temporary memory to save image data : size will be WIDTH*HEIGHT*3 59 | integer temp_BMP_L [0 : WIDTH*HEIGHT - 1]; 60 | integer temp_BMP_R [0 : WIDTH*HEIGHT - 1]; 61 | integer org_L[0 : WIDTH*HEIGHT - 1]; // temporary storage for R component 62 | integer org_R [0 : WIDTH*HEIGHT - 1]; // temporary storage for G component 63 | //integer org_B [0 : WIDTH*HEIGHT - 1]; // temporary storage for B component 64 | // counting variables 65 | integer i, j; 66 | // temporary signals for calculation: details in the paper. 67 | integer temp0,temp1;//,tempG0,tempG1,tempB0,tempB1; // temporary variables in contrast and brightness operation 68 | 69 | integer value,value1,value2,value4;// temporary variables in invert and threshold operation 70 | reg [ 8:0] row; // row index of the image 71 | reg [8:0] col; // column index of the Left image 72 | integer window = 7; 73 | integer x,y; // column index of the Right image 74 | reg [4:0] offset, best_offset, best_offset_1; 75 | localparam [4:0] maxoffset = 10; // Maximum extent where to look for the same pixel 76 | reg offsetfound; 77 | reg offsetping; 78 | reg compare,SSD_calc; 79 | reg [20:0] ssd, ssd_1; // sum of squared difference 80 | reg [20:0] prev_ssd, prev_ssd_1; 81 | reg [18:0] data_count; // data counting for entire pixels of the image 82 | //-------------------------------------------------// 83 | // -------- Reading data from input file ----------// 84 | //-------------------------------------------------// 85 | initial begin 86 | $readmemh(INFILE_L,total_memory_L,0,sizeOfLengthReal-1); // read file from INFILE 87 | $readmemh(INFILE_R,total_memory_R,0,sizeOfLengthReal-1); // read file from INFILE 88 | end 89 | // use 3 intermediate signals RGB to save image data 90 | always@(start) begin 91 | if(start == 1'b1) begin 92 | for(i=0; i