├── .DS_Store
├── .gitignore
├── .idea
    └── vcs.xml
├── LICENSE
├── README.md
├── epipolar_geometry.py
├── example
    ├── .DS_Store
    ├── features_poster.txt
    ├── poster1.jpg
    ├── poster2.JPG
    └── poster3.JPG
├── image_sequence.py
├── img
    ├── .DS_Store
    └── demo.png
├── model.py
├── rec_model_cloud.txt
├── reconstructed_model.stl
└── uncalibrated_rec.py


/.DS_Store:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/nbarba/py3DRec/369f26d807631e21b405d19697fe158158784ffb/.DS_Store


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
  1 | # Byte-compiled / optimized / DLL files
  2 | __pycache__/
  3 | *.py[cod]
  4 | *$py.class
  5 | 
  6 | # C extensions
  7 | *.so
  8 | 
  9 | # Distribution / packaging
 10 | .Python
 11 | env/
 12 | build/
 13 | develop-eggs/
 14 | dist/
 15 | downloads/
 16 | eggs/
 17 | .eggs/
 18 | lib/
 19 | lib64/
 20 | parts/
 21 | sdist/
 22 | var/
 23 | wheels/
 24 | *.egg-info/
 25 | .installed.cfg
 26 | *.egg
 27 | 
 28 | # PyInstaller
 29 | #  Usually these files are written by a python script from a template
 30 | #  before PyInstaller builds the exe, so as to inject date/other infos into it.
 31 | *.manifest
 32 | *.spec
 33 | 
 34 | # Installer logs
 35 | pip-log.txt
 36 | pip-delete-this-directory.txt
 37 | 
 38 | # Unit test / coverage reports
 39 | htmlcov/
 40 | .tox/
 41 | .coverage
 42 | .coverage.*
 43 | .cache
 44 | nosetests.xml
 45 | coverage.xml
 46 | *.cover
 47 | .hypothesis/
 48 | 
 49 | # Translations
 50 | *.mo
 51 | *.pot
 52 | 
 53 | # Django stuff:
 54 | *.log
 55 | local_settings.py
 56 | 
 57 | # Flask stuff:
 58 | instance/
 59 | .webassets-cache
 60 | 
 61 | # Scrapy stuff:
 62 | .scrapy
 63 | 
 64 | # Sphinx documentation
 65 | docs/_build/
 66 | 
 67 | # PyBuilder
 68 | target/
 69 | 
 70 | # Jupyter Notebook
 71 | .ipynb_checkpoints
 72 | 
 73 | # pyenv
 74 | .python-version
 75 | 
 76 | # celery beat schedule file
 77 | celerybeat-schedule
 78 | 
 79 | # SageMath parsed files
 80 | *.sage.py
 81 | 
 82 | # dotenv
 83 | .env
 84 | 
 85 | # virtualenv
 86 | .venv
 87 | venv/
 88 | ENV/
 89 | 
 90 | # Spyder project settings
 91 | .spyderproject
 92 | .spyproject
 93 | 
 94 | # Rope project settings
 95 | .ropeproject
 96 | 
 97 | # mkdocs documentation
 98 | /site
 99 | 
100 | # mypy
101 | .mypy_cache/
102 | 


--------------------------------------------------------------------------------
/.idea/vcs.xml:
--------------------------------------------------------------------------------
1 | <?xml version="1.0" encoding="UTF-8"?>
2 | <project version="4">
3 |   <component name="VcsDirectoryMappings">
4 |     <mapping directory="$PROJECT_DIR$" vcs="Git" />
5 |   </component>
6 | </project>


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | MIT License
 2 | 
 3 | Copyright (c) 2017 Nikolaos Barmpalios
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy
 6 | of this software and associated documentation files (the "Software"), to deal
 7 | in the Software without restriction, including without limitation the rights
 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # py3DRec
 2 | 
 3 | This repository contains a revised implementation of the structure from motion algorithm used in [Barbalios et al., "3D Human Face Modelling From Uncalibrated Images Using Spline Based Deformation", VISSAP 2008](http://poseidon.csd.auth.gr/papers/PUBLISHED/CONFERENCE/pdf/VISAPP_final.pdf). Given at least three views of an object, and identified feature points across these views, the algorithm generates a 3D reconstruction of the object.
 4 | 
 5 | 
 6 | ## Overview
 7 | 
 8 | The algorithm takes a series of features (i.e. 2D coordinates), tracked through a sequence of images (i.e. taken with a common handheld camera), and returns the 3D coordinates of these features in the metric space. To make this happen, the algorithm consists of the following steps: 
 9 | * Selecting two views (images), e.g. i-th and j-th view, the fundamental matrix and the epipoles are computed from the corresponding 2D features of the i-th and j-th view. For best results, the fartherst views in the sequence are selected.
10 | * Estimating the projection matrices for the i-th and j-th view. To do so, the i-th view is assumed to be aligned with the world frame, and the projection matrix for the j-th view can be deduced using the fundamental matrix, the epipole and the reference frame of the reconstruction.
11 | * Triangulation of the 2D features of the i-th and j-th views to get an initial estimate of the 3D point coordinates of the 2D features. 
12 | * Estimating the projection matrices for all the remaining views, using the 3D points we got from triangulation.
13 | * Bundle adjustment, i.e. an overall optimization to refine the 3D points and the projection matrices by minimizing the reprojection error of the 3D points back to each view. 
14 | * Self-calibration to estimate the camera intrisic parameters for each view, and transform the 3D point coordinates from projective to metric space. 
15 | * Delaunay triangulation of the 3D points, to get a 3D structure. 
16 | 
17 | Note: The algorithm used in the above referenced paper included an additional step, where the 3D points were used to deform a generic face model.
18 | 
19 | ![Overview](./img/demo.png)
20 | 
21 | ## Prerequisites 
22 | The following python packages are required to run the projet: 
23 | * [NumPy](http://www.numpy.org)
24 | * [pandas](http://pandas.pydata.org) 
25 | * [SciPy](https://www.scipy.org) 
26 | * [numpy-stl](https://pypi.python.org/pypi/numpy-stl)
27 | * [pillow](https://pillow.readthedocs.io/en/4.3.x/)
28 | 
29 | 
30 | ## Usage 
31 | ### Code Structure 
32 | 
33 | The code consists of four classes, each one designed for a specific task: 
34 | * __ImageSequence:__ holds all the image sequence related stuff, like sequence length, width and height of each image, 2D feature coordinates across all images. It also contains a method (i.e. show()), to visualize the image sequence with the 2D features highlighted. 
35 | * __EpipolarGeometry:__ implements epipolar geometry related operations, such as computing the fundamental matrix, finding the epipoles, triangulation, computing homographies, and the reprojection error for one or multiple views.
36 | * __UncalibratedReconstruction:__ the main class that implements the 3D modeling algorithm. Constructor arguments include: 
37 |   * _sequence length:_ the length of the image sequence
38 |   * _width:_ the width of images in the sequence
39 |   * _length:_ the length of the images in the sequence
40 |   * _triang_method:_ triangulation method (0: standard triangulation, 1: polynomial triangulation)
41 |   * _opt_triang:_ optimize triangulation result (i.e. 3D points)
42 |   * _opt_f:_ optimize fundamental matrix estimation
43 |   * _self_foc:_ for self-calibration, defines the type of focal length expected across views (0: fixed, 1: varying)
44 | * __RecModel:__ holds the result of the reconstruction, i.e. projection matrices, rotation matrices, translation vectors, camera intrisic parameters vectors and the 3D structure point coordinates. It also contains a method, named export_stl_file, that performs Delaunay triangulation and saves the result in stl format. 
45 | 
46 | ## Example
47 | A toy-example is provided in the examples folder. It is a sequence of pictures of a poster on a wall (3 images). The 2D features were automatically selected using the SIFT algorithm, and are given in a txt file that has the following format: 
48 | | x coordinates | y coordinates | feature id  | image id |.
49 | 
50 | To run the example, use the following command: 
51 | ```
52 | python uncalibrated_rec.py --input_file=./example/features_poster.txt --show
53 | ```
54 | The first argument (i.e. --input_file) defines the txt file with the features, the second flag (--show) displays the image sequence together with the 2D features (remove this argument to not show the sequence). 
55 | 
56 | After the algorithm is executed, two files should be generated: 
57 | * rec_model_cloud.txt : contains the 3D homogeneous coordinates of the reconstructed 3D model.
58 | * reconstructed_model.stl: an stl file of the reconstructed 3D model
59 | 
60 | 
61 | ## License
62 | 
63 | This project is licensed under the MIT License - see the [LICENSE.md](LICENSE.md) file for details
64 | 
65 | 


--------------------------------------------------------------------------------
/epipolar_geometry.py:
--------------------------------------------------------------------------------
  1 | import numpy as np
  2 | from numpy.linalg import inv
  3 | from numpy.linalg import svd
  4 | from numpy.linalg import eig
  5 | from scipy.optimize import leastsq,least_squares,fmin
  6 | 
  7 | 
  8 | 
  9 | class EpipolarGeometry: 
 10 | 	'''
 11 | 	Class that implements basic epipolar geometry operations 
 12 | 	'''
 13 | 
 14 | 
 15 | 	def fundamental_matrix(self,view1_feat2D,view2_feat2D,optimize):
 16 | 		'''
 17 | 		Method to computes the fundamental matrix betwen two images. 
 18 | 		Args: 
 19 | 			view1_feat2D: 2D feature coordinates in view 1
 20 | 			view2_feat2D: 2D feature coordinates in view 2
 21 | 			optimize:  optimize the F estimation using non-linear least squares 
 22 | 		Returns:
 23 | 			the fundamendal matrix F
 24 | 		'''
 25 | 		number_of_features=view1_feat2D.shape[1]
 26 | 		
 27 | 		#convert to homogeneous coordinates
 28 | 		view1_feat2D=np.vstack((view1_feat2D,np.ones(number_of_features)))
 29 | 		view2_feat2D=np.vstack((view2_feat2D,np.ones(number_of_features)))
 30 | 
 31 | 		# create a homogeneous system Af=0 (f : elements of fundamental matrix), using mFm'=0
 32 | 		A=np.zeros((number_of_features,9))
 33 | 		A[:,0]=np.transpose(view2_feat2D[0,:])*(view1_feat2D[0,:])
 34 | 		A[:,1]=np.transpose(view2_feat2D[0,:])*(view1_feat2D[1,:])
 35 | 		A[:,2]=np.transpose(view2_feat2D[0,:])
 36 | 		A[:,3]=np.transpose(view1_feat2D[0,:])*(view1_feat2D[1,:])
 37 | 		A[:,4]=np.transpose(view2_feat2D[1,:])*(view1_feat2D[1,:])
 38 | 		A[:,5]=np.transpose(view2_feat2D[1,:])
 39 | 		A[:,6]=np.transpose(view1_feat2D[0,:])
 40 | 		A[:,7]=np.transpose(view1_feat2D[1,:])
 41 | 		A[:,8]=np.ones(number_of_features)
 42 | 
 43 | 
 44 | 		# use the eigenvector that corresponds to the smallest eigenvalue as initial estimate for F.
 45 | 		U, s, Vh = svd(A)
 46 | 		V=np.transpose(Vh)
 47 | 
 48 | 		# fundamental matrix is now the eigenvector corresponding to the smallest eigen value.
 49 | 		F=np.reshape(V[:,8],(3,3))
 50 | 
 51 | 		# make sure that fundamental matrix is of rank 2
 52 | 		U,s,V = svd(F);
 53 | 		s[2]=0
 54 | 		S = np.diag(s)
 55 | 		F=np.dot(U, np.dot(S, V))
 56 | 
 57 | 		if optimize==1:
 58 | 			#Optimize initial estimate using the algebraic error
 59 | 			f=np.append(np.concatenate((F[0,:],F[1,:]),axis=0),F[2,0])/F[2,2]
 60 | 	
 61 | 			result=least_squares(self.fundamental_matrix_error,f, args=(view1_feat2D[0:2,:],view2_feat2D[0:2,:]))
 62 | 			f=result.x
 63 | 
 64 | 			F=np.asarray([np.transpose(f[0:3]), np.transpose(f[3:6]), [f[6],-(-f[0]*f[4]+f[6]*f[2]*f[4]+f[3]*f[1]-f[6]*f[1]*f[5])/(-f[3]*f[2]+f[0]*f[5]),1]]);
 65 | 			F=F/sum(sum(F))*9;
 66 | 
 67 | 		return F	
 68 | 
 69 | 
 70 | 	def fundamental_matrix_error(self,f,view1_feat2D,view2_feat2D):
 71 | 		'''
 72 | 		Method to compute the fundamental matrix error based on the epipolar constraint (mFm'=0)
 73 | 		Args: 
 74 | 			f : a vector with the elements of the fundamental matrix
 75 | 			view1_feat2D: 2D feature coordinates in view 1
 76 |  			view2_feat2D: 2D feature coordinates in view 2
 77 |  		Returns: 
 78 |  			the error based on mFm'=0
 79 | 
 80 | 		'''
 81 | 		F=np.asarray([np.transpose(f[0:3]), np.transpose(f[3:6]), [f[6],-(-f[0]*f[4]+f[6]*f[2]*f[4]+f[3]*f[1]-f[6]*f[1]*f[5])/(-f[3]*f[2]+f[0]*f[5]),1]]);
 82 | 
 83 | 		number_of_features=view1_feat2D.shape[1]
 84 | 		#convert to homogeneous coordinates
 85 | 		view1_feat2D=np.vstack((view1_feat2D,np.ones(number_of_features)))
 86 | 		view2_feat2D=np.vstack((view2_feat2D,np.ones(number_of_features)))
 87 | 		#compute error
 88 | 		error=np.zeros((number_of_features,1))
 89 | 		for i in range(0,number_of_features):
 90 | 			error[i]=np.dot(view2_feat2D[:,i],np.dot(F,np.transpose(view1_feat2D[:,i])));
 91 | 
 92 | 		return error.flatten()
 93 | 
 94 | 	def get_epipole(self,F):
 95 | 		'''
 96 | 		Method to return the epipole from the fundamental matrix
 97 | 		Args: 
 98 | 			F: the fundamental matrix
 99 | 		Retruns: 
100 | 			the epipole
101 | 		'''
102 | 		u,v=eig(F)
103 | 		dd=np.square(u);
104 | 		min_index=np.argmin(np.abs(dd))	# SOS: These are complex numbers, so compare their modulus
105 | 		epipole=v[:,min_index]
106 | 		epipole=epipole/epipole[2]
107 | 		return epipole.real
108 | 
109 | 	def compute_homography(self,epipole,F):
110 | 		'''
111 | 		Method that computes the homograhpy [epipole]x[F]
112 | 		Args: 
113 | 			epipole: the epipole
114 | 			F : the fundamental matrix
115 | 		Returns: 
116 | 			the homography H
117 | 		'''
118 | 		e_12=np.asarray([[0,-epipole[2], epipole[1]],[epipole[2],0,-epipole[0]],[-epipole[1],epipole[0],0]]);
119 | 		H=np.dot(e_12,F)
120 | 		H=H*np.sign(np.trace(H))
121 | 		return H
122 | 
123 | 
124 | 	def triangulate_points(self,view1_feat2D,view2_feat2D,P1,P2):
125 | 		'''
126 | 		Method that triangulates using two projection matrices, and the
127 | 		normalized 2D features coordinates.
128 | 		Args: 
129 | 			view1_feat2D: 2D feature coordinates in view 1
130 | 			view2_feat2D: 2D feature coordinates in view 2
131 | 			P1: projection matrix for view 1
132 | 			P2: projection matrix for view 2
133 | 		Returns: 
134 | 			the 3D points
135 | 		'''
136 | 		A=np.zeros((4,4));
137 | 		A[0,:]=P1[2,:]*view1_feat2D[0]-P1[0,:]
138 | 		A[1,:]=P1[2,:]*view1_feat2D[1]-P1[1,:]
139 | 		A[2,:]=P2[2,:]*view2_feat2D[0]-P2[0,:]
140 | 		A[3,:]=P2[2,:]*view2_feat2D[1]-P2[1,:]
141 | 
142 | 		U, s, Vh = svd(A)
143 | 		V=np.transpose(Vh)
144 | 		feat3D=V[:,V.shape[0]-1]
145 | 		feat3D=feat3D/feat3D[3]
146 | 		return feat3D	
147 | 
148 | 
149 | 	def compute_L_R(self,view_feat2D,epipole):
150 | 		'''
151 | 		Utility method used in polynomial triangulation
152 | 		'''
153 | 		L=np.eye(3);
154 | 		L[0:2,2]=-view_feat2D; 
155 | 
156 | 		th = np.arctan(-(epipole[1]-epipole[2]*view_feat2D[1])/(epipole[0]-epipole[2]*view_feat2D[0]));
157 | 		R=np.eye(3)
158 | 		R[0,0]=np.cos(th)
159 | 		R[1,1]=np.cos(th)
160 | 		R[0,1]=-np.sin(th)
161 | 		R[1,0]=np.sin(th)
162 | 		return L,R
163 | 
164 | 	def polynomial_triangulation(self,feat1,feat2,epipole_1,epipole_2,F,P1,P2):
165 | 		'''
166 | 		Method to perform 'polynomial' triangulation, as suggested in 
167 | 		R. Hartley and P. Sturm, "Triangulation", Computer Vision and Image Understanding, 68(2):146-157, 1997
168 | 		The method searches for the epipolar line which fits best to both feature points, and then project both points on tis line. Triangulation
169 | 		is then performed on these new points. (This now happens through SVD since the error will be zero)
170 | 		Args: 
171 | 			feat1: 2D feature coordinates in view 1
172 | 			feat2: 2D feature coordinates in view 2
173 | 			epipole_1: the epipole of view 1
174 | 			epipole_2: the epipole of view 2
175 | 			F: the fundamental matrix 
176 | 			P1: projection matrix for view 1
177 | 			P2: projection matrix for view 2
178 | 		Returns: 
179 | 			the 3D points
180 | 		'''
181 | 		L1,R1=self.compute_L_R(feat1,epipole_1)
182 | 		L2,R2=self.compute_L_R(feat2,epipole_2)
183 | 
184 | 		F1=R2.dot(np.transpose(inv(L2))).dot(F).dot(inv(L1)).dot(np.transpose(R1))
185 | 		a=F1[1,1];
186 | 		b=F1[1,2];
187 | 		c=F1[2,1];
188 | 		d=F1[2,2];
189 | 		f1=-F1[1,0]/b;
190 | 		f2=-F1[0,1]/c;
191 | 
192 | 		p=np.zeros(7);
193 | 		p[0]=-2*np.power(f1,4)*np.power(a,2)*c*d+2*np.power(f1,4)*a*np.power(c,2)*b;
194 | 		p[1]=2*np.power(a,4)+2*np.power(f2,4)*np.power(c,4)-2*np.power(f1,4)*np.power(a,2)*np.power(d,2)+2*np.power(f1,4)*np.power(b,2)*np.power(c,2)+4*np.power(a,2)*np.power(f2,2)*np.power(c,2);
195 | 		p[2]=8*np.power(f2,4)*d*np.power(c,3)+8*b*np.power(a,3)+8*b*a*np.power(f2,2)*np.power(c,2)+8*np.power(f2,2)*d*c*np.power(a,2)-4*np.power(f1,2)*np.power(a,2)*c*d+4*np.power(f1,2)*a*np.power(c,2)*b-2*np.power(f1,4)*b*np.power(d,2)*a+2*np.power(f1,4)*np.power(b,2)*d*c;
196 | 		p[3]=-4*np.power(f1,2)*np.power(a,2)*np.power(d,2)+4*np.power(f1,2)*np.power(b,2)*np.power(c,2)+4*np.power(b,2)*np.power(f2,2)*np.power(c,2)+16*b*a*np.power(f2,2)*d*c+12*np.power(f2,4)*np.power(d,2)*np.power(c,2)+12*np.power(b,2)*np.power(a,2)+4*np.power(f2,2)*np.power(d,2)*np.power(a,2);
197 | 		p[4]=8*np.power(f2,4)*np.power(d,3)*c+8*np.power(b,2)*np.power(f2,2)*d*c+8*np.power(f2,2)*np.power(d,2)*b*a-4*np.power(f1,2)*b*np.power(d,2)*a+4*np.power(f1,2)*np.power(b,2)*d*c+2*a*np.power(c,2)*b+8*np.power(b,3)*a-2*np.power(a,2)*c*d;
198 | 		p[5]=4*np.power(b,2)*np.power(f2,2)*np.power(d,2)+2*np.power(f2,4)*np.power(d,4)+2*np.power(b,2)*np.power(c,2)-2*np.power(a,2)*np.power(d,2)+2*np.power(b,4);
199 | 		p[6]=-2*b*d*(a*d-b*c);
200 | 		
201 | 		r=np.roots(p)
202 | 		y=np.polyval(p,r)
203 | 		
204 | 		aa=max(y)
205 | 		i=0;
206 | 		ii=0;
207 | 		
208 | 		for i in range(0,6):
209 | 			if ((np.imag(r[i])==0) and (abs(y[i])<aa)):
210 | 				aa=abs(y[i]);
211 | 				ii=i;
212 | 			i=i+1;
213 | 
214 | 		t=r[ii].real;
215 | 
216 | 		#get the refined feature coordinates
217 | 		u=np.asarray([f1,1/t,1]);
218 | 		v=np.asarray([-1/f1,t,1]);
219 | 		s=inv([[u[0],-v[0]],[u[1],-v[1]]]).dot([1/f1,0])
220 | 		refined_feat1=np.asarray([u[0]*s[0],u[1]*s[0],1])
221 | 
222 | 		u=F1.dot([0,t,1]);
223 | 		u=u/u[2];
224 | 		v=[1/u[0],-1/u[1],1];
225 | 		s=inv([[u[0],-v[0]],[u[1],-v[1]]]).dot([1/f2,0]);
226 | 		refined_feat2=np.asarray([u[0]*s[0],u[1]*s[0],1]);
227 | 
228 | 		refined_feat1=inv(L1).dot(np.transpose(R1)).dot(refined_feat1);
229 | 		refined_feat2=inv(L2).dot(np.transpose(R2)).dot(refined_feat2);
230 | 
231 | 		# normalize to get homogeneous coordinates
232 | 		refined_feat1=refined_feat1/refined_feat1[2]
233 | 		refined_feat2=refined_feat2/refined_feat2[2]
234 | 
235 | 		return self.triangulate_points(refined_feat1,refined_feat2,P1,P2);
236 | 
237 | 
238 | 	def compute_reprojection_error_point(self,P,point3D,view_feat2D):
239 | 		'''
240 | 		Method to compute the reprojection error of a 3D point on a particular view
241 | 		Args: 
242 | 			P: the projection matrix of the view
243 | 			point3D: the 3D point coordinates
244 | 			view_feat2D: 2D feature coordinates for that point in that view
245 | 		Returns: 
246 | 			the reprojection error
247 | 		'''
248 | 		return np.power((P[0:2,:].dot(point3D)/P[2,:].dot(point3D)-view_feat2D),2)
249 | 
250 | 	def compute_reprojection_error_points(self,P,feat3D,view_feat2D):
251 | 		'''
252 | 		Method that returns the reprojection error for a cloud of 3D points.
253 | 		Args: 
254 | 			P: the projection matrix of the view
255 | 			feat3D: the 3D point cloud coordinates
256 | 			view_feat2D: 2D feature coordinates for all the 3D points in that view
257 | 		Returns: 
258 | 			the reprojection error
259 | 		'''
260 | 		reproj_feature=P.dot(np.transpose(feat3D))
261 | 		reproj_feature=reproj_feature/reproj_feature[2,:]
262 | 		error=sum(sum(np.power((view_feat2D-reproj_feature[0:2,:]),2)))
263 | 		return error
264 | 
265 | 	def refine_projection_matrix(self,VV,feat3D,view_feat2D):
266 | 		'''
267 | 		Utility method, used to optimize the projection matrix estimation
268 | 		Args: 
269 | 			VV: vector containing elements of the projection matrix of the i-th view
270 | 			feat3D: 3D point cloud coordinates
271 | 			view_feat2D: 2D feature coordinate for all the feature points in the i-th view 
272 | 		'''
273 | 		#construct the projection matrix
274 | 		P=np.zeros(shape=[3,4]);
275 | 		P[0,:]=VV[0:4]
276 | 		P[1,:]=VV[4:8]
277 | 		P[2,:]=np.append(np.append(VV[8:10],1),VV[10])
278 | 		return self.compute_reprojection_error_points(P,feat3D,view_feat2D)
279 | 	
280 | 
281 | 	def overall_reprojection_error(self,X,feat2d):
282 | 		'''
283 | 		Method to compute the overall reprojection error by reprojecting the 3D model back to each images. This is 
284 | 		used in bundle adustment to optimize the structure (i.e. 3D points), and the motion (i.e. projection matrices)
285 | 		The 
286 | 		Args: 
287 | 			X: vector containing the projection matrices and the 3D coordinates (all concatenated in one vector)
288 | 			feat2D: 2D feature coordinates for all the features for all views
289 | 		Returns: 
290 | 			the overall reprojection error
291 | 		'''
292 | 		number_of_features=feat2d.shape[2]
293 | 		sequence_length=feat2d.shape[0]
294 | 		error_vector=[];
295 | 		for img_number in range(0,sequence_length):
296 | 			for feat_id in range(0,number_of_features):
297 | 				img_feat=feat2d[img_number,:,feat_id]
298 | 				
299 | 				a=np.transpose(X[(0+img_number*11):(4+img_number*11)])
300 | 				a1=np.transpose(X[(4+img_number*11):(8+img_number*11)])
301 | 				b=np.append(X[(0+sequence_length*11+feat_id*3):(3+sequence_length*11+feat_id*3)],1)
302 | 				c= np.append(X[8+img_number*11:10+img_number*11],1)
303 | 				c= np.append(c,X[10+img_number*11]);
304 | 				c=np.transpose(c);
305 | 				
306 | 				error_x=img_feat[0]-a.dot(b)/c.dot(b)
307 | 				error_y=img_feat[1]-a1.dot(b)/c.dot(b)
308 | 			
309 | 				error_vector.append(error_x)
310 | 				error_vector.append(error_y)
311 | 
312 | 		return np.asarray(error_vector)	


--------------------------------------------------------------------------------
/example/.DS_Store:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/nbarba/py3DRec/369f26d807631e21b405d19697fe158158784ffb/example/.DS_Store


--------------------------------------------------------------------------------
/example/features_poster.txt:
--------------------------------------------------------------------------------
  1 | x y feature_id image_id image_filename
  2 | 967.340000 1439.560000 1.000000 1.000000 ./example/poster1.jpg
  3 | 931.960000 1422.820000 2.000000 1.000000 ./example/poster1.jpg
  4 | 881.720000 1389.140000 3.000000 1.000000 ./example/poster1.jpg
  5 | 890.600000 1382.520000 4.000000 1.000000 ./example/poster1.jpg
  6 | 923.440000 1374.760000 5.000000 1.000000 ./example/poster1.jpg
  7 | 993.280000 1367.840000 6.000000 1.000000 ./example/poster1.jpg
  8 | 492.140000 1356.100000 7.000000 1.000000 ./example/poster1.jpg
  9 | 481.320000 1355.960000 8.000000 1.000000 ./example/poster1.jpg
 10 | 488.780000 1346.800000 9.000000 1.000000 ./example/poster1.jpg
 11 | 504.260000 1319.400000 10.000000 1.000000 ./example/poster1.jpg
 12 | 531.580000 1320.120000 11.000000 1.000000 ./example/poster1.jpg
 13 | 489.940000 1319.440000 12.000000 1.000000 ./example/poster1.jpg
 14 | 1041.860000 1276.880000 13.000000 1.000000 ./example/poster1.jpg
 15 | 938.960000 1278.340000 14.000000 1.000000 ./example/poster1.jpg
 16 | 724.500000 1174.720000 15.000000 1.000000 ./example/poster1.jpg
 17 | 838.700000 1149.220000 16.000000 1.000000 ./example/poster1.jpg
 18 | 497.980000 1147.860000 17.000000 1.000000 ./example/poster1.jpg
 19 | 742.580000 1075.860000 18.000000 1.000000 ./example/poster1.jpg
 20 | 710.560000 1076.880000 19.000000 1.000000 ./example/poster1.jpg
 21 | 803.400000 1053.860000 20.000000 1.000000 ./example/poster1.jpg
 22 | 641.440000 1063.440000 21.000000 1.000000 ./example/poster1.jpg
 23 | 817.880000 1033.860000 22.000000 1.000000 ./example/poster1.jpg
 24 | 734.280000 1039.760000 23.000000 1.000000 ./example/poster1.jpg
 25 | 846.240000 995.580000 24.000000 1.000000 ./example/poster1.jpg
 26 | 721.900000 1004.620000 25.000000 1.000000 ./example/poster1.jpg
 27 | 707.920000 1005.360000 26.000000 1.000000 ./example/poster1.jpg
 28 | 755.020000 979.400000 27.000000 1.000000 ./example/poster1.jpg
 29 | 631.280000 988.740000 28.000000 1.000000 ./example/poster1.jpg
 30 | 684.700000 966.700000 29.000000 1.000000 ./example/poster1.jpg
 31 | 1023.540000 776.020000 30.000000 1.000000 ./example/poster1.jpg
 32 | 888.060000 792.240000 31.000000 1.000000 ./example/poster1.jpg
 33 | 947.360000 774.980000 32.000000 1.000000 ./example/poster1.jpg
 34 | 1000.020000 768.100000 33.000000 1.000000 ./example/poster1.jpg
 35 | 652.980000 803.680000 34.000000 1.000000 ./example/poster1.jpg
 36 | 992.580000 757.520000 35.000000 1.000000 ./example/poster1.jpg
 37 | 902.160000 750.000000 36.000000 1.000000 ./example/poster1.jpg
 38 | 688.100000 761.620000 37.000000 1.000000 ./example/poster1.jpg
 39 | 705.200000 739.400000 38.000000 1.000000 ./example/poster1.jpg
 40 | 997.420000 693.200000 39.000000 1.000000 ./example/poster1.jpg
 41 | 983.500000 686.900000 40.000000 1.000000 ./example/poster1.jpg
 42 | 943.220000 681.820000 41.000000 1.000000 ./example/poster1.jpg
 43 | 659.560000 718.240000 42.000000 1.000000 ./example/poster1.jpg
 44 | 898.820000 678.900000 43.000000 1.000000 ./example/poster1.jpg
 45 | 946.800000 661.320000 44.000000 1.000000 ./example/poster1.jpg
 46 | 983.840000 655.400000 45.000000 1.000000 ./example/poster1.jpg
 47 | 874.200000 672.580000 46.000000 1.000000  ./example/poster1.jpg
 48 | 935.240000 662.700000 47.000000 1.000000 ./example/poster1.jpg
 49 | 914.420000 657.320000 48.000000 1.000000 ./example/poster1.jpg
 50 | 1007.580000 631.320000 49.000000 1.000000 ./example/poster1.jpg
 51 | 950.740000 620.120000 50.000000 1.000000 ./example/poster1.jpg
 52 | 870.220000 614.260000 51.000000 1.000000 ./example/poster1.jpg
 53 | 920.160000 513.300000 52.000000 1.000000 ./example/poster1.jpg
 54 | 592.040000 560.860000 53.000000 1.000000 ./example/poster1.jpg
 55 | 545.170732 244.682927 54.000000 1.000000 ./example/poster1.jpg
 56 | 1310.829268 5.853659 55.000000 1.000000 ./example/poster1.jpg
 57 | 449.170732 1502.048780 56.000000 1.000000 ./example/poster1.jpg
 58 | 1280.390244 1520.780488 57.000000 1.000000 ./example/poster1.jpg
 59 | 505.365854 361.756098 58.000000 1.000000 ./example/poster1.jpg
 60 | 608.390244 424.975610 59.000000 1.000000 ./example/poster1.jpg
 61 | 758.243902 392.195122 60.000000 1.000000 ./example/poster1.jpg
 62 | 779.317073 382.829268 61.000000 1.000000 ./example/poster1.jpg
 63 | 945.560976 345.365854 62.000000 1.000000 ./example/poster1.jpg
 64 | 966.634146 338.341463 63.000000 1.000000 ./example/poster1.jpg
 65 | 1149.268293 296.195122 64.000000 1.000000 ./example/poster1.jpg
 66 | 1191.414634 485.853659 65.000000 1.000000 ./example/poster1.jpg
 67 | 1214.829268 780.878049 66.000000 1.000000 ./example/poster1.jpg
 68 | 673.951220 324.292683 67.000000 1.000000 ./example/poster1.jpg
 69 | 816.780488 282.146341 68.000000 1.000000 ./example/poster1.jpg
 70 | 837.853659 275.121951 69.000000 1.000000 ./example/poster1.jpg
 71 | 1001.756098 232.975610 70.000000 1.000000 ./example/poster1.jpg
 72 | 1025.170732 223.609756 71.000000 1.000000 ./example/poster1.jpg
 73 | 1196.097561 176.780488 72.000000 1.000000 ./example/poster1.jpg
 74 | 587.317073 867.512195 73.000000 1.000000 ./example/poster1.jpg
 75 | 568.585366 609.951220 74.000000 1.000000 ./example/poster1.jpg
 76 | 1064.960000 1413.880000 1.000000 2.000000 ./example/poster2.jpg
 77 | 1019.440000 1397.960000 2.000000 2.000000 ./example/poster2.jpg
 78 | 959.220000 1365.980000 3.000000 2.000000 ./example/poster2.jpg
 79 | 970.180000 1357.340000 4.000000 2.000000 ./example/poster2.jpg
 80 | 1012.780000 1350.600000 5.000000 2.000000 ./example/poster2.jpg
 81 | 1103.180000 1342.240000 6.000000 2.000000 ./example/poster2.jpg
 82 | 379.160000 1338.640000 7.000000 2.000000 ./example/poster2.jpg
 83 | 361.180000 1335.880000 8.000000 2.000000 ./example/poster2.jpg
 84 | 374.480000 1328.160000 9.000000 2.000000 ./example/poster2.jpg
 85 | 403.120000 1300.580000 10.000000 2.000000 ./example/poster2.jpg
 86 | 447.920000 1298.900000 11.000000 2.000000 ./example/poster2.jpg
 87 | 378.900000 1297.860000 12.000000 2.000000 ./example/poster2.jpg
 88 | 1167.880000 1255.040000 13.000000 2.000000 ./example/poster2.jpg
 89 | 1039.920000 1254.720000 14.000000 2.000000 ./example/poster2.jpg
 90 | 755.720000 1147.060000 15.000000 2.000000 ./example/poster2.jpg
 91 | 917.520000 1124.960000 16.000000 2.000000 ./example/poster2.jpg
 92 | 416.180000 1107.980000 17.000000 2.000000 ./example/poster2.jpg
 93 | 795.500000 1045.600000 18.000000 2.000000 ./example/poster2.jpg
 94 | 748.660000 1044.660000 19.000000 2.000000 ./example/poster2.jpg
 95 | 881.780000 1027.140000 20.000000 2.000000 ./example/poster2.jpg
 96 | 651.520000 1026.200000 21.000000 2.000000 ./example/poster2.jpg
 97 | 904.340000 1007.960000 22.000000 2.000000 ./example/poster2.jpg
 98 | 787.760000 1007.680000 23.000000 2.000000 ./example/poster2.jpg
 99 | 948.860000 971.740000 24.000000 2.000000 ./example/poster2.jpg
100 | 777.500000 970.280000 25.000000 2.000000 ./example/poster2.jpg
101 | 757.540000 970.080000 26.000000 2.000000 ./example/poster2.jpg
102 | 826.060000 947.360000 27.000000 2.000000 ./example/poster2.jpg
103 | 647.700000 946.260000 28.000000 2.000000 ./example/poster2.jpg
104 | 729.900000 928.540000 29.000000 2.000000 ./example/poster2.jpg
105 | 1203.840000 776.120000 30.000000 2.000000 ./example/poster2.jpg
106 | 1036.140000 774.460000 31.000000 2.000000 ./example/poster2.jpg
107 | 1111.360000 765.420000 32.000000 2.000000 ./example/poster2.jpg
108 | 1175.900000 765.400000 33.000000 2.000000 ./example/poster2.jpg
109 | 711.280000 754.780000 34.000000 2.000000 ./example/poster2.jpg
110 | 1167.400000 754.640000 35.000000 2.000000 ./example/poster2.jpg
111 | 1056.860000 734.820000 36.000000 2.000000 ./example/poster2.jpg
112 | 768.060000 715.760000 37.000000 2.000000 ./example/poster2.jpg
113 | 797.140000 695.440000 38.000000 2.000000 ./example/poster2.jpg
114 | 1179.760000 694.240000 39.000000 2.000000 ./example/poster2.jpg
115 | 1165.480000 685.500000 40.000000 2.000000 ./example/poster2.jpg
116 | 1115.940000 674.800000 41.000000 2.000000 ./example/poster2.jpg
117 | 732.420000 665.800000 42.000000 2.000000 ./example/poster2.jpg
118 | 1060.200000 664.900000 43.000000 2.000000 ./example/poster2.jpg
119 | 1124.580000 656.000000 44.000000 2.000000 ./example/poster2.jpg
120 | 1168.520000 655.500000 45.000000 2.000000 ./example/poster2.jpg
121 | 1031.220000 655.120000 46.000000 2.000000 ./example/poster2.jpg
122 | 1108.920000 655.100000 47.000000 2.000000 ./example/poster2.jpg
123 | 1083.900000 646.300000 48.000000 2.000000 ./example/poster2.jpg
124 | 1198.100000 636.320000 49.000000 2.000000 ./example/poster2.jpg
125 | 1132.320000 616.560000 50.000000 2.000000 ./example/poster2.jpg
126 | 1031.240000 596.800000 51.000000 2.000000 ./example/poster2.jpg
127 | 1096.000000 506.900000 52.000000 2.000000 ./example/poster2.jpg
128 | 639.100000 486.560000 53.000000 2.000000 ./example/poster2.jpg
129 | 378.926829 111.219512 54.000000 2.000000 ./example/poster2.jpg
130 | 1376.390244 97.170732 55.000000 2.000000 ./example/poster2.jpg
131 | 299.317073 1504.390244 56.000000 2.000000 ./example/poster2.jpg
132 | 1418.536585 1473.951220 57.000000 2.000000 ./example/poster2.jpg
133 | 362.536585 237.658537 58.000000 2.000000 ./example/poster2.jpg
134 | 615.414634 345.365854 59.000000 2.000000 ./example/poster2.jpg
135 | 833.170732 345.365854 60.000000 2.000000 ./example/poster2.jpg
136 | 863.609756 345.365854 61.000000 2.000000 ./example/poster2.jpg
137 | 1081.365854 345.365854 62.000000 2.000000 ./example/poster2.jpg
138 | 1109.463415 343.024390 63.000000 2.000000 ./example/poster2.jpg
139 | 1324.878049 345.365854 64.000000 2.000000 ./example/poster2.jpg
140 | 1411.512195 532.682927 65.000000 2.000000 ./example/poster2.jpg
141 | 1418.536585 801.951220 66.000000 2.000000 ./example/poster2.jpg
142 | 631.804878 242.341463 67.000000 2.000000 ./example/poster2.jpg
143 | 840.195122 242.341463 68.000000 2.000000 ./example/poster2.jpg
144 | 870.634146 240.000000 69.000000 2.000000 ./example/poster2.jpg
145 | 1079.024390 237.658537 70.000000 2.000000 ./example/poster2.jpg
146 | 1111.804878 237.658537 71.000000 2.000000 ./example/poster2.jpg
147 | 1320.195122 232.975610 72.000000 2.000000 ./example/poster2.jpg
148 | 601.365854 806.634146 73.000000 2.000000 ./example/poster2.jpg
149 | 606.048780 537.365854 74.000000 2.000000 ./example/poster2.jpg
150 | 1154.020000 1391.540000 1.000000 3.000000 ./example/poster3.jpg
151 | 1118.100000 1375.540000 2.000000 3.000000 ./example/poster3.jpg
152 | 1068.260000 1342.700000 3.000000 3.000000 ./example/poster3.jpg
153 | 1078.400000 1335.660000 4.000000 3.000000 ./example/poster3.jpg
154 | 1115.580000 1330.880000 5.000000 3.000000 ./example/poster3.jpg
155 | 1190.940000 1327.440000 6.000000 3.000000 ./example/poster3.jpg
156 | 529.840000 1290.180000 7.000000 3.000000 ./example/poster3.jpg
157 | 512.280000 1287.340000 8.000000 3.000000 ./example/poster3.jpg
158 | 525.260000 1279.580000 9.000000 3.000000 ./example/poster3.jpg
159 | 558.320000 1253.900000 10.000000 3.000000 ./example/poster3.jpg
160 | 602.300000 1253.520000 11.000000 3.000000 ./example/poster3.jpg
161 | 534.480000 1248.800000 12.000000 3.000000 ./example/poster3.jpg
162 | 1249.240000 1248.740000 13.000000 3.000000 ./example/poster3.jpg
163 | 1144.720000 1242.720000 14.000000 3.000000 ./example/poster3.jpg
164 | 908.000000 1123.960000 15.000000 3.000000 ./example/poster3.jpg
165 | 1052.040000 1114.460000 16.000000 3.000000 ./example/poster3.jpg
166 | 593.800000 1059.080000 17.000000 3.000000 ./example/poster3.jpg
167 | 953.280000 1029.420000 18.000000 3.000000 ./example/poster3.jpg
168 | 914.160000 1026.100000 19.000000 3.000000 ./example/poster3.jpg
169 | 1030.500000 1019.800000 20.000000 3.000000 ./example/poster3.jpg
170 | 827.280000 998.720000 21.000000 3.000000 ./example/poster3.jpg
171 | 1053.100000 1003.460000 22.000000 3.000000 ./example/poster3.jpg
172 | 952.580000 993.300000 23.000000 3.000000 ./example/poster3.jpg
173 | 1095.440000 973.640000 24.000000 3.000000 ./example/poster3.jpg
174 | 948.060000 956.600000 25.000000 3.000000 ./example/poster3.jpg
175 | 930.120000 954.500000 26.000000 3.000000 ./example/poster3.jpg
176 | 993.080000 939.380000 27.000000 3.000000 ./example/poster3.jpg
177 | 834.320000 921.380000 28.000000 3.000000 ./example/poster3.jpg
178 | 911.240000 912.100000 29.000000 3.000000 ./example/poster3.jpg
179 | 1324.880000 819.980000 30.000000 3.000000 ./example/poster3.jpg
180 | 1191.000000 800.080000 31.000000 3.000000 ./example/poster3.jpg
181 | 1253.220000 800.680000 32.000000  3.000000 ./example/poster3.jpg
182 | 1303.560000 807.560000 33.000000 3.000000 ./example/poster3.jpg
183 | 916.400000 743.800000 34.000000 3.000000 ./example/poster3.jpg
184 | 1299.000000 797.140000 35.000000 3.000000 ./example/poster3.jpg
185 | 1211.260000 766.140000 36.000000 3.000000 ./example/poster3.jpg
186 | 969.820000 713.220000 37.000000 3.000000 ./example/poster3.jpg
187 | 997.440000 697.960000 38.000000 3.000000 ./example/poster3.jpg
188 | 1311.740000 743.880000 39.000000 3.000000 ./example/poster3.jpg
189 | 1302.280000 734.540000 40.000000 3.000000 ./example/poster3.jpg
190 | 1264.700000 719.100000 41.000000 3.000000 ./example/poster3.jpg
191 | 946.080000 661.640000 42.000000 3.000000 ./example/poster3.jpg
192 | 1221.740000 702.880000 43.000000 3.000000 ./example/poster3.jpg
193 | 1273.120000 702.400000 44.000000 3.000000 ./example/poster3.jpg
194 | 1307.680000 708.160000 45.000000 3.000000 ./example/poster3.jpg
195 | 1198.980000 690.380000 46.000000 3.000000 ./example/poster3.jpg
196 | 1260.960000 699.980000 47.000000 3.000000 ./example/poster3.jpg
197 | 1241.560000 689.040000 48.000000 3.000000 ./example/poster3.jpg
198 | 1332.320000 694.020000 49.000000 3.000000 ./example/poster3.jpg
199 | 1282.340000 668.140000 50.000000 3.000000 ./example/poster3.jpg
200 | 1204.200000 637.580000 51.000000 3.000000 ./example/poster3.jpg
201 | 1260.900000 564.480000 52.000000 3.000000 ./example/poster3.jpg
202 | 872.960000 476.040000 53.000000 3.000000 ./example/poster3.jpg
203 | 535.804878 40.975610 54.000000 3.000000 ./example/poster3.jpg
204 | 1411.512195 232.975610 55.000000 3.000000 ./example/poster3.jpg
205 | 439.804878 1459.902439 56.000000 3.000000 ./example/poster3.jpg
206 | 1425.560976 1455.219512 57.000000 3.000000 ./example/poster3.jpg
207 | 542.829268 169.756098 58.000000 3.000000 ./example/poster3.jpg
208 | 835.512195 331.317073 59.000000 3.000000 ./example/poster3.jpg
209 | 1029.853659 371.121951 60.000000 3.000000 ./example/poster3.jpg
210 | 1053.268293 375.804878 61.000000 3.000000 ./example/poster3.jpg
211 | 1235.902439 410.926829 62.000000 3.000000 ./example/poster3.jpg
212 | 1256.975610 413.268293 63.000000 3.000000 ./example/poster3.jpg
213 | 1427.902439 448.390244 64.000000 3.000000 ./example/poster3.jpg
214 | 1505.170732 631.024390 65.000000 3.000000 ./example/poster3.jpg
215 | 1474.731707 865.170732 66.000000 3.000000 ./example/poster3.jpg
216 | 812.097561 235.317073 67.000000 3.000000 ./example/poster3.jpg
217 | 997.073171 270.439024 68.000000 3.000000 ./example/poster3.jpg
218 | 1022.829268 277.463415 69.000000 3.000000 ./example/poster3.jpg
219 | 1200.780488 312.585366 70.000000 3.000000 ./example/poster3.jpg
220 | 1221.853659 314.926829 71.000000 3.000000 ./example/poster3.jpg
221 | 1390.439024 345.365854 72.000000 3.000000 ./example/poster3.jpg
222 | 812.097561 778.536585 73.000000  3.000000 ./example/poster3.jpg
223 | 842.536585 516.292683 74.000000 3.000000 ./example/poster3.jpg


--------------------------------------------------------------------------------
/example/poster1.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/nbarba/py3DRec/369f26d807631e21b405d19697fe158158784ffb/example/poster1.jpg


--------------------------------------------------------------------------------
/example/poster2.JPG:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/nbarba/py3DRec/369f26d807631e21b405d19697fe158158784ffb/example/poster2.JPG


--------------------------------------------------------------------------------
/example/poster3.JPG:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/nbarba/py3DRec/369f26d807631e21b405d19697fe158158784ffb/example/poster3.JPG


--------------------------------------------------------------------------------
/image_sequence.py:
--------------------------------------------------------------------------------
 1 | import numpy as np;
 2 | import pandas as pd
 3 | from PIL import Image, ImageDraw, ImageFont
 4 | 
 5 | 
 6 | class ImageSequence:
 7 |     '''
 8 |     Class to hold the necessary information about the image sequence
 9 |     '''
10 | 
11 |     def __init__(self, filename):
12 |         self.load_features(filename)
13 | 
14 |     def feat_2d(self):
15 |         return self._feat_2d
16 | 
17 |     def length(self):
18 |         return self._length
19 | 
20 |     def number_of_features(self):
21 |         return self._number_of_features
22 | 
23 |     def width(self):
24 |         return self._width
25 | 
26 |     def height(self):
27 |         return self._height
28 | 
29 |     def load_features(self, filename):
30 |         '''
31 |         Method that loads a txt file containing 2D coordinates of image features. The format of each line should be:
32 |         [x y feature_number image_number image_filename]
33 |         '''
34 |         features_df = pd.read_csv(filename, delimiter=r"\s+", index_col=False)
35 |         # get length of sequence and number of features
36 |         self._length = int(features_df['image_id'].max())
37 |         self._number_of_features = int(features_df['feature_id'].max())
38 | 
39 |         # get the 2d features
40 |         self.feat_2d = np.zeros(shape=[self._length, 4, self._number_of_features])
41 |         for i in range(1, self._length + 1):
42 |             self.feat_2d[i - 1, :, :] = np.transpose(features_df.loc[features_df['image_id'] == i].values)[0:4]
43 | 
44 |         # keep the image filenames
45 |         self._image_filenames = features_df.image_filename.unique()
46 | 
47 |         # get the image sequence width and height
48 |         image = Image.open(self._image_filenames[0])
49 |         self._width = 1024  # image.width
50 |         self._height = 768  # image.height
51 | 
52 |     def get_normalized_coordinates(self):
53 |         '''
54 |         Method to normalize the coordinates to the range [-1,1]
55 |         '''
56 |         mm = (self._width + self._height) / 2;
57 | 
58 |         rows = (self.feat_2d[:, 0] - np.ones(self.number_of_features) * self._width) / mm
59 |         cols = (self.feat_2d[:, 1] - np.ones(self.number_of_features) * self._height) / mm
60 |         return np.dstack((rows, cols)).swapaxes(1, 2)
61 | 
62 |     def show(self):
63 |         '''
64 |         Method to display the sequence, with the 2D features superimposed
65 |         '''
66 |         font = ImageFont.truetype('/Library/fonts/arial.ttf', 30)
67 |         for i in range(0, self.length):
68 |             filename = self._image_filenames[i]
69 |             image = Image.open(filename)
70 |             draw = ImageDraw.Draw(image)
71 |             for j in range(0, self.number_of_features):
72 |                 x = self.feat_2d[i, :, j][0];
73 |                 y = image.height - self.feat_2d[i, :, j][1]
74 |                 draw.text((x, y), "+" + str(j), font=font, fill=(0, 255, 0))
75 | 
76 |             image.show()
77 | 


--------------------------------------------------------------------------------
/img/.DS_Store:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/nbarba/py3DRec/369f26d807631e21b405d19697fe158158784ffb/img/.DS_Store


--------------------------------------------------------------------------------
/img/demo.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/nbarba/py3DRec/369f26d807631e21b405d19697fe158158784ffb/img/demo.png


--------------------------------------------------------------------------------
/model.py:
--------------------------------------------------------------------------------
 1 | import numpy as np;
 2 | from numpy.linalg import inv
 3 | from numpy.linalg import svd
 4 | from numpy.linalg import eig
 5 | from numpy.linalg import det
 6 | from scipy.optimize import leastsq,least_squares,fmin
 7 | import pandas as pd
 8 | import numpy as np
 9 | import time
10 | import random
11 | from stl import mesh
12 | from scipy.spatial import Delaunay
13 | 
14 | 
15 | class RecModel:
16 | 	'''
17 | 	Class to hold all the result of a reconstruction
18 | 	'''
19 | 	def __init__(self):
20 | 		self._P = 0			#projection matrices for each view
21 | 		self._R = 0			#rotation matrices for each view
22 | 		self._t = 0			#translation vectors for each view
23 | 		self._points3D=0	#3D points in metric space
24 | 		self._Tm=0			#Transformation matrix from projective to metric
25 | 
26 | 
27 | 	@property
28 | 	def P(self):
29 | 		return self._P
30 | 
31 | 	@P.setter
32 | 	def P(self,val):
33 | 		self._R,self._t=extract_rotation_translation(P)
34 | 		self._P=val
35 | 
36 | 	@property
37 | 	def points3D(self):
38 | 		return self._points3D
39 | 
40 | 	@points3D.setter
41 | 	def points3D(self,val):
42 | 		self._points3D=val
43 | 
44 | 
45 | 	def extract_rotation_translation(self, P):  
46 | 		'''
47 | 		Method to extract the rotation matrices and translation vectors from the projection matrices
48 | 		'''
49 | 		sequence_length=P.shape[2]
50 | 		R=np.zeros((3,3,self._sequence_length))
51 | 		t=np.zeros((3,self._sequence_length))
52 | 
53 | 		for i in range(1,self._sequence_length):   
54 | 			PP=inv(K[:,:,i]).dot(P[:,:,i]);
55 | 			R[:,:,i]=np.transpose(PP[0:3,0:3])  
56 | 			t[:,i]=-inv(np.transpose(R[:,:,i])).dot(PP[0:3,3])
57 | 		return R,t
58 | 
59 | 	def export_stl_file(self,filename):
60 | 		'''
61 | 		Method that generates an .stl file with the generated model
62 | 		'''
63 | 		vertices=self.points3D[:,0:3];
64 | 
65 | 		faces=Delaunay(vertices[:,0:2],incremental=1);
66 | 		faces= faces.simplices;
67 | 		
68 | 		wireframe = mesh.Mesh(np.zeros(faces.shape[0], dtype=mesh.Mesh.dtype))
69 | 		for i, f in enumerate(faces):
70 | 		    for j in range(3):
71 | 		        wireframe.vectors[i][j] = vertices[f[j],:]
72 | 
73 | 		wireframe.save(filename)
74 | 
75 | 


--------------------------------------------------------------------------------
/rec_model_cloud.txt:
--------------------------------------------------------------------------------
 1 | 6.658528004086457556e-05,-7.875401786596030471e-04,-4.776641953853161260e-03,1.000000000000000000e+00
 2 | 1.092307801037924864e-04,-7.735604747344070184e-04,-4.812959968716376546e-03,1.000000000000000000e+00
 3 | 1.695252664014383311e-04,-7.413431756486327847e-04,-4.862667044136712718e-03,1.000000000000000000e+00
 4 | 1.587921102931981519e-04,-7.315209703512190161e-04,-4.856302168357570781e-03,1.000000000000000000e+00
 5 | 1.193772157462488477e-04,-7.188731624472650310e-04,-4.824926833447320054e-03,1.000000000000000000e+00
 6 | 3.604358044877856068e-05,-7.008578583005102428e-04,-4.757655841368236757e-03,1.000000000000000000e+00
 7 | 6.924235600404160139e-04,-7.661432694672266897e-04,-5.302251420555560875e-03,1.000000000000000000e+00
 8 | 7.081456310979056821e-04,-7.659693654421349113e-04,-5.315725103293646517e-03,1.000000000000000000e+00
 9 | 6.974246941263369031e-04,-7.550527246614792409e-04,-5.308194391941706107e-03,1.000000000000000000e+00
10 | 6.756256479198401860e-04,-7.193188254511575543e-04,-5.288593972225304890e-03,1.000000000000000000e+00
11 | 6.355992842856576612e-04,-7.137225177403058902e-04,-5.257256502043373102e-03,1.000000000000000000e+00
12 | 6.963974146942620777e-04,-7.193501037321257561e-04,-5.307748200017457611e-03,1.000000000000000000e+00
13 | -2.039018372059222623e-05,-5.912866616924863971e-04,-4.722551108590994015e-03,1.000000000000000000e+00
14 | 1.006714835029537376e-04,-6.042812770306952826e-04,-4.819228361421979204e-03,1.000000000000000000e+00
15 | 3.717225189685878332e-04,-5.061678542278023928e-04,-5.048788477400239366e-03,1.000000000000000000e+00
16 | 2.246358846310142141e-04,-4.633291826191610670e-04,-4.929899907094581479e-03,1.000000000000000000e+00
17 | 6.851706326051504748e-04,-4.947803934975076928e-04,-5.301294542131129717e-03,1.000000000000000000e+00
18 | 3.474070761147301763e-04,-3.816021852047234478e-04,-5.029864539414129025e-03,1.000000000000000000e+00
19 | 3.903858407660827146e-04,-3.856651771045775190e-04,-5.062555062886837064e-03,1.000000000000000000e+00
20 | 2.690540764325101853e-04,-3.505440599121865909e-04,-4.967366592199070133e-03,1.000000000000000000e+00
21 | 4.827035774837975060e-04,-3.736905398413766064e-04,-5.136453524698043720e-03,1.000000000000000000e+00
22 | 2.507502950341740085e-04,-3.248009106940910823e-04,-4.950882684223347451e-03,1.000000000000000000e+00
23 | 3.588275766284330297e-04,-3.373054706950706313e-04,-5.037470897509793537e-03,1.000000000000000000e+00
24 | 2.147835339217130145e-04,-2.762870128464616393e-04,-4.919176730732268896e-03,1.000000000000000000e+00
25 | 3.745290658148693066e-04,-2.936704890193752855e-04,-5.046852846320558014e-03,1.000000000000000000e+00
26 | 3.929743168338855524e-04,-2.955832439980104142e-04,-5.061940360782568767e-03,1.000000000000000000e+00
27 | 3.314284159218329842e-04,-2.609733608379346289e-04,-5.013697983693454936e-03,1.000000000000000000e+00
28 | 4.965801905002092953e-04,-2.793724397560231080e-04,-5.144753192736172681e-03,1.000000000000000000e+00
29 | 4.240068174169345878e-04,-2.487403942125147437e-04,-5.084781534670488462e-03,1.000000000000000000e+00
30 | 7.403514434816054394e-07,-9.782741243938423731e-06,-4.738765455784329367e-03,1.000000000000000000e+00
31 | 1.619034884003386703e-04,-2.836829699572080571e-05,-4.865367073301432414e-03,1.000000000000000000e+00
32 | 9.035707323174677313e-05,-8.305681787558168508e-06,-4.809675317152794713e-03,1.000000000000000000e+00
33 | 2.776347191692414302e-05,-4.260085850549169477e-07,-4.762043367328390453e-03,1.000000000000000000e+00
34 | 4.657886527645255042e-04,-4.357089534847972847e-05,-5.109571097013277004e-03,1.000000000000000000e+00
35 | 3.686404364135253340e-05,1.179795537248050385e-05,-4.768727581774749417e-03,1.000000000000000000e+00
36 | 1.448999934317434532e-04,2.201321297540003222e-05,-4.855420658339360505e-03,1.000000000000000000e+00
37 | 4.182077506042501712e-04,9.732797463565452960e-06,-5.072507970058860033e-03,1.000000000000000000e+00
38 | 3.950005679551650008e-04,3.743872253896039451e-05,-5.051799456300513913e-03,1.000000000000000000e+00
39 | 3.055518277908653841e-05,8.691750192584235491e-05,-4.768432043837956234e-03,1.000000000000000000e+00
40 | 4.693562958102799409e-05,9.533438557276923428e-05,-4.778289984352130856e-03,1.000000000000000000e+00
41 | 9.519661975140544276e-05,1.021690370026732004e-04,-4.816645898927935168e-03,1.000000000000000000e+00
42 | 4.571048031621252465e-04,6.496872293667835436e-05,-5.101504988915408524e-03,1.000000000000000000e+00
43 | 1.492783239425512262e-04,1.072296539223990716e-04,-4.859286766903127795e-03,1.000000000000000000e+00
44 | 9.051797000664450689e-05,1.264766971879299057e-04,-4.811388919150815999e-03,1.000000000000000000e+00
45 | 4.667323758055161174e-05,1.322923926098711815e-04,-4.779209201719352568e-03,1.000000000000000000e+00
46 | 1.792050870160200298e-04,1.153890975275458292e-04,-4.881382992582766929e-03,1.000000000000000000e+00
47 | 1.046624162405941790e-04,1.253410674907158475e-04,-4.823701087513850884e-03,1.000000000000000000e+00
48 | 1.297598460882725660e-04,1.325960675140948624e-04,-4.843535753068006766e-03,1.000000000000000000e+00
49 | 1.881029881939585088e-05,1.598782438422779832e-04,-4.759905225568371312e-03,1.000000000000000000e+00
50 | 8.589759275489007454e-05,1.752721321909959137e-04,-4.811064311641260853e-03,1.000000000000000000e+00
51 | 1.843366238749458499e-04,1.858551909806048269e-04,-4.888645770804531804e-03,1.000000000000000000e+00
52 | 1.238049135705764027e-04,3.048336448786388925e-04,-4.858638650984635314e-03,1.000000000000000000e+00
53 | 5.531132535884604939e-04,2.682665886707769694e-04,-5.208121175789331460e-03,1.000000000000000000e+00
54 | 6.803009491040514292e-04,7.484206740512284279e-04,-5.800206177359697700e-03,1.000000000000000000e+00
55 | -3.452974979596911656e-04,9.262179266764898958e-04,-4.924285408348871677e-03,1.000000000000000000e+00
56 | 7.517968974346065508e-04,-9.577487226388334491e-04,-5.321751478398790633e-03,1.000000000000000000e+00
57 | -2.834437040147317417e-04,-8.315256984333880066e-04,-4.508196686642778998e-03,1.000000000000000000e+00
58 | 7.240056481015720883e-04,5.640214828087748928e-04,-5.691683030864534351e-03,1.000000000000000000e+00
59 | 5.445714626537852786e-04,4.468902633233923054e-04,-5.320772697449033077e-03,1.000000000000000000e+00
60 | 3.376193947098130054e-04,4.760458743416579670e-04,-5.150249399462548949e-03,1.000000000000000000e+00
61 | 3.085488803792759511e-04,4.824055416626456586e-04,-5.128969734468556869e-03,1.000000000000000000e+00
62 | 9.642836149949101916e-05,5.137181894229466512e-04,-4.956196138708402789e-03,1.000000000000000000e+00
63 | 6.988981708545168839e-05,5.203988759909922417e-04,-4.935191297023701872e-03,1.000000000000000000e+00
64 | -1.451048300263578245e-04,5.504843430850018789e-04,-4.759947705212606676e-03,1.000000000000000000e+00
65 | -1.882121639422611539e-04,3.174068736720455199e-04,-4.609367747728889721e-03,1.000000000000000000e+00
66 | -2.168773681122131061e-04,-1.656221386572242869e-05,-4.583587872539577994e-03,1.000000000000000000e+00
67 | 4.692993671781916132e-04,5.920062539841192822e-04,-5.446170062609071751e-03,1.000000000000000000e+00
68 | 2.680631316428236204e-04,6.264494962085198626e-04,-5.273107581611331113e-03,1.000000000000000000e+00
69 | 2.391097899648426896e-04,6.318774618757538788e-04,-5.247186393032251747e-03,1.000000000000000000e+00
70 | 2.846942185114809384e-05,6.661085189815830771e-04,-5.072379668334592653e-03,1.000000000000000000e+00
71 | -3.624311023625717676e-06,6.736742267174610768e-04,-5.050233769387536646e-03,1.000000000000000000e+00
72 | -2.061072763409636181e-04,7.086674446092095393e-04,-4.864749663287814239e-03,1.000000000000000000e+00
73 | 5.572225716997091785e-04,-1.197189383442980363e-04,-5.187181384830158101e-03,1.000000000000000000e+00
74 | 5.840900916401071778e-04,2.051987344248867872e-04,-5.216529575888964251e-03,1.000000000000000000e+00
75 | 


--------------------------------------------------------------------------------
/reconstructed_model.stl:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/nbarba/py3DRec/369f26d807631e21b405d19697fe158158784ffb/reconstructed_model.stl


--------------------------------------------------------------------------------
/uncalibrated_rec.py:
--------------------------------------------------------------------------------
  1 | import numpy as np
  2 | from numpy.linalg import inv
  3 | from numpy.linalg import svd
  4 | from numpy.linalg import eig
  5 | from numpy.linalg import det
  6 | from scipy.optimize import leastsq, least_squares, fmin
  7 | import pandas as pd
  8 | import numpy as np
  9 | import time
 10 | import random
 11 | import os.path
 12 | import argparse
 13 | 
 14 | from epipolar_geometry import EpipolarGeometry
 15 | from model import RecModel
 16 | from image_sequence import ImageSequence
 17 | 
 18 | 
 19 | class UncalibratedReconstruction:
 20 |     '''
 21 |     Class that contains high level methods to perform 3D reconstruction from a sequence of uncalibrated images.
 22 |     '''
 23 | 
 24 |     def __init__(self, sequence_length, width, height, triang_method=0, opt_triang=0, opt_f=1, self_foc=0):
 25 |         '''
 26 |         Constructor
 27 |         Args:
 28 |             sequence_length: number of images (views)
 29 |             width: width of the images
 30 |             height: height of the images
 31 |             triang_method: triangulation method (0: standard, 1: polynomial)
 32 |             opt_triang: optimize initial 3D point estimate
 33 |             opt_f: optimize fundamental matrix estimation
 34 |             self_foc: for self-calibration, type of focal length expected across views (0: fixed, 1: varying )
 35 | 
 36 |         '''
 37 |         self._eg_utils = EpipolarGeometry()
 38 | 
 39 |         # things that are needed throught the class
 40 |         self._sequence_length = sequence_length
 41 |         self._width = width
 42 |         self._height = height
 43 |         self._mm = (width + height) / 2
 44 | 
 45 |         self._triangulation_method = triang_method
 46 |         self._optimize_triangulation = opt_triang
 47 |         self._optimize_f = opt_f
 48 |         self._self_foc = self_foc
 49 | 
 50 |     def two_view_geometry_computation(self, view1_feat2D, view2_feat2D):
 51 |         '''
 52 |         Method to compute the fundamental matrix and epipoles for two views
 53 |         Args:
 54 |             view1_feat2D: 2D feature coordinates in view 1
 55 |             view2_feat2D: 2D feature coordinates in view 2
 56 |         Returns:
 57 |             F: the fundamental matrix
 58 |             epipole_1: view1 epipole
 59 |             epipole_2: view2 epipoles
 60 | 
 61 |         '''
 62 |         F = self._eg_utils.fundamental_matrix(view1_feat2D, view2_feat2D, self._optimize_f)
 63 |         epipole_1 = self._eg_utils.get_epipole(F)
 64 |         epipole_2 = self._eg_utils.get_epipole(np.transpose(F))
 65 |         return F, epipole_1, epipole_2
 66 | 
 67 |     def compute_reference_frame(self, epipole, F):
 68 |         '''
 69 |         Method to compute the reference frame of the reconstruction (i.e. plane at infinity in an affine or metric space).
 70 |         Args:
 71 |             epipole: the epipole
 72 |             F: the fundamental matrix
 73 |         Returns:
 74 |             p: the reference plane
 75 |             h: the homography [e]xF
 76 | 
 77 |         '''
 78 |         H = self._eg_utils.compute_homography(epipole, F)  # compute the homography [e]xF
 79 |         # get the reference plane
 80 |         p = np.sum(np.divide(np.eye(3) - H, np.transpose(np.asarray([epipole, epipole, epipole]))), axis=0) / 3
 81 |         # adjust reference plane to make the first two projection matrices as equal as possible
 82 |         p = fmin(self.init_plane, np.append(p, 1), xtol=1e-25, ftol=1e-25, args=(H.real, epipole.real))
 83 |         p = p[0:3]
 84 |         return p, H
 85 | 
 86 |     def init_plane(self, p, H, epi):
 87 |         '''
 88 |         Error function to make the difference between the first two projection matrices as small as possible
 89 |         Note: assuming that the two views used for the initial reconstruction are not too far apart (this their projection matrices are almost equal), has proven to give good results
 90 |         Args:
 91 |             p: the reference plane (i.e. plane at infinity)
 92 |             H: homography [e]x[F]
 93 |             epi: epipola
 94 |         Returns:
 95 |             error: difference between two projection matrices
 96 |         '''
 97 |         epi = np.reshape(epi, (3, 1))
 98 |         p = np.reshape(p, (1, 4))
 99 | 
100 |         t = p[0, 0:3]
101 |         t = np.reshape(t, (1, 3))
102 | 
103 |         error = sum(sum(abs(H + epi.dot(t) - p[0, 3] * np.eye(3))))
104 | 
105 |         return error
106 | 
107 |     def estimate_initial_projection_matrices(self, H, epipole_2, p):
108 |         '''
109 |         Method to estimate the projection matrices for the two views (i.e. P1=[I | 0], P2=[H+epi1|e])
110 |         Args:
111 |             H: homography [e]x[F]
112 |             epipole_2: epipole in the 2nd view
113 |             p: the reference plane of the reconstruction (i.e. plane at infinity)
114 |         Returns:
115 |             P: projection matrices for these two views
116 | 
117 |         '''
118 |         P = np.zeros((3, 4, self._sequence_length))
119 |         P[:, :, 0] = [[1, 0, 0, 0], [0, 1, 0, 0],
120 |                       [0, 0, 1, 0]]  # P1=[I | 0], i.e. frist frame aligned with world frame
121 |         epi_tmp = np.reshape(epipole_2, (3, 1))  # P2=[H+epi1|e]
122 |         P[:, :3, 1] = H + epi_tmp.dot(np.reshape(p, (1, 3)))
123 |         P[:, 3, 1] = epipole_2
124 |         P[:, :, 1] = P[:, :, 1] / P[2, 2, 1]
125 |         return P
126 | 
127 |     def get_initial_structure(self, feat_2D, P, epipole_1, epipole_2, F):
128 |         '''
129 |         Method to get an initial 3D structure (i.e. 3D point cloud), from the first two projection matrices through triangulation.
130 |         Args:
131 |             feat_2D: 2D feature coordinates for all images
132 |             P: projection matrices for all views (only the first two views are used)
133 |             epipole_1: view 1 epipole
134 |             epipole_2: view 2 epipole
135 |             F: fundamental matrix
136 |         Returns:
137 |             points3D: 3D point cloud
138 |         '''
139 |         number_of_features = feat_2D.shape[2]
140 |         points3D = np.zeros(shape=[number_of_features, 4])
141 |         for i in range(0, number_of_features):
142 | 
143 |             if (self._triangulation_method == 0):
144 |                 x = self._eg_utils.triangulate_points(feat_2D[0, :, i], feat_2D[1, :, i], P[:, :, 0], P[:, :, 1])
145 |                 x = x[0:3]
146 |             elif (self._triangulation_method == 1):
147 |                 x = self._eg_utils.polynomial_triangulation(feat_2D[1, :, i], feat_2D[1, :, i], epipole_1, epipole_2, F,
148 |                                                             P[:, :, 0], P[:, :, 1])
149 |                 x = x[0:3] / x[3]  # normalize
150 | 
151 |             if (self._optimize_triangulation == 1):
152 |                 # refine 3D point estimation (due to noise, lines of sight may not intersect perfectly). Minimizations should be carried out in the images
153 |                 # and not in the projective 3D space, thus the reprojection error is used.
154 |                 x = fmin(self.refine_3d_point, x, xtol=1e-25, ftol=1e-25, full_output=0,
155 |                          args=(P[:, :, 0], P[:, :, 1], feat_2D[0, :, i], feat_2D[1, :, i]))
156 | 
157 |             points3D[i, :] = np.append(x, 1)
158 | 
159 |         return points3D
160 | 
161 |     def refine_3d_point(self, point3D, P1, P2, view1_feat2D, view2_feat2D):
162 |         '''
163 |         Method to compute the reprojection error of a 3D point in two views
164 |         Args:
165 |             point3D: 3D point cloud
166 |             P1: projection matrix of view 1
167 |             P2: projection matrix of view 2
168 |             view1_feat2D: 2D feature coordinates in view 1
169 |             view2_feat2D: 2D feature coordinates in view 1
170 |         Returns:
171 |             error: the reprojection error
172 | 
173 |         '''
174 |         point3D = np.append(point3D, 1)
175 |         error = sum(
176 |             self.compute_reprojection_error_point(P1, point3D, view1_feat2D) + self.compute_reprojection_error_point(P2,
177 |                                                                                                                      point3D,
178 |                                                                                                                      view2_feat2D))
179 |         # sdfds
180 |         return error
181 | 
182 |     def projective_pose_estimation(self, feat_2D, P, points3D):
183 |         '''
184 |         Method to add views using an initial 3D structure, i.e. compute the projection matrices for all the additional views (the first two are already
185 |         estimated in previous steps)
186 |         Args:
187 |             feat_2D: 2D feature coordinates for all images
188 |             P: projection matrices
189 |             points3d: 3D point cloud
190 |         Returns:
191 |             P: projection matrices for all views
192 |         '''
193 |         number_of_features = feat_2D.shape[2]
194 | 
195 |         AA = np.zeros(shape=[2 * number_of_features, 12])
196 | 
197 |         for i in range(2, self._sequence_length):
198 |             for j in range(0, number_of_features):
199 |                 AA[2 * j, 0:4] = points3D[j]
200 |                 AA[2 * j, 8:12] = -feat_2D[i, 0, j] * points3D[j]
201 |                 AA[2 * j + 1, 4:8] = points3D[j]
202 |                 AA[2 * j + 1, 8:12] = -feat_2D[i, 1, j] * points3D[j]
203 | 
204 |             U, s, Vh = svd(AA)
205 |             V = np.transpose(Vh)
206 | 
207 |             VV = V[0:12, 11]
208 |             VV = VV / VV[10]
209 |             VV = np.delete(VV, 10)
210 | 
211 |             # refine the estimate for the i-th projection matrix
212 |             result = least_squares(self._eg_utils.refine_projection_matrix, VV, args=(points3D, feat_2D[i, :, :]))
213 |             VV = result.x
214 | 
215 |             Pr = np.zeros(shape=[3, 4])
216 |             Pr[0, :] = VV[0:4]
217 |             Pr[1, :] = VV[4:8]
218 |             Pr[2, :] = np.append(np.append(VV[8:10], 1), VV[10])
219 |             P[:, :, i] = Pr
220 | 
221 |         return P
222 | 
223 |     def bundle_adjustment(self, feat_2D, P, feat3D):
224 |         '''
225 |         Method to refine structure and motion, i.e. refine the projection matrices and 3D points using the reprojection error
226 |         Args:
227 |             feat_2D: 2D feature coordinates for all images
228 |             P: projection matrices
229 |             points3d: 3D point cloud
230 |         Returns:
231 |             P: the refined projection matrices
232 |             feat3D: the refined 3D point cloud
233 |             error: the reprojection error
234 |         '''
235 |         number_of_features = feat_2D.shape[2]
236 | 
237 |         # The vector to be optimized
238 |         X = np.reshape(P[:, :, 0], (1, 12))
239 | 
240 |         # Append the projection matrices...
241 |         for i in range(1, self._sequence_length):
242 |             X = np.append(X, np.reshape(P[:, :, i], (1, 12)))
243 |         X = np.delete(X, [10, 22, (self._sequence_length - 1) * 12 + 10])
244 | 
245 |         # ...and then append the 3D points
246 |         X = np.append(X, np.reshape(feat3D[:, 0:3], number_of_features * self._sequence_length))
247 | 
248 |         # Optimize using Levenberg-Marquardt
249 |         result = least_squares(self._eg_utils.overall_reprojection_error, X, max_nfev=1000, method='lm',
250 |                                args=([feat_2D]))
251 |         X = result.x
252 |         error = np.power(sum(self._eg_utils.overall_reprojection_error(X, feat_2D)), 2)
253 | 
254 |         # get the refined projection matrices from the optimal vector
255 |         for i in range(0, self._sequence_length):
256 |             P[:, :, i] = np.reshape(X[0 + i * 11:12 + i * 11], (3, 4))
257 |             P[2, 3, i] = P[2, 2, i]
258 |             P[2, 2, i] = 1
259 | 
260 |         # get the refined 3D coordinates from the optimal vector
261 |         feat3D[:, 0:3] = np.reshape(
262 |             X[self._sequence_length * 11:self._sequence_length * 11 + self._sequence_length * number_of_features * 3],
263 |             (number_of_features, 3))
264 | 
265 |         Tp1 = np.vstack([P[:, :, 0], [0, 0, 0, 1]])
266 |         for i in range(0, self._sequence_length):
267 |             P[:, :, i] = P[:, :, i].dot(inv(Tp1))
268 | 
269 |         feat3D = Tp1.dot(np.transpose(feat3D))
270 |         feat3D = np.transpose(feat3D / feat3D[3, :])
271 | 
272 |         return P, feat3D, error
273 | 
274 |     def self_calibration(self, P):
275 |         '''
276 |         Self calibration using the procedure described in
277 |         M. Pollefeys, R. Koch and L. Van Gool, "Self-Calibration and Metric Reconstruction in spite of Varying and Unknown Internal Camera Parameters", Proc. International Conference on Computer Vision, Narosa Publishing House, pp.90-95, 1998.
278 |         Args:
279 |             P: projection matrices
280 |         Returns:
281 |             Tm: transformation matrix that will transform from the projective space to metric space
282 |             K: camera intrisic parameters for each view
283 |             error: the reprojection error
284 | 
285 |         '''
286 |         # setup the system of equations
287 |         AAA = np.zeros(shape=[4 * self._sequence_length - 4, 6])
288 |         for i in range(0, self._sequence_length - 1):
289 |             P_tmp = P[:, :, i + 1]
290 |             AAA[0 + 4 * i, :] = [(-np.power(P_tmp[1, 1], 2) + np.power(P_tmp[0, 1], 2) - np.power(P_tmp[1, 0],
291 |                                                                                                   2) + np.power(
292 |                 P_tmp[0, 0], 2)), (-2 * P_tmp[1, 0] * P_tmp[1, 3] + 2 * P_tmp[0, 0] * P_tmp[0, 3]),
293 |                                  (-2 * P_tmp[1, 1] * P_tmp[1, 3] + 2 * P_tmp[0, 1] * P_tmp[0, 3]),
294 |                                  (2 * P_tmp[0, 2] * P_tmp[0, 3] - 2 * P_tmp[1, 2] * P_tmp[1, 3]),
295 |                                  (-np.power(P_tmp[1, 3], 2) + np.power(P_tmp[0, 3], 2)),
296 |                                  (-np.power(P_tmp[1, 2], 2) + np.power(P_tmp[0, 2], 2))]
297 |             AAA[1 + 4 * i, :] = [(P_tmp[1, 0] * P_tmp[0, 0] + P_tmp[1, 1] * P_tmp[0, 1]),
298 |                                  (P_tmp[1, 0] * P_tmp[0, 3] + P_tmp[1, 3] * P_tmp[0, 0]),
299 |                                  (P_tmp[1, 1] * P_tmp[0, 3] + P_tmp[1, 3] * P_tmp[0, 1]),
300 |                                  (P_tmp[1, 2] * P_tmp[0, 3] + P_tmp[1, 3] * P_tmp[0, 2]), P_tmp[1, 3] * P_tmp[0, 3],
301 |                                  P_tmp[1, 2] * P_tmp[0, 2]]
302 |             AAA[2 + 4 * i, :] = [(P_tmp[2, 0] * P_tmp[0, 0] + P_tmp[2, 1] * P_tmp[0, 1]),
303 |                                  (P_tmp[2, 0] * P_tmp[0, 3] + P_tmp[2, 3] * P_tmp[0, 0]),
304 |                                  (P_tmp[2, 1] * P_tmp[0, 3] + P_tmp[2, 3] * P_tmp[0, 1]),
305 |                                  (P_tmp[2, 2] * P_tmp[0, 3] + P_tmp[2, 3] * P_tmp[0, 2]), P_tmp[2, 3] * P_tmp[0, 3],
306 |                                  P_tmp[2, 2] * P_tmp[0, 2]]
307 |             AAA[3 + 4 * i, :] = [(P_tmp[2, 0] * P_tmp[1, 0] + P_tmp[2, 1] * P_tmp[1, 1]),
308 |                                  (P_tmp[2, 0] * P_tmp[1, 3] + P_tmp[2, 3] * P_tmp[1, 0]),
309 |                                  (P_tmp[2, 1] * P_tmp[1, 3] + P_tmp[2, 3] * P_tmp[1, 1]),
310 |                                  (P_tmp[2, 2] * P_tmp[1, 3] + P_tmp[2, 3] * P_tmp[1, 2]), P_tmp[2, 3] * P_tmp[1, 3],
311 |                                  P_tmp[2, 2] * P_tmp[1, 2]]
312 | 
313 |         U, s, Vh = svd(AAA)
314 |         V = np.transpose(Vh)
315 |         x = V[0:5, 5] / V[5, 5]
316 |         jj = np.sign(x[0])
317 |         b = x * np.sign(x[0])
318 | 
319 |         # initial estimate of the absolute conic
320 |         W = np.asarray([[b[0], 0, 0, b[1]], [0, b[0], 0, b[2]], [0, 0, 1, b[3]], [b[1], b[2], b[3], b[4]]])
321 | 
322 |         # initial estimate of the focal lengths
323 |         y = np.ones(shape=[self._sequence_length, 1])
324 |         for i in range(0, self._sequence_length):
325 |             y[i] = np.sqrt(np.abs(P[0, :, i].dot(W).dot(np.transpose(P[0, :, i]))) / (
326 |                 P[2, :, i].dot(W).dot(np.transpose(P[2, :, i]))))
327 | 
328 |         if (self._self_foc == 0):
329 |             # optimize for fixed focal lengths
330 |             pp2 = np.asarray([-b[1] / b[0], -b[2] / b[0], -b[3]])
331 |             x = np.hstack((sum(y) / y.shape[0], pp2))
332 |             x = fmin(self.fixed_f_error, x, args=(P, self._sequence_length))
333 |             error = self.fixed_f_error(x, P, self._sequence_length)
334 | 
335 |             # fill out the camera instrisic parameters.
336 |             K = np.zeros((3, 3, self._sequence_length))
337 |             for i in range(0, self._sequence_length):
338 |                 K[:, :, i] = np.eye(3)
339 |                 K[0, 0, i] = x[0] * self._mm
340 |                 K[1, 1, i] = x[0] * self._mm
341 |                 K[0, 2, i] = self._width
342 |                 K[1, 2, i] = self._height
343 | 
344 |             inf_plane = x[1:4]
345 | 
346 |             # construct the transformation matrix that will take us from the projective space to to metric
347 |             a = inv([[x[0], 0, 0], [0, x[0], 0], [0, 0, 1]])
348 |             a = np.asarray(a) * jj
349 |             tmp = np.asarray([0, 0, 0])
350 |             tmp = np.reshape(tmp, (3, 1))
351 |             Tm = np.append(a, tmp, 1)
352 |             Tm = np.vstack((Tm, np.append(inf_plane, 1)))
353 | 
354 |         else:
355 |             # optimize for varying focal lenghts (to do)
356 |             print("not yet supported")
357 | 
358 |         return Tm, K, error
359 | 
360 |     def fixed_f_error(self, x, P, n):
361 |         '''
362 |         Error function for the self-calibration error when the focal lengths are fixed (i.e. we
363 |         assume the same focal lengths accross the image sequence)
364 |         '''
365 |         K1 = np.eye(3)
366 |         K1[0, 0] = x[0]
367 |         K1[1, 1] = x[0]
368 |         pp = np.asarray(x[1:4])
369 |         pp = np.reshape(pp, (3, 1))
370 | 
371 |         W = np.asarray(np.append(K1.dot(np.transpose(K1)), -K1.dot(np.transpose(K1)).dot(pp), 1))
372 |         tmp = np.append((np.transpose(-pp).dot(K1).dot(np.transpose(K1))),
373 |                         np.transpose(pp).dot(K1).dot(np.transpose(K1)).dot(pp))
374 |         W = np.vstack((W, tmp))
375 | 
376 |         a = K1.dot(np.transpose(K1))
377 |         b = a / np.sqrt(np.trace(np.transpose(a).dot(a)))
378 |         error = 0
379 |         for i in range(0, n - 1):
380 |             P_tmp = P[:, :, i + 1]
381 |             c = P_tmp.dot(W).dot(np.transpose(P_tmp))
382 |             d = c / np.sqrt(np.trace(np.transpose(c) * c))
383 |             error = error + np.trace(np.transpose((b - d)).dot(b - d))
384 | 
385 |         return error
386 | 
387 |     def convert_to_metric_space(self, Tm, feat3D, P, K):
388 |         '''
389 |         Transform the 3D points and projective matrices to the metric space
390 |         Args:
391 |             Tm: transformation matrix for transformic from projective to metric space
392 |             feat3D: 3D point cloud
393 |             P: projection matrices (for all views)
394 |             K: camera intrisic parameters (for all views)
395 |         Returns:
396 |             feat3D: 3D point cloud in metric space
397 |             P: projectio matrices in metric space
398 |         '''
399 |         # transform the projective 3d coordinates to metric
400 |         InvT = Tm.dot(np.eye(4))
401 |         a = inv(InvT)
402 |         InvT = InvT * a[3, 3]
403 | 
404 |         feat3D = InvT.dot(np.transpose(feat3D))
405 |         feat3D = np.transpose(feat3D / feat3D[3, :])
406 | 
407 |         # Rescale the projection matrices to width,height (no -1, 1)
408 |         tmp = np.eye(3)
409 |         tmp[0, 0] = self._mm
410 |         tmp[1, 1] = self._mm
411 |         tmp[0, 2] = self._width
412 |         tmp[1, 2] = self._height
413 |         for i in range(0, self._sequence_length):
414 |             P[:, :, i] = P[:, :, i].dot(inv(InvT))
415 |             P[:, :, i] = tmp.dot(P[:, :, i])
416 |             a = det(inv(K[:, :, i]).dot(P[0:3, 0:3, i]))
417 |             P[:, :, i] = P[:, :, i] * np.sign(a) / np.power(abs(a), 0.333)
418 | 
419 |         return feat3D, P
420 | 
421 | 
422 | def main(input_file, show):
423 |     print("--------------------------------")
424 |     print(" Uncalibrated 3D Reconstruction ")
425 |     print("")
426 |     print("--------------------------------")
427 | 
428 |     sequence = ImageSequence(input_file)
429 | 
430 |     start_time = time.time()
431 |     rec_engine = UncalibratedReconstruction(sequence.length, sequence.width, sequence.height)
432 | 
433 |     # normalize coordinates
434 |     norm_feat_2d = sequence.get_normalized_coordinates()
435 | 
436 |     print("> Estimating fundamental matrix....")
437 |     F, epipole_1, epipole_2 = rec_engine.two_view_geometry_computation(norm_feat_2d[0], norm_feat_2d[1])
438 | 
439 |     print("> Computing reference plane....")
440 |     # Step 2: compute the reconstruction reference plane using the epipole in the second image
441 |     p, H = rec_engine.compute_reference_frame(epipole_2, F)
442 | 
443 |     print("> Estimating projection matrices for first two views....")
444 |     # Step 3: Estimate projection matrices for the first two views
445 |     P = rec_engine.estimate_initial_projection_matrices(H, epipole_2, p)
446 | 
447 |     print("> 3D point estimate triangulation....")
448 |     # Step 4: triangulate points to get an initial estimate of the 3D point cloud
449 |     feat3D = rec_engine.get_initial_structure(norm_feat_2d, P, epipole_1, epipole_2, F)
450 | 
451 |     print("> Estimating projection matrices for additional views....")
452 |     # Step 5: Use the 3D point estimates to estimate the projection matrices of the remaining views
453 |     P = rec_engine.projective_pose_estimation(norm_feat_2d, P, feat3D)
454 | 
455 |     print("> Bundle Adjustment....")
456 |     # Step 5: Optimize 3D points and projection matrices using the reprojection error
457 |     P, feat3D, error = rec_engine.bundle_adjustment(norm_feat_2d, P, feat3D)
458 |     print("  - Bundle adjustment error: ", error)
459 | 
460 |     print("> Self-calibration")
461 |     # Step 6: Self-calibration
462 |     Tm, K, error = rec_engine.self_calibration(P)
463 |     print("  - Self-calibration error: ", error)
464 |     print("  - Tranformation Matrix (Projective -> Metric): ")
465 | 
466 |     print("> Converting to metric space")
467 |     metric_feat3D, metric_P = rec_engine.convert_to_metric_space(Tm, feat3D, P, K)
468 | 
469 |     print("> Saving model...")
470 | 
471 |     recModel = RecModel()
472 |     recModel.P = P
473 |     recModel.points3D = metric_feat3D
474 |     recModel.Tm = Tm
475 | 
476 |     np.savetxt('rec_model_cloud.txt', metric_feat3D, delimiter=',')
477 |     print("	 - 3D point cloud saved in rec_model_cloud.txt ")
478 | 
479 |     recModel.export_stl_file('reconstructed_model.stl')
480 |     print("  - STL model saved in reconstructed_model.stl")
481 | 
482 |     if (show == True):
483 |         sequence.show()
484 | 
485 |     print("> 3D reconstruction completed in " + str(round(time.time() - start_time, 1)) + " sec!")
486 | 
487 | 
488 | if __name__ == "__main__":
489 |     parser = argparse.ArgumentParser(description='3D Reconstruction from uncalibrated images')
490 |     parser.add_argument('--input_file', metavar='path', required=True,
491 |                         help='Input file containing image point correspondences')
492 |     parser.add_argument('--show', required=False, action="store_true",
493 |                         help="Display the image sequence with the 2D features")
494 |     args = parser.parse_args()
495 |     main(input_file=args.input_file, show=args.show)
496 | 


--------------------------------------------------------------------------------