├── .gitignore ├── compile.m ├── Params3D.mat ├── example.jpg ├── example.Z.png ├── Documentation.pdf ├── GetBottomPoint.m ├── partsGibbsSampler.mexmaci ├── getListLargeObjects.m ├── edgeLength.m ├── RH2LH.m ├── LH2RH.m ├── getOccludedControlPoints.m ├── LM3DgetTexturedMesh.m ├── readNmap.m ├── readYmap.m ├── readZmap.m ├── LMplot3Dmesh.m ├── getPlanes.m ├── Contents.m ├── rq.m ├── findHorizonLine.m ├── getListGroundObjects.m ├── getLMcontact.m ├── isThereAnyPointBelowEdge.m ├── edgeVerticality.m ├── readXmap.m ├── relativeError.m ├── writeNmap.m ├── writeYmap.m ├── writeZmap.m ├── getFootprint.m ├── setLMpolygon3D.m ├── setLMcontact.m ├── getContactPoints.m ├── LM3Ddatabase.m ├── getLMpolygon3D.m ├── addimagesize.m ├── RectAreas.m ├── writeXmap.m ├── projectOntoGroundPlane.m ├── LM3Dstats.m ├── style.css ├── ObjectID2Index.m ├── intersectPolygon.m ├── roc.m ├── getEdges.m ├── depthLinearRegression.m ├── plotScene3D.m ├── getPartOfParents.m ├── relativeOverlap.m ├── demoGraphs.m ├── isThereAnyPointBelow.m ├── ReadObjectHeights.m ├── LM3Dinstall.m ├── getObjectMesh.m ├── isCleanObject.m ├── groundObjectWithMesh.m ├── plotNmap.m ├── Project3D2D.m ├── plotLineBmap.m ├── getHorizonLine.m ├── README.txt ├── PolyAreas.m ├── depthThumbnail.m ├── writeXML3D.m ├── LICENSE.txt ├── includeGroundTruthEdgeLabels.m ├── LM3DpolyEdgeTypesThumbnail.m ├── getaproximagesize.m ├── decomposeP.m ├── plotOutputEdgeTypes.m ├── myLMaddtags.m ├── getCameraMatrix.m ├── inferParts.m ├── plotSupport.m ├── plotParts.m ├── splitPolygon.m ├── setCameraMatrix.m ├── getWidthMap.m ├── InflateSceneNew.m ├── getHeightMap.m ├── SupportDist.m ├── addVerticalPoint.m ├── detectShallowConcave.m ├── plotEdgeLabels.m ├── imageWarp.m ├── removeMissedParts.m ├── samplePoly.m ├── standingFoldedCardWithMesh.m ├── logpYCV0.m ├── avg_error.m ├── inferSupport.m ├── ConvertOld2New.m ├── LM3D_ObjectHeights.txt ├── LM3Dgenerate3D.m ├── getObjectDimensions.m ├── detectOccluded.m ├── plotXmap.m ├── plotZmap.m ├── collectPartsEvidence.m ├── plotYmap.m ├── AddAnglePolygonType.m ├── LMsceneThumbnail.m ├── bpFactorGraph.m ├── subDivideMesh.m ├── LMplot3Dscene.m ├── getDepthMap.m ├── myLMimscale.m ├── getObject3D.m ├── supportGraph.m ├── insertGroundPlane.m ├── getTextures.m ├── generateO3D.m ├── collectEdgeEvidence.m ├── learnPartsParams.m ├── showPartsTree.m ├── LM2VRML.m ├── getMesh.m ├── CostFunctionLM3D_ObjectHeights.m ├── CostFunctionLM3D_old.m ├── inferPartsPosterior.m ├── Recover3DSceneComponents.m ├── partsGibbsSampler.cpp ├── showFootprint.m ├── plotPolyEdgeTypes.m ├── partsGraph.m ├── demo.m ├── LM2VRMLfill.m ├── contactPointsNew.m ├── getPolyEdgeTypes.m ├── InflateSceneXML.m ├── getXYZmaps.m └── inferEdgeTypesNew.m /.gitignore: -------------------------------------------------------------------------------- 1 | *.mexmaci 2 | *~ 3 | -------------------------------------------------------------------------------- /compile.m: -------------------------------------------------------------------------------- 1 | function compile 2 | 3 | mex rayTrace.cpp 4 | -------------------------------------------------------------------------------- /Params3D.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brussell123/LabelMe3dToolbox/HEAD/Params3D.mat -------------------------------------------------------------------------------- /example.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brussell123/LabelMe3dToolbox/HEAD/example.jpg -------------------------------------------------------------------------------- /example.Z.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brussell123/LabelMe3dToolbox/HEAD/example.Z.png -------------------------------------------------------------------------------- /Documentation.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brussell123/LabelMe3dToolbox/HEAD/Documentation.pdf -------------------------------------------------------------------------------- /GetBottomPoint.m: -------------------------------------------------------------------------------- 1 | function [xx,yy] = GetBottomPoint(X,Y) 2 | [yy,low_ndx] = max(Y); 3 | xx = X(low_ndx); 4 | -------------------------------------------------------------------------------- /partsGibbsSampler.mexmaci: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brussell123/LabelMe3dToolbox/HEAD/partsGibbsSampler.mexmaci -------------------------------------------------------------------------------- /getListLargeObjects.m: -------------------------------------------------------------------------------- 1 | function list = getListLargeObjects 2 | 3 | list = {'building','wall','fence','house','church','castle'}; 4 | -------------------------------------------------------------------------------- /edgeLength.m: -------------------------------------------------------------------------------- 1 | function len = edgeLength(X,Y,edges) 2 | 3 | e1 = edges(1,:); 4 | e2 = edges(2,:); 5 | len = ((X(e1)-X(e2)).^2+(Y(e1)-Y(e2)).^2).^0.5; 6 | -------------------------------------------------------------------------------- /RH2LH.m: -------------------------------------------------------------------------------- 1 | function [x,y] = RH2LH(x,y,imageSize) 2 | 3 | x = (imageSize(2)+1)/2-x; 4 | y = (imageSize(1)+1)/2-y; 5 | % $$$ x = x+(imageSize(2)+1)/2; 6 | % $$$ y = (imageSize(1)+1)/2-y; 7 | -------------------------------------------------------------------------------- /LH2RH.m: -------------------------------------------------------------------------------- 1 | function [x,y] = LH2RH(x,y,imageSize) 2 | 3 | x = (imageSize(2)+1)/2-x; 4 | y = (imageSize(1)+1)/2-y; 5 | % $$$ x = x-(imageSize(2)+1)/2; 6 | % $$$ y = (imageSize(1)+1)/2-y; 7 | 8 | -------------------------------------------------------------------------------- /getOccludedControlPoints.m: -------------------------------------------------------------------------------- 1 | function annotation = getOccludedControlPoints(annotation,sup) 2 | 3 | return; 4 | 5 | % If a point is close to its support object, then it is not occluded. 6 | -------------------------------------------------------------------------------- /LM3DgetTexturedMesh.m: -------------------------------------------------------------------------------- 1 | function mesh = LM3DgetTexturedMesh(annotation,img) 2 | 3 | mesh = getSceneMesh(annotation); 4 | mesh = subDivideMesh(mesh,2); 5 | mesh = getTextures(mesh,annotation,img); 6 | -------------------------------------------------------------------------------- /readNmap.m: -------------------------------------------------------------------------------- 1 | function Nmap = readNmap(fname) 2 | 3 | % Read from PNG file 4 | 5 | png = uint32(imread(fname)); 6 | Nmap = zeros(size(png,1),size(png,2),'uint32'); 7 | Nmap = png(:,:,1)+256*png(:,:,2)+256*256*png(:,:,3); 8 | -------------------------------------------------------------------------------- /readYmap.m: -------------------------------------------------------------------------------- 1 | function Ymap = readYmap(fname) 2 | 3 | % Read from PNG file 4 | 5 | png = uint32(imread(fname)); 6 | Ymap = zeros(size(png,1),size(png,2),'uint32'); 7 | Ymap = png(:,:,1)+256*png(:,:,2)+256*256*png(:,:,3); 8 | -------------------------------------------------------------------------------- /readZmap.m: -------------------------------------------------------------------------------- 1 | function Zmap = readZmap(fname) 2 | 3 | % Read from PNG file 4 | 5 | png = uint32(imread(fname)); 6 | Zmap = zeros(size(png,1),size(png,2),'uint32'); 7 | Zmap = png(:,:,1)+256*png(:,:,2)+256*256*png(:,:,3); 8 | -------------------------------------------------------------------------------- /LMplot3Dmesh.m: -------------------------------------------------------------------------------- 1 | function LMplot3Dmesh(mesh) 2 | 3 | trisurf(mesh.faces',mesh.vertices(1,:),mesh.vertices(2,:),mesh.vertices(3,:)); 4 | axis equal; 5 | xlabel('X'); 6 | ylabel('Y'); 7 | zlabel('Z'); 8 | cameratoolbar('setmodeGUI','orbit'); 9 | -------------------------------------------------------------------------------- /getPlanes.m: -------------------------------------------------------------------------------- 1 | function PI = getPlanes(plane) 2 | 3 | pix = str2num(char({plane(:).pix})); 4 | piy = str2num(char({plane(:).piy})); 5 | piz = str2num(char({plane(:).piz})); 6 | piw = str2num(char({plane(:).piw})); 7 | PI = [pix(:) piy(:) piz(:) piw(:)]'; 8 | -------------------------------------------------------------------------------- /Contents.m: -------------------------------------------------------------------------------- 1 | % Contents.m 2 | % 3 | % This file lists the contents of the LabelMe3D toolbox. 4 | % 5 | % Labeling geometric properties 6 | % guiEdgeLabeling.m 7 | % includeGroundTruthEdgeLabels.m 8 | % plotEdgeLabels.m 9 | % plotOutputEdgeTypes.m 10 | -------------------------------------------------------------------------------- /rq.m: -------------------------------------------------------------------------------- 1 | function [U,Q] = rq(S) 2 | 3 | S = S'; 4 | [Q,U] = qr(S(end:-1:1,end:-1:1)); 5 | Q = Q'; 6 | Q = Q(end:-1:1,end:-1:1); 7 | U = U'; 8 | U = U(end:-1:1,end:-1:1); 9 | 10 | if det(Q)<0 11 | U(:,1) = -U(:,1); 12 | Q(1,:) = -Q(1,:); 13 | end 14 | 15 | return 16 | -------------------------------------------------------------------------------- /findHorizonLine.m: -------------------------------------------------------------------------------- 1 | function Hy = findHorizonLine(annotation,img) 2 | 3 | % For now, have the user select the horizon line. This will need to be 4 | % found automatically. 5 | figure; 6 | imshow(img); 7 | title('Select horizon line'); 8 | [Hx Hy] = ginput(1); % get horizon from user 9 | -------------------------------------------------------------------------------- /getListGroundObjects.m: -------------------------------------------------------------------------------- 1 | function list = getListGroundObjects 2 | 3 | list = {'floor', 'ground', 'grass', 'field', 'street', 'road', ... 4 | 'sidewalk', 'path', 'sea', 'water', 'sand', 'carpet', 'beach', 'highway', ... 5 | 'lake', 'pond', 'ocean', 'river', 'snow'}; 6 | -------------------------------------------------------------------------------- /getLMcontact.m: -------------------------------------------------------------------------------- 1 | function [x,y] = getLMcontact(polygon) 2 | % Inputs: 3 | % polygon 4 | % 5 | % Outputs: 6 | % x 7 | % y 8 | 9 | x = []; y = []; 10 | if isfield(polygon,'contact') 11 | x = str2num(char({polygon.contact.x})); 12 | y = str2num(char({polygon.contact.y})); 13 | end 14 | -------------------------------------------------------------------------------- /isThereAnyPointBelowEdge.m: -------------------------------------------------------------------------------- 1 | function pb = isThereAnyPointBelowEdge(X,Y,edges) 2 | 3 | pb = isThereAnyPointBelow(X,Y); 4 | xmid = 0.5*(X(edges(1,:))+X(edges(2,:))); 5 | ymid = 0.5*(Y(edges(1,:))+Y(edges(2,:))); 6 | pbmid = isThereAnyPointBelow(xmid,ymid); 7 | pb = pb(edges(1,:))|pb(edges(2,:))|pbmid; 8 | -------------------------------------------------------------------------------- /edgeVerticality.m: -------------------------------------------------------------------------------- 1 | function theta = edgeVerticality(X,Y,edges) 2 | % Measure verticality of edge: 3 | % theta = 0 -> horizontal edge 4 | % theta = 1 -> vertical edge 5 | 6 | e1 = edges(1,:); 7 | e2 = edges(2,:); 8 | theta = mod(atan2(Y(e2)-Y(e1),X(e2)-X(e1)),pi); 9 | theta = min(theta,abs(pi-theta))/0.5/pi; 10 | -------------------------------------------------------------------------------- /readXmap.m: -------------------------------------------------------------------------------- 1 | function Xmap = readXmap(fname) 2 | 3 | % Read from PNG file 4 | 5 | png = int32(imread(fname)); 6 | sgn = int32(-2*floor(double(png(:,:,3))/128)+1); 7 | Xmap = zeros(size(png,1),size(png,2),'int32'); 8 | Xmap = png(:,:,1)+256*png(:,:,2)+256*256*mod(png(:,:,3),128); 9 | Xmap = Xmap.*sgn; 10 | -------------------------------------------------------------------------------- /relativeError.m: -------------------------------------------------------------------------------- 1 | function err = relativeError(z,zhat) 2 | % err = relativeError(z,zhat) 3 | % 4 | % Compute absolute relative error. 5 | % 6 | % Inputs: 7 | % z - System depth. 8 | % zhat - Ground truth depth. 9 | % 10 | % Outputs: 11 | % err - Absolute relative error. 12 | 13 | err = mean(abs(zhat-z)./zhat); 14 | -------------------------------------------------------------------------------- /writeNmap.m: -------------------------------------------------------------------------------- 1 | function writeNmap(Nmap,fname) 2 | 3 | % Write to PNG file 4 | 5 | MAX_DEPTH = 16777215; 6 | 7 | Nmap = max(min(Nmap,MAX_DEPTH),0); 8 | 9 | png = zeros(size(Nmap,1),size(Nmap,2),3,'uint8'); 10 | png(:,:,1) = mod(Nmap,256); 11 | png(:,:,2) = mod(bitshift(Nmap,-8),256); 12 | png(:,:,3) = mod(bitshift(Nmap,-16),256); 13 | 14 | imwrite(png,fname,'png'); 15 | -------------------------------------------------------------------------------- /writeYmap.m: -------------------------------------------------------------------------------- 1 | function writeYmap(Ymap,fname) 2 | 3 | % Write to PNG file 4 | 5 | MAX_DEPTH = 16777215; 6 | 7 | Ymap = max(min(Ymap,MAX_DEPTH),0); 8 | 9 | png = zeros(size(Ymap,1),size(Ymap,2),3,'uint8'); 10 | png(:,:,1) = mod(Ymap,256); 11 | png(:,:,2) = mod(bitshift(Ymap,-8),256); 12 | png(:,:,3) = mod(bitshift(Ymap,-16),256); 13 | 14 | imwrite(png,fname,'png'); 15 | -------------------------------------------------------------------------------- /writeZmap.m: -------------------------------------------------------------------------------- 1 | function writeZmap(Zmap,fname) 2 | 3 | % Write to PNG file 4 | 5 | MAX_DEPTH = 16777215; 6 | 7 | Zmap = max(min(Zmap,MAX_DEPTH),0); 8 | 9 | png = zeros(size(Zmap,1),size(Zmap,2),3,'uint8'); 10 | png(:,:,1) = mod(Zmap,256); 11 | png(:,:,2) = mod(bitshift(Zmap,-8),256); 12 | png(:,:,3) = mod(bitshift(Zmap,-16),256); 13 | 14 | imwrite(png,fname,'png'); 15 | -------------------------------------------------------------------------------- /getFootprint.m: -------------------------------------------------------------------------------- 1 | function ycc = getFootprint(xc,yc,x) 2 | % ycc = getFootprint(xc,yc,x) 3 | % 4 | % Interpolate contact points. 5 | % 6 | % Inputs: 7 | % xc - X coordinates of contact points 8 | % yc - Y coordinates of contact points 9 | % x - X coordinate to interpolate 10 | % 11 | % Output: 12 | % ycc - Y coordinate of interpolated contact point 13 | 14 | ycc = interp1(xc,yc,x,'linear','extrap'); 15 | -------------------------------------------------------------------------------- /setLMpolygon3D.m: -------------------------------------------------------------------------------- 1 | function polygon3D = setLMpolygon3D(x,y,z) 2 | 3 | for i = 1:length(x) 4 | polygon3D.pt(i).x = num2str(x(i)); 5 | polygon3D.pt(i).y = num2str(y(i)); 6 | polygon3D.pt(i).z = num2str(z(i)); 7 | % $$$ polygon3D.pt(i).x = num2str(round(x(i))); 8 | % $$$ polygon3D.pt(i).y = num2str(round(y(i))); 9 | % $$$ polygon3D.pt(i).z = num2str(round(z(i))); 10 | end 11 | -------------------------------------------------------------------------------- /setLMcontact.m: -------------------------------------------------------------------------------- 1 | function polygon = setLMcontact(x,y,polygon) 2 | % Inputs: 3 | % x 4 | % y 5 | % polygon 6 | % 7 | % Outputs: 8 | % polygon 9 | 10 | % Need to have unique x, else interp1 does not work. Need to handle 11 | % this in a better way. 12 | [x,j] = unique(x); 13 | y = y(j); 14 | 15 | for i = 1:length(x) 16 | polygon.contact(i).x = num2str(x(i)); 17 | polygon.contact(i).y = num2str(y(i)); 18 | end 19 | -------------------------------------------------------------------------------- /getContactPoints.m: -------------------------------------------------------------------------------- 1 | function [x,y] = getContactPoints(polygon) 2 | % [x,y] = getContactPoints(polygon) 3 | % 4 | % Get contact points for polygon. 5 | % 6 | % Inputs: 7 | % polygon - LabelMe polygon structure. 8 | % 9 | % Outputs: 10 | % x,y - Arrays of contact points. 11 | 12 | x = []; 13 | y = []; 14 | if isfield(polygon,'contact') 15 | x = str2num(char({polygon.contact.x})); 16 | y = str2num(char({polygon.contact.y})); 17 | end 18 | -------------------------------------------------------------------------------- /LM3Ddatabase.m: -------------------------------------------------------------------------------- 1 | function DB = LM3Ddatabase(HOMEANNOTATIONS,HOMEIMAGES,folderlist,filelist) 2 | 3 | switch nargin 4 | case 1 5 | DB = LMdatabase(HOMEANNOTATIONS); 6 | case 2 7 | DB = LMdatabase(HOMEANNOTATIONS,HOMEIMAGES); 8 | case 3 9 | DB = LMdatabase(HOMEANNOTATIONS,HOMEIMAGES,folderlist); 10 | case 4 11 | DB = LMdatabase(HOMEANNOTATIONS,HOMEIMAGES,folderlist,filelist); 12 | otherwise 13 | error('Invalid number of arguments.'); 14 | end 15 | -------------------------------------------------------------------------------- /getLMpolygon3D.m: -------------------------------------------------------------------------------- 1 | function [x,y,z] = getLMpolygon3D(polygon) 2 | % 3 | % Utility function that gives the coordinates of the polygon's vertices 4 | 5 | x = str2num(char({polygon.pt.x})); % get X polygon coordinates 6 | y = str2num(char({polygon.pt.y})); % get Y polygon coordinates 7 | z = str2num(char({polygon.pt.z})); % get Y polygon coordinates 8 | 9 | x = x(:)'; 10 | y = y(:)'; 11 | z = z(:)'; 12 | 13 | if nargout==1 14 | x = [x; y; z]; 15 | end 16 | -------------------------------------------------------------------------------- /addimagesize.m: -------------------------------------------------------------------------------- 1 | function D = addimagesize(D, HOMEIMAGES) 2 | 3 | n = length(D); 4 | 5 | for i = 1:n 6 | if mod(i,10)==1; disp(n-i); end 7 | 8 | 9 | info = imfinfo(strrep(strrep(fullfile(HOMEIMAGES, D(i).annotation.folder, D(i).annotation.filename), '\', '/'), '%20', ' ')); 10 | nrows = info.Height; 11 | ncols = info.Width; 12 | 13 | D(i).annotation.imagesize.nrows = nrows; 14 | D(i).annotation.imagesize.ncols = ncols; 15 | end 16 | 17 | -------------------------------------------------------------------------------- /RectAreas.m: -------------------------------------------------------------------------------- 1 | function [int,union,A1,A2] = RectAreas(ro,rp) 2 | % Fast intersection computation. Used as a bound on the normalized 3 | % intersection area. Normalize by the area of the smaller rectangle. 4 | % min_x,min_y,max_x,max_y 5 | 6 | tt = 0; 7 | wo = ro(3)-ro(1); 8 | ho = ro(4)-ro(2); 9 | wp = rp(3)-rp(1); 10 | hp = rp(4)-rp(2); 11 | int = rectint([ro(1) ro(2) wo ho],[rp(1) rp(2) wp hp]); 12 | A1 = wo*ho; 13 | A2 = wp*hp; 14 | union = A1+A2-int; 15 | -------------------------------------------------------------------------------- /writeXmap.m: -------------------------------------------------------------------------------- 1 | function png = writeXmap(Xmap,fname) 2 | 3 | % Write to PNG file 4 | 5 | MAX_WIDTH = 8388607; 6 | 7 | Xmap = max(min(Xmap,MAX_WIDTH),-MAX_WIDTH); 8 | Xmap = double(Xmap); 9 | XmapSign = 1-min(sign(Xmap)+1,1); 10 | Xmap = abs(Xmap); 11 | 12 | png = zeros(size(Xmap,1),size(Xmap,2),3,'uint8'); 13 | png(:,:,1) = mod(Xmap,256); 14 | png(:,:,2) = mod(bitshift(Xmap,-8),256); 15 | png(:,:,3) = mod(bitshift(Xmap,-16),256)+128*XmapSign; 16 | 17 | imwrite(png,fname,'png'); 18 | -------------------------------------------------------------------------------- /projectOntoGroundPlane.m: -------------------------------------------------------------------------------- 1 | function [X,isAdded,nPlane,x,y] = projectOntoGroundPlane(x,y,annotation) 2 | 3 | imageSize = [str2num(annotation.imagesize.nrows) str2num(annotation.imagesize.ncols)]; 4 | [x,y] = LH2RH(x,y,imageSize); 5 | P = getCameraMatrix(annotation,'RH'); 6 | 7 | H = P(:,[1 3 4]); 8 | X = H\[x(:) y(:) ones(length(x),1)]'; 9 | X = [X(1,:)./X(3,:); zeros(1,length(x)); X(2,:)./X(3,:)]; 10 | isAdded = zeros(1,length(x)); 11 | nPlane = cell(1,length(x)); 12 | nPlane(:) = {1}; 13 | -------------------------------------------------------------------------------- /LM3Dstats.m: -------------------------------------------------------------------------------- 1 | function [countObj,countImg] = LM3Dstats(HOMEANNOTATIONS) 2 | 3 | DB = LM3Ddatabase(HOMEANNOTATIONS); 4 | 5 | countImg = 0; 6 | countObj = 0; 7 | for i = 1:length(DB) 8 | annotation = DB(i).annotation; 9 | cc = 0; 10 | for j = 1:length(annotation.object) 11 | if isfield(annotation.object(j),'world3d') && isfield(annotation.object(j).world3d,'polygon3d') 12 | countObj = countObj+1; 13 | cc = 1; 14 | end 15 | end 16 | countImg = countImg+cc; 17 | end 18 | -------------------------------------------------------------------------------- /style.css: -------------------------------------------------------------------------------- 1 | .matlab { 2 | color: #0000CC; 3 | background-color: #99FFFF; 4 | border: 1px none #FFFFFF; 5 | left: auto; 6 | font-family: Verdana, Arial, Helvetica, sans-serif; 7 | font-size: 12px; 8 | 9 | 10 | } 11 | .section { 12 | font-family: Verdana, Arial, Helvetica, sans-serif; 13 | font-size: 16px; 14 | font-weight: bold; 15 | color: #000000; 16 | } 17 | .chapter { 18 | font-family: Verdana, Arial, Helvetica, sans-serif; 19 | font-size: 36px; 20 | font-weight: bold; 21 | 22 | } 23 | -------------------------------------------------------------------------------- /ObjectID2Index.m: -------------------------------------------------------------------------------- 1 | function n = ObjectID2Index(annotation,ID) 2 | % Inputs: 3 | % annotation - LabelMe annotation structure 4 | % ID - String or cell array of object IDs. 5 | % 6 | % Outputs: 7 | % n - Integer indices into LabelMe annotation structure such that 8 | % annotation.object(n).id == ID 9 | 10 | if isstr(ID) 11 | ID = {ID}; 12 | end 13 | 14 | % Get list of object IDs: 15 | objIDs = {annotation.object(:).id}; 16 | 17 | n = zeros(1,length(ID)); 18 | for i = 1:length(ID) 19 | n(i) = find(ismember(objIDs,ID{i})); 20 | end 21 | -------------------------------------------------------------------------------- /intersectPolygon.m: -------------------------------------------------------------------------------- 1 | function yc = intersectPolygon(x,y,xc) 2 | % finds all the vertical intersection points 'yc' 3 | % of the polygon [x,y] 4 | 5 | yc = []; 6 | x = [x; x(1)]; 7 | y = [y; y(1)]; 8 | 9 | n = length(x); 10 | 11 | j = find((x(1:n-1)<=xc & x(2:n)>=xc) | (x(2:n)<=xc & x(1:n-1)>=xc) ); 12 | for k = 1:length(j) 13 | i = j(k); 14 | if x(i+1)==x(i) 15 | yc = [yc y(i+1) y(i)]; % vertically aligned, so add both points 16 | else 17 | yc = [yc (y(i+1)-y(i))/(x(i+1)-x(i))*(xc-x(i))+y(i)]; 18 | end 19 | end 20 | -------------------------------------------------------------------------------- /roc.m: -------------------------------------------------------------------------------- 1 | function [pd,pf,areaROC,Scores] = roc(Scores,GT) 2 | 3 | GT = GT>0; 4 | 5 | [Scores,i] = sort(Scores,'descend'); 6 | GT = GT(i); 7 | 8 | pd = cumsum(GT); 9 | pf = cumsum(~GT); 10 | 11 | % Get unique scores: 12 | [s,i] = unique(Scores,'last'); 13 | i = i(end:-1:1); 14 | pd = pd(i)/sum(GT); 15 | pf = pf(i)/sum(~GT); 16 | Scores = Scores(i); 17 | 18 | pd = [0 pd 1]; 19 | pf = [0 pf 1]; 20 | Scores = [inf Scores -inf]; 21 | 22 | k = unique(convhull([pf 1],[pd 0])); 23 | pd = pd(k(1:end-1)); 24 | pf = pf(k(1:end-1)); 25 | Scores = Scores(k(1:end-1)); 26 | 27 | areaROC = polyarea([pf 1],[pd 0]); 28 | -------------------------------------------------------------------------------- /getEdges.m: -------------------------------------------------------------------------------- 1 | function edges = getEdges(polygon,j) 2 | % Inputs: 3 | % polygon 4 | % j 5 | % 6 | % Outputs: 7 | % edges 8 | 9 | [X,Y] = getLMpolygon(polygon); 10 | allEdges = [[1:length(X)]; [2:length(X) 1]]; 11 | if nargin < 2 12 | j = 1:size(allEdges,2); 13 | end 14 | edges = [X(allEdges(1,j)) Y(allEdges(1,j)) X(allEdges(2,j)) Y(allEdges(2,j))]; 15 | 16 | return; 17 | 18 | % $$$ allEdges = [[1:length(polygon.pt)]; [2:length(polygon.pt) 1]]; 19 | % $$$ 20 | % $$$ if nargin < 2 21 | % $$$ j = 1:size(allEdges,2); 22 | % $$$ end 23 | % $$$ 24 | % $$$ [X,Y] = getLMpolygon(polygon); 25 | % $$$ edges = [X(allEdges(1,j)) Y(allEdges(1,j)) X(allEdges(2,j)) Y(allEdges(2,j))]; 26 | -------------------------------------------------------------------------------- /depthLinearRegression.m: -------------------------------------------------------------------------------- 1 | function [alpha,beta] = depthLinearRegression(z,zhat) 2 | % [alpha,beta] = depthLinearRegression(z,zhat) 3 | % 4 | % Find linear transformation of system depth outputs that minimizes 5 | % relative error criterion: 6 | % 7 | % min sum(abs(a*z+b-zhat)./zhat) 8 | % a,b 9 | % 10 | % Inputs: 11 | % z - System depths 12 | % zhat - Corresponding ground truth depths 13 | % 14 | % Outputs: 15 | % a,b - Linear parameters 16 | 17 | % For now, we minimize squared relative error. Need to minimize absolute 18 | % relative error. 19 | N = length(z); 20 | alpha = (sum(z)*sum(1./zhat)+N*sum(z))/(sum(z.^2./zhat)*sum(1./zhat)+sum(z)*sum(z./zhat)); 21 | beta = (alpha*sum(z./zhat)-N)/sum(1./zhat); 22 | -------------------------------------------------------------------------------- /plotScene3D.m: -------------------------------------------------------------------------------- 1 | function plotScene3D(annotation) 2 | 3 | clf; 4 | range = [inf -inf inf -inf inf -inf]; 5 | for i = 1:length(annotation.object) 6 | if ~isempty(annotation.object(i).mesh) 7 | X = annotation.object(i).mesh.v3D(1,:); 8 | Y = annotation.object(i).mesh.v3D(2,:); 9 | Z = annotation.object(i).mesh.v3D(3,:); 10 | plot3([X X(1)],[Z Z(1)],[Y Y(1)],'b'); 11 | hold on; 12 | range([1 3 5]) = min(range([1 3 5]),[min(X) min(Z) min(Y)]); 13 | range([2 4 6]) = max(range([2 4 6]),[max(X) max(Z) max(Y)]); 14 | end 15 | end 16 | range(3) = min(range(3),0); 17 | axis(range); 18 | % $$$ axis equal; 19 | xlabel('x (cm)'); 20 | ylabel('z (cm)'); 21 | zlabel('y (cm)'); 22 | rotate3d 23 | -------------------------------------------------------------------------------- /getPartOfParents.m: -------------------------------------------------------------------------------- 1 | function k = getPartOfParents(annotation,i) 2 | % Gets list of indices of parents for part. 3 | % 4 | % Inputs: 5 | % annotation - LabelMe annotation structure 6 | % i - Object index into annotation.object 7 | % 8 | % Outputs: 9 | % k - Vector of indices into annotation.object. First element is parent 10 | % of part. Last element is the top-most parent of part. 11 | 12 | k = []; 13 | if isfield(annotation.object(i),'world3d') && isfield(annotation.object(i).world3d,'parentid') && ~isempty(annotation.object(i).world3d.parentid) 14 | pp = annotation.object(i).world3d.parentid; 15 | k = ObjectID2Index(annotation,pp); 16 | end 17 | 18 | if ~isempty(k) 19 | k = [k getPartOfParents(annotation,k)]; 20 | end 21 | 22 | return; 23 | -------------------------------------------------------------------------------- /relativeOverlap.m: -------------------------------------------------------------------------------- 1 | function [R1,R2] = relativeOverlap(X1,Y1,X2,Y2) 2 | % r = relativeOverlap(X1,Y1,X2,Y2) 3 | % Compute relative overlap between two polygons. 4 | % 5 | % Inputs: 6 | % X1,Y1 - Polygon 1. 7 | % X2,Y2 - Polygon 2. 8 | % 9 | % Outputs: 10 | % R1 - Relative overlap of polygon 1: area(P1 \cup P2)/area(P1). 11 | % R2 - Relative overlap of polygon 2: area(P1 \cup P2)/area(P2). 12 | 13 | % Do fast approximation of intersection area: 14 | Rn = [min(X1) min(Y1) max(X1) max(Y1)]; 15 | Rm = [min(X2) min(Y2) max(X2) max(Y2)]; 16 | if Rn(3)=.9); 13 | DB = DB(ImsNdxLabeled); 14 | DB = LMaddtags(DB,'tags.txt'); 15 | 16 | %% Learn parts and support graphs: 17 | objectClasses = LMobjectnames(DB,'name'); 18 | 19 | Ppart = partsGraph(DB, objectClasses); 20 | Psup = supportGraph(DB, objectClasses, Ppart); 21 | save graphsPartSuport objectClasses Ppart Psup DB 22 | 23 | showPartsTree(Ppart, objectClasses, 'building', DB) 24 | showPartsTree(Psup*3, objectClasses, 'road', DB) 25 | -------------------------------------------------------------------------------- /isThereAnyPointBelow.m: -------------------------------------------------------------------------------- 1 | function pb = isThereAnyPointBelow(x,y) 2 | % 3 | % A point can not be a contact point if there is another point in the same 4 | % boundary that is bellow the current point and in the same vertical 5 | % 6 | % this routine will return a indicator variable indicating if there is 7 | % anything bellow it 8 | 9 | x = [x;x(1)]; 10 | y = [y;y(1)]; 11 | np = length(x); 12 | pb = zeros(np-1,1); 13 | 14 | for n = 1:np-1 15 | yy = [];m=0; 16 | for i = 1:np-1 17 | if min(x(i),x(i+1))<=x(n)+0.001 & max(x(i),x(i+1))>x(n)-0.001 & i~=n 18 | % Insert point 19 | m = m+1; 20 | yy(m) = (y(i+1)-y(i))/(x(i+1)-x(i))*(x(n)-x(i))+y(i); 21 | end 22 | end 23 | if length(yy)>0 24 | pb(n) = y(n)0 23 | params.objNames{end+1} = tline1; 24 | params.mu_obj(end+1) = str2num(stats{1}); 25 | end 26 | if length(stats)>1 27 | params.sig_obj(end+1) = str2num(stats{2}); 28 | else 29 | params.sig_obj(end+1) = 0; 30 | end 31 | end 32 | fclose(fp); 33 | -------------------------------------------------------------------------------- /LM3Dinstall.m: -------------------------------------------------------------------------------- 1 | function LM3Dinstall(destImages, destAnnotations, folder) 2 | % 3 | % LM3Dinstall(destImages, destAnnotations) 4 | % 5 | % To download a single folder add a third argument: 6 | % LM3Dinstall(destImages, destAnnotations, folder) 7 | 8 | HOMEANNOTATIONS = 'http://labelme.csail.mit.edu/Annotations3D'; 9 | HOMEIMAGES = 'http://labelme.csail.mit.edu/Images'; 10 | 11 | % Build index 12 | disp('Reading folder list') 13 | if nargin == 3 14 | folder = folderlist(HOMEANNOTATIONS, folder); 15 | else 16 | folder = folderlist(HOMEANNOTATIONS); 17 | end 18 | 19 | % create list of folders to copy, copy and rename folders in XML: 20 | LMinstall(folder, destImages, destAnnotations, HOMEIMAGES, HOMEANNOTATIONS) 21 | 22 | % make sure folder names are coherent 23 | ADMINrenamefolder(destAnnotations); 24 | 25 | 26 | 27 | -------------------------------------------------------------------------------- /getObjectMesh.m: -------------------------------------------------------------------------------- 1 | function mesh = getObjectMesh(mesh,i) 2 | % Retrieve mesh for object. 3 | % 4 | % Inputs: 5 | % mesh - Mesh structure. 6 | % i - Object index. 7 | % 8 | % Outputs: 9 | % mesh 10 | 11 | % Get indices of faces belonging to object: 12 | nFaces = find(mesh.objndx==i); 13 | 14 | % Get indices of vertices belonging to object: 15 | nVertices = unique(mesh.faces(:,nFaces)); 16 | 17 | % Get mapping from old vertices to new vertices: 18 | mapVertices = zeros(1,size(mesh.vertices,2),'int32'); 19 | mapVertices(nVertices) = [1:length(nVertices)]; 20 | 21 | % Set output: 22 | mesh.vertices = mesh.vertices(:,nVertices); 23 | mesh.faces = mapVertices(mesh.faces(:,nFaces)); 24 | mesh.objndx = mesh.objndx(nFaces); 25 | if isfield(mesh,'textures') 26 | texture = mesh.textures{i}; 27 | mesh.textures = cell(1,length(mesh.textures)); 28 | mesh.textures{i} = texture; 29 | mesh.tx = mesh.tx(:,nVertices); 30 | end 31 | -------------------------------------------------------------------------------- /isCleanObject.m: -------------------------------------------------------------------------------- 1 | function valid = isCleanObject(X,Y,imageSize) 2 | % valid = isCleanObject(X,Y,imageSize) 3 | % 4 | % Returns true if the polygon is not cropped and is not very small. 5 | % 6 | % Inputs: 7 | % X - X coordinates of polygon 8 | % Y - Y coordinates of polygon 9 | % imageSize - [nrows ncols] of image 10 | % 11 | % Outputs: 12 | % valid - 1 if polygon satisfies conditions. 13 | 14 | % Parameters: 15 | minHeightPixels = 0.02;%0.1; % Minimum height of object in pixels 16 | cropDist = 0.01; % Minimum distance to image boundary before considered 17 | % as cropped object 18 | 19 | minHeightPixels = round(imageSize(1)*minHeightPixels); 20 | cropDist = (imageSize(1)^2+imageSize(2)^2)^0.5*cropDist; 21 | 22 | valid = 0; 23 | if ((max(Y)-min(Y))>=minHeightPixels) && (min(Y)>cropDist) && (min(X)>cropDist) && (max(Y)<(imageSize(1)-cropDist)) && (max(X)<(imageSize(2)-cropDist)) 24 | valid = 1; 25 | end 26 | -------------------------------------------------------------------------------- /groundObjectWithMesh.m: -------------------------------------------------------------------------------- 1 | function [X,Y,Z,x,y,tri,boundary] = groundObjectWithMesh(x,y,ncols,nrows,CAM_H,Hy,F) 2 | % [X,Y,Z,TX,TY,tri,boundary] = groundObjectWithMesh(x,y,ncols,nrows,CAM_H,Hy,F) 3 | % 4 | % This function estimates the depth of an horizontal shape lying on the ground 5 | % directly standing on the ground. 6 | % Input: 7 | % x,y: image polygon coordinates 8 | % ncols, nrows: image size 9 | % CAM_H, Hy, F: camera parameters 10 | % 11 | % Output: 12 | % X,Y,Z: world coordinates of the polygon 13 | % TX, TY: image coordinates, for the texture mesh 14 | % tri: triangular mesh indices 15 | % boundary: indices to points in the boundary 16 | 17 | % Parameters: 18 | npts = 11; 19 | 20 | if nargout>5 21 | % Create mesh x,y: 22 | [x,y,tri,boundary] = getMesh(x,y,npts); 23 | end 24 | 25 | % Convert image coordinates into world coordinates: 26 | [X,Y,Z] = Image2WorldCoords(x,y,[],[],[nrows ncols],CAM_H,Hy,F); 27 | -------------------------------------------------------------------------------- /plotNmap.m: -------------------------------------------------------------------------------- 1 | function plotNmap(Nmap) 2 | % Input: 3 | % Nmap - uint32 map of object indices. 4 | 5 | [nrows,ncols,cc] = size(Nmap); 6 | maxVal = inf*ones(1,1,'uint32'); 7 | valid = find(Nmap(:)~=maxVal); 8 | ndx = unique(Nmap(valid)); 9 | 10 | % Get colors for map: 11 | colors = uint8(255*jet(double(max(ndx)+1))); 12 | 13 | outR = zeros(nrows,ncols,'uint8'); 14 | outG = zeros(nrows,ncols,'uint8'); 15 | outB = zeros(nrows,ncols,'uint8'); 16 | 17 | for i = 1:length(ndx) 18 | nn = find(Nmap(valid)==ndx(i)); 19 | outR(valid(nn)) = colors(ndx(i)+1,1); 20 | outG(valid(nn)) = colors(ndx(i)+1,2); 21 | outB(valid(nn)) = colors(ndx(i)+1,3); 22 | end 23 | 24 | outImg = reshape([outR outG outB],[nrows ncols 3]); 25 | % $$$ colorbar('YTick',255*ytick,'YTickLabel',yticklabels,'YLim',255*ylim); 26 | % $$$ colorbar('YTick',[0 double(max(ndx))],'YLim',[0 double(max(ndx))]); 27 | % $$$ h = colorbar; 28 | % $$$ set(h,'YLim',[0 double(max(ndx))]); 29 | 30 | imshow(outImg); 31 | -------------------------------------------------------------------------------- /Project3D2D.m: -------------------------------------------------------------------------------- 1 | function [x,y] = Project3D2D(varargin) 2 | % Project 3D points to 2D points. 3 | % 4 | % Example 1: 5 | % 6 | % annotation; % Annotation structure 7 | % X = rand(3,10); % Generate 10 random 3D points 8 | % [x,y] = Project3D2D(X,annotation); 9 | % 10 | % Example 2: 11 | % 12 | % P = getCameraMatrix(annotation); 13 | % imageSize = [str2num(annotation.imagesize.nrows) str2num(annotation.imagesize.ncols)]; 14 | % [x,y] = Project3D2D(X,P,imageSize); 15 | 16 | X = varargin{1}; 17 | switch nargin 18 | case 2 19 | annotation = varargin{2}; 20 | P = getCameraMatrix(annotation); 21 | imageSize = [str2num(annotation.imagesize.nrows) str2num(annotation.imagesize.ncols)]; 22 | case 3 23 | P = varargin{2}; 24 | imageSize = varargin{3}; 25 | end 26 | 27 | if size(X,1) < 4 28 | X = [X; ones(1,size(X,2))]; 29 | end 30 | x = P*X; 31 | 32 | if nargout==1 33 | x = [x(1,:)./x(3,:); x(2,:)./x(3,:)]; 34 | else 35 | y = x(2,:)./x(3,:); 36 | x = x(1,:)./x(3,:); 37 | end 38 | -------------------------------------------------------------------------------- /plotLineBmap.m: -------------------------------------------------------------------------------- 1 | function seg = plotLineBmap(seg,edges,color,sz) 2 | % Plots lines on a bitmap image. 3 | 4 | if nargin < 4 5 | sz = 2; 6 | end 7 | 8 | N = ceil(max(((edges(:,3)-edges(:,1)).^2+(edges(:,4)-edges(:,2)).^2).^0.5))+10; 9 | alpha = linspace(0,1,N); 10 | sx = zeros(size(edges,1),N); 11 | sy = zeros(size(edges,1),N); 12 | for i = 1:N 13 | sx(:,i) = (1-alpha(i))*edges(:,1)+alpha(i)*edges(:,3); 14 | sy(:,i) = (1-alpha(i))*edges(:,2)+alpha(i)*edges(:,4); 15 | end 16 | s = round([sx(:) sy(:)]); 17 | s = max(s,1); 18 | s(:,1) = min(s(:,1),size(seg,2)); 19 | s(:,2) = min(s(:,2),size(seg,1)); 20 | s = unique(s,'rows'); 21 | ndx = sub2ind(size(seg),s(:,2),s(:,1)); 22 | mask = zeros(size(seg,1),size(seg,2)); 23 | mask(ndx) = 1; 24 | cB = strel('disk',sz); 25 | mask = imdilate(mask,cB); 26 | ndx = find(mask(:)); 27 | 28 | seg(ndx) = round(255*color(1)); 29 | seg(ndx+size(seg,1)*size(seg,2)) = round(255*color(2)); 30 | seg(ndx+2*size(seg,1)*size(seg,2)) = round(255*color(3)); 31 | 32 | -------------------------------------------------------------------------------- /getHorizonLine.m: -------------------------------------------------------------------------------- 1 | function [x,y] = getHorizonLine(annotation,imageCoordsType) 2 | % Gets horizon line passing through X and Z axes for image. 3 | % 4 | % Inputs: 5 | % annotation - LabelMe annotation structure. 6 | % imageCoordsType - 'lh' or 'rh' (left-hand or right-hand coordinates) 7 | % 8 | % Outputs: 9 | % x - X coordinates for horizon line 10 | % y - Y coordinates for horizon line 11 | 12 | if nargin < 2 13 | imageCoordsType = 'lh'; 14 | end 15 | 16 | x = []; y = []; 17 | P = getCameraMatrix(annotation); 18 | if ~isempty(P) && isfield(annotation,'imagesize') 19 | imageSize = [str2num(annotation.imagesize.nrows) str2num(annotation.imagesize.ncols)]; 20 | 21 | % $$$ lh = cross([Project3D2D([1 0 0 0]',annotation); 1],[Project3D2D([0 0 1 0]',annotation); 1]); 22 | lh = cross(P(:,1),P(:,3)); 23 | x1 = cross(lh,cross([1 1 1],[1 2 1])'); 24 | x2 = cross(lh,cross([imageSize(2) 1 1],[imageSize(2) 2 1])'); 25 | x = [x1(1)/x1(3) x2(1)/x2(3)]; 26 | y = [x1(2)/x1(3) x2(2)/x2(3)]; 27 | end 28 | -------------------------------------------------------------------------------- /README.txt: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | LabelMe3D Matlab toolbox 6 | 7 | 8 | CITATION: 9 | 10 | B. C. Russell and A. Torralba 11 | Building a Database of 3D Scenes from User Annotations 12 | IEEE Conference on Computer Vision and Pattern Recognition (CVPR), 2009 13 | 14 | 15 | DESCRIPTION: 16 | 17 | Here you will find the LabelMe3D Matlab toolbox. Given a LabelMe 18 | annotation file, the toolbox contains source code to infer world 19 | 3D coordinates for labeled objects in the scene and recover the 20 | camera parameters for the image. Moreover, there are functions for 21 | visualizing the inferred 3D information. 22 | 23 | 24 | QUICK START: 25 | 26 | demo.m - This file has a demonstration of the main features of the 27 | toolbox. It is best to read and step through the demonstration 28 | line-by-line. 29 | 30 | Documentation.pdf - This file documents the scene geometry and format 31 | of the annotation XML files. 32 | 33 | 34 | (c) 2013, MIT Computer Science and Artificial Intelligence Laboratory 35 | -------------------------------------------------------------------------------- /PolyAreas.m: -------------------------------------------------------------------------------- 1 | function [A,ua,area1,area2] = PolyAreas(X1,Y1,X2,Y2) 2 | % Compute intersection and union areas, as well as areas of individual 3 | % polygons 4 | % 5 | % [Aint,Aunion,A1,A2] = PolyAreas(X1,Y1,X2,Y2) 6 | 7 | max_res = 100; 8 | 9 | min_x = min([X1(:); X2(:)]); 10 | max_x = max([X1(:); X2(:)]); 11 | min_y = min([Y1(:); Y2(:)]); 12 | max_y = max([Y1(:); Y2(:)]); 13 | 14 | X1 = X1-min_x; 15 | X2 = X2-min_x; 16 | Y1 = Y1-min_y; 17 | Y2 = Y2-min_y; 18 | max_x = max_x - min_x; 19 | max_y = max_y - min_y; 20 | 21 | max_dim = max(max_x,max_y); 22 | 23 | X1 = X1*max_res/max_dim+1; 24 | X2 = X2*max_res/max_dim+1; 25 | Y1 = Y1*max_res/max_dim+1; 26 | Y2 = Y2*max_res/max_dim+1; 27 | 28 | max_x = ceil(max_x+1); 29 | max_y = ceil(max_y+1); 30 | 31 | M1 = poly2mask(double(X1),double(Y1),max_res,max_res); 32 | M2 = poly2mask(double(X2),double(Y2),max_res,max_res); 33 | 34 | A = sum(sum(double(M1&M2))); 35 | ua = sum(sum(double((M1+M2)>0))); 36 | area1 = sum(double(M1(:))); 37 | area2 = sum(double(M2(:))); 38 | -------------------------------------------------------------------------------- /depthThumbnail.m: -------------------------------------------------------------------------------- 1 | function thumbnail = depthThumbnail(annotation, HOMEIMAGES, img, Zmap) 2 | % 3 | % To call the function, first inflate a scene: 4 | % [annotation,Zmap]=InflateScene(DB(ndx).annotation,Hy,sup,parts,Cpts,img); 5 | % thumbnail=depthThumbnail(DB(ndx).annotation, HOMEIMAGES, [], Zmap); 6 | % figure 7 | % imshow(thumbnail) 8 | 9 | 10 | MIN_DEPTH = 2e2; % 1m 11 | MAX_DEPTH = 2e4; % 1km 12 | 13 | thumb = LMsceneThumbnail(annotation, HOMEIMAGES, img, 'lines'); 14 | [nrows ncols cc] = size(thumb) 15 | 16 | thumb = thumb(:, fix(ncols/2)+2:ncols,:); 17 | 18 | 19 | mask = Zmap<500000; 20 | Zmap = log10(min(max(double(Zmap),MIN_DEPTH),MAX_DEPTH)); 21 | Zmap = uint8(256*(Zmap-log10(MIN_DEPTH)) / (log10(MAX_DEPTH) - log10(MIN_DEPTH))); 22 | 23 | Zmap = Zmap.*uint8(mask); 24 | cmap = fliplr(jet(256)); 25 | cmap(1,:) = [0 0 0]; 26 | 27 | depthmap = ind2rgb(Zmap, cmap); 28 | depthmap = imresize(depthmap, [size(thumb,1) size(thumb,2)], 'bilinear'); 29 | depthmap = uint8(255*depthmap); 30 | 31 | thumbnail = [thumb 255*ones(nrows,3,3) depthmap]; 32 | -------------------------------------------------------------------------------- /writeXML3D.m: -------------------------------------------------------------------------------- 1 | function writeXML3D(annotation,fname) 2 | % writeXML3D(annotation,fname) 3 | % 4 | % Write XML file with 3D information. 5 | % 6 | % Inputs: 7 | % annotation - LabelMe annotation structure. 8 | % fname - Filename. 9 | 10 | if isfield(annotation,'object') 11 | if isfield(annotation.object,'mesh') 12 | annotation.object = rmfield(annotation.object,'mesh'); 13 | end 14 | for ii = 1:length(annotation.object) 15 | if isfield(annotation.object(ii),'polygon') 16 | if isfield(annotation.object(ii).polygon,'edges') 17 | annotation.object(ii).polygon = rmfield(annotation.object(ii).polygon,'edges'); 18 | end 19 | if isfield(annotation.object(ii).polygon,'pAttached') 20 | annotation.object(ii).polygon = rmfield(annotation.object(ii).polygon,'pAttached'); 21 | end 22 | if isfield(annotation.object(ii).polygon,'pAttachedObject') 23 | annotation.object(ii).polygon = rmfield(annotation.object(ii).polygon,'pAttachedObject'); 24 | end 25 | end 26 | end 27 | end 28 | 29 | 30 | v.annotation = annotation; 31 | writeXML(fname,v); 32 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (C) 2013 MIT, Computer Science and Artificial Intelligence Laboratory. Bryan Russell, Antonio Torralba 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 4 | 5 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 8 | -------------------------------------------------------------------------------- /includeGroundTruthEdgeLabels.m: -------------------------------------------------------------------------------- 1 | function annotation = includeGroundTruthEdgeLabels(annotation,edgeLabels,img) 2 | 3 | CONTACT_EDGE = 1; 4 | Nobjects = length(annotation.object); 5 | largeObjects = getListLargeObjects; 6 | for i = 1:Nobjects 7 | if ismember(strtrim(lower(annotation.object(i).name)),largeObjects) 8 | Npoints = length(annotation.object(i).polygon.pt); 9 | edges = edgeLabels.object(i).edges; 10 | edgeType = edgeLabels.object(i).edgeType; 11 | n = find(edgeType==CONTACT_EDGE); 12 | c = edges(:,n); 13 | contactPoints = zeros(1,Npoints); 14 | contactPoints(unique(c(:))) = 1; 15 | for j = 1:Npoints 16 | annotation.object(i).polygon.pt(j).contact = contactPoints(j); 17 | end 18 | 19 | % $$$ clf; 20 | % $$$ imshow(img); 21 | % $$$ hold on; 22 | % $$$ [X,Y] = getLMpolygon(annotation.object(i).polygon); 23 | % $$$ plot([X; X(1)],[Y; Y(1)],'b','LineWidth',4); 24 | % $$$ for j = 1:length(X) 25 | % $$$ if annotation.object(i).polygon.pt(j).contact 26 | % $$$ plot(X(j),Y(j),'ro','LineWidth',4); 27 | % $$$ end 28 | % $$$ end 29 | % $$$ pause; 30 | end 31 | end 32 | -------------------------------------------------------------------------------- /LM3DpolyEdgeTypesThumbnail.m: -------------------------------------------------------------------------------- 1 | function thumb = LM3DpolyEdgeTypesThumbnail(annotation,img) 2 | % Inputs: 3 | % annotation 4 | % img 5 | % 6 | % Outputs: 7 | % thumb 8 | 9 | % Thumbnail size (height) 10 | tY = 96; 11 | tY = 160; 12 | 13 | % Scale annotation and image: 14 | [annotation,img] = LMimscale(annotation,img,(tY+1)/size(img,1),'bilinear'); 15 | 16 | % Get polygon/edge types: 17 | thumb3D = plotPolyEdgeTypes(annotation,img,'valid'); 18 | 19 | % sort layers 20 | annotation = LMsortlayers(annotation, img); 21 | 22 | if size(img,3) < 3 23 | img = repmat(img(:,:,1), [1 1 3]); 24 | end 25 | 26 | % get segmentation masks 27 | [mask, cc, maskpol, classpol] = LMobjectmask(annotation, [size(img,1) size(img,2)]); 28 | 29 | % Thumbnails with masks 30 | seg = 128*ones(size(img)); 31 | if size(mask,3)>0 32 | M = double(colorSegments(mask, 'donotsort'))/255; 33 | if size(M,1)>0 34 | M = M + .5*repmat(sum(mask,3)==0, [1 1 3]); 35 | M = M / max(M(:)); 36 | seg = M .* repmat(mean(double(img),3)/2+128, [1 1 3]); 37 | end 38 | end 39 | 40 | seg = uint8(seg); 41 | thumb = [seg 255*ones([size(img,1),2,size(img,3)]) thumb3D]; 42 | 43 | if nargout == 0 44 | figure 45 | imshow(thumb) 46 | end 47 | 48 | -------------------------------------------------------------------------------- /getaproximagesize.m: -------------------------------------------------------------------------------- 1 | function [ncols, nrows] = getaproximagesize(annotation) 2 | % 3 | % THIS IS AN APPROXIMATION 4 | % 5 | % Script that return the image size. First it looks to see if there is a 6 | % field in the annotation that provides the image size. If the field is not 7 | % present, it will return the size estimated using the coordinates of the 8 | % annotated objects. This is faster than reading the image, but it might 9 | % return underestimates of the real image size if not all the objects are 10 | % annotated. 11 | 12 | ncols = 0; 13 | nrows = 0; 14 | 15 | if isfield(annotation,'imagesize') 16 | ncols = annotation.imagesize.ncols; 17 | nrows = annotation.imagesize.nrows; 18 | else 19 | % Estimate image size. This information is not 20 | % available in the annotation and reading the image is too slow, so 21 | % we estimate it by looking at the polygons: 22 | if isfield(annotation,'object') 23 | Nobj = length(annotation.object); 24 | 25 | for j = 1:Nobj 26 | [X,Y] = getLMpolygon(annotation.object(j).polygon); 27 | ncols = max(ncols, max(X)); 28 | nrows = max(nrows, max(Y)); 29 | end 30 | end 31 | end 32 | -------------------------------------------------------------------------------- /decomposeP.m: -------------------------------------------------------------------------------- 1 | function [K, R, t] = decomposeP(P) 2 | %VGG_KR_FROM_P Extract K, R from camera matrix. 3 | % 4 | % [K,R,t] = VGG_KR_FROM_P(P [,noscale]) finds K, R, t such that P = K*R*[eye(3) -t]. 5 | % It is det(R)==1. 6 | % K is scaled so that K(3,3)==1 and K(1,1)>0. Optional parameter noscale prevents this. 7 | % 8 | % Works also generally for any P of size N-by-(N+1). 9 | % Works also for P of size N-by-N, then t is not computed. 10 | 11 | 12 | % Author: Andrew Fitzgibbon 13 | % Modified by werner. 14 | % Date: 15 May 98 15 | 16 | 17 | 18 | N = size(P,1); 19 | H = P(:,1:N); 20 | 21 | [K,R] = rq(H); 22 | 23 | K = K*sign(K(1)); 24 | if prod(sign(diag(K))) < 0 25 | D = diag(-1*sign(diag(K))); 26 | else 27 | D = diag(sign(diag(K))); 28 | end 29 | K = K*D; 30 | R = D*R; 31 | 32 | K = K/K(end); 33 | 34 | % $$$ if nargin < 2 35 | % $$$ K = K / K(N,N); 36 | % $$$ if K(1,1) < 0 37 | % $$$ D = diag([-1 -1 ones(1,N-2)]); 38 | % $$$ K = K * D; 39 | % $$$ R = D * R; 40 | % $$$ 41 | % $$$ % test = K*R; 42 | % $$$ % vgg_assert0(test/test(1,1) - H/H(1,1), 1e-07) 43 | % $$$ end 44 | % $$$ end 45 | 46 | if nargout > 2 47 | t = -P(:,1:N)\P(:,end); 48 | end 49 | 50 | return 51 | -------------------------------------------------------------------------------- /plotOutputEdgeTypes.m: -------------------------------------------------------------------------------- 1 | function plotOutputEdgeTypes(annotation,img,etype) 2 | 3 | if nargin < 3 4 | etype = [1 2 3]; 5 | end 6 | 7 | % $$$ edgeOptions = {'Attached edge','Contact edge','Occlusion edge'}; 8 | edgeOptions = {'Contact','Attached','Occlusion'}; 9 | Njet = length(edgeOptions); 10 | edgeTypeColors = jet; 11 | edgeTypeColors = edgeTypeColors(round((size(edgeTypeColors,1)-1)*[0:Njet-1]/(Njet-1))+1,:); 12 | 13 | clf; 14 | h = imshow(img); 15 | set(h,'AlphaData',0.5); 16 | hold on; 17 | h = zeros(1,Njet); 18 | for i = 1:length(annotation.object) 19 | [X,Y] = getLMpolygon(annotation.object(i).polygon); 20 | edges = annotation.object(i).polygon.edges; 21 | Nedges = size(edges,2); 22 | edgeType = annotation.object(i).polygon.edgeType; 23 | 24 | edgeType(~ismember(edgeType,etype)) = 0; 25 | edgeType = 2*(edgeType==1) + 1*(edgeType==2) + 3*(edgeType==3); 26 | for j = 1:Nedges 27 | if edgeType(j)>0 28 | e = edges(:,j); 29 | h(edgeType(j)) = plot([X(e(1)) X(e(2))],[Y(e(1)) Y(e(2))],'Color',edgeTypeColors(edgeType(j),:),'LineWidth',4); 30 | end 31 | end 32 | end 33 | 34 | if sum(h>0)>1 35 | legend(h(h>0),edgeOptions(h>0),'Location','NorthEastOutside'); 36 | else 37 | legend(edgeOptions(h>0),'Location','NorthEastOutside'); 38 | end 39 | -------------------------------------------------------------------------------- /myLMaddtags.m: -------------------------------------------------------------------------------- 1 | function annotation = myLMaddtags(annotation,tagsfile) 2 | 3 | if nargin < 2 4 | tagsfile = 'tags.txt'; 5 | end 6 | 7 | % Below is code extracted from LMaddtags.m 8 | 9 | D.annotation = annotation; 10 | 11 | [Tag, Descriptions] = loadtags(tagsfile); 12 | Ntags = length(Tag); 13 | 14 | [labelmeDescriptions,counts,imagendx,objectndx,descriptionndx] = LMobjectnames(D); 15 | ndxtag = zeros(length(labelmeDescriptions),1); 16 | for i = 1:length(labelmeDescriptions) 17 | for k = 1:Ntags 18 | j = strmatch(lower(labelmeDescriptions{i}), Descriptions{k}, 'exact'); 19 | if ~isempty(j) 20 | ndxtag(i) = k; 21 | break 22 | end 23 | end 24 | end 25 | for i = 1:length(labelmeDescriptions) 26 | if ndxtag(i)>0 27 | j = find(descriptionndx==i); 28 | for k = 1:length(j) 29 | D(imagendx(j(k))).annotation.object(objectndx(j(k))).name = Tag{ndxtag(i)}; 30 | end 31 | else 32 | j = find(descriptionndx==i); 33 | for k = 1:length(j) 34 | D(imagendx(j(k))).annotation.object(objectndx(j(k))).name = 'unmatched'; 35 | end 36 | end 37 | end 38 | 39 | for i = 1:length(annotation.object) 40 | annotation.object(i).originalname = annotation.object(i).name; 41 | annotation.object(i).name = D.annotation.object(i).name; 42 | end 43 | -------------------------------------------------------------------------------- /getCameraMatrix.m: -------------------------------------------------------------------------------- 1 | function P = getCameraMatrix(annotation,type) 2 | % function P = getCameraMatrix(annotation) 3 | % 4 | % Gets camera matrix for 3D scene. 5 | % 6 | % Inputs: 7 | % annotation - LabelMe3D annotation structure 8 | % 9 | % Outputs: 10 | % P - 3x4 camera matrix. 11 | 12 | if nargin < 2 13 | type = 'LH'; 14 | end 15 | 16 | P = []; 17 | 18 | if isfield(annotation,'camera') && isfield(annotation.camera,'pmatrix') && isfield(annotation.camera.pmatrix,'p11') 19 | pmatrix = annotation.camera.pmatrix; 20 | if isstr(pmatrix.p11) 21 | P = [str2num(pmatrix.p11) str2num(pmatrix.p12) str2num(pmatrix.p13) str2num(pmatrix.p14); ... 22 | str2num(pmatrix.p21) str2num(pmatrix.p22) str2num(pmatrix.p23) str2num(pmatrix.p24); ... 23 | str2num(pmatrix.p31) str2num(pmatrix.p32) str2num(pmatrix.p33) str2num(pmatrix.p34)]; 24 | else 25 | P = [pmatrix.p11 pmatrix.p12 pmatrix.p13 pmatrix.p14; ... 26 | pmatrix.p21 pmatrix.p22 pmatrix.p23 pmatrix.p24; ... 27 | pmatrix.p31 pmatrix.p32 pmatrix.p33 pmatrix.p34]; 28 | end 29 | end 30 | 31 | if ~isempty(P) && strcmp(type,'RH') 32 | imageSize = [str2num(annotation.imagesize.nrows) str2num(annotation.imagesize.ncols)]; 33 | T = [-1 0 (imageSize(2)+1)/2; 0 -1 (imageSize(1)+1)/2; 0 0 1]; 34 | P = T*P; 35 | end 36 | -------------------------------------------------------------------------------- /inferParts.m: -------------------------------------------------------------------------------- 1 | function parts = inferParts(annotation,Ppart,objectClasses) 2 | 3 | if ~isfield(annotation,'object') 4 | parts = []; 5 | return; 6 | end 7 | 8 | % Parameters: 9 | thresh_par = 0.3; % Threshold for support 10 | 11 | Nobjects = length(annotation.object); 12 | 13 | % Collect polygons: 14 | for i = 1:Nobjects 15 | n = strmatch(strtrim(lower(annotation.object(i).name)), objectClasses, 'exact'); 16 | if ~isempty(n) 17 | ndx(i) = n; 18 | else 19 | ndx(i) = 0; 20 | end 21 | [X{i},Y{i}] = getLMpolygon(annotation.object(i).polygon); 22 | end 23 | 24 | parts = zeros(1,Nobjects); 25 | P = zeros(1,Nobjects); 26 | for i = 1:Nobjects 27 | for j = 1:Nobjects 28 | if (i~=j) && ndx(i) && ndx(j) 29 | [A,ua,area1,area2] = PolyAreas(X{i},Y{i},X{j},Y{j}); 30 | pp = Ppart(ndx(i),ndx(j))*A/(area2+eps); 31 | if pp > P(j) 32 | P(j) = pp; 33 | parts(j) = i; 34 | end 35 | end 36 | end 37 | end 38 | 39 | % Threshold support relations: 40 | parts(P<=thresh_par) = 0; 41 | 42 | % Show outputs: 43 | [v,n] = sort(P,'descend'); 44 | for i = n 45 | if parts(i) 46 | disp(sprintf('%s is a part of %s (with probability = %1.2f)', ... 47 | annotation.object(i).name, ... 48 | annotation.object(parts(i)).name,P(i))); 49 | end 50 | end 51 | -------------------------------------------------------------------------------- /plotSupport.m: -------------------------------------------------------------------------------- 1 | function plotSupport(annotation,img,sup) 2 | 3 | % Visualize support relations: 4 | Nobjects = length(annotation.object); 5 | parents = find(sup==0); 6 | Nsubplot = length(unique(sup))+2; 7 | Nx = ceil(sqrt(Nsubplot)); 8 | Ny = ceil(Nsubplot/Nx); 9 | 10 | %% Resize image and annotation for space savings: 11 | [annotation,img] = LMimscale(annotation,img,min(640/size(img,2),1)); 12 | for i = 1:Nobjects 13 | [X{i},Y{i}] = getLMpolygon(annotation.object(i).polygon); 14 | end 15 | 16 | figure; 17 | subplot(Ny,Nx,1); 18 | LMplot(annotation,img); 19 | legend hide; 20 | 21 | k = 2; 22 | i = 1; 23 | while i <= length(parents) 24 | if ismember(parents(i),sup) 25 | subplot(Ny,Nx,k); 26 | imshow(img); 27 | hold on; 28 | Xi = X{parents(i)}; 29 | Yi = Y{parents(i)}; 30 | plot([Xi; Xi(1)],[Yi; Yi(1)],'r','LineWidth',4); 31 | children = find(sup==parents(i)); 32 | for j = 1:length(children) 33 | Xi = X{children(j)}; 34 | Yi = Y{children(j)}; 35 | plot([Xi; Xi(1)],[Yi; Yi(1)],'b','LineWidth',4); 36 | end 37 | parents = [parents children(~ismember(children,parents))]; 38 | title('Support relations'); 39 | k = k+1; 40 | end 41 | i = i+1; 42 | end 43 | 44 | % $$$ subplot(Ny,Nx,k); 45 | % $$$ showTreeObj(sup,{annotation.object(:).name}); 46 | % $$$ axis off; 47 | % $$$ title('Support tree'); 48 | -------------------------------------------------------------------------------- /plotParts.m: -------------------------------------------------------------------------------- 1 | function plotParts(annotation,img,parts) 2 | 3 | % Visualize support relations: 4 | Nobjects = length(annotation.object); 5 | parents = find(parts==0); 6 | Nsubplot = length(unique(parts))+2; 7 | Nx = ceil(sqrt(Nsubplot)); 8 | Ny = ceil(Nsubplot/Nx); 9 | 10 | %% Resize image and annotation for space savings: 11 | [annotation,img] = LMimscale(annotation,img,min(640/size(img,2),1)); 12 | for i = 1:Nobjects 13 | [X{i},Y{i}] = getLMpolygon(annotation.object(i).polygon); 14 | end 15 | 16 | figure; 17 | subplot(Ny,Nx,1); 18 | LMplot(annotation,img); 19 | legend hide; 20 | 21 | k = 2; 22 | i = 1; 23 | while i <= length(parents) 24 | if ismember(parents(i),parts) 25 | subplot(Ny,Nx,k); 26 | imshow(img); 27 | hold on; 28 | Xi = X{parents(i)}; 29 | Yi = Y{parents(i)}; 30 | plot([Xi; Xi(1)],[Yi; Yi(1)],'r','LineWidth',4); 31 | children = find(parts==parents(i)); 32 | for j = 1:length(children) 33 | Xi = X{children(j)}; 34 | Yi = Y{children(j)}; 35 | plot([Xi; Xi(1)],[Yi; Yi(1)],'b','LineWidth',4); 36 | end 37 | parents = [parents children(~ismember(children,parents))]; 38 | title('Part relations'); 39 | k = k+1; 40 | end 41 | i = i+1; 42 | end 43 | 44 | % $$$ subplot(Ny,Nx,k); 45 | % $$$ showTreeObj(parts,{annotation.object(:).name}); 46 | % $$$ axis off; 47 | % $$$ title('Support tree'); 48 | -------------------------------------------------------------------------------- /splitPolygon.m: -------------------------------------------------------------------------------- 1 | function [Txn, Tyn, Xn, Yn, Zn] = splitPolygon(Tx,Ty,X,Y,Z) 2 | % 3 | 4 | %keyboard 5 | 6 | Tx = [Tx; Tx(1)]; 7 | Ty = [Ty; Ty(1)]; 8 | X = [X; X(1)]; 9 | Y = [Y; Y(1)]; 10 | Z = [Z; Z(1)]; 11 | 12 | 13 | 14 | [h,xnn]=hist(Tx,unique(Tx)); 15 | j = find(h==2); 16 | xc = xnn(j); 17 | 18 | 19 | Xn{1} = X; 20 | Yn{1} = Y; 21 | Zn{1} = Z; 22 | Txn{1} = Tx; 23 | Tyn{1} = Ty; 24 | nc = length(xc); 25 | 26 | for m = 1:nc 27 | j1 = find(Txn{m}<=xc(m)); 28 | j2 = find(Txn{m}>=xc(m)); 29 | 30 | %keyboard 31 | 32 | Xc = Xn{m}; 33 | Yc = Yn{m}; 34 | Zc = Zn{m}; 35 | Txc = Txn{m}; 36 | Tyc = Tyn{m}; 37 | 38 | Xn{m} = Xc(j1); 39 | Yn{m} = Yc(j1); 40 | Zn{m} = Zc(j1); 41 | Txn{m} = Txc(j1); 42 | Tyn{m} = Tyc(j1); 43 | 44 | Xn{m+1} = Xc(j2); 45 | Yn{m+1} = Yc(j2); 46 | Zn{m+1} = Zc(j2); 47 | Txn{m+1} = Txc(j2); 48 | Tyn{m+1} = Tyc(j2); 49 | end 50 | 51 | 52 | if 0 53 | figure 54 | subplot(221) 55 | plot3(X,Y,Z,'o-') 56 | subplot(222) 57 | for m = 1:nc 58 | plot3(Xn{m},Yn{m},Zn{m},'o-') 59 | hold on 60 | end 61 | subplot(223) 62 | plot(Tx,Ty,'o-') 63 | subplot(224) 64 | for m = 1:nc 65 | plot(Txn{m},Tyn{m},'o-') 66 | hold on 67 | end 68 | end 69 | -------------------------------------------------------------------------------- /setCameraMatrix.m: -------------------------------------------------------------------------------- 1 | function annotation = setCameraMatrix(annotation,P,units) 2 | % Inputs: 3 | % annotation - LabelMe annotation structure 4 | % P - Camera matrix 5 | % units - World units (e.g. 'centimeters', 'inches', 'none') 6 | % 7 | % Outputs: 8 | % annotation 9 | 10 | if nargin < 3 11 | units = 'none'; 12 | end 13 | 14 | % Store world units: 15 | annotation.camera.units = units; 16 | 17 | % Store camera matrix: 18 | annotation.camera.pmatrix.p11 = num2str(P(1,1)); 19 | annotation.camera.pmatrix.p12 = num2str(P(1,2)); 20 | annotation.camera.pmatrix.p13 = num2str(P(1,3)); 21 | annotation.camera.pmatrix.p14 = num2str(P(1,4)); 22 | annotation.camera.pmatrix.p21 = num2str(P(2,1)); 23 | annotation.camera.pmatrix.p22 = num2str(P(2,2)); 24 | annotation.camera.pmatrix.p23 = num2str(P(2,3)); 25 | annotation.camera.pmatrix.p24 = num2str(P(2,4)); 26 | annotation.camera.pmatrix.p31 = num2str(P(3,1)); 27 | annotation.camera.pmatrix.p32 = num2str(P(3,2)); 28 | annotation.camera.pmatrix.p33 = num2str(P(3,3)); 29 | annotation.camera.pmatrix.p34 = num2str(P(3,4)); 30 | 31 | % If necessary, remove old extra fields: 32 | if isfield(annotation.camera,'Hy') 33 | annotation.camera = rmfield(annotation.camera,'Hy'); 34 | end 35 | if isfield(annotation.camera,'CAM_H') 36 | annotation.camera = rmfield(annotation.camera,'CAM_H'); 37 | end 38 | if isfield(annotation.camera,'F') 39 | annotation.camera = rmfield(annotation.camera,'F'); 40 | end 41 | -------------------------------------------------------------------------------- /getWidthMap.m: -------------------------------------------------------------------------------- 1 | function [Xmap,objectWidthMaps] = getWidthMap(annotation,img,Zmap,objectDepthMaps) 2 | % Xmap = getWidthMap(annotation,img) 3 | % 4 | % Computes width map given the computed 3D mesh. 5 | % 6 | % Inputs: 7 | % annotation - LabelMe annotation structure 8 | % img - Corresponding image 9 | % 10 | % Outputs: 11 | % Xmap - Width map 12 | % objectWidthMaps - 3D matrix containing widths of the individual 13 | % objects. 14 | 15 | if nargin < 4 16 | [Zmap,objectDepthMaps] = getDepthMap(annotation,img); 17 | end 18 | 19 | MAX_WIDTH = 16777215; 20 | Nobjects = length(annotation.object); 21 | [nrows,ncols,dim] = size(img); 22 | 23 | [mx,my] = meshgrid([1:ncols],[1:nrows]); 24 | Xmap = MAX_WIDTH*ones(nrows,ncols,'uint32'); 25 | if nargout>=2 26 | objectWidthMaps = MAX_WIDTH*ones(nrows,ncols,Nobjects,'uint32'); 27 | end 28 | for i = 1:Nobjects 29 | if isfield(annotation.object(i),'mesh') && ~isempty(annotation.object(i).mesh) 30 | [X,Y] = getLMpolygon(annotation.object(i).polygon); 31 | v2D = annotation.object(i).mesh.v2D; 32 | v3D = annotation.object(i).mesh.v3D; 33 | n = find(inpolygon(mx,my,X,Y)); 34 | x = uint32(griddata(v2D(1,:),v2D(2,:),v3D(1,:),mx(n),my(n))); 35 | if nargout>=2 36 | objectWidthMaps(n+(i-1)*nrows*ncols) = x; 37 | end 38 | z = objectDepthMaps(:,:,i); 39 | j = find(Zmap(n)==z(n)); 40 | Xmap(n(j)) = x(j); 41 | end 42 | end 43 | 44 | if nargout==0 45 | plotDepthMap(Xmap); 46 | end 47 | -------------------------------------------------------------------------------- /InflateSceneNew.m: -------------------------------------------------------------------------------- 1 | function annotation = InflateSceneNew(annotation) 2 | % annotation = InflateScene(annotation) 3 | % 4 | % Given extracted image information, inflate scene. 5 | % 6 | % Inputs: 7 | % annotation - LabelMe annotation structure. 8 | % 9 | % Outputs: 10 | % annotation - LabelMe annotation with "mesh" field for each object 11 | 12 | % $$$ % Insert ground plane: 13 | % $$$ annotation = insertGroundPlane(annotation); 14 | 15 | % Get indices of non-deleted objects: 16 | notDeleted = find(~isdeleted(annotation))'; 17 | 18 | Nobjects = length(annotation.object); % Number of objects 19 | 20 | if isfield(annotation.object,'mesh') 21 | annotation.object = rmfield(annotation.object,'mesh'); 22 | end 23 | 24 | for i = notDeleted%1:Nobjects 25 | polyType = annotation.object(i).polygon.polyType; 26 | if strcmp(polyType,'ground') || strcmp(polyType,'standing') 27 | [X,Y,Z,TX,TY,tri,boundary,CptsNdx,valid] = getObject3D(annotation,i,'trimesh'); 28 | if valid 29 | annotation.object(i).mesh.v3D = [X; Y; Z]; 30 | annotation.object(i).mesh.v2D = [TX; TY]; 31 | annotation.object(i).mesh.tri = tri; 32 | annotation.object(i).mesh.boundary = boundary; 33 | % $$$ annotation.object(i).mesh.type = polytype; 34 | % $$$ if strcmp(polytype,'foldedcard') 35 | if strcmp(polyType,'standing') 36 | annotation.object(i).mesh.CptsNdx = CptsNdx; 37 | end 38 | end 39 | end 40 | end 41 | -------------------------------------------------------------------------------- /getHeightMap.m: -------------------------------------------------------------------------------- 1 | function [Ymap,objectHeightMaps] = getHeightMap(annotation,img,Zmap,objectDepthMaps) 2 | % Zmap = getDepthMap(annotation,img) 3 | % 4 | % Computes depth map given the computed 3D mesh. 5 | % 6 | % Inputs: 7 | % annotation - LabelMe annotation structure 8 | % img - Corresponding image 9 | % 10 | % Outputs: 11 | % Ymap - Depth map 12 | % objectHeightMaps - 3D matrix containing depths of the individual 13 | % objects. 14 | 15 | if nargin < 4 16 | [Zmap,objectDepthMaps] = getDepthMap(annotation,img); 17 | end 18 | 19 | MAX_HEIGHT = 16777215; 20 | Nobjects = length(annotation.object); 21 | [nrows,ncols,dim] = size(img); 22 | 23 | [mx,my] = meshgrid([1:ncols],[1:nrows]); 24 | Ymap = MAX_HEIGHT*ones(nrows,ncols,'uint32'); 25 | if nargout>=2 26 | objectHeightMaps = MAX_HEIGHT*ones(nrows,ncols,Nobjects,'uint32'); 27 | end 28 | for i = 1:Nobjects 29 | if isfield(annotation.object(i),'mesh') && ~isempty(annotation.object(i).mesh) 30 | [X,Y] = getLMpolygon(annotation.object(i).polygon); 31 | v2D = annotation.object(i).mesh.v2D; 32 | v3D = annotation.object(i).mesh.v3D; 33 | n = find(inpolygon(mx,my,X,Y)); 34 | y = uint32(griddata(v2D(1,:),v2D(2,:),v3D(2,:),mx(n),my(n))); 35 | if nargout>=2 36 | objectHeightMaps(n+(i-1)*nrows*ncols) = y; 37 | end 38 | z = objectDepthMaps(:,:,i); 39 | j = find(Zmap(n)==z(n)); 40 | Ymap(n(j)) = y(j); 41 | end 42 | end 43 | 44 | if nargout==0 45 | plotDepthMap(Ymap); 46 | end 47 | -------------------------------------------------------------------------------- /SupportDist.m: -------------------------------------------------------------------------------- 1 | function [dist,xout,yout,alpha] = SupportDist(X,Y,xx,yy) 2 | 3 | X(end+1) = X(1); 4 | Y(end+1) = Y(1); 5 | 6 | %% First, test if inside polygon: 7 | ndx = find((yy>min(Y(2:end),Y(1:end-1))) & ... 8 | (yy<=max(Y(2:end),Y(1:end-1))) & ... 9 | (xx<=max(X(2:end),X(1:end-1))) & ... 10 | (Y(2:end)~=Y(1:end-1))); 11 | if ~isempty(ndx) 12 | xinter = (yy-Y(ndx)).*(X(ndx+1)-X(ndx))./(Y(ndx+1)-Y(ndx))+X(ndx); 13 | counter = sum(X(ndx)==X(ndx+1) | xx<= xinter); 14 | else 15 | counter = 0; 16 | end 17 | 18 | if mod(counter,2) 19 | %% We're inside of polygon; return 0 distance: 20 | dist = 0; 21 | xout = xx; 22 | yout = yy; 23 | alpha = 0; 24 | return; 25 | end 26 | 27 | %% Compute distance to inside of polygon: 28 | Xa = X(1:end-1); Xb = X(2:end); 29 | Ya = Y(1:end-1); Yb = Y(2:end); 30 | alpha = xx*Xa-Xa.*Xb+Xb.^2-xx*Xb+yy*Ya-Ya.*Yb+Yb.^2-yy*Yb; 31 | alpha = alpha./((Xa-Xb).^2+(Ya-Yb).^2+eps); 32 | alpha = max(min(alpha,1),0); 33 | 34 | dist = sqrt((alpha.*Xa+(1-alpha).*Xb-xx).^2 + ... 35 | (alpha.*Ya+(1-alpha).*Yb-yy).^2); 36 | [dist,min_ndx] = min(dist); 37 | xout = alpha(min_ndx)*Xa(min_ndx)+(1-alpha(min_ndx))*Xb(min_ndx); 38 | yout = alpha(min_ndx)*Ya(min_ndx)+(1-alpha(min_ndx))*Yb(min_ndx); 39 | alpha = alpha(min_ndx); 40 | 41 | % For support, ,let's measure just the vertical distance between 42 | % polygons. 43 | %dist = abs(max(Y)-yy); 44 | -------------------------------------------------------------------------------- /addVerticalPoint.m: -------------------------------------------------------------------------------- 1 | function [X,Y,CptsNdx] = addVerticalPoint(X,Y,xc,CptsNdx); 2 | % 3 | % Add points in the polygon that are the intersection of the edges of the 4 | % polygon with the vertical lines defines by xc 5 | % 6 | % Also, propagate the contact point label to the new points 7 | % 8 | % xc is a vector 9 | 10 | 11 | bb = [min(X) max(X)]; 12 | delta = (bb(2)-bb(1))/100; 13 | nc = length(xc); 14 | ndx = []; 15 | 16 | for m = 1:nc 17 | X = [X; X(1)]; 18 | Y = [Y; Y(1)]; 19 | if nargin >3; CptsNdx = [CptsNdx; CptsNdx(1)]; end 20 | np = length(X)-1; 21 | % Search points (only add the point, if there is no point nearby) 22 | Xn = []; 23 | Yn = []; 24 | Cn = []; 25 | for i = 1:np 26 | if min(X(i),X(i+1))+delta<=xc(m) & max(X(i),X(i+1))-delta>xc(m) 27 | % Insert point 28 | xx = xc(m); 29 | yy = (Y(i+1)-Y(i))/(X(i+1)-X(i))*(xc(m)-X(i))+Y(i); 30 | Xn = [Xn; X(i); xx]; 31 | Yn = [Yn; Y(i); yy]; 32 | if nargin > 3 33 | Cn = [Cn; CptsNdx(i); CptsNdx(i)*CptsNdx(i+1)]; % the new point is a contact point if it is betwen two contact points 34 | end 35 | else 36 | Xn = [Xn; X(i)]; 37 | Yn = [Yn; Y(i)]; 38 | if nargin >3 39 | Cn = [Cn; CptsNdx(i)]; 40 | end 41 | end 42 | end 43 | X = Xn; 44 | Y = Yn; 45 | if nargin >3 46 | CptsNdx = Cn; 47 | end 48 | end 49 | -------------------------------------------------------------------------------- /detectShallowConcave.m: -------------------------------------------------------------------------------- 1 | function n = detectShallowConcave(polygon,noPointBelow) 2 | 3 | [X,Y] = getLMpolygon(polygon); 4 | edges = polygon.edges; 5 | 6 | if nargin < 2 7 | % $$$ pb = isThereAnyPointBelow(X,Y); 8 | % $$$ noPointBelow = 1-double(pb(edges(1,:))|pb(edges(2,:))); 9 | noPointBelow = 1-double(isThereAnyPointBelowEdge(X,Y,edges)); 10 | end 11 | 12 | Nedges = size(edges,2); 13 | k = 0; 14 | n = []; 15 | if Nedges < 3 16 | return; 17 | end 18 | 19 | for i = 2:Nedges-1 20 | if noPointBelow(i) 21 | yleft = Y(edges(1,i-1)); 22 | y1 = Y(edges(1,i)); 23 | y2 = Y(edges(2,i)); 24 | yright = Y(edges(2,i+1)); 25 | 26 | % $$$ if (yleft>y1) && (yleft>y2) && (yright>y1) && (yright>y2) 27 | if (yleft>y1) && (yright>y2) 28 | k = k+1; 29 | n(k,:) = [i-1 i i+1]; 30 | end 31 | end 32 | end 33 | 34 | if noPointBelow(1) 35 | yleft = Y(edges(1,end)); 36 | y1 = Y(edges(1,1)); 37 | y2 = Y(edges(2,1)); 38 | yright = Y(edges(2,2)); 39 | 40 | % $$$ if (yleft>y1) && (yleft>y2) && (yright>y1) && (yright>y2) 41 | if (yleft>y1) && (yright>y2) 42 | k = k+1; 43 | n(k,:) = [Nedges 1 2]; 44 | end 45 | end 46 | 47 | if noPointBelow(Nedges) 48 | yleft = Y(edges(1,end-1)); 49 | y1 = Y(edges(1,end)); 50 | y2 = Y(edges(2,end)); 51 | yright = Y(edges(2,1)); 52 | 53 | % $$$ if (yleft>y1) && (yleft>y2) && (yright>y1) && (yright>y2) 54 | if (yleft>y1) && (yright>y2) 55 | k = k+1; 56 | n(k,:) = [Nedges-1 Nedges 1]; 57 | end 58 | end 59 | -------------------------------------------------------------------------------- /plotEdgeLabels.m: -------------------------------------------------------------------------------- 1 | function plotEdgeLabels(annotation,img,edgeLabels,labelingType,i) 2 | 3 | Nobjects = length(annotation.object); 4 | 5 | if nargin < 5 6 | i = 1:Nobjects; 7 | end 8 | 9 | switch labelingType 10 | case 'basic' 11 | edgeOptions = {'Contact edge','Occlusion edge',sprintf('Occlusion edge\nnon-owner'),'Attached edge','Other edge'}; 12 | case 'visibility' 13 | edgeOptions = {'Visible','partially-visible','invisible'}; 14 | case 'simplified' 15 | edgeOptions = {'Contact','Attached','Occlusion'}; 16 | end 17 | 18 | Njet = length(edgeOptions); 19 | edgeTypeColors = jet; 20 | edgeTypeColors = edgeTypeColors(round((size(edgeTypeColors,1)-1)*[0:Njet-1]/(Njet-1))+1,:); 21 | 22 | imshow(img); 23 | hold on; 24 | h = zeros(1,Njet); 25 | for j = i 26 | edges = edgeLabels.object(j).edges; 27 | switch labelingType 28 | case 'basic' 29 | edgeType = edgeLabels.object(j).edgeType; 30 | case 'visibility' 31 | edgeType = edgeLabels.object(j).visibility; 32 | case 'simplified' 33 | edgeType = edgeLabels.object(j).edgeType; 34 | edgeType(ismember(edgeType,[2 3 5])) = 3; 35 | edgeType(edgeType==4) = 2; 36 | end 37 | [X,Y] = getLMpolygon(annotation.object(j).polygon); 38 | for k = 1:size(edges,2) 39 | e = edges(:,k); 40 | h(edgeType(k)) = plot([X(e(1)) X(e(2))],[Y(e(1)) Y(e(2))],'Color',edgeTypeColors(edgeType(k),:),'LineWidth',4); 41 | end 42 | end 43 | 44 | if sum(h>0)>1 45 | legend(h(h>0),edgeOptions(h>0),'Location','NorthEastOutside'); 46 | else 47 | legend(edgeOptions(h>0),'Location','NorthEastOutside'); 48 | end 49 | -------------------------------------------------------------------------------- /imageWarp.m: -------------------------------------------------------------------------------- 1 | function imgWarp = imageWarp(Ox, Oy, imgOriginal, Wx, Wy, imgWarp, boundary) 2 | % 3 | % Input: 4 | % Ox, Oy: landmarks in input image 5 | % imgOriginal: original image to be warped 6 | % Wx, Wy: landmarks in warped image 7 | % imgWard: target image 8 | % boundary: indices of landmarks defining the boundary of the polygon 9 | % 10 | % Output: 11 | % imgWarp 12 | 13 | 14 | % April, 15, 2002 15 | % A. Torralba 16 | % torralba@ai.mit.edu 17 | 18 | [nrows, ncols, cc] = size(imgWarp); 19 | 20 | % coordinates in warped image 21 | [denseWx, denseWy] = meshgrid(floor(min(Wx)):ceil(max(Wx)), floor(min(Wy)):ceil(max(Wy))); 22 | i = inpolygon(denseWx, denseWy, Wx(boundary), Wy(boundary)); 23 | denseWx = denseWx(i); % remove points outside polygon 24 | denseWy = denseWy(i); 25 | 26 | % project coordinates into original image 27 | denseOx = griddata(Wx, Wy, Ox, denseWx, denseWy, 'linear'); denseOx = denseOx(:); 28 | denseOy = griddata(Wx, Wy, Oy, denseWx, denseWy, 'linear'); denseOy = denseOy(:); 29 | %j=find(isnan(denseOx+denseOy)==0); 30 | 31 | % find points projected inside the target image 32 | j = find(denseWx>0 & denseWx0 & denseWy0 18 | mask = zeros(nrows,ncols,Nobj,'single'); 19 | objarea = zeros(Nobj,1); 20 | for k = 1:Nobj 21 | [X,Y] = getLMpolygon(annotation.object(jobj(k)).polygon); 22 | mask(:,:,k) = poly2mask(X,Y,nrows,ncols); 23 | end 24 | intersection = reshape(mask,[nrows*ncols Nobj]); 25 | intersection = intersection'*intersection; 26 | objarea = eps+diag(intersection); 27 | 28 | % Row i is normalized by the area of object i: intersection(i,j) = 29 | % proportion of object i occupied by object j 30 | intersection = intersection./repmat(objarea, [1 Nobj]); 31 | intersection = intersection .*(intersection>.05); 32 | 33 | % OnTop(i,j) = 1 => object i is on top of object j 34 | OnTop = intersection>0.9; % If polygon i is included in j, then obj i is on top of j 35 | 36 | OnTop = logical(OnTop-diag(ones(1,length(jobj)))); 37 | [i,j] = find(OnTop); 38 | i = unique(jobj(i)); 39 | if ~isempty(i) 40 | for k = i 41 | annotation.object(k).polygon.polyType = ''; 42 | end 43 | end 44 | end 45 | 46 | -------------------------------------------------------------------------------- /samplePoly.m: -------------------------------------------------------------------------------- 1 | function [x,y] = samplePoly(X,Y,S) 2 | % Evenly sample around polygon in clockwise fashion. 3 | 4 | if nargin < 3 5 | S = 2000; % Number of sample points 6 | end 7 | x = zeros(S,1); 8 | y = zeros(S,1); 9 | N = length(X); 10 | 11 | % Make input polygon clockwise: 12 | if sum(X.*[Y(2:end); Y(1)]-[X(2:end); X(1)].*Y)<=0 13 | % Counter-clockwise, so flip order: 14 | X = flipud(X); 15 | Y = flipud(Y); 16 | end 17 | 18 | % Compute polygon spacing: 19 | L = sum(sqrt((X-[X(2:end); X(1)]).^2+(Y-[Y(2:end); Y(1)]).^2))/S; 20 | 21 | % Get top-mid point: 22 | x(1) = (min(X)+max(X))/2; 23 | c = find((X<=x(1))&(x(1)<=[X(2:end); X(1)])); 24 | n = mod(c,N)+1; 25 | a = (x(1)-X(n))./(X(c)-X(n)); 26 | [y(1),m] = max(a.*Y(c)+(1-a).*Y(n)); 27 | 28 | % Current, next, alpha, and distance vars: 29 | c = c(m); 30 | n = n(m); 31 | a = 1-a(m); 32 | d = sqrt((X(c)-X(n))^2+(Y(c)-Y(n))^2); 33 | 34 | % Loop through polygon and get evenly spaced points: 35 | for i = 2:S 36 | len = L; 37 | while len > 0 38 | l = a*d+len; 39 | if l<=d 40 | a = l/d; 41 | x(i) = (1-a)*X(c)+a*X(n); 42 | y(i) = (1-a)*Y(c)+a*Y(n); 43 | break; 44 | else 45 | len = l-d; 46 | c = n; 47 | n = mod(c,N)+1; 48 | a = 0; 49 | d = sqrt((X(c)-X(n))^2+(Y(c)-Y(n))^2); 50 | end 51 | end 52 | end 53 | 54 | return; 55 | 56 | ff = figure; 57 | plot([X; X(1)],[Y; Y(1)],'b','LineWidth',4); 58 | hold on; 59 | plot(x(1),y(1),'go','LineWidth',4); 60 | plot(x(2),y(2),'ro','LineWidth',4); 61 | plot(x(3:end),y(3:end),'bo','LineWidth',4); 62 | axis ij; 63 | 64 | pause; 65 | 66 | close(ff); 67 | -------------------------------------------------------------------------------- /standingFoldedCardWithMesh.m: -------------------------------------------------------------------------------- 1 | function [X,Y,Z,x,y,CptsNdx,tri,boundary] = standingFoldedCardWithMesh(x,y,CptsNdx,ncols,nrows,CAM_H,Hy,F) 2 | % [X,Y,Z,x,y,CptsNdx,tri,boundary] = standingFoldedCardWithMesh(x,y,CptsNdx,ncols,nrows,CAM_H,Hy,F) 3 | % 4 | % This function estimates the depth of a shape that is like a folded card 5 | % directly standing on the ground. For objects with a unique contact point, 6 | % it assumes that the object is a flat surface parallel to the camera 7 | % plane. 8 | % 9 | % Input: 10 | % x,y: image polygon coordinates 11 | % CptsNdx = [0 0 1 1 1 0 ...] binary indicator vector for contact 12 | % points, same length than x and y 13 | % ncols, nrows: image size 14 | % CAM_H, Hy, F: camera parameters 15 | % 16 | % Output: meshes 17 | % X,Y,Z: world coordinates of the polygon 18 | % x,y: new polygon coordinates (with inserted points) 19 | % CptsNdx: indicator of contact points 20 | % tri: triangular mesh indices 21 | % boundary: indices to points in the boundary 22 | 23 | % Parameters: 24 | npts = 11; 25 | 26 | % Get contact points: 27 | [x,y,CptsNdx] = addVerticalPoint(x,y,unique(x),CptsNdx); 28 | j = find(CptsNdx); 29 | xc = x(j); 30 | yc = y(j); 31 | 32 | % Need to have unique xc, else interp1 does not work. Need to handle 33 | % this in a better way. 34 | [xc,i] = unique(xc); 35 | yc = yc(i); 36 | 37 | if nargout>6 38 | % Create mesh x,y: 39 | [x,y,tri,boundary] = getMesh(x,y,npts); 40 | end 41 | 42 | % Convert image coordinates into world coordinates: 43 | [X,Y,Z] = Image2WorldCoords(x,y,xc,yc,[nrows ncols],CAM_H,Hy,F); 44 | -------------------------------------------------------------------------------- /logpYCV0.m: -------------------------------------------------------------------------------- 1 | function p = logpYCV0(v0,yc,objh,objv,mu_obj,sig_obj,w_c,mu_c,cov_c) 2 | % p = logpYCV0(v0,yc,objh,objv,mu_obj,sig_obj,w_c,mu_c,cov_c) 3 | % 4 | % Log-likelihood for camera parameters. 5 | % 6 | % Inputs: 7 | % v0 - Horizon line 8 | % yc - Camera height 9 | % objh - Object height (in pixels) 10 | % objv - Position of foot of object (in pixels) 11 | % mu_obj - Mean object height (in meters) 12 | % sig_obj - Standard deviation object height (in meters) 13 | % w_c - Prior mixture of Gaussians weights 14 | % mu_c - Prior mixture of Gaussians means 15 | % cov_c - Prior mixture of Gaussians covariances 16 | % 17 | % Outputs: 18 | % p = -log(P(c,obj;mu, sig)) 19 | 20 | % Parameters: 21 | noiseSigma = 0.02; 22 | 23 | p = zeros(size(v0)); 24 | 25 | % Derek's learned prior on camera parameters: 26 | if nargin >= 9 27 | for j = 1:numel(v0) 28 | p(j) = 0; 29 | for m = 1:numel(w_c) 30 | p(j) = p(j) + w_c(m)*mvnpdf([v0(j) yc(j)], mu_c(m, :), cov_c(:, :, m)); 31 | end 32 | end 33 | p = log(max(p, 1E-40)); 34 | end 35 | 36 | Nobj = length(objh); 37 | Npts = numel(v0); 38 | dims = size(v0); 39 | v0 = repmat(reshape(v0,Npts,1),1,Nobj); 40 | yc = repmat(reshape(yc,Npts,1),1,Nobj); 41 | objh = repmat(reshape(objh,1,Nobj),Npts,1); 42 | objv = repmat(reshape(objv,1,Nobj),Npts,1); 43 | mu_obj = repmat(reshape(mu_obj,1,Nobj),Npts,1); 44 | sig_obj = repmat(reshape(sig_obj,1,Nobj),Npts,1); 45 | 46 | mu = mu_obj.*(v0-objv)./yc; 47 | sigma = sig_obj.*(v0-objv)./yc + noiseSigma; 48 | p = normpdf(objh,mu,sigma); 49 | p = sum(log(max(p,1E-40)),2); 50 | 51 | % Prior on camera height: 52 | p = p+log(normpdf(yc(:,1),1.7,0.5)); 53 | 54 | p = reshape(p,dims); 55 | -------------------------------------------------------------------------------- /avg_error.m: -------------------------------------------------------------------------------- 1 | function [z F] = avg_error(Z, G); 2 | % Z is our guess 3 | % G is ground truth 4 | 5 | % Parameters: 6 | F = 0; 7 | 8 | [nrows,ncols,d] = size(Z); 9 | 10 | % rescale G (note that all images in this dataset have the same size): 11 | GT = imresize(G, [nrows ncols]); 12 | 13 | 14 | E = 0; 15 | count = 0; 16 | for i = 1:nrows 17 | for j = 1:ncols 18 | Z(i,j) = double(Z(i,j))/100; 19 | if (Z(i,j) > 0) 20 | % just to avoid divide-by-zeros 21 | count = count+1; 22 | if (GT(i,j) == 0) 23 | GT(i,j) = 0.1; 24 | end; 25 | E = E + abs(double(Z(i,j))-GT(i,j))/GT(i,j); 26 | F = F + double(Z(i,j))-GT(i,j); 27 | end; 28 | end; 29 | end; 30 | z = E/count; 31 | 32 | return; 33 | 34 | % $$$ % Z is our guess 35 | % $$$ % G is ground truth 36 | % $$$ F = 0; 37 | % $$$ 38 | % $$$ s = size(Z); 39 | % $$$ h = s(1); 40 | % $$$ w = s(2); 41 | % $$$ 42 | % $$$ %rescale G, since it has a funky dimension 43 | % $$$ % note that all images in this dataset have the same size 44 | % $$$ GT = imresize(G, s); 45 | % $$$ 46 | % $$$ 47 | % $$$ E = 0; 48 | % $$$ count = 0; 49 | % $$$ for i = 1:h 50 | % $$$ for j = 1:w 51 | % $$$ Z(i,j) = double(Z(i,j))/100; 52 | % $$$ if (Z(i,j) > 0) 53 | % $$$ 54 | % $$$ % just to avoid divide-by-zeros 55 | % $$$ count = count+1; 56 | % $$$ if (GT(i,j) == 0) 57 | % $$$ GT(i,j) = 0.1; 58 | % $$$ end; 59 | % $$$ E = E + abs(double(Z(i,j))-GT(i,j))/GT(i,j); 60 | % $$$ F = F + double(Z(i,j))-GT(i,j); 61 | % $$$ % E = E + abs(double(Z(i,j))-GT(i,j))/GT(i,j); 62 | % $$$ end; 63 | % $$$ end; 64 | % $$$ end; 65 | % $$$ z = E/count; 66 | -------------------------------------------------------------------------------- /inferSupport.m: -------------------------------------------------------------------------------- 1 | function sup = inferSupport(annotation,Psup,objectClasses,j) 2 | 3 | if ~isfield(annotation,'object') 4 | sup = []; 5 | return; 6 | end 7 | 8 | % Parameters: 9 | thresh_sup = 0.0001; % Threshold for support 10 | lambda = 0.1; % Exponential distribution parameter 11 | 12 | Nobjects = length(annotation.object); 13 | 14 | if nargin < 4 15 | j = 1:Nobjects; 16 | end 17 | 18 | % Collect polygons: 19 | for i = 1:Nobjects 20 | n = strmatch(strtrim(lower(annotation.object(i).name)), objectClasses, 'exact'); 21 | if ~isempty(n) 22 | ndx(i) = n; 23 | else 24 | ndx(i) = 0; 25 | end 26 | [X{i},Y{i}] = getLMpolygon(annotation.object(i).polygon); 27 | [x(i),y(i)] = GetBottomPoint(X{i},Y{i}); 28 | end 29 | 30 | Bottom = {'floor', 'ground', 'grass', 'field', 'street', 'road', ... 31 | 'sidewalk', 'path', 'sea', 'water'}; 32 | 33 | sup = zeros(1,length(j)); 34 | P = zeros(1,length(j)); 35 | for m = 1:Nobjects 36 | for i = 1:length(j) 37 | n = j(i); 38 | if (m~=n) && ndx(m) && ndx(n) && ~ismember(strtrim(lower(annotation.object(n).name)),Bottom) 39 | pp = Psup(ndx(m),ndx(n))*exp(-lambda*SupportDist(X{m},Y{m},x(n),y(n))); 40 | if pp > P(i) 41 | P(i) = pp; 42 | sup(i) = m; 43 | end 44 | end 45 | end 46 | end 47 | %keyboard 48 | 49 | % Threshold support relations: 50 | sup(P<=thresh_sup) = 0; 51 | 52 | % Show outputs: 53 | [v,n] = sort(P,'descend'); 54 | for i = n 55 | if sup(i) 56 | disp(sprintf('%s supports %s (with probability = %1.2f)', ... 57 | annotation.object(sup(i)).name, ... 58 | annotation.object(j(i)).name,P(i))); 59 | end 60 | end 61 | -------------------------------------------------------------------------------- /ConvertOld2New.m: -------------------------------------------------------------------------------- 1 | function annotation = ConvertOld2New(annotation,validObjects) 2 | % Convert XML file to new format (this needs to be removed at some point) 3 | 4 | for i = validObjects%1:length(annotation.object) 5 | annotation.object(i).world3d.stale = '0'; 6 | annotation.object(i).world3d.type = ''; 7 | 8 | % annotation.object.polygon.polyType -> annotation.object.world3d.type 9 | if isfield(annotation.object(i),'polygon') && isfield(annotation.object(i).polygon,'polyType') 10 | switch annotation.object(i).polygon.polyType 11 | case 'standing' 12 | annotation.object(i).world3d.type = 'standingplanes'; 13 | case 'part' 14 | annotation.object(i).world3d.type = 'part'; 15 | case 'ground' 16 | annotation.object(i).world3d.type = 'groundplane'; 17 | otherwise 18 | annotation.object(i).world3d.type = 'none'; 19 | end 20 | annotation.object(i).polygon = rmfield(annotation.object(i).polygon,'polyType'); 21 | end 22 | 23 | % annotation.object.polygon.contact -> annotation.object.world3d.contact 24 | if isfield(annotation.object(i),'polygon') && isfield(annotation.object(i).polygon,'contact') 25 | annotation.object(i).world3d.contact = annotation.object(i).polygon.contact; 26 | annotation.object(i).polygon = rmfield(annotation.object(i).polygon,'contact'); 27 | end 28 | 29 | % annotation.object.partof -> annotation.object.world3d.partof 30 | if isfield(annotation.object(i),'partof') && ~isempty(annotation.object(i).partof) 31 | annotation.object(i).world3d.parentid = annotation.object(i).partof; 32 | end 33 | end 34 | if isfield(annotation.object,'partof') 35 | annotation.object = rmfield(annotation.object,'partof'); 36 | end 37 | 38 | % Convert image size to strings: 39 | annotation.imagesize.nrows = num2str(annotation.imagesize.nrows); 40 | annotation.imagesize.ncols = num2str(annotation.imagesize.ncols); 41 | -------------------------------------------------------------------------------- /LM3D_ObjectHeights.txt: -------------------------------------------------------------------------------- 1 | car 2 | 1.511999 0.191003 3 | personupright 4 | 1.700000 0.103308 5 | chair 6 | 1.098513 0.125670 7 | table 8 | 1.046356 0.211309 9 | tree trunk 10 | 13.240456 3.321042 11 | pasperson 12 | 2.236451 0.232761 13 | wheel 14 | 0.622199 0.026055 15 | person 16 | 1.734205 0.171112 17 | carside az0deg az0deg 18 | 1.461481 0.104969 19 | carfrontal az270deg az270deg 20 | 1.462282 0.177415 21 | carfrontal az90deg az90deg 22 | 1.430517 0.081373 23 | paspersonfrontal 24 | 2.293040 0.201341 25 | parkingmeter 26 | 1.365936 0.088687 27 | firehydrant 28 | 0.876598 0.099877 29 | man 30 | 1.796425 0.107447 31 | pedestrian 32 | 1.612473 0.236328 33 | pasbus 34 | 4.313055 0.261817 35 | pasmotorbike 36 | 1.512634 0.039726 37 | pasbicycle 38 | 1.369863 0.040745 39 | truck 40 | 2.613873 0.633883 41 | carside az180deg az180deg 42 | 1.442110 0.078583 43 | paspersonleft 44 | 2.307535 0.074461 45 | van 46 | 1.954150 0.158620 47 | bench 48 | 0.865274 0.030130 49 | pashorse 50 | 1.991942 0.357689 51 | woman 52 | 1.676187 0.039322 53 | paspersonright 54 | 2.628259 0.031976 55 | motorbike 56 | 1.579409 0.110136 57 | fence 58 | 1.723895 0.501567 59 | bus 60 | 3.919032 0.439907 61 | carside az330deg az330deg 62 | 1.426478 0.031946 63 | carfrontal az60deg az60deg 64 | 1.463580 0.189617 65 | cone 66 | 0.729092 0.131516 67 | bicycle 68 | 1.077182 0.070234 69 | pasmotorbikeright 70 | 1.657141 0.047287 71 | carfrontal az300deg az300deg 72 | 1.507455 0.150731 73 | sofa 74 | 1.310730 0.042400 75 | carfrontal az240deg az240deg 76 | 1.504584 0.164045 77 | carside az30deg az30deg 78 | 1.466239 0.028673 79 | motorcycle 80 | 1.398771 0.115146 81 | carfrontal az120deg az120deg 82 | 1.402152 0.150583 83 | parking meter 84 | 1.306206 0.084008 85 | pasbustrunc 86 | 4.049580 0.210172 87 | trash can 88 | 1.115519 0.111831 89 | truck rear 90 | 3.091754 0.594528 91 | van rear 92 | 1.892583 0.046863 93 | fire hydrant 94 | 0.915414 0.016812 95 | -------------------------------------------------------------------------------- /LM3Dgenerate3D.m: -------------------------------------------------------------------------------- 1 | function caught_errs = LM3Dgenerate3D(HOMEANNOTATIONS,HOMEIMAGES) 2 | % Runs LabelMe3D algorithm over annotations folder to generate 3D 3 | % outputs. 4 | % 5 | % WARNING: This function will overwrite your XML annotations. 6 | % 7 | % Example: Run over all XML annotations inside HOMEANNOTATIONS folder: 8 | % 9 | % HOMEANNOTATIONS = '/path/to/Annotations'; 10 | % HOMEIMAGES = '/path/to/Images'; 11 | % LM3Dgenerate3D(HOMEANNOTATIONS,HOMEIMAGES) 12 | 13 | % LabelMe3D structures: 14 | load Params3D.mat; 15 | 16 | % Make sure that folder names are consistent: 17 | ADMINrenamefolder(HOMEANNOTATIONS); 18 | 19 | % Get list of folders: 20 | folders = strread(genpath(HOMEANNOTATIONS),'%s','delimiter',pathsep); 21 | 22 | caught_errs = []; 23 | for i = 1:length(folders) 24 | % Get list of XML files in folder: 25 | files = dir(fullfile(folders{i},'*.xml')); 26 | for j = 1:length(files) 27 | fname = fullfile(folders{i},files(j).name); 28 | try 29 | [annotation,img] = LMread(fname,HOMEIMAGES); 30 | imageSize = [size(img,1) size(img,2)]; 31 | [annotation,valid] = Recover3DSceneComponents(annotation,Params3D,imageSize); 32 | if valid 33 | % Write XML file: 34 | writeXML3D(annotation,fname); 35 | end 36 | catch me 37 | display(sprintf('%s failed; continuing...',fname)); 38 | if isempty(caught_errs) 39 | caught_errs.fname = fname; 40 | else 41 | caught_errs(end+1).fname = fname; 42 | end 43 | caught_errs(end).me = me; 44 | end 45 | end 46 | end 47 | 48 | return; 49 | 50 | 51 | addpath '~/work/MatlabLibraries/LabelMeToolbox'; 52 | addpath '~/work/MatlabLibraries/LabelMe3dToolbox'; 53 | 54 | HOMEIMAGES = '/Users/brussell/work/Datasets/LabelMe3D/Images'; 55 | HOMEANNOTATIONS = '/Users/brussell/work/Datasets/LabelMe3D/Annotations'; 56 | 57 | caught_errs = LM3Dgenerate3D(HOMEANNOTATIONS,HOMEIMAGES); 58 | -------------------------------------------------------------------------------- /getObjectDimensions.m: -------------------------------------------------------------------------------- 1 | function [Xmax,Ymax,Zmax,valid] = getObjectDimensions(annotation,i) 2 | % Inputs: 3 | % annotation 4 | % i 5 | % 6 | % Outputs: 7 | % Xmax 8 | % Ymax 9 | % Zmax 10 | % valid 11 | 12 | Xmax = 0; Ymax = 0; Zmax = 0; valid = 0; 13 | [X,Y,Z,mx,my,tri,boundary,CptsNdx,valid_i] = getObject3D(annotation,i,'poly'); 14 | if valid_i && ~isempty(X) 15 | Xmax = max(X)-min(X); 16 | Ymax = max(Y)-min(Y); 17 | Zmax = max(Z)-min(Z); 18 | valid = 1; 19 | end 20 | valid = logical(valid); 21 | 22 | return; 23 | 24 | notDeleted = find(~isdeleted(annotation))'; 25 | Nobjects = length(annotation.object); 26 | Xmax = zeros(1,Nobjects); 27 | Ymax = zeros(1,Nobjects); 28 | Zmax = zeros(1,Nobjects); 29 | valid = logical(zeros(1,Nobjects)); 30 | for i = notDeleted%1:Nobjects 31 | [X,Y,Z,mx,my,tri,boundary,CptsNdx,valid_i] = getObject3D(annotation,i,'poly'); 32 | if valid_i && ~isempty(X) 33 | Xmax(i) = max(X)-min(X); 34 | Ymax(i) = max(Y)-min(Y); 35 | Zmax(i) = max(Z)-min(Z); 36 | valid(i) = 1; 37 | end 38 | end 39 | 40 | return; 41 | 42 | Nobjects = length(annotation.object); 43 | Xmax = zeros(1,Nobjects); 44 | Ymax = zeros(1,Nobjects); 45 | Zmax = zeros(1,Nobjects); 46 | valid = logical(ones(1,Nobjects)); 47 | for i = 1:Nobjects 48 | j = getPartOfParents(annotation,i); 49 | j = j(end); 50 | if strcmp(annotation.object(i).polygon.polyType,'standing') || strcmp(annotation.object(i).polygon.polyType,'ground') 51 | [X,Y,Z] = getObject3D(annotation,i,'poly'); 52 | Xmax(i) = max(X)-min(X); 53 | Ymax(i) = max(Y)-min(Y); 54 | Zmax(i) = max(Z)-min(Z); 55 | elseif strcmp(annotation.object(i).polygon.polyType,'part') && strcmp(annotation.object(j).polygon.polyType,'standing') 56 | [mx,my] = getLMpolygon(annotation.object(i).polygon); 57 | [X,Y,Z] = getObject3D(annotation,j,'mymesh',mx,my); 58 | Xmax(i) = max(X)-min(X); 59 | Ymax(i) = max(Y)-min(Y); 60 | Zmax(i) = max(Z)-min(Z); 61 | else 62 | valid(i) = 0; 63 | end 64 | end 65 | -------------------------------------------------------------------------------- /detectOccluded.m: -------------------------------------------------------------------------------- 1 | function occluded = detectOccluded(annotation,img,Ppart,objectClasses) 2 | % Detect which objects are occluded. 3 | 4 | Nobjects = length(annotation.object); 5 | [nrows,ncols,dim] = size(img); 6 | 7 | % First, detect parts so that they don't occlude the object they belong 8 | % to: 9 | parts = inferParts(annotation,Ppart,objectClasses); 10 | 11 | % Sort layers: 12 | [a,i,layers] = LMsortlayers(annotation,img); 13 | % $$$ a == annotation(i) 14 | 15 | % Get polygon masks: 16 | for j = 1:Nobjects 17 | [X{j},Y{j}] = getLMpolygon(annotation.object(j).polygon); 18 | % $$$ [X{j},Y{j}] = samplePoly(X{j},Y{j}); 19 | end 20 | 21 | occluded = logical(zeros(1,Nobjects)); 22 | for j = 1:Nobjects 23 | for k = 1:Nobjects 24 | if (j~=k) && (layers(i==k)>=layers(i==j)) && (parts(k)~=j) && ... 25 | (any(inpolygon(X{j},Y{j},X{k},Y{k})) || ... 26 | any(inpolygon(X{k},Y{k},X{j},Y{j}))) 27 | occluded(j) = 1; 28 | break; 29 | end 30 | end 31 | end 32 | 33 | return; 34 | 35 | figure; 36 | for j = 1:length(occluded) 37 | if ~occluded(j) && ~parts(j) 38 | clf; 39 | imshow(img); 40 | hold on; 41 | plot([X{j}; X{j}(1)],[Y{j}; Y{j}(1)],'b','LineWidth',4); 42 | pause; 43 | end 44 | end 45 | 46 | im_counts = LMcountobject(DB); 47 | [v,n] = sort(im_counts,'descend'); 48 | figure; 49 | c = 1; 50 | imnum = 1105;%1079;%1052;%1035;%1018;%1000; 51 | while c<=64 52 | annotation = DB(n(imnum)).annotation; 53 | img = LMimread(DB,n(imnum),HOMEIMAGES); 54 | sup = inferSupport(annotation,Psup,objectClasses); 55 | unocc = find(~detectOccluded(annotation,img,Ppart,objectClasses) & ... 56 | ~inferParts(annotation,Ppart,objectClasses) & ... 57 | sup); 58 | unocc = unocc(sup(sup(unocc))==0); 59 | 60 | j = 1; 61 | while (j<=length(unocc))&&(c<=64) 62 | imgCrop = LMobjectcrop(img,annotation,unocc(j),2); 63 | subplottight(8,8,c); 64 | imshow(imgCrop); 65 | drawnow; 66 | j = j+1; 67 | c = c+1; 68 | end 69 | imnum = imnum+1 70 | end 71 | -------------------------------------------------------------------------------- /plotXmap.m: -------------------------------------------------------------------------------- 1 | function Xmap = plotXmap(Xmap,lim,units) 2 | % Xmap = plotXmap(Xmap,lim) 3 | % 4 | % Output pretty width map figure. If no output argument is specified, 5 | % then a figure is created showing the width map 6 | % 7 | % Inputs: 8 | % Xmap - Width map in absolute coordinates. 9 | % lim - Min and max width limits (optional) 10 | % units - Use 'metric' or 'english' units (optional, 'metric' is default) 11 | % 12 | % Outputs: 13 | % Xmap - Output width map, with colors indicating absolute distance 14 | 15 | MAXVAL = 8388607;%16777215; 16 | if (nargin < 2) || isempty(lim) 17 | v = double(max(abs(Xmap(Xmap~=MAXVAL)))); 18 | v = 1000; 19 | MIN_DEPTH = -v; 20 | MAX_DEPTH = v; 21 | else 22 | MIN_DEPTH = lim(1); 23 | MAX_DEPTH = lim(2); 24 | end 25 | if nargin < 3 26 | units = 'metric'; 27 | end 28 | 29 | ytick = round(linspace(MIN_DEPTH,MAX_DEPTH,7)); 30 | yticklabels = []; 31 | for i = 1:length(ytick) 32 | switch units 33 | case 'metric' 34 | yticklabels{i} = sprintf('%dcm',ytick(i)); 35 | case 'english' 36 | yticklabels{i} = sprintf('%0.2fft',ytick(i)*unitsratio('feet','cm')); 37 | otherwise 38 | error('Invalid units'); 39 | end 40 | end 41 | 42 | % Get valid regions: 43 | Zinvalid = find(Xmap==MAXVAL); 44 | 45 | Xmap = min(max(double(Xmap),MIN_DEPTH),MAX_DEPTH); 46 | 47 | % Adjust Xmap range to lie between 0 and 1: 48 | Xmap = Xmap-MIN_DEPTH; 49 | ytick = ytick-MIN_DEPTH; 50 | Xmap = Xmap/(MAX_DEPTH-MIN_DEPTH); 51 | ytick = ytick/(MAX_DEPTH-MIN_DEPTH); 52 | 53 | % Pass Xmap through -jet colormap: 54 | cm = uint8(255*fliplr(jet(64))); 55 | z = round(Xmap*(size(cm,1)-1))+1; 56 | Xmap(:,:,1) = cm(z); 57 | Xmap(:,:,2) = cm(z+size(cm,1)); 58 | Xmap(:,:,3) = cm(z+2*size(cm,1)); 59 | Xmap = uint8(Xmap); 60 | 61 | % Make invalid regions black: 62 | Xmap(Zinvalid) = 0; 63 | Xmap(Zinvalid+size(Xmap,1)*size(Xmap,2)) = 0; 64 | Xmap(Zinvalid+2*size(Xmap,1)*size(Xmap,2)) = 0; 65 | 66 | if ~nargout 67 | imagesc(Xmap,[0 255]); 68 | colormap(double(cm)/255); 69 | axis image off; 70 | colorbar('YTick',255*ytick,'YTickLabel',yticklabels); 71 | end 72 | -------------------------------------------------------------------------------- /plotZmap.m: -------------------------------------------------------------------------------- 1 | function Zmap = plotZmap(Zmap,lim,units) 2 | % Zmap = plotZmap(Zmap,lim) 3 | % 4 | % Output pretty depth map figure. If no output argument is specified, 5 | % then a figure is created showing the depth map 6 | % 7 | % Inputs: 8 | % Zmap - Depth map in absolute coordinates. 9 | % lim - Min and max depth limits (optional) 10 | % units - Use 'metric' or 'english' units (optional, 'metric' is default) 11 | % 12 | % Outputs: 13 | % Zmap - Output depth map, with colors indicating absolute distance 14 | 15 | MAXVAL = 16777215; 16 | if (nargin < 2) || isempty(lim) 17 | % MAX_DEPTH = 16777215; % max uint24 18 | MIN_DEPTH = 1e2; % 1m 19 | MAX_DEPTH = 1e5; % 1km 20 | else 21 | MIN_DEPTH = lim(1); 22 | MAX_DEPTH = lim(2); 23 | end 24 | if nargin < 3 25 | units = 'metric'; 26 | end 27 | 28 | ytick = [0:7]; 29 | switch units 30 | case 'metric' 31 | yticklabels = {'1cm','10cm','1m','10m','100m','1km','10km','100km'}; 32 | case 'english' 33 | yticklabels = {'0.03ft','0.33ft','3.28ft','32.81ft','328.08ft','3280.80ft','32808ft','100km'}; 34 | otherwise 35 | error('Invalid units'); 36 | end 37 | 38 | ylim = [log10(MIN_DEPTH) log10(MAX_DEPTH)]; 39 | 40 | % Get valid regions: 41 | Zinvalid = find(Zmap==MAXVAL); 42 | 43 | % Make Zmap lie within MIN_DEPTH/MAX_DEPTH range: 44 | Zmap = log10(min(max(double(Zmap),MIN_DEPTH),MAX_DEPTH)); 45 | 46 | % Adjust Zmap range to lie between 0 and 1: 47 | Zmap = Zmap-ylim(1); 48 | ytick = ytick-ylim(1); 49 | ylim = ylim-ylim(1); 50 | Zmap = Zmap/ylim(2); 51 | ytick = ytick/ylim(2); 52 | ylim = ylim/ylim(2); 53 | 54 | % Pass Zmap through -jet colormap: 55 | cm = uint8(255*fliplr(jet(64))); 56 | z = round(Zmap*(size(cm,1)-1))+1; 57 | Zmap(:,:,1) = cm(z); 58 | Zmap(:,:,2) = cm(z+size(cm,1)); 59 | Zmap(:,:,3) = cm(z+2*size(cm,1)); 60 | Zmap = uint8(Zmap); 61 | 62 | % Make invalid regions black: 63 | Zmap(Zinvalid) = 0; 64 | Zmap(Zinvalid+size(Zmap,1)*size(Zmap,2)) = 0; 65 | Zmap(Zinvalid+2*size(Zmap,1)*size(Zmap,2)) = 0; 66 | 67 | if ~nargout 68 | % figure; 69 | imagesc(Zmap,[0 255]); 70 | % imagesc(Zmap,[0 1]); 71 | colormap(double(cm)/255); 72 | axis image off; 73 | colorbar('YTick',255*ytick,'YTickLabel',yticklabels,'YLim',255*ylim); 74 | end 75 | -------------------------------------------------------------------------------- /collectPartsEvidence.m: -------------------------------------------------------------------------------- 1 | function [evidence,objectClasses] = collectPartsEvidence(DB,objectClasses) 2 | % [evidence,objectClasses] = collectPartsEvidence(DB) 3 | % or 4 | % evidence = collectPartsEvidence(annotation,objectClasses,validObjects) 5 | % 6 | % Collect evidence for attachment (part-of) relationships. 7 | % 8 | % Inputs: 9 | % DB (annotation) - LabelMe DB structure. 10 | % objectClasses - Cell array of object classes. 11 | % 12 | % Outputs: 13 | % evidence(i).relativeOverlap - Relative overlap. 14 | % evidence(i).relativeArea - Relative area. 15 | % evidence(i).objNdx - Object class index. 16 | 17 | if ~isfield(DB,'annotation') 18 | DB.annotation = DB; 19 | end 20 | if nargin < 2 21 | objectClasses = unique(lower(LMobjectnames(DB,'name'))); 22 | end 23 | 24 | Nobjects = length(objectClasses); 25 | Nimages = length(DB); 26 | for i = 1:Nimages 27 | if isfield(DB(i).annotation,'object') 28 | if isfield(DB(i).annotation,'imagesize') 29 | nrows = DB(i).annotation.imagesize.nrows; 30 | ncols = DB(i).annotation.imagesize.ncols; 31 | else 32 | [ncols,nrows] = getaproximagesize(DB(i).annotation); 33 | end 34 | 35 | % Get object indices and relative area: 36 | Nobj = length(DB(i).annotation.object); 37 | objNdx = zeros(1,Nobj,'uint16'); 38 | relativeArea = zeros(1,Nobj,'single'); 39 | clear X Y; 40 | for j = 1:Nobj 41 | k = strmatch(strtrim(lower(DB(i).annotation.object(j).name)),objectClasses,'exact'); 42 | if ~isempty(k) 43 | objNdx(j) = k; 44 | end 45 | [X{j},Y{j}] = getLMpolygon(DB(i).annotation.object(j).polygon); 46 | relativeArea(j) = min(max(polyarea(X{j},Y{j})/ncols/nrows,0),1); 47 | end 48 | 49 | % Collect relative overlaps: 50 | R = single(eye(Nobj)); 51 | for m = 1:Nobj-1 52 | for n = m+1:Nobj 53 | [Rm,Rn] = relativeOverlap(X{m},Y{m},X{n},Y{n}); 54 | R(m,n) = Rm; 55 | R(n,m) = Rn; 56 | end 57 | end 58 | 59 | % Store evidence into structure: 60 | evidence(i).relativeOverlap = R; 61 | evidence(i).relativeArea = relativeArea; 62 | evidence(i).objNdx = objNdx; 63 | else 64 | evidence(i).relativeOverlap = []; 65 | evidence(i).relativeArea = []; 66 | evidence(i).objNdx = []; 67 | end 68 | end 69 | -------------------------------------------------------------------------------- /plotYmap.m: -------------------------------------------------------------------------------- 1 | function Ymap = plotYmap(Ymap,lim,units) 2 | % Ymap = plotDepthMap(Ymap,lim) 3 | % 4 | % Output pretty height map figure. If no output argument is specified, 5 | % then a figure is created showing the height map 6 | % 7 | % Inputs: 8 | % Ymap - Height map in absolute coordinates. 9 | % lim - Min and max height limits (optional) 10 | % units - Use 'metric' or 'english' units (optional, 'metric' is default) 11 | % 12 | % Outputs: 13 | % Ymap - Output height map, with colors indicating absolute distance 14 | 15 | MAXVAL = 16777215; 16 | if (nargin < 2) || isempty(lim) 17 | % MAX_DEPTH = 16777215; % max uint24 18 | MIN_DEPTH = 1e1;%1e2; % 1m 19 | MAX_DEPTH = 1e4;%1e5; % 1km 20 | else 21 | MIN_DEPTH = lim(1); 22 | MAX_DEPTH = lim(2); 23 | end 24 | if nargin < 3 25 | units = 'metric'; 26 | end 27 | 28 | ytick = [0:7]; 29 | switch units 30 | case 'metric' 31 | yticklabels = {'1cm','10cm','1m','10m','100m','1km','10km','100km'}; 32 | case 'english' 33 | yticklabels = {'0.03ft','0.33ft','3.28ft','32.81ft','328.08ft','3280.80ft','32808ft','100km'}; 34 | otherwise 35 | error('Invalid units'); 36 | end 37 | 38 | ylim = [log10(MIN_DEPTH) log10(MAX_DEPTH)]; 39 | 40 | % Get valid regions: 41 | Zinvalid = find(Ymap==MAXVAL); 42 | 43 | % Make Ymap lie within MIN_DEPTH/MAX_DEPTH range: 44 | Ymap = log10(min(max(double(Ymap),MIN_DEPTH),MAX_DEPTH)); 45 | 46 | % Adjust Ymap range to lie between 0 and 1: 47 | Ymap = Ymap-ylim(1); 48 | ytick = ytick-ylim(1); 49 | ylim = ylim-ylim(1); 50 | Ymap = Ymap/ylim(2); 51 | ytick = ytick/ylim(2); 52 | ylim = ylim/ylim(2); 53 | 54 | % Pass Ymap through -jet colormap: 55 | cm = uint8(255*fliplr(jet(64))); 56 | z = round(Ymap*(size(cm,1)-1))+1; 57 | Ymap(:,:,1) = cm(z); 58 | Ymap(:,:,2) = cm(z+size(cm,1)); 59 | Ymap(:,:,3) = cm(z+2*size(cm,1)); 60 | Ymap = uint8(Ymap); 61 | 62 | % Make invalid regions black: 63 | Ymap(Zinvalid) = 0; 64 | Ymap(Zinvalid+size(Ymap,1)*size(Ymap,2)) = 0; 65 | Ymap(Zinvalid+2*size(Ymap,1)*size(Ymap,2)) = 0; 66 | 67 | if ~nargout 68 | % figure; 69 | imagesc(Ymap,[0 255]); 70 | % imagesc(Ymap,[0 1]); 71 | colormap(double(cm)/255); 72 | axis image off; 73 | colorbar('YTick',255*ytick,'YTickLabel',yticklabels,'YLim',255*ylim); 74 | end 75 | -------------------------------------------------------------------------------- /AddAnglePolygonType.m: -------------------------------------------------------------------------------- 1 | function annotation = AddAnglePolygonType(annotation,validObjects) 2 | % Inputs: 3 | % annotation - LabelMe annotation structure 4 | % validObjects - Indices of valid objects to compute. 5 | % 6 | % Outputs: 7 | % annotation - LabelMe annotation structure with added angle polygon types. 8 | 9 | % Get indices of objects that have been labeled as an angle constraint: 10 | nAngle = []; 11 | for i = validObjects 12 | strs = regexp(lower(annotation.object(i).originalname),' ','split'); 13 | if ismember('angle',strs) 14 | [x,y] = getLMpolygon(annotation.object(i).polygon); 15 | if length(x)==3 16 | nAngle(end+1) = i; 17 | end 18 | end 19 | end 20 | 21 | if ~isempty(nAngle) 22 | % Get objects that are not deleted and that are not angles: 23 | n = setdiff(find(isdeleted(annotation)==0)',nAngle); 24 | 25 | % Get ground and standing objects: 26 | nGround = []; 27 | nStanding = []; 28 | for i = n 29 | if isfield(annotation.object(i),'world3d') 30 | switch annotation.object(i).world3d.type 31 | case 'standingplanes' 32 | nStanding(end+1) = i; 33 | case 'groundplane' 34 | nGround(end+1) = i; 35 | end 36 | end 37 | end 38 | 39 | % Intersect angles with ground and standing objects: 40 | nStandingGround = [nStanding nGround]; 41 | for i = nAngle 42 | [Xi,Yi] = getLMpolygon(annotation.object(i).polygon); 43 | for j = nStandingGround 44 | [Xj,Yj] = getLMpolygon(annotation.object(j).polygon); 45 | if all(inpolygon(Xi,Yi,Xj,Yj)) 46 | % Set annotation structure with angle type: 47 | annotation.object(i).world3d.type = 'angle'; 48 | annotation.object(i).world3d.stale = '0'; 49 | annotation.object(i).world3d.angle = '90 degrees'; 50 | annotation.object(i).world3d.rootid = annotation.object(j).id; 51 | if strcmp(annotation.object(j).world3d.type,'groundplane') 52 | annotation.object(i).world3d.planeindex = '0'; 53 | elseif strcmp(annotation.object(j).world3d.type,'standingplanes') 54 | %%% NEED TO SET THIS BASED ON WHICH STANDING PLANE ANGLE 55 | %%% BELONGS TO: 56 | annotation.object(i).world3d.planeindex = '0'; 57 | end 58 | 59 | break; 60 | end 61 | end 62 | end 63 | end 64 | -------------------------------------------------------------------------------- /LMsceneThumbnail.m: -------------------------------------------------------------------------------- 1 | function thumb = LMsceneThumbnail(annotation, HOMEIMAGES, img, style) 2 | % thumb = LMsceneThumbnail(annotation, HOMEIMAGES, img, style) 3 | % 4 | % This function returns a thumbnail with the style used on the online tool. 5 | % 6 | % There are two styles 7 | % style = 'shaded' [default], 'lines' 8 | % 9 | % HOMEIMAGES = 'http://labelme.csail.mit.edu/Images'; 10 | 11 | % Thumbnail size (height) 12 | tY = 96; 13 | tY = 160; 14 | 15 | D.annotation = annotation; 16 | if nargin < 3 17 | img = LMimread(D, 1, HOMEIMAGES); 18 | end 19 | if nargin > 3 20 | if isempty(img) 21 | img = LMimread(D, 1, HOMEIMAGES); 22 | end 23 | end 24 | 25 | if nargin < 3 26 | style = 'shaded'; 27 | end 28 | 29 | % Re-scale image 30 | [annotation, img] = LMimscale(annotation, img, (tY+1)/size(img,1), 'bilinear'); 31 | [annotation, img] = LMimcrop(annotation, img, [1 size(img,2) 1 tY]); 32 | 33 | % sort layers 34 | annotation = LMsortlayers(annotation, img); 35 | 36 | if size(img,3) < 3 37 | img = repmat(img(:,:,1), [1 1 3]); 38 | end 39 | 40 | % get segmentation masks 41 | [mask, cc, maskpol, classpol] = LMobjectmask(annotation, [size(img,1) size(img,2)]); 42 | 43 | % Thumbnails 44 | switch style 45 | case 'shaded' 46 | % Thumbnails with masks 47 | seg = 128*ones(size(img)); 48 | if size(mask,3)>0 49 | M = double(colorSegments(mask, 'donotsort'))/255; 50 | if size(M,1)>0 51 | M = M + .5*repmat(sum(mask,3)==0, [1 1 3]); 52 | M = M / max(M(:)); 53 | seg = M .* repmat(mean(double(img),3)/2+128, [1 1 3]); 54 | end 55 | end 56 | case 'lines' 57 | % Thumbnails with polygons 58 | seg = img; 59 | if size(mask,3)>0 60 | colors = hsv(size(mask,3)); 61 | % black boundary 62 | Th = 3; cB = strel('disk',2); cL = strel('disk',1); 63 | for n = 1:size(mask,3) 64 | boundary = bwperim(mask(:,:,n),8); 65 | boundaryB = imdilate(boundary, cB); 66 | seg = seg.*uint8(repmat(1-boundaryB, [1 1 3])); 67 | boundaryL = double(imdilate(boundary, cL)); 68 | for m = 1:3; seg(:,:,m) = seg(:,:,m)+uint8(255*boundaryL*colors(n,m)); end 69 | end 70 | end 71 | end 72 | 73 | seg = uint8(seg); 74 | thumb = [img 255*ones([size(img,1),2,size(img,3)]) seg]; 75 | 76 | if nargout == 0 77 | figure 78 | imshow(thumb) 79 | end 80 | 81 | -------------------------------------------------------------------------------- /bpFactorGraph.m: -------------------------------------------------------------------------------- 1 | function beliefs = bpFactorGraph(nodes,factors) 2 | % Run belief propagation given factor graph. 3 | % 4 | % Inputs: 5 | % nodes(i).msgIn - Messages from factors. 6 | % nodes(i).msgOut - Messages to factors. 7 | % 8 | % factors(i).pot; 9 | % factors(i).ndxNodes; 10 | % factors(i).ndxMsgs; 11 | 12 | % Parameters: 13 | Niterations = 10; 14 | 15 | Nnodes = length(nodes); 16 | Nfactors = length(factors); 17 | for i = 1:Niterations 18 | % Pass messages from nodes to factors: 19 | for j = 1:Nnodes 20 | msg = log(nodes(j).msgIn); 21 | m = msg; 22 | m(isinf(msg)) = 0; 23 | msg = repmat(sum(msg,2),1,size(msg,2))-m; 24 | 25 | % Normalize messages: 26 | msg = exp(msg-repmat(max(msg,[],1),size(msg,1),1)); 27 | msg = msg./repmat(sum(msg,1),size(msg,1),1); 28 | 29 | % Set outgoing messages: 30 | nodes(j).msgOut = msg; 31 | end 32 | 33 | % Pass messages from factors to nodes: 34 | for j = 1:Nfactors 35 | pot = factors(j).pot; 36 | ndxNodes = factors(j).ndxNodes; 37 | ndxMsgs = factors(j).ndxMsgs; 38 | 39 | switch ndims(pot) 40 | case 2 41 | if prod(size(pot))==length(pot) 42 | nodes(ndxNodes).msgIn(:,ndxMsgs) = pot; 43 | else 44 | m1 = nodes(ndxNodes(1)).msgOut(:,ndxMsgs(1)); 45 | m2 = nodes(ndxNodes(2)).msgOut(:,ndxMsgs(2)); 46 | m = pot*m2; 47 | nodes(ndxNodes(1)).msgIn(:,ndxMsgs(1)) = m/sum(m); 48 | m = m1'*pot; 49 | nodes(ndxNodes(2)).msgIn(:,ndxMsgs(2)) = m/sum(m); 50 | end 51 | case 3 52 | m1 = nodes(ndxNodes(1)).msgOut(:,ndxMsgs(1)); 53 | m2 = nodes(ndxNodes(2)).msgOut(:,ndxMsgs(2)); 54 | m3 = nodes(ndxNodes(3)).msgOut(:,ndxMsgs(3)); 55 | m = zeros(size(pot,1),1); 56 | for s = 1:size(pot,1) 57 | m(s) = m2'*squeeze(pot(s,:,:))*m3; 58 | end 59 | nodes(ndxNodes(1)).msgIn(:,ndxMsgs(1)) = m/sum(m); 60 | m = zeros(size(pot,2),1); 61 | for s = 1:size(pot,2) 62 | m(s) = m1'*squeeze(pot(:,s,:))*m3; 63 | end 64 | nodes(ndxNodes(2)).msgIn(:,ndxMsgs(2)) = m/sum(m); 65 | m = zeros(size(pot,3),1); 66 | for s = 1:size(pot,3) 67 | m(s) = m1'*squeeze(pot(:,:,s))*m2; 68 | end 69 | nodes(ndxNodes(3)).msgIn(:,ndxMsgs(3)) = m/sum(m); 70 | otherwise 71 | error('BP can only handle 2 or 3 node factors for now.'); 72 | end 73 | end 74 | end 75 | 76 | % Compute beliefs: 77 | for j = 1:Nnodes 78 | msg = sum(log(nodes(j).msgIn),2); 79 | 80 | % Normalize messages: 81 | msg = exp(msg-max(msg)); 82 | beliefs(j).bel = msg/sum(msg); 83 | end 84 | -------------------------------------------------------------------------------- /subDivideMesh.m: -------------------------------------------------------------------------------- 1 | function mesh = subDivideMesh(mesh,levels) 2 | 3 | if nargin < 2 4 | levels = 1; 5 | end 6 | 7 | for i = 1:levels 8 | Nvertices = size(mesh.vertices,2); 9 | Nfaces = size(mesh.faces,2); 10 | v1 = [(mesh.vertices(1,mesh.faces(1,:))+mesh.vertices(1,mesh.faces(2,:)))/2; ... 11 | (mesh.vertices(2,mesh.faces(1,:))+mesh.vertices(2,mesh.faces(2,:)))/2; ... 12 | (mesh.vertices(3,mesh.faces(1,:))+mesh.vertices(3,mesh.faces(2,:)))/2]; 13 | 14 | v2 = [(mesh.vertices(1,mesh.faces(2,:))+mesh.vertices(1,mesh.faces(3,:)))/2; ... 15 | (mesh.vertices(2,mesh.faces(2,:))+mesh.vertices(2,mesh.faces(3,:)))/2; ... 16 | (mesh.vertices(3,mesh.faces(2,:))+mesh.vertices(3,mesh.faces(3,:)))/2]; 17 | 18 | v3 = [(mesh.vertices(1,mesh.faces(3,:))+mesh.vertices(1,mesh.faces(1,:)))/2; ... 19 | (mesh.vertices(2,mesh.faces(3,:))+mesh.vertices(2,mesh.faces(1,:)))/2; ... 20 | (mesh.vertices(3,mesh.faces(3,:))+mesh.vertices(3,mesh.faces(1,:)))/2]; 21 | 22 | [v,ii,jj] = unique([v1 v2 v3]','rows' ); 23 | 24 | m1 = int32(jj(1:Nfaces)+Nvertices)'; 25 | m2 = int32(jj(Nfaces+1:2*Nfaces)+Nvertices)'; 26 | m3 = int32(jj(2*Nfaces+1:3*Nfaces)+Nvertices)'; 27 | 28 | tri1 = [mesh.faces(1,:); m1; m3]; 29 | tri2 = [mesh.faces(2,:); m2; m1]; 30 | tri3 = [m1; m2; m3]; 31 | tri4 = [m2; mesh.faces(3,:); m3]; 32 | 33 | mesh.vertices = [mesh.vertices v']; % the new vertices 34 | mesh.faces = [tri1 tri2 tri3 tri4]; % the new faces 35 | mesh.objndx = repmat(mesh.objndx,1,4); 36 | end 37 | 38 | % $$$ [vertices,faces,ndx] = subDivideMesh_helper(mesh.vertices,mesh.faces); 39 | % $$$ 40 | % $$$ mesh.vertices = vertices; 41 | % $$$ mesh.faces = faces; 42 | % $$$ mesh.objndx = mesh.objndx(ndx); 43 | 44 | % Invalidate texture coordinates: 45 | if isfield(mesh,'tx') 46 | mesh = rmfield(mesh,'tx'); 47 | end 48 | 49 | 50 | return; 51 | 52 | i = 6; 53 | mesh_i = getObjectMesh(mesh,i); 54 | 55 | 56 | addpath '~/work/MatlabLibraries/LabelMeToolbox'; 57 | addpath '~/work/MatlabLibraries/LabelMe3dToolbox'; 58 | addpath './NewCodeToInclude'; 59 | 60 | load foo.mat; 61 | 62 | mesh_out = subDivideMesh(mesh_i); 63 | 64 | figure; 65 | trisurf(mesh_out.faces',mesh_out.vertices(1,:),mesh_out.vertices(2,:),mesh_out.vertices(3,:)); 66 | axis equal; 67 | xlabel('X'); 68 | ylabel('Y'); 69 | zlabel('Z'); 70 | cameratoolbar('setmodeGUI','orbit'); 71 | 72 | figure; 73 | trisurf(mesh_i.faces',mesh_i.vertices(1,:),mesh_i.vertices(2,:),mesh_i.vertices(3,:)); 74 | axis equal; 75 | xlabel('X'); 76 | ylabel('Y'); 77 | zlabel('Z'); 78 | cameratoolbar('setmodeGUI','orbit'); 79 | -------------------------------------------------------------------------------- /LMplot3Dscene.m: -------------------------------------------------------------------------------- 1 | function LMplot3Dscene(annotation,img,varargin) 2 | % Displays the 3D scene. 3 | % 4 | % The following plots 3D polygons: 5 | % 6 | % LMplot3Dscene(annotation) 7 | % 8 | % The following plots a 3D textured model: 9 | % 10 | % LMplot3Dscene(annotation,img) 11 | % 12 | % You can specify the min/max depth range: 13 | % 14 | % minDepth = 1; maxDepth = 10000; 15 | % LMplot3Dscene(annotation,img,minDepth,maxDepth) 16 | 17 | if nargin > 1 18 | PlotTexturedScene(annotation,img,varargin{:}); 19 | return; 20 | end 21 | 22 | P = getCameraMatrix(annotation,'RH'); 23 | if isempty(P) 24 | display('Cannot plot since there is no camera matrix associated with this annotation file.'); 25 | return; 26 | end 27 | [K,R,C] = decomposeP(P); 28 | 29 | Ncolors = 10; 30 | colors = hsv(Ncolors); 31 | plot3(C(1),C(2),C(3),'ko'); 32 | hold on; 33 | plot3(0,0,0,'ko'); 34 | plot3([C(1); 0],[C(2); 0],[C(3); 0],'k'); 35 | objStrs = []; 36 | h = []; 37 | for i = 1:length(annotation.object) 38 | if isfield(annotation.object(i).world3d,'polygon3d') && ~isempty(annotation.object(i).world3d.polygon3d) 39 | [X,Y,Z] = getLMpolygon3D(annotation.object(i).world3d.polygon3d); 40 | hi = plot3([X; X([2:end 1])]',[Y; Y([2:end 1])]',[Z; Z([2:end 1])]','Color',colors(mod(i-1,Ncolors)+1,:)); 41 | h(end+1) = hi(1); 42 | objStrs{end+1} = annotation.object(i).name; 43 | end 44 | end 45 | axis equal; 46 | xlabel('X'); 47 | ylabel('Y'); 48 | zlabel('Z'); 49 | legend(h,objStrs); 50 | cameratoolbar('setmodeGUI','orbit'); 51 | 52 | function PlotTexturedScene(annotation,img,minDepth,maxDepth) 53 | 54 | % Get depth maps: 55 | [Xmap,Ymap,Zmap,Nmap] = getXYZmaps(annotation); 56 | Xmap = double(Xmap); 57 | Ymap = double(Ymap); 58 | Zmap = double(Zmap); 59 | 60 | % Get min/max depth to display: 61 | MAX_DEPTH = 16777215; 62 | zz = Zmap(Zmap~=MAX_DEPTH); 63 | if nargin < 4 64 | maxDepth = prctile(zz,98); 65 | % $$$ maxDepth = 10000; 66 | end 67 | if nargin < 3 68 | minDepth = prctile(zz,2); 69 | % $$$ minDepth = 1; 70 | end 71 | 72 | % Show warped image: 73 | seg = getPolyEdgeTypes(annotation,img,'occluded_edges'); 74 | mask = (Zmap>minDepth).*(ZmapminDepth).*(Zmap0); 77 | % $$$ mask = (Zmap>minDepth).*(Zmap=2 32 | objectDepthMaps = MAX_DEPTH*ones(nrows,ncols,Nobjects,'uint32'); 33 | end 34 | for i = 1:Nobjects 35 | if isfield(annotation.object(i),'mesh') && ~isempty(annotation.object(i).mesh) 36 | [X,Y] = getLMpolygon(annotation.object(i).polygon); 37 | v2D = annotation.object(i).mesh.v2D; 38 | v3D = annotation.object(i).mesh.v3D; 39 | n = find(inpolygon(mx,my,X,Y)); 40 | % $$$ z = uint32(griddata(v2D(1,:),v2D(2,:),v3D(3,:),mx(n),my(n))); 41 | 42 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 43 | % This part should be abstracted since it also appears in 44 | % InflateScenesNew 45 | if ismember(i,groundObjects) 46 | [xx,yy,z] = Image2WorldCoords(mx(n),my(n),[],[],[nrows ncols],CAM_H,Hy,F); 47 | z = uint32(z); 48 | else 49 | % Get contact points: 50 | CptsNdx = [annotation.object(i).polygon.pt.contact]'; 51 | [x,y,CptsNdx] = addVerticalPoint(X,Y,unique(X),CptsNdx); 52 | j = find(CptsNdx); 53 | xc = x(j); 54 | yc = y(j); 55 | 56 | % Need to have unique xc, else interp1 does not work. Need to handle 57 | % this in a better way. 58 | [xc,j] = unique(xc); 59 | yc = yc(j); 60 | 61 | % Convert image coordinates into world coordinates: 62 | [xx,yy,z] = Image2WorldCoords(mx(n),my(n),xc,yc,[nrows ncols],CAM_H,Hy,F); 63 | z = uint32(z); 64 | end 65 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 66 | 67 | if nargout>=2 68 | objectDepthMaps(n+(i-1)*nrows*ncols) = z; 69 | end 70 | Zmap(n) = min(Zmap(n),z); 71 | end 72 | end 73 | 74 | if nargout==0 75 | plotDepthMap(Zmap); 76 | end 77 | -------------------------------------------------------------------------------- /myLMimscale.m: -------------------------------------------------------------------------------- 1 | function [annotation,img] = myLMimscale(annotation,img,scale,method) 2 | 3 | % Store the original numerical format of the image and turn it into 'double'. 4 | imgtype = whos('img'); 5 | img = single(img); 6 | 7 | if nargin < 4 8 | method = 'nearest'; 9 | end 10 | 11 | if scale ~= 1 12 | if nargout > 1 13 | % Image resampling: 14 | img = imresizefast(img, scale, method); 15 | end 16 | 17 | % add/modify image size field 18 | annotation.imagesize.nrows = num2str(size(img,1)); 19 | annotation.imagesize.ncols = num2str(size(img,2)); 20 | 21 | % $$$ [annotation,img] = LMimscale(annotation,img,scale); 22 | % $$$ if isnumeric(annotation.imagesize.nrows) 23 | % $$$ annotation.imagesize.nrows = num2str(annotation.imagesize.nrows); 24 | % $$$ end 25 | % $$$ if isnumeric(annotation.imagesize.ncols) 26 | % $$$ annotation.imagesize.ncols = num2str(annotation.imagesize.ncols); 27 | % $$$ end 28 | 29 | if isfield(annotation, 'object') 30 | for i = 1:length(annotation.object) 31 | [x,y] = getLMpolygon(annotation.object(i).polygon); 32 | x = round(x*scale); 33 | y = round(y*scale); 34 | annotation.object(i).polygon.x = x(:); 35 | annotation.object(i).polygon.y = y(:); 36 | 37 | if isfield(annotation.object(i).polygon,'contact') 38 | [xc,yc] = getContactPoints(annotation.object(i).polygon); 39 | xc = round(xc*scale); 40 | yc = round(yc*scale); 41 | for j = 1:length(xc) 42 | annotation.object(i).polygon.contact(j).x = num2str(xc(j)); 43 | annotation.object(i).polygon.contact(j).y = num2str(yc(j)); 44 | end 45 | end 46 | end 47 | end 48 | 49 | if isfield(annotation,'camera') 50 | if isfield(annotation.camera,'Hy') 51 | Hy = str2num(annotation.camera.Hy); 52 | annotation.camera.Hy = num2str(Hy*scale); 53 | end 54 | if isfield(annotation.camera,'F') 55 | F = str2num(annotation.camera.F); 56 | annotation.camera.F = num2str(F*scale); 57 | end 58 | end 59 | end 60 | 61 | if nargout > 1 62 | % return the image in its original numeric format 63 | img = feval(imgtype.class, img); 64 | end 65 | 66 | function img = imresizefast(img, scaling, method, init); 67 | 68 | if nargin<4 69 | init = 0; 70 | end 71 | 72 | if scaling > .5 73 | img = imresize(img, scaling, method); 74 | else 75 | c = size(img,3); 76 | for n = 1:c 77 | img(:,:,n) = conv2(img(:,:,n), [1 2 1; 2 4 2; 1 2 1]/16, 'same'); 78 | end 79 | img = img(init+1:2:end, init+1:2:end, :); 80 | %img = convn(img, [1 2 1]/4, 'same'); 81 | %img = img(:,init+1:2:end,:); 82 | img = imresizefast(img, 2*scaling, method, 1-init); 83 | end 84 | -------------------------------------------------------------------------------- /getObject3D.m: -------------------------------------------------------------------------------- 1 | function [X,Y,Z,mx,my,tri,boundary,CptsNdx,valid] = getObject3D(annotation,i,meshtype,mx,my) 2 | % Inputs: 3 | % annotation - LabelMe annotation structure. 4 | % i - Object index 5 | % meshtype - 'grid' or 'trimesh' or 'poly' or 'mymesh' 6 | % 7 | % Outputs: 8 | % X 9 | % Y 10 | % Z 11 | % mx 12 | % my 13 | % tri 14 | % boundary 15 | % CptsNdx 16 | % valid - Indicates whether 3D points are valid. 17 | 18 | npts = 11; % Trimesh parameter 19 | 20 | % Get camera matrix: 21 | P = getCameraMatrix(annotation,'RH'); 22 | Hy = str2num(annotation.camera.Hy); 23 | 24 | nrows = str2num(annotation.imagesize.nrows); 25 | ncols = str2num(annotation.imagesize.ncols); 26 | 27 | % Get polygon: 28 | [x,y] = getLMpolygon(annotation.object(i).polygon); 29 | 30 | % Get mesh for object: 31 | switch meshtype 32 | case 'grid' 33 | % $$$ [mx,my] = meshgrid([1:ncols],[1:nrows]); 34 | % $$$ n = find(inpolygon(mx,my,x,y)); 35 | bb = [max(1,floor(min(x))) min(ncols,ceil(max(x))) max(1,floor(min(y))) min(nrows,ceil(max(y)))]; 36 | [mx,my] = meshgrid([bb(1):bb(2)],[bb(3):bb(4)]); 37 | n = inpolygon(mx,my,x,y); 38 | mx = mx(n); 39 | my = my(n); 40 | tri = []; 41 | boundary = []; 42 | case 'trimesh' 43 | [mx,my] = addVerticalPoint(x,y,unique(x)); 44 | [mx,my,tri,boundary] = getMesh(mx,my,npts); 45 | case 'poly' 46 | [mx,my] = addVerticalPoint(x,y,unique(x)); 47 | tri = []; 48 | boundary = []; 49 | case 'mymesh' 50 | tri = []; 51 | boundary = []; 52 | otherwise 53 | error('Invalid meshtype'); 54 | end 55 | 56 | % Get 3D coordinates: 57 | X = []; Y = []; Z = []; valid = 0; 58 | CptsNdx = []; 59 | switch annotation.object(i).polygon.polyType 60 | case 'ground' 61 | [X,Y,Z,valid] = Image2WorldCoords(mx,my,[],[],P); 62 | case 'standing' 63 | % Get contact points: 64 | [xc,yc] = getLMcontact(annotation.object(i).polygon); 65 | % Objects above the horizon line can not be inflated: 66 | if (max(y)>Hy) && ~isempty(xc) 67 | % Convert image coordinates into world coordinates: 68 | [X,Y,Z,valid] = Image2WorldCoords(mx,my,xc,yc,P); 69 | end 70 | case 'part' 71 | % Get parent object: 72 | parents = getPartOfParents(annotation,i); 73 | if ~isempty(parents) 74 | parentNdx = parents(end); 75 | switch annotation.object(parentNdx).polygon.polyType 76 | case 'ground' 77 | [X,Y,Z,valid] = Image2WorldCoords(mx,my,[],[],P); 78 | case 'standing' 79 | % Get contact points: 80 | [xc,yc] = getLMcontact(annotation.object(parentNdx).polygon); 81 | [x,y] = getLMpolygon(annotation.object(parentNdx).polygon); 82 | % Objects above the horizon line can not be inflated: 83 | if (max(y)>Hy) && ~isempty(xc) 84 | % Convert image coordinates into world coordinates: 85 | [X,Y,Z,valid] = Image2WorldCoords(mx,my,xc,yc,P); 86 | end 87 | end 88 | end 89 | end 90 | -------------------------------------------------------------------------------- /supportGraph.m: -------------------------------------------------------------------------------- 1 | function [Psup,objectClasses] = supportGraph(DB, objectClasses, Ppart) 2 | % 3 | % Inputs: 4 | % DB - LabelMe DB structure. 5 | % 6 | % Outputs: 7 | % Psup(i,j) - Probability of object class Oi supports Oi. 8 | % objectClasses - Cell array of object classes corresponding to the 9 | % rows/cols of Psup. 10 | 11 | % Parameters: 12 | thresh_par = 0.5; % parts threshold (to collect local evidences) 13 | thresh_sup = 25; % support threshold (to collect local evidences) 14 | alpha = 5; % pseudocounts 15 | 16 | % Only use images that are mostly labeled: 17 | %DB = DB(LMlabeledarea(DB)>=thresh_labeled_area); 18 | 19 | % Extract object classes: 20 | if nargin == 1 21 | objectClasses = LMobjectnames(DB,'name'); 22 | end 23 | objectClasses = lower(objectClasses); 24 | Nobjects = length(objectClasses); 25 | Nimages = length(DB); 26 | 27 | % Compute part probabilities 28 | PC_s = zeros([Nobjects, Nobjects]); % P(overlap | no part) 29 | Counts = zeros([Nobjects, Nobjects]); % P(overlap | no part) 30 | for i = 1:Nimages 31 | display(sprintf('%d out of %d',i,Nimages)); 32 | 33 | %keyboard 34 | 35 | if isfield(DB(i).annotation,'object') 36 | % Get object indices for this image 37 | Nobj = length(DB(i).annotation.object); 38 | clear ndx X Y x y 39 | for j = 1:Nobj 40 | ndx(j) = strmatch(strtrim(lower(DB(i).annotation.object(j).name)), objectClasses, 'exact'); 41 | [X{j},Y{j}] = getLMpolygon(DB(i).annotation.object(j).polygon); 42 | [x(j),y(j)] = GetBottomPoint(X{j},Y{j}); 43 | end 44 | 45 | % Get Pso for this image 46 | for m = 1:Nobj 47 | for cl = unique(ndx) % loop on classes in this image (each class counts only once as background) 48 | if Ppart(cl,ndx(m))<.9 49 | n = find(ndx==cl); 50 | 51 | clear C 52 | for k = 1:length(n) 53 | C(k) = inpolygon(x(m),y(m), X{n(k)},Y{n(k)}); 54 | end 55 | C = sum(C)>0; 56 | 57 | n = n(1); 58 | PC_s(ndx(n),ndx(m)) = PC_s(ndx(n),ndx(m))+C; 59 | Counts(ndx(n),ndx(m)) = Counts(ndx(n),ndx(m))+1; 60 | end 61 | end 62 | end 63 | end 64 | end 65 | 66 | Psup = PC_s./(Counts+eps); 67 | Psup = Psup .* (Counts>5); 68 | Psup = Psup - diag(diag(Psup)); 69 | 70 | % Just for visualization: 71 | [i,j] = find(Psup>.5 & Counts>10); %only show results when there are enough samples 72 | v = Psup(sub2ind(size(Psup),i,j)); 73 | [v,n] = sort(v, 'descend'); 74 | for k=n' 75 | disp(sprintf('%s supports %s (with probability = %1.2f, estimated from %d samples)', objectClasses{i(k)}, objectClasses{j(k)}, Psup(i(k),j(k)), Counts(i(k),j(k)))) 76 | end 77 | -------------------------------------------------------------------------------- /insertGroundPlane.m: -------------------------------------------------------------------------------- 1 | function annotation = insertGroundPlane(annotation) 2 | % insertGroundPlane - Inserts a new object called "ground plane". 3 | 4 | % Get indices of non-deleted objects: 5 | notDeleted = find(~isdeleted(annotation))'; 6 | 7 | % Get last object ID: 8 | maxID = annotation.object(end).id; 9 | 10 | % Get horizon line: 11 | [xh,yh] = getHorizonLine(annotation); 12 | 13 | % Get image size: 14 | imageSize = [str2num(annotation.imagesize.nrows) str2num(annotation.imagesize.ncols)]; 15 | 16 | annotation.object(end+1).name = 'ground'; 17 | annotation.object(end).deleted = '0'; 18 | annotation.object(end).verified = '0'; 19 | annotation.object(end).date = datestr(now); 20 | annotation.object(end).id = num2str(str2num(maxID)+1); 21 | annotation.object(end).polygon.pt = []; 22 | annotation.object(end).polygon.pt(1).x = num2str(xh(1)); 23 | annotation.object(end).polygon.pt(1).y = num2str(yh(1)+1); 24 | annotation.object(end).polygon.pt(2).x = num2str(xh(2)); 25 | annotation.object(end).polygon.pt(2).y = num2str(yh(2)+1); 26 | annotation.object(end).polygon.pt(3).x = num2str(imageSize(2)); 27 | annotation.object(end).polygon.pt(3).y = num2str(imageSize(1)); 28 | annotation.object(end).polygon.pt(4).x = num2str(1); 29 | annotation.object(end).polygon.pt(4).y = num2str(imageSize(1)); 30 | 31 | % Insert 3D information: 32 | annotation.object(end).world3d.type = 'groundplane'; 33 | annotation.object(end).world3d.stale = '0'; 34 | 35 | [x,y] = getLMpolygon(annotation.object(end).polygon); 36 | X = projectOntoGroundPlane(x,y,annotation); 37 | annotation.object(end).world3d.polygon3d = setLMpolygon3D(X(1,:),X(2,:),X(3,:)); 38 | 39 | % Get plane parameters: 40 | annotation.object(end).world3d.plane.pix = '0'; 41 | annotation.object(end).world3d.plane.piy = '1'; 42 | annotation.object(end).world3d.plane.piz = '0'; 43 | annotation.object(end).world3d.plane.piw = '0'; 44 | 45 | % Change all ground objects to be attached to ground plane: 46 | for i = notDeleted 47 | if strcmp(annotation.object(i).world3d.type,'groundplane') 48 | annotation.object(i).world3d.type = 'part'; 49 | annotation.object(i).world3d.parentid = annotation.object(end).id; 50 | annotation.object(i).world3d.rootid = annotation.object(end).id; 51 | for j = 1:length(annotation.object(i).world3d.polygon3d.pt) 52 | annotation.object(i).world3d.polygon3d.pt(j).added = '0'; 53 | annotation.object(i).world3d.polygon3d.pt(j).planeindex.index = '0'; 54 | end 55 | annotation.object(i).world3d = rmfield(annotation.object(i).world3d,'plane'); 56 | annotation.object(i).world3d = orderfields(annotation.object(i).world3d,{'type','stale','parentid','rootid','polygon3d'}); 57 | end 58 | end 59 | 60 | % Update rootid tags: 61 | for i = notDeleted 62 | if strcmp(annotation.object(i).world3d.type,'part') 63 | % Get root object index: 64 | k = getPartOfParents(annotation,i); 65 | nRoot = k(end); 66 | 67 | % Set root ID: 68 | annotation.object(i).world3d.rootid = annotation.object(nRoot).id; 69 | end 70 | end 71 | 72 | return; 73 | -------------------------------------------------------------------------------- /getTextures.m: -------------------------------------------------------------------------------- 1 | function [mesh,tn] = getTextures(mesh,annotation,img) 2 | % Gets textures for the objects. 3 | % 4 | % Inputs: 5 | % mesh - Triangular mesh (note: mesh.vertices has 3xN entries) 6 | % annotation - LabelMe annotation structure 7 | % img - Image 8 | % 9 | % Outputs: 10 | % mesh.textures - Cell array of object textures. 11 | % mesh.tx - 2xN texture coordinates corresponding to 3D vertices. 12 | % tn - 1xN texture indices 13 | % 14 | % 15 | % Example: 16 | % 17 | % mesh = getTextures(mesh,annotation,img); 18 | % 19 | % Display mesh points on texture: 20 | % 21 | % i - Desired object index. 22 | % n = find(mesh.objndx==i); 23 | % figure; 24 | % imshow(mesh.textures{i}); 25 | % hold on; 26 | % triplot(mesh.faces(:,n)',mesh.tx(1,:),mesh.tx(2,:)); 27 | 28 | P = getCameraMatrix(annotation); 29 | imageSize = [size(img,1) size(img,2)]; 30 | 31 | maxRes = 360;%640; 32 | if max(imageSize)>maxRes 33 | scaleFactor = maxRes/max(imageSize); 34 | P = [scaleFactor 0 0; 0 scaleFactor 0; 0 0 1]*P; 35 | img = imresize(img,scaleFactor,'bicubic'); 36 | imageSize = [size(img,1) size(img,2)]; 37 | end 38 | 39 | [K,R,C] = decomposeP(P); 40 | 41 | % Ray-cast: 42 | [x,y] = meshgrid(1:imageSize(2),1:imageSize(1)); 43 | x = [x(:) y(:) ones(prod(size(x)),1)]'; 44 | [X,isValid,faceNdx] = rayTrace(single(P(:,1:3)\x),single(C),mesh.vertices,mesh.faces); 45 | 46 | % Get object map: 47 | isValid = reshape(isValid,imageSize); 48 | faceNdx = reshape(faceNdx,imageSize); 49 | objMap = zeros(imageSize); 50 | objMap(isValid) = mesh.objndx(faceNdx(isValid)); 51 | 52 | % Get textures: 53 | N = size(mesh.vertices,2); 54 | objNdx = unique(mesh.objndx); 55 | mesh.textures = cell(1,length(annotation.object)); 56 | mesh.tx = zeros(2,N,'single'); 57 | tn = zeros(1,N); 58 | for i = objNdx 59 | % Project mesh points: 60 | n = mesh.faces(:,find(mesh.objndx==i)); 61 | n = double(unique(n(:)))'; 62 | [xi,yi] = Project3D2D(mesh.vertices(:,n),P,imageSize); 63 | 64 | % Get bounding box: 65 | bb = [max(1,floor(min(xi))) min(imageSize(2),ceil(max(xi))) max(1,floor(min(yi))) min(imageSize(1),ceil(max(yi)))]; 66 | 67 | % Get crops: 68 | imgCrop = img(bb(3):bb(4),bb(1):bb(2),:); 69 | objMapCrop = objMap(bb(3):bb(4),bb(1):bb(2)); 70 | 71 | % $$$ figure; 72 | % $$$ imshow(imgCrop); 73 | % $$$ figure; 74 | % $$$ imagesc(objMapCrop); 75 | 76 | % Get average color within object: 77 | imgCrop_r = double(squeeze(imgCrop(:,:,1))); 78 | imgCrop_g = double(squeeze(imgCrop(:,:,2))); 79 | imgCrop_b = double(squeeze(imgCrop(:,:,3))); 80 | colorAvg = uint8([mean(imgCrop_r(objMapCrop==i)) mean(imgCrop_g(objMapCrop==i)) mean(imgCrop_b(objMapCrop==i))]); 81 | 82 | % Fill non-object pixels with average color: 83 | imgCrop_r(objMapCrop~=i) = colorAvg(1); 84 | imgCrop_g(objMapCrop~=i) = colorAvg(2); 85 | imgCrop_b(objMapCrop~=i) = colorAvg(3); 86 | imgCrop = uint8(reshape([imgCrop_r imgCrop_g imgCrop_b],size(imgCrop))); 87 | 88 | % Get texture coordinates: 89 | mesh.tx(:,n) = [xi-bb(1); yi-bb(3)]; 90 | tn(n) = i; 91 | mesh.textures{i} = imgCrop; 92 | 93 | % $$$ figure; 94 | % $$$ imshow(imgCrop); 95 | % $$$ hold on; 96 | % $$$ plot(xi,yi,'r.'); 97 | end 98 | -------------------------------------------------------------------------------- /generateO3D.m: -------------------------------------------------------------------------------- 1 | function generateO3D(mesh,outfname) 2 | % Inputs: 3 | % mesh 4 | % outfname 5 | 6 | objNdx = 1:max(mesh.objndx);%unique(mesh.objndx); 7 | Nobjects = length(objNdx); 8 | fp = fopen(outfname,'w'); 9 | fprintf(fp,'%d',Nobjects); 10 | for i = 1:Nobjects 11 | mesh_i = getObjectMesh(mesh,objNdx(i)); 12 | if ~isempty(mesh_i.vertices) 13 | mesh_i.tx(1,:) = mesh_i.tx(1,:)/size(mesh_i.textures{objNdx(i)},2); 14 | mesh_i.tx(2,:) = 1-mesh_i.tx(2,:)/size(mesh_i.textures{objNdx(i)},1); 15 | end 16 | if ~isempty(mesh_i.vertices) 17 | fprintf(fp,',%d,%d,',size(mesh_i.vertices,2),size(mesh_i.faces,2)); 18 | fprintf(fp,'%.4f,%.4f,%.4f,',mesh_i.vertices); 19 | fprintf(fp,'%.4f,%.4f,',mesh_i.tx); 20 | fprintf(fp,'%d,%d,%d,',mesh_i.faces(:,1:end-1)-1); 21 | fprintf(fp,'%d,%d,%d',mesh_i.faces(:,end)-1); 22 | else 23 | fprintf(fp,',%d,%d',0,0); 24 | end 25 | end 26 | fclose(fp); 27 | 28 | return; 29 | 30 | function generateO3D_old(annotation,img,outfname) 31 | % Inputs: 32 | % annotation 33 | % img 34 | % outfname 35 | 36 | % Parameters: 37 | scaling = 1/100; % Convert to meters 38 | 39 | % Get meshes: 40 | [X,Y,Z,TX,TY,tri,boundary,textures,objNames] = getTextures(annotation,img); 41 | 42 | Nobjects = length(X); 43 | fp = fopen(outfname,'w'); 44 | fprintf(fp,'%d',Nobjects); 45 | for i = 1:Nobjects 46 | fprintf(fp,',%d,%d,',length(X{i}),length(tri{i})); 47 | for j = 1:length(X{i}) 48 | fprintf(fp,'%.4f,%.4f,%.4f,',X{i}(j)*scaling,Y{i}(j)*scaling,-Z{i}(j)*scaling); 49 | end 50 | for j = 1:length(TX{i}) 51 | fprintf(fp,'%.4f,%.4f,',TX{i}(j),1-TY{i}(j)); 52 | end 53 | for j = 1:size(tri{i},1)-1 54 | fprintf(fp,'%d,%d,%d,',tri{i}(j,:)); 55 | end 56 | j = size(tri{i},1); 57 | fprintf(fp,'%d,%d,%d',tri{i}(j,:)); 58 | end 59 | fclose(fp); 60 | 61 | return; 62 | 63 | % Demo script: 64 | 65 | addpath '~/work/MatlabLibraries/LabelMeToolbox'; 66 | addpath '~/work/MatlabLibraries/LabelMe3dToolbox'; 67 | 68 | fname_jpg = 'example.jpg'; 69 | fname_xml = 'example.out.xml'; 70 | fname_Z = 'example.Z.png'; 71 | img = imread(fname_jpg); 72 | annotation = getfield(loadXML(fname_xml),'annotation'); 73 | Zmap = readZmap(fname_Z); 74 | figure; LMplot(annotation,img); title('Image with annotations'); 75 | figure; plotZmap(Zmap); title('Depth map'); 76 | 77 | % Create object meshes (we need this for VRML and book outputs): 78 | annotation = InflateSceneNew(annotation); 79 | 80 | OUTDIR = './VRML'; 81 | % Create VRML file: 82 | vrmlfilename = 'example.wrl'; 83 | imwrite(img,fullfile(OUTDIR,'example.jpg'),'jpg','quality',95); 84 | LM2VRMLfill(annotation,OUTDIR,vrmlfilename,'example.jpg',img); 85 | 86 | o3dfilename = strrep(fname_jpg,'.jpg','_o3dmesh.txt'); 87 | generateO3D(annotation,img,o3dfilename); 88 | 89 | % Parameters: 90 | scaling = 1/100; % Convert to meters 91 | 92 | % Get camera parameters: 93 | fov = rad2deg(atan2(sqrt(size(img,1)^2+size(img,2)^2)/2,str2num(annotation.camera.F))); 94 | cameraEye = [0 scaling*str2num(annotation.camera.CAM_H) 0]; 95 | sceneUp = [0 1 0]; 96 | C = 100; % Direction constant 97 | cameraTarget = scaling*[0 str2num(annotation.camera.CAM_H)-C*(str2num(annotation.camera.Hy)-size(img,1)/2)/str2num(annotation.camera.F) -C]; 98 | -------------------------------------------------------------------------------- /collectEdgeEvidence.m: -------------------------------------------------------------------------------- 1 | function [evidence,annotation] = collectEdgeEvidence(annotation,validObjects) 2 | % Inputs: 3 | % annotation 4 | % validObjects - Vector of desired object indices to compute. 5 | % 6 | % Outputs: 7 | % annotation.object(i).polygon.edges 8 | % evidence.object(i).pBelow 9 | % evidence.object(i).pOri 10 | % evidence.object(i).pLen 11 | % evidence.object(i).pGround 12 | 13 | % Parameters: 14 | Nsamples = 10; 15 | 16 | if nargin < 2 17 | validObjects = 1:length(annotation.object); 18 | end 19 | 20 | Nobjects = length(annotation.object); 21 | if isfield(annotation,'imagesize') 22 | nrows = annotation.imagesize.nrows; 23 | ncols = annotation.imagesize.ncols; 24 | else 25 | [ncols,nrows] = getaproximagesize(annotation); 26 | end 27 | 28 | notDeleted = find(~isdeleted(annotation))'; 29 | 30 | % Get ground object indices: 31 | groundObjects = getListGroundObjects; 32 | groundObjects = sprintf('%s,', groundObjects{:}); groundObjects = groundObjects(1:end-1); 33 | groundObjects = LMobjectindex(annotation, groundObjects, 'exact'); 34 | groundObjects = groundObjects(ismember(groundObjects,notDeleted)); 35 | 36 | % Get polygons and add edges: 37 | for i = notDeleted 38 | [X{i},Y{i}] = getLMpolygon(annotation.object(i).polygon); 39 | end 40 | 41 | evidence = []; 42 | alpha = linspace(1,0,Nsamples); 43 | for i = validObjects 44 | if strcmp(annotation.object(i).polygon.polyType,'standing') 45 | if ~isfield(annotation.object(i).polygon,'edges') 46 | annotation.object(i).polygon.edges = [[1:length(X{i})]; [2:length(X{i}) 1]]; 47 | end 48 | 49 | otherGroundObjects = setdiff(groundObjects,i); 50 | edges = annotation.object(i).polygon.edges; 51 | Nedges = size(edges,2); 52 | 53 | % Do not consider as contact edge if there is a point below: 54 | pBelow = 1-double(isThereAnyPointBelowEdge(X{i},Y{i},edges)); 55 | 56 | % Measure edge verticality: 57 | pOri = 1-edgeVerticality(X{i},Y{i},edges)'; 58 | 59 | % Measure edge length width respect to object width: 60 | len = edgeLength(X{i},Y{i},edges)'; 61 | objWidth = max(X{i})-min(X{i}); 62 | pLen = min(len/objWidth,1); 63 | 64 | % Compute distance to ground object: 65 | pGround = zeros(1,Nedges); 66 | for j = 1:Nedges 67 | % Evenly sample over edge: 68 | e = edges(:,j); 69 | x = alpha*X{i}(e(1))+(1-alpha)*X{i}(e(2)); 70 | y = alpha*Y{i}(e(1))+(1-alpha)*Y{i}(e(2)); 71 | dGround = inf*ones(1,Nsamples); 72 | 73 | % Compute distance of edge samples to edges on other polygons: 74 | for ii = notDeleted 75 | if any(ii==otherGroundObjects) 76 | d = zeros(1,Nsamples); 77 | for jj = 1:Nsamples 78 | d(jj) = min(SupportDist(X{ii},Y{ii},x(jj),y(jj))/ncols,1); 79 | end 80 | dGround = min(d,dGround); 81 | end 82 | end 83 | 84 | % Compute probability of edge type by taking mean over edge 85 | % samples: 86 | pGround(j) = 1-mean(dGround); 87 | end 88 | 89 | % Store evidence: 90 | evidence.object(i).pBelow = pBelow; 91 | evidence.object(i).pOri = pOri; 92 | evidence.object(i).pLen = pLen; 93 | evidence.object(i).pGround = pGround; 94 | end 95 | end 96 | -------------------------------------------------------------------------------- /learnPartsParams.m: -------------------------------------------------------------------------------- 1 | function [par,N] = learnPartsParams(evidence,objectClasses) 2 | % Inputs: 3 | % evidence - Structure of evidence. 4 | % objectClasses - Cell array of object classes. 5 | % 6 | % Outputs: 7 | % P(i,j) - Probability that object class i is attached to object class 8 | % j. If j=length(objectClasses)+1, then this is probability of not being 9 | % attached to any object class. 10 | 11 | % Parameters: 12 | Niterations = 10; % Number of Gibbs sampling iterations 13 | alpha_r = 20; % Beta distribution prior for relative overlap 14 | alpha_a = 2;%0.8; % Beta distribution prior for relative area 15 | eta_a = 5; % Beta prior for no attachment 16 | eta_o = 0.5; % Dirichlet prior for which object attached to 17 | 18 | % Get number of objects and seed random number generator: 19 | Nobjects = length(objectClasses); 20 | rand('twister',sum(100*clock)); 21 | 22 | % Collect data point information: 23 | imgNdx = []; 24 | objNdx = []; 25 | for i = 1:length(evidence) 26 | if ~isempty(evidence(i).objNdx) 27 | n = find(evidence(i).objNdx>0); 28 | imgNdx = [imgNdx i*ones(1,length(n),'int32')]; 29 | objNdx = [objNdx int32(n)]; 30 | end 31 | end 32 | 33 | % Run Gibbs sampler: 34 | Ndata = length(imgNdx); 35 | N = zeros(Nobjects,Nobjects+2,'int32'); % Last two cols are detect counts 36 | q = -1*ones(1,Ndata,'int32'); 37 | for i = 1:Niterations 38 | display(sprintf('%d out of %d',i,Niterations)); 39 | rp = int32(randperm(Ndata)); 40 | [N,q] = partsGibbsSampler(evidence,imgNdx,objNdx,N,q,rp,alpha_r,alpha_a,eta_a,eta_o); 41 | end 42 | 43 | [Ndata sum(sum(N(:,1:end-1))) sum(sum(N(:,end-1:end)))] 44 | 45 | % Compute output parameters: 46 | P = double(N); 47 | P(:,end-1) = (P(:,end-1)+eta_a)./(P(:,end-1)+P(:,end)+eta_a); 48 | P(:,1:end-2) = P(:,1:end-2)+eta_o; 49 | P(:,1:end-2) = P(:,1:end-2)./repmat(sum(P(:,1:end-2),2),1,Nobjects); 50 | P(:,end) = []; 51 | 52 | par.P = P; 53 | par.alpha_r = alpha_r; 54 | par.alpha_a = alpha_a; 55 | par.eta_a = eta_a; 56 | par.eta_o = eta_o; 57 | 58 | return; 59 | 60 | %%%% TEST %%%% 61 | 62 | [v,n] = sort(Ppart.P(:,end)); 63 | for i = 1:200 64 | display(sprintf('%s: %f',objectClasses{n(i)},1-v(i))); 65 | [v2,n2] = sort(Ppart.P(n(i),1:Nobjects),'descend'); 66 | for j = 1:5 67 | if v2(j) > 10*eta_o/Nobjects 68 | display(sprintf('-> %s: %f',objectClasses{n2(j)},v2(j))); 69 | end 70 | end 71 | pause; 72 | end 73 | 74 | % license plate: 435 75 | [v,n] = sort(P(435,1:Nobjects),'descend'); 76 | 77 | % Visualize beta distribution 78 | figure; 79 | alpha_a = 10;%0.8; 80 | x = [0:0.01:1]; 81 | p = alpha_a*x.^(alpha_a-1); 82 | plot(x,p); 83 | axis([0 1 0 max(p)]); 84 | 85 | % Collect evidence statistics: 86 | c = 793; 87 | for i = 1:length(evidence) 88 | objNdx = evidence(i).objNdx; 89 | relativeArea = evidence(i).relativeArea; 90 | n = find(objNdx==c); 91 | for j = 1:length(n) 92 | relativeOverlap = evidence(i).relativeOverlap(n(j),:); 93 | [v,k] = sort(relativeOverlap,'descend'); 94 | v(k==n(j)) = []; 95 | k(k==n(j)) = []; 96 | display(sprintf('%s: %f; %s: %f',objectClasses{objNdx(k(1))},v(1),objectClasses{objNdx(k(2))},v(2))); 97 | pause; 98 | end 99 | end 100 | -------------------------------------------------------------------------------- /showPartsTree.m: -------------------------------------------------------------------------------- 1 | function showPartsTree(Ppart, partObjectClasses, object, DB) 2 | % 3 | % showPartsTree(Ppart, partObjectClasses) 4 | % 5 | % showPartsTree(Ppart, partObjectClasses, object) 6 | 7 | th = .75; 8 | 9 | ndx = strmatch(lower(object), lower(partObjectClasses), 'exact'); 10 | 11 | % search children (parts of object) 12 | children = find(Ppart(ndx,:)>th); 13 | [foo,j] = sort(Ppart(ndx,children), 'descend'); 14 | children = children(j); 15 | 16 | % search (object belong to) 17 | parents = find(Ppart(:,ndx)>th); 18 | [foo,j] = sort(Ppart(parents,ndx), 'descend'); 19 | parents = parents(j); 20 | 21 | N = max(length(parents), length(children)); 22 | xo = 1; yo = N/2; 23 | 24 | [foo,n] = sort(Ppart(ndx,:), 'descend'); 25 | for k=children 26 | disp(sprintf('%s is part of %s (with probability = %1.2f)', partObjectClasses{k}, object, Ppart(ndx,k))) 27 | end 28 | 29 | figure 30 | hold on 31 | axis([-1 3 0 N+1]) 32 | for c = 1:length(children) 33 | xc = 2; yc = N/2-(length(children)+1)/2+c; 34 | plot([xo xc], [yo yc], 'k') 35 | end 36 | for c = 1:length(parents) 37 | xc = 0; yc = N/2-(length(parents)+1)/2+c; 38 | plot([xo xc], [yo yc], 'k') 39 | end 40 | for c = 1:length(children) 41 | xc = 2; yc = N/2-(length(children)+1)/2+c; 42 | text (xc, yc, strrep(partObjectClasses{children(c)}, '_', ' '), 'HorizontalAlignment', 'center', 'VerticalAlignment', 'middle', 'BackgroundColor', [1 1 0], 'EdgeColor','k') 43 | end 44 | for c = 1:length(parents) 45 | xc = 0; yc = N/2-(length(parents)+1)/2+c; 46 | text (xc, yc, strrep(partObjectClasses{parents(c)}, '_', ' '), 'HorizontalAlignment', 'center', 'VerticalAlignment', 'middle', 'BackgroundColor', [1 .2 0], 'EdgeColor','k') 47 | end 48 | text (xo, yo, object, 'HorizontalAlignment', 'center', 'VerticalAlignment', 'middle', 'BackgroundColor', [1 .7 0], 'EdgeColor','k') 49 | axis('off') 50 | 51 | 52 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 53 | if nargin > 3 54 | % Show examples 55 | [foo,j] = LMquery(DB, 'object.name', object); 56 | [foo,jp] = LMquery(DB(j), 'object.name', partObjectClasses{children(1)}); 57 | 58 | figure 59 | for m = 1:min(4, length(jp)) 60 | annotation = DB(j(jp(m))).annotation; 61 | 62 | % Get object mask 63 | img = LMimread(DB, j(jp(m)), 'http://labelme.csail.mit.edu/Images'); 64 | [nrows, ncols, cc] = size(img); 65 | [annotation,img] = LMimscale(annotation, img, 320/ncols); 66 | [nrows, ncols] = getaproximagesize(annotation); 67 | mask = LMobjectmask(annotation, [ncols nrows], object); 68 | 69 | subplot(2,4,2*m-1) 70 | LMplot(DB, img); 71 | legend off 72 | 73 | subplot(2,4,2*m) 74 | imagesc(1-mask); colormap(gray(256)); axis('equal'); axis('off') 75 | hold on 76 | 77 | % Get part polygons 78 | colors = hsv(length(children)); 79 | for p = 1:length(children) 80 | jc = LMobjectindex(annotation, partObjectClasses{children(p)}); 81 | for k = 1:length(jc) 82 | [X,Y] = getLMpolygon(annotation.object(jc(k)).polygon); 83 | plot([X; X(1)], [Y; Y(1)], 'LineWidth', 2, 'color', colors(p,:)) 84 | end 85 | end 86 | end 87 | end 88 | 89 | 90 | 91 | 92 | -------------------------------------------------------------------------------- /LM2VRML.m: -------------------------------------------------------------------------------- 1 | function LM2VRML(annotation,mesh,vrmlfile,vrmlfolder) 2 | % Creates VRML 3D file of LabelMe3D scene. 3 | % 4 | % Inputs: 5 | % annotation - LabelMe annotation structure. 6 | % mesh - 3D triangular mesh structure. 7 | % vrmlfile - VRML filename (e.g. 'example.wrl') 8 | % vrmlfolder - Folder to dump textures to 9 | 10 | % VRML Parameters: 11 | fieldOfView = 0.78; 12 | 13 | if nargin < 4 14 | vrmlfolder = pwd; 15 | end 16 | 17 | % Save textures 18 | relativeFolder = strrep(vrmlfile, '.wrl', ''); 19 | if isfield(mesh,'textures') 20 | mkdir(fullfile(vrmlfolder,relativeFolder)); 21 | for ii = 1:length(mesh.textures) 22 | if ~isempty(mesh.textures{ii}) 23 | texturefilename{ii} = sprintf('object_%03d.jpg', ii); 24 | imwrite(mesh.textures{ii},fullfile(vrmlfolder,relativeFolder,texturefilename{ii})); 25 | end 26 | end 27 | else 28 | colors = hsv(length(annotation.object)); 29 | end 30 | 31 | % Get initial camera position: 32 | P = getCameraMatrix(annotation,'RH'); 33 | [K,R,initialCamera] = decomposeP(P); 34 | 35 | % Dump VRML file: 36 | fp = fopen(fullfile(vrmlfolder,vrmlfile),'w'); 37 | fprintf(fp,'#VRML V2.0 utf8\n\nNavigationInfo {\n headlight TRUE\n type [\"EXAMINE\", \"ANY\"]\n}\n\n'); 38 | fprintf(fp,'Viewpoint {\n position %.2f %.2f %.2f\n orientation 0 1 0 3.14159\n fieldOfView %.2f\n description "Original"\n}\n\n',initialCamera(1), initialCamera(2), initialCamera(3), fieldOfView); 39 | fprintf(fp,'DEF Back1 Background {\n groundColor [1 1 1]\n skyColor [1 1 1]\n}\n\n'); 40 | 41 | for ii = 1:length(annotation.object) 42 | isValidObject = any(mesh.objndx==ii); 43 | if isValidObject 44 | fprintf(fp,'# %s\nShape{\n',annotation.object(ii).name); 45 | if isfield(mesh,'textures') 46 | fprintf(fp,' appearance Appearance {\n texture ImageTexture { url \"%s\" }\n }\n', fullfile(relativeFolder,texturefilename{ii})); 47 | else 48 | fprintf(fp,' appearance Appearance {\n material Material {\n diffuseColor %.2f %.2f %.2f\n }\n }\n',colors(ii,:)); 49 | end 50 | 51 | % Get object mesh and texture coordinates: 52 | mesh_i = getObjectMesh(mesh,ii); 53 | 54 | fprintf(fp,' geometry IndexedFaceSet {\n coord Coordinate {\n point [\n'); 55 | for jj = 1:size(mesh_i.vertices,2) 56 | fprintf(fp,' %.4f %.4f %.4f,\n',mesh_i.vertices(:,jj)); 57 | end 58 | fprintf(fp,' ]\n }\n'); 59 | 60 | %% Get coordIndex: 61 | fprintf(fp,' coordIndex [\n'); 62 | for j = 1:size(mesh_i.faces,2) 63 | fprintf(fp,' %d %d %d -1,\n',mesh_i.faces(:,j)-1); 64 | end 65 | fprintf(fp,' ]\n'); 66 | 67 | if isfield(mesh,'textures') 68 | %% Get texture coordinates: 69 | fprintf(fp,' texCoord TextureCoordinate {\n point [\n'); 70 | for jj = 1:size(mesh_i.tx,2) 71 | fprintf(fp,' %.4f %.4f,\n',mesh_i.tx(1,jj)/size(mesh_i.textures{ii},2),1-mesh_i.tx(2,jj)/size(mesh_i.textures{ii},1)); 72 | end 73 | fprintf(fp,' ]\n'); 74 | fprintf(fp,' }\n'); 75 | 76 | %% Get texture coordinate indices: 77 | fprintf(fp,' texCoordIndex [\n'); 78 | for j = 1:size(mesh_i.faces,2) 79 | fprintf(fp,' %d %d %d -1,\n',mesh_i.faces(:,j)-1); 80 | end 81 | fprintf(fp,' ]\n'); 82 | end 83 | 84 | fprintf(fp,' solid FALSE\n'); 85 | fprintf(fp,' }\n'); 86 | fprintf(fp,'}\n\n'); 87 | end 88 | end 89 | fclose(fp); 90 | -------------------------------------------------------------------------------- /getMesh.m: -------------------------------------------------------------------------------- 1 | function [xx,yy,tri,j] = getMesh(x,y,npts) 2 | % 3 | % Input: 4 | % x,y: 2D polygon 5 | % npts: density of the grid 6 | % 7 | % Output: 8 | % xx,yy,tri: triangular mesh 9 | % j: indices of the boundary points in the mesh 10 | % 11 | % The triangular mesh can be plotted with the standard matlab command: 12 | % triplot(tri,xx,yy) 13 | % 14 | % Example: 15 | % x = [0 10 10 6 2]'; 16 | % y = [0 0 5 3 6]' 17 | % getMesh(x,y,10); 18 | 19 | 20 | if nargin==2 21 | npts = 10; 22 | end 23 | 24 | 25 | % Horizontal sections vertices 26 | xm = unique([x(:)' linspace(min(x),max(x),npts)]); 27 | 28 | yy = []; xx = []; 29 | for i = 1:length(xm) 30 | yc = intersectPolygon(x,y,xm(i)); 31 | N = length(yc); 32 | if N>0 33 | for m = 1:N/2 34 | h = linspace(yc(2*m-1),yc(2*m),floor(npts/N)+2); 35 | yy = [yy h]; 36 | xx = [xx xm(i)*ones(size(h))]; 37 | end 38 | end 39 | end 40 | 41 | % make sure that all the boundary points are part of the mesh (this might 42 | % generate triangles with area zero. So we will remove them later). 43 | xx = [x' xx]; 44 | yy = [y' yy]; 45 | 46 | % Remove repeated points 47 | pp = unique([xx; yy]','rows'); 48 | xx = pp(:,1)'; 49 | yy = pp(:,2)'; 50 | 51 | % Create triangular mesh 52 | tri = delaunay(xx,yy,{'Qbb','Qc','QJ','Pp'}); 53 | % $$$ tri = delaunay(xx,yy); 54 | 55 | % remove triangles outside the polygon 56 | np = size(tri,1); 57 | cx = mean(xx(tri),2); 58 | cy = mean(yy(tri),2); 59 | inside = inpolygon(cx,cy,x,y); 60 | tri = tri(find(inside==1),:); 61 | 62 | % remove triangles with zero area 63 | areaBB = (max(xx(tri),[],2) - min(xx(tri),[],2)) .* (max(yy(tri),[],2) - min(yy(tri),[],2)); 64 | tri = tri(find(areaBB>0),:); 65 | 66 | % recover indices boundary 67 | j = zeros(1,length(x)); 68 | for i = 1:length(x) 69 | d = (xx-x(i)).^2+(yy-y(i)).^2; 70 | [foo, t] = min(d); 71 | j(i) = t(1); 72 | end 73 | 74 | if nargout==0 75 | figure 76 | plot(x,y,'go-') 77 | hold on 78 | triplot(tri,xx,yy) 79 | end 80 | 81 | return; 82 | 83 | 84 | % $$$ function [xx,yy,tri,j] = getMesh_v02(x,y,npts) 85 | % $$$ % 86 | % $$$ % Input: 87 | % $$$ % x,y: 2D polygon 88 | % $$$ % 89 | % $$$ % Output: 90 | % $$$ % xx,yy,tri: triangular mesh 91 | % $$$ % j: indices of the boundary points in the mesh 92 | % $$$ % 93 | % $$$ % The triangular mesh can be plotted with the standard matlab command: 94 | % $$$ % triplot(tri,xx,yy) 95 | % $$$ % 96 | % $$$ % Example: 97 | % $$$ % x = [0 10 10 6 2]'; 98 | % $$$ % y = [0 0 5 3 6]' 99 | % $$$ % getMesh(x,y); 100 | % $$$ 101 | % $$$ % Horizontal sections vertices 102 | % $$$ xm = unique(x); 103 | % $$$ 104 | % $$$ yc1 = unique(intersectPolygon(x,y,xm(1))); 105 | % $$$ xc1 = xm(1)*ones(1,length(yc1)); 106 | % $$$ xx = xc1; yy = yc1; tri = []; 107 | % $$$ for i = 2:length(xm) 108 | % $$$ yc2 = unique(intersectPolygon(x,y,xm(i))); 109 | % $$$ xc2 = xm(i)*ones(1,length(yc2)); 110 | % $$$ 111 | % $$$ % Triangulate planar section: 112 | % $$$ tri = [tri; delaunay([xc1 xc2],[yc1 yc2],{'Qbb','Qc','QJ','Pp'})]; 113 | % $$$ % $$$ tri = delaunay([xc1 xc2],[yc1 yc2],{'Qt','Qbb','Qc','QJ'}); 114 | % $$$ 115 | % $$$ xx = [xx xc2]; 116 | % $$$ yy = [yy yc2]; 117 | % $$$ xc1 = xc2; 118 | % $$$ yc1 = yc2; 119 | % $$$ end 120 | % $$$ j = 1:length(xx); 121 | % $$$ 122 | % $$$ if nargout==0 123 | % $$$ figure 124 | % $$$ plot(x,y,'go-') 125 | % $$$ hold on 126 | % $$$ triplot(tri,xx,yy) 127 | % $$$ end 128 | % $$$ 129 | % $$$ 130 | % $$$ return; 131 | % $$$ 132 | % $$$ 133 | -------------------------------------------------------------------------------- /CostFunctionLM3D_ObjectHeights.m: -------------------------------------------------------------------------------- 1 | function [F,J] = CostFunctionLM3D_ObjectHeights(params,h,cy,mu,sigma,Hy,fhat,py,sigma_N) 2 | % Inputs: 3 | % params - Camera parameters to optimize 4 | % h - 1xN object heights (in pixels) 5 | % cy - 1xN y-coordinate of contact point 6 | % mu - 1xN mean of real-world object height (in centimeters) 7 | % sigma - 1xN std dev of real-world object height (in centimeters) 8 | % f - Focal length 9 | % py - Y coordinate of principal point 10 | % sigma_N - std dev of pixel labeling error 11 | 12 | % Hyper-parameters: 13 | lambda_f = 0.1; % Regularization weight for focal length 14 | Cy_mean = 170; 15 | Cy_std = 50; 16 | 17 | % Get parameters to optimize: 18 | Cy = params(1); % Camera height 19 | t = params(2); % Camera pitch angle 20 | f = params(3); % Focal length 21 | 22 | % Get y-coordinate of horizon line: 23 | vy = py-f*tan(t); 24 | 25 | % Number of objects 26 | N = length(h); 27 | 28 | F = []; J = []; 29 | 30 | if length(h) > 0 31 | % Height error (in pixels), weighted by uncertainty in real-world 32 | % object height and labeling error (in pixels): 33 | w = 1./(sigma.*(vy-cy)./Cy + sigma_N); % Standard deviation 34 | F = w.*(mu.*(vy-cy)./Cy - h); 35 | J = Jacobian_ObjectHeights(Cy,t,f,py,h,cy,mu,sigma,sigma_N)'; 36 | end 37 | 38 | if length(Hy) > 0 39 | Hy = mean(Hy,1); 40 | 41 | Nv = length(Hy); 42 | 43 | % Horizon line error (in pixels), weighted by labeling error (in pixels): 44 | F = [F (vy-Hy)/sigma_N]; 45 | J = [J; [zeros(Nv,1) -f*(tan(t)^2+1)/sigma_N*ones(Nv,1) -tan(t)/sigma_N*ones(Nv,1)]]; 46 | end 47 | 48 | % Regularization on camera height: 49 | F(end+1) = (Cy-Cy_mean)/Cy_std; 50 | J(end+1,:) = [1/Cy_std 0 0]; 51 | 52 | % Regularization on focal length: 53 | F(end+1) = lambda_f*(f-fhat); 54 | J(end+1,:) = [0 0 lambda_f]; 55 | 56 | return; 57 | 58 | function J = Jacobian_ObjectHeights(Cy,t,f,py,h,cy,mu,sigma,sigma_N) 59 | % This function was generated by the Symbolic Math Toolbox version 5.2. 60 | % 13-Feb-2013 11:39:27 61 | 62 | t2 = tan(t); 63 | t3 = f.*t2; 64 | t4 = cy - py + t3; 65 | t5 = 1./Cy.^2; 66 | t6 = 1./Cy; 67 | t8 = sigma.*t4.*t6; 68 | t7 = sigma_N - t8; 69 | t9 = 1./t7; 70 | t10 = mu.*t4.*t6; 71 | t11 = h + t10; 72 | t12 = 1./t7.^2; 73 | t13 = t2.^2; 74 | t14 = t13 + 1; 75 | J = [mu.*t4.*t5.*t9 + sigma.*t11.*t12.*t4.*t5;- f.*mu.*t14.*t6.*t9 - f.*sigma.*t11.*t12.*t14.*t6;- mu.*t2.*t6.*t9 - sigma.*t11.*t12.*t2.*t6]; 76 | 77 | return; 78 | 79 | 80 | 81 | % $$$ % Derivation for Jacobian_ObjectHeights 82 | % $$$ 83 | % $$$ % Camera parameters: 84 | % $$$ syms f px py 85 | % $$$ syms Cy t; 86 | % $$$ 87 | % $$$ % Data: 88 | % $$$ syms h cy mu sigma sigma_N 89 | % $$$ 90 | % $$$ % Get camera calibration matrix: 91 | % $$$ K = [f 0 px; 0 f py; 0 0 1]; 92 | % $$$ 93 | % $$$ % Camera rotation (assume rotation through X axis): 94 | % $$$ N = [0 0 0; 0 0 -1; 0 1 0]; 95 | % $$$ R = eye(3) + sin(t)*N + (1-cos(t))*N*N; 96 | % $$$ 97 | % $$$ % Get camera center: 98 | % $$$ C = [0; Cy; 0]; 99 | % $$$ 100 | % $$$ % Get camera matrix: 101 | % $$$ P = K*R*[eye(3) -C]; 102 | % $$$ 103 | % $$$ % Get horizon line: 104 | % $$$ hl = cross(P(:,1),P(:,3)); 105 | % $$$ 106 | % $$$ % Get y-coordinate of horizon line: 107 | % $$$ Hy = cross(hl,cross([1 1 1],[1 2 1])); % Intersect vertical line with horizon 108 | % $$$ Hy = Hy(2)/Hy(3); 109 | % $$$ 110 | % $$$ Hy = py-f*tan(t); % This is resulting Y coordinate 111 | % $$$ 112 | % $$$ % Height error, weighted by uncertainty in object height and labeling error: 113 | % $$$ w = 1./(sigma.*(Hy-cy)./Cy+sigma_N); % Standard deviation 114 | % $$$ F = w*(mu.*(Hy-cy)./Cy - h); 115 | % $$$ 116 | % $$$ % Jacobian: 117 | % $$$ J = [diff(F,'Cy'); diff(F,'t'); diff(F,'f')]; 118 | % $$$ 119 | % $$$ matlabFunction(J,'file','Jacobian_ObjectHeights.m'); 120 | -------------------------------------------------------------------------------- /CostFunctionLM3D_old.m: -------------------------------------------------------------------------------- 1 | function [F,J] = CostFunctionLM3D_old(params,h,b_y,mu,sigma,K,sigma_N) 2 | % Inputs: 3 | % params - Camera parameters to optimize 4 | % h - 1xN object heights (in pixels) 5 | % b_y - 1xN y-coordinate of contact point 6 | % mu - 1xN mean of real-world object height (in centimeters) 7 | % sigma - 1xN std of real-world object height (in centimeters) 8 | % K - 3x3 camera calibration matrix 9 | % sigma_N - std of pixel labeling error 10 | 11 | % Get parameters to optimize: 12 | Cy = params(1); % Camera height 13 | t = params(2); % Camera pitch angle 14 | 15 | % Get camera rotation matrix: 16 | nx = 1; %wx/t; 17 | nz = 0; %wz/t; 18 | N = [0 -nz 0; nz 0 -nx; 0 nx 0]; 19 | R = eye(3) + sin(t)*N + (1-cos(t))*N*N; 20 | 21 | % Get camera center: 22 | C = [0; Cy; 0]; 23 | 24 | % Get camera matrix: 25 | P = K*R*[eye(3) -C]; 26 | 27 | % Get horizon line: 28 | hl = cross(P(:,1),P(:,3)); 29 | 30 | % Get foot vanishing point: 31 | vy = P(:,2); 32 | 33 | N = length(h); % Number of objects 34 | F = zeros(1,N); 35 | J = zeros(N,2); 36 | for i = 1:N 37 | b_x = 0; 38 | b = [b_x; b_y(i); 1]; 39 | pp = cross(hl,cross(vy,b)); 40 | ppx = pp(1)/pp(3); 41 | ppy = pp(2)/pp(3); 42 | u = sqrt((ppx-b_x)^2 + (ppy-b_y(i))^2); % distance from horizon line to 43 | % object foot (in pixels) 44 | F(i) = (Cy*h(i)-u*mu(i))/(u*sigma(i)+sigma_N*Cy); 45 | J(i,:) = Jacobian(Cy,t,h(i),0,b_y(i),mu(i),sigma(i),sigma_N,K(1),K(7),K(8)); 46 | 47 | % Should optimize (need to re-compute std): 48 | % h = (v-b)*H/Cy 49 | end 50 | 51 | % Regularization on camera height: 52 | F(end+1) = (Cy-170)/50; 53 | J(end+1,:) = [1/50 0]; 54 | 55 | return; 56 | 57 | function J = Jacobian(Cy,t,h,b_x,b_y,mu,sigma,sigma_N,f,px,py) 58 | J = [-1/2/(((-cos(t)*px*sin(t)*b_y+py*cos(t)*px*sin(t)+cos(t)^2*f*px+f*b_x-f*px)/cos(t)/(f*cos(t)+py*sin(t)-sin(t)*b_y)-b_x)^2+((-f*sin(t)+py*cos(t))/cos(t)-b_y)^2)^(1/2)*mu*(2*((-cos(t)*px*sin(t)*b_y+py*cos(t)*px*sin(t)+cos(t)^2*f*px+f*b_x-f*px)/cos(t)/(f*cos(t)+py*sin(t)-sin(t)*b_y)-b_x)*((sin(t)^2*px*b_y-cos(t)^2*px*b_y-py*sin(t)^2*px+py*cos(t)^2*px-2*cos(t)*f*px*sin(t))/cos(t)/(f*cos(t)+py*sin(t)-sin(t)*b_y)+(-cos(t)*px*sin(t)*b_y+py*cos(t)*px*sin(t)+cos(t)^2*f*px+f*b_x-f*px)/cos(t)^2/(f*cos(t)+py*sin(t)-sin(t)*b_y)*sin(t)-(-cos(t)*px*sin(t)*b_y+py*cos(t)*px*sin(t)+cos(t)^2*f*px+f*b_x-f*px)/cos(t)/(f*cos(t)+py*sin(t)-sin(t)*b_y)^2*(-f*sin(t)+py*cos(t)-b_y*cos(t)))+2*((-f*sin(t)+py*cos(t))/cos(t)-b_y)*((-f*cos(t)-py*sin(t))/cos(t)+(-f*sin(t)+py*cos(t))/cos(t)^2*sin(t)))/((((-cos(t)*px*sin(t)*b_y+py*cos(t)*px*sin(t)+cos(t)^2*f*px+f*b_x-f*px)/cos(t)/(f*cos(t)+py*sin(t)-sin(t)*b_y)-b_x)^2+((-f*sin(t)+py*cos(t))/cos(t)-b_y)^2)^(1/2)*sigma+sigma_N*Cy)-1/2*(Cy*h-(((-cos(t)*px*sin(t)*b_y+py*cos(t)*px*sin(t)+cos(t)^2*f*px+f*b_x-f*px)/cos(t)/(f*cos(t)+py*sin(t)-sin(t)*b_y)-b_x)^2+((-f*sin(t)+py*cos(t))/cos(t)-b_y)^2)^(1/2)*mu)/((((-cos(t)*px*sin(t)*b_y+py*cos(t)*px*sin(t)+cos(t)^2*f*px+f*b_x-f*px)/cos(t)/(f*cos(t)+py*sin(t)-sin(t)*b_y)-b_x)^2+((-f*sin(t)+py*cos(t))/cos(t)-b_y)^2)^(1/2)*sigma+sigma_N*Cy)^2/(((-cos(t)*px*sin(t)*b_y+py*cos(t)*px*sin(t)+cos(t)^2*f*px+f*b_x-f*px)/cos(t)/(f*cos(t)+py*sin(t)-sin(t)*b_y)-b_x)^2+((-f*sin(t)+py*cos(t))/cos(t)-b_y)^2)^(1/2)*sigma*(2*((-cos(t)*px*sin(t)*b_y+py*cos(t)*px*sin(t)+cos(t)^2*f*px+f*b_x-f*px)/cos(t)/(f*cos(t)+py*sin(t)-sin(t)*b_y)-b_x)*((sin(t)^2*px*b_y-cos(t)^2*px*b_y-py*sin(t)^2*px+py*cos(t)^2*px-2*cos(t)*f*px*sin(t))/cos(t)/(f*cos(t)+py*sin(t)-sin(t)*b_y)+(-cos(t)*px*sin(t)*b_y+py*cos(t)*px*sin(t)+cos(t)^2*f*px+f*b_x-f*px)/cos(t)^2/(f*cos(t)+py*sin(t)-sin(t)*b_y)*sin(t)-(-cos(t)*px*sin(t)*b_y+py*cos(t)*px*sin(t)+cos(t)^2*f*px+f*b_x-f*px)/cos(t)/(f*cos(t)+py*sin(t)-sin(t)*b_y)^2*(-f*sin(t)+py*cos(t)-b_y*cos(t)))+2*((-f*sin(t)+py*cos(t))/cos(t)-b_y)*((-f*cos(t)-py*sin(t))/cos(t)+(-f*sin(t)+py*cos(t))/cos(t)^2*sin(t))); h/((((-cos(t)*px*sin(t)*b_y+py*cos(t)*px*sin(t)+cos(t)^2*f*px+f*b_x-f*px)/cos(t)/(f*cos(t)+py*sin(t)-sin(t)*b_y)-b_x)^2+((-f*sin(t)+py*cos(t))/cos(t)-b_y)^2)^(1/2)*sigma+sigma_N*Cy)-(Cy*h-(((-cos(t)*px*sin(t)*b_y+py*cos(t)*px*sin(t)+cos(t)^2*f*px+f*b_x-f*px)/cos(t)/(f*cos(t)+py*sin(t)-sin(t)*b_y)-b_x)^2+((-f*sin(t)+py*cos(t))/cos(t)-b_y)^2)^(1/2)*mu)/((((-cos(t)*px*sin(t)*b_y+py*cos(t)*px*sin(t)+cos(t)^2*f*px+f*b_x-f*px)/cos(t)/(f*cos(t)+py*sin(t)-sin(t)*b_y)-b_x)^2+((-f*sin(t)+py*cos(t))/cos(t)-b_y)^2)^(1/2)*sigma+sigma_N*Cy)^2*sigma_N]; 59 | -------------------------------------------------------------------------------- /inferPartsPosterior.m: -------------------------------------------------------------------------------- 1 | function annotation = inferPartsPosterior(annotation,Ppart,objectClasses,validObjects) 2 | % annotation = inferPartsPosterior(annotation,Ppart,objectClasses,validObjects) 3 | % Infer posterior of attachment relationship. 4 | % 5 | % Inputs: 6 | % annotation - LabelMe annotation structure. 7 | % Ppart - Learned part parameters. 8 | % objectClasses - Cell array of object classes. 9 | % validObjects - Vector of desired object indices to compute. 10 | % 11 | % Outputs: 12 | % annotation.object(i).polygon.polyType - Polygon type ('part', 13 | % 'standing','ground','') 14 | % annotation.object(i).partof - ID of parent part-of polygon. 15 | % annotation.object(i).polygon.pAttached - Probability that polygon is 16 | % attached. 17 | % annotation.object(i).polygon.pAttachedObject - Probabilities for 18 | % part-of parents. 19 | 20 | % Parameters: 21 | threshPart = 0.4872; % Threshold for part-of relationship 22 | 23 | if nargin < 4 24 | validObjects = 1:length(annotation.object); 25 | end 26 | 27 | notDeleted = (~isdeleted(annotation))'; 28 | 29 | % Collect parts evidence: 30 | evidence = collectPartsEvidence(annotation,objectClasses); 31 | 32 | % Get ground object indices: 33 | groundObjects = getListGroundObjects; 34 | groundObjects = sprintf('%s,', groundObjects{:}); groundObjects = groundObjects(1:end-1); 35 | groundObjects = LMobjectindex(annotation,groundObjects,'exact'); 36 | 37 | Nobjects = length(annotation.object); 38 | for i = validObjects%1:Nobjects 39 | s = evidence.objNdx(i); 40 | 41 | if ismember(i,groundObjects) 42 | % This is a hack for now: ground object edges are automatically 43 | % considered attached: 44 | annotation.object(i).polygon.pAttached = 1; 45 | annotation.object(i).polygon.polyType = 'ground'; 46 | elseif s>0 47 | % Histogram object classes in this image: 48 | h = hist(evidence.objNdx(setdiff(find(notDeleted),i)),[1:length(objectClasses)]); 49 | % $$$ h = hist(evidence.objNdx(setdiff([1:Nobjects],i)),[1:length(objectClasses)]); 50 | 51 | % Compute posterior multinomial: 52 | mult = zeros(1,Nobjects+1); 53 | mult(i) = log(0); 54 | mult(~notDeleted) = log(0); % Remove deleted objects from consideration 55 | mult(end) = log(Ppart.P(s,end)); 56 | % $$$ for j = setdiff(1:Nobjects,i) 57 | for j = setdiff(find(notDeleted),i) 58 | t = evidence.objNdx(j); 59 | if t>0 60 | mult(j) = -log(Ppart.alpha_a)-(Ppart.alpha_a-1)*log(evidence.relativeArea(j)+eps)+log(Ppart.alpha_r)+(Ppart.alpha_r-1)*log(evidence.relativeOverlap(i,j)+eps)+log(1-Ppart.P(s,end))+log(Ppart.P(s,t))-log(h(t)); 61 | else 62 | % Polygon is not in objectClasses: 63 | mult(j) = log(0); 64 | end 65 | end 66 | mult = exp(mult-max(mult)); 67 | mult = mult/sum(mult); 68 | 69 | % Assign probability of being attached: 70 | annotation.object(i).polygon.pAttached = sum(mult(1:Nobjects)); 71 | 72 | % Assign probability of which object polygon is attached to: 73 | annotation.object(i).polygon.pAttachedObject = mult(1:Nobjects); 74 | 75 | % Determine if object is part-of: 76 | if any(annotation.object(i).polygon.pAttached > threshPart) 77 | annotation.object(i).polygon.polyType = 'part'; 78 | [vv,pp] = max(annotation.object(i).polygon.pAttachedObject); 79 | annotation.object(i).partof = annotation.object(pp).id; 80 | else 81 | annotation.object(i).polygon.polyType = 'standing'; 82 | end 83 | else 84 | % Polygon is not in objectClasses: 85 | annotation.object(i).polygon.pAttached = 0; 86 | annotation.object(i).polygon.polyType = ''; 87 | end 88 | end 89 | 90 | % Get which standing object each attached object is attached to: 91 | allID = {annotation.object(:).id}; 92 | for i = validObjects%1:length(annotation.object) 93 | if strcmp(annotation.object(i).polygon.polyType,'part') 94 | pp = find(ismember(allID,annotation.object(i).partof)); 95 | while strcmp(annotation.object(pp).polygon.polyType,'part') 96 | pp = find(ismember(allID,annotation.object(pp).partof)); 97 | end 98 | annotation.object(i).partof = annotation.object(pp).id; 99 | end 100 | end 101 | -------------------------------------------------------------------------------- /Recover3DSceneComponents.m: -------------------------------------------------------------------------------- 1 | function [annotation,valid] = Recover3DSceneComponents(annotation,Params3D,imageSize) 2 | % Inputs: 3 | % annotation - LabelMe annotation structure. 4 | % Params3D - LabelMe3D parameters. 5 | % imageSize - Image size [nrows ncols] 6 | % 7 | % Outputs: 8 | % annotation - LabelMe annotation structure, augmented with 3D information. 9 | % valid - Indicates whether output is valid. 10 | 11 | valid = 0; 12 | if (nargin < 3) && ~isfield(annotation,'imagesize') 13 | % If 'imagesize' field does not exist, then look for image dimensions 14 | % on LabelMe server. If image does not exist, then you need to pass in 15 | % the imageSize as the third argument to this function. If you have 16 | % the original image, then an easy way to set the 'imagesize' field is 17 | % as follows: 18 | % 19 | % DB.annotation.imagesize.nrows = size(img,1); 20 | % DB.annotation.imagesize.ncols = size(img,2); 21 | 22 | info = imfinfo(fullfile('http://labelme.csail.mit.edu/Images',annotation.folder,annotation.filename)); 23 | annotation.imagesize.nrows = info.Height; 24 | annotation.imagesize.ncols = info.Width; 25 | end 26 | if ~isfield(annotation,'imagesize') 27 | annotation.imagesize.nrows = imageSize(1); 28 | annotation.imagesize.ncols = imageSize(2); 29 | end 30 | if isstr(annotation.imagesize.nrows) 31 | annotation.imagesize.nrows = str2num(annotation.imagesize.nrows); 32 | end 33 | if isstr(annotation.imagesize.ncols) 34 | annotation.imagesize.ncols = str2num(annotation.imagesize.ncols); 35 | end 36 | imageSize = [annotation.imagesize.nrows annotation.imagesize.ncols]; 37 | 38 | if nargin < 2 39 | load Params3D.mat; 40 | end 41 | 42 | validObjects = []; 43 | for i = 1:length(annotation.object) 44 | if ~isfield(annotation.object(i),'world3d') || ~isfield(annotation.object(i).world3d,'stale') || strcmp(annotation.object(i).world3d.stale,'1') 45 | validObjects(end+1) = i; 46 | end 47 | end 48 | 49 | % Add object IDs: 50 | for i = 1:length(annotation.object) 51 | annotation.object(i).id = num2str(i-1); 52 | end 53 | maxID = annotation.object(end).id; 54 | 55 | % Get valid object indices: 56 | dd = isdeleted(annotation); 57 | validObjects = intersect(validObjects,find(dd==0)'); 58 | 59 | display(sprintf('Computing geometry for %d objects.',length(validObjects))); 60 | 61 | % Clean up object names: 62 | annotation = myLMaddtags(annotation); 63 | 64 | if length(validObjects)>0 65 | % Step 1: Infer polygon types (groundplane|standingplanes|part): 66 | annotation = inferPartsPosterior(annotation,Params3D.Ppart,Params3D.objectClasses,validObjects); 67 | 68 | % Step 2: Infer contact edges for "standingplanes" objects: 69 | annotation = inferEdgeTypesNew(annotation,validObjects); 70 | 71 | % Make sure all standing objects make contact with the ground: 72 | for i = validObjects 73 | if strcmp(annotation.object(i).polygon.polyType,'standing') && isempty(getLMcontact(annotation.object(i).polygon)) 74 | annotation.object(i).polygon.polyType = ''; 75 | end 76 | end 77 | 78 | % Convert XML file to new format (this needs to be removed at some point): 79 | annotation = ConvertOld2New(annotation,validObjects); 80 | 81 | % Infer angle polygon type: 82 | annotation = AddAnglePolygonType(annotation,validObjects); 83 | 84 | % Step 3: Recover camera parameters: 85 | % $$$ annotation = getviewpoint(annotation,Params3D.ObjectHeights); 86 | annotation = getviewpoint_ObjectHeights(annotation); 87 | else 88 | display('Skipping part-of and edge type inference...'); 89 | end 90 | 91 | % Get indices of non-deleted objects: 92 | notDeleted = find(~isdeleted(annotation))'; 93 | 94 | for i = notDeleted 95 | if strcmp(annotation.object(i).world3d.type,'groundplane') 96 | valid = 1; 97 | end 98 | end 99 | 100 | % Restore original object names and remove "originalname" field: 101 | for i = 1:length(annotation.object) 102 | annotation.object(i).name = annotation.object(i).originalname; 103 | end 104 | annotation.object = rmfield(annotation.object,'originalname'); 105 | 106 | % Remove points that have been added before: 107 | for i = notDeleted 108 | switch annotation.object(i).world3d.type 109 | case {'standingplanes','part'} 110 | if isfield(annotation.object(i).world3d,'polygon3d') 111 | nAdded = logical(str2num(char({annotation.object(i).world3d.polygon3d.pt.added}))); 112 | annotation.object(i).polygon.pt(nAdded) = []; 113 | annotation.object(i).world3d.polygon3d.pt(nAdded) = []; 114 | end 115 | end 116 | end 117 | 118 | % Inflate the 3D scene and store in annotation structure: 119 | annotation = InflateSceneXML(annotation,notDeleted); 120 | -------------------------------------------------------------------------------- /partsGibbsSampler.cpp: -------------------------------------------------------------------------------- 1 | // partsGibbsSampler.cpp 2 | 3 | #include 4 | #include 5 | #include 6 | #include "mex.h" 7 | 8 | int MultiRand(double *p,int dim) { 9 | double r = ((double)rand() / ((double)(RAND_MAX)+(double)(1))); 10 | double tot = p[0]; 11 | for(int i = 0; i < dim-1; i++) { 12 | if(tot>r) return i; 13 | tot += p[i+1]; 14 | } 15 | return dim-1; 16 | } 17 | 18 | // Inputs: evidence,imgNdx,objNdx,N,q,rp,alpha_r,alpha_a,eta_a,eta_o 19 | // Outputs: N,q 20 | void mexFunction(int nlhs,mxArray *plhs[],int nrhs,const mxArray *prhs[]) { 21 | // Initialize random number generator: 22 | srand(time(0)); 23 | 24 | if(nrhs != 10) { 25 | mexErrMsgTxt("Error: 10 args. needed."); 26 | return; 27 | } 28 | 29 | const mxArray *evidence = prhs[0]; 30 | int *imgNdx = (int*)mxGetData(prhs[1]); 31 | int *objNdx = (int*)mxGetData(prhs[2]); 32 | int *Nin = (int*)mxGetData(prhs[3]); 33 | int *qin = (int*)mxGetData(prhs[4]); 34 | int *rp = (int*)mxGetData(prhs[5]); 35 | double alpha_r = (double)(*mxGetPr(prhs[6])); 36 | double alpha_a = (double)(*mxGetPr(prhs[7])); 37 | double eta_a = (double)(*mxGetPr(prhs[8])); 38 | double eta_o = (double)(*mxGetPr(prhs[9])); 39 | 40 | int Nobjects = mxGetM(prhs[3]); 41 | int Ndata = mxGetNumberOfElements(prhs[4]); 42 | 43 | // Allocate outputs: 44 | int dims[2]; 45 | dims[0] = Nobjects; dims[1] = Nobjects+2; 46 | plhs[0] = mxCreateNumericArray(2,dims,mxINT32_CLASS,mxREAL); 47 | int *N = (int*)mxGetData(plhs[0]); 48 | dims[0] = 1; dims[1] = Ndata; 49 | plhs[1] = mxCreateNumericArray(2,dims,mxINT32_CLASS,mxREAL); 50 | int *q = (int*)mxGetData(plhs[1]); 51 | 52 | // Allocate temporary memory: 53 | double *mult = (double*)mxCalloc(Ndata,sizeof(double)); 54 | int *Cobj = (int*)mxCalloc(Nobjects,sizeof(int)); 55 | 56 | // Initialize outputs: 57 | for(int i = 0; i < Ndata; i++) q[i] = qin[i]; 58 | for(int i = 0; i < Nobjects*(Nobjects+2); i++) N[i] = Nin[i]; 59 | 60 | for(int i = 0; i < Ndata; i++) { 61 | // Get pointers to sample: 62 | int j = rp[i]-1; 63 | int ii = imgNdx[j]-1; 64 | int oo = objNdx[j]-1; 65 | 66 | // Get object class of sample: 67 | int Nobj = mxGetNumberOfElements(mxGetField(evidence,ii,"objNdx")); 68 | unsigned short *evObjNdx = (unsigned short*)mxGetData(mxGetField(evidence,ii,"objNdx")); 69 | float *relativeOverlap = (float*)mxGetData(mxGetField(evidence,ii,"relativeOverlap")); 70 | float *relativeArea = (float*)mxGetData(mxGetField(evidence,ii,"relativeArea")); 71 | int oj = (int)evObjNdx[oo]-1; 72 | 73 | // Decrement counts: 74 | if(q[j]>-1) { 75 | if(q[j]>0) { 76 | int op = (int)evObjNdx[q[j]-1]-1; 77 | N[oj+op*Nobjects]--; 78 | N[oj+(Nobjects+1)*Nobjects]--; 79 | } 80 | else N[oj+Nobjects*Nobjects]--; 81 | } 82 | 83 | // Histogram object classes in this image: 84 | int Uobj = 0; 85 | int totobj = 0; 86 | for(int k = 0; k < Nobj; k++) { 87 | int op = (int)evObjNdx[k]-1; 88 | if(k!=oo) { 89 | if(Cobj[op]==0) { 90 | Uobj++; 91 | totobj += N[oj+op*Nobjects]; 92 | } 93 | Cobj[op]++; 94 | } 95 | } 96 | 97 | // Compute posterior: 98 | mult[0] = log(double(N[oj+Nobjects*Nobjects])+eta_a); 99 | for(int k = 0; k < Nobj; k++) { 100 | int op = (int)evObjNdx[k]-1; 101 | if(k==oo) mult[k+1] = log(0); 102 | else mult[k+1] = log(double(N[oj+(Nobjects+1)*Nobjects])+1) + log(double(N[oj+op*Nobjects])+eta_o) - log(double(totobj)+Uobj*eta_o) - log(double(Cobj[op])) - log(alpha_a)-(alpha_a-1)*log(relativeArea[k]) + log(alpha_r)+(alpha_r-1)*log(relativeOverlap[oo+Nobj*k]); 103 | } 104 | 105 | // Normalize multinomial: 106 | double maxval = mult[0]; 107 | for(int k = 1; k < Nobj+1; k++) { 108 | if(mult[k]>maxval) maxval = mult[k]; 109 | } 110 | double tot = 0; 111 | for(int k = 0; k < Nobj+1; k++) { 112 | mult[k] = exp(mult[k]-maxval); 113 | tot += mult[k]; 114 | } 115 | for(int k = 0; k < Nobj+1; k++) mult[k] /= tot; 116 | 117 | // Sample from posterior: 118 | int s = MultiRand(mult,Nobj+1); 119 | 120 | // Update counts: 121 | q[j] = s; // s is in {0,...,Nobj_j} 122 | if(q[j]>0) { 123 | int op = (int)evObjNdx[q[j]-1]-1; 124 | N[oj+op*Nobjects]++; 125 | N[oj+(Nobjects+1)*Nobjects]++; 126 | } 127 | else N[oj+Nobjects*Nobjects]++; 128 | 129 | // Reset histogram of object classes in this image: 130 | for(int k = 0; k < Nobj; k++) { 131 | if(k!=oo) Cobj[(int)evObjNdx[k]-1]--; 132 | } 133 | } 134 | 135 | // Free temporary memory: 136 | mxFree(mult); 137 | mxFree(Cobj); 138 | } 139 | -------------------------------------------------------------------------------- /showFootprint.m: -------------------------------------------------------------------------------- 1 | function showFootprint(annotation,img,HOME3D) 2 | 3 | Hy = annotation.camera.Hy; 4 | Nobjects = length(annotation.object); 5 | 6 | % Plot ground objects on image: 7 | clf; 8 | imshow(img); 9 | hold on; 10 | plot([1 size(img,2)],[Hy Hy],'r','LineWidth',4); 11 | for i = 1:Nobjects 12 | if isfield(annotation.object(i),'mesh') && ~isempty(annotation.object(i).mesh) && strcmp(annotation.object(i).mesh.type,'ground') 13 | % Get coordinates: 14 | TX = annotation.object(i).mesh.v2D(1,:); 15 | TY = annotation.object(i).mesh.v2D(2,:); 16 | b = annotation.object(i).mesh.boundary; 17 | plot([TX(b) TX(b(1))],[TY(b) TY(b(1))],'b','LineWidth',4); 18 | end 19 | end 20 | 21 | if nargin > 2 22 | print('-djpeg',fullfile(HOME3D,annotation.folder,[annotation.filename '_groundObjects.jpg'])); 23 | else 24 | pause; 25 | end 26 | 27 | % Plot foldedcard objects on image: 28 | clf; 29 | imshow(img); 30 | hold on; 31 | plot([1 size(img,2)],[Hy Hy],'r','LineWidth',4); 32 | colors = {'r','b','c','m'}; 33 | objColors = cell(1,Nobjects); 34 | for i = 1:Nobjects 35 | if isfield(annotation.object(i),'mesh') && ~isempty(annotation.object(i).mesh) && strcmp(annotation.object(i).mesh.type,'foldedcard') %&& sum(annotation.object(i).mesh.CptsNdx)>1 36 | TX = annotation.object(i).mesh.v2D(1,:); 37 | TY = annotation.object(i).mesh.v2D(2,:); 38 | b = annotation.object(i).mesh.boundary; 39 | rp = randperm(length(colors)); 40 | objColors{i} = colors{rp(1)}; 41 | plot([TX(b) TX(b(1))],[TY(b) TY(b(1))],objColors{i},'LineWidth',4); 42 | end 43 | end 44 | for i = 1:Nobjects 45 | if isfield(annotation.object(i),'mesh') && ~isempty(annotation.object(i).mesh) && strcmp(annotation.object(i).mesh.type,'foldedcard') %&& sum(annotation.object(i).mesh.CptsNdx)>1 46 | TX = annotation.object(i).mesh.v2D(1,:); 47 | TY = annotation.object(i).mesh.v2D(2,:); 48 | b = annotation.object(i).mesh.boundary; 49 | c = find(annotation.object(i).mesh.CptsNdx); 50 | x = TX(b(c)); 51 | y = TY(b(c)); 52 | [v,j] = sort(x); 53 | if length(j) > 1 54 | plot(x(j),y(j),'g','LineWidth',4); 55 | else 56 | plot(x(j),y(j),'g.','MarkerSize',30); 57 | end 58 | end 59 | end 60 | 61 | if nargin > 2 62 | print('-djpeg',fullfile(HOME3D,annotation.folder,[annotation.filename '_foldedCards.jpg'])); 63 | else 64 | pause; 65 | end 66 | 67 | % Plot footprint: 68 | clf; 69 | objName = []; 70 | range = [inf -inf inf -inf]; 71 | for i = 1:Nobjects 72 | if isfield(annotation.object(i),'mesh') && ~isempty(annotation.object(i).mesh) 73 | % Get coordinates: 74 | X = annotation.object(i).mesh.v3D(1,:); 75 | Y = annotation.object(i).mesh.v3D(2,:); 76 | Z = annotation.object(i).mesh.v3D(3,:); 77 | TX = annotation.object(i).mesh.v2D(1,:); 78 | TY = annotation.object(i).mesh.v2D(2,:); 79 | b = annotation.object(i).mesh.boundary; 80 | 81 | switch annotation.object(i).mesh.type 82 | case 'ground' 83 | plot([X(b) X(b(1))],[Z(b) Z(b(1))],'k','LineWidth',4); 84 | hold on; 85 | objName{end+1} = annotation.object(i).name; 86 | range([1 3]) = min(range([1 3]),[min(X) min(Z)]); 87 | range([2 4]) = max(range([2 4]),[max(X) max(Z)]); 88 | case 'foldedcard' 89 | % if sum(annotation.object(i).mesh.CptsNdx)>1 90 | x = X(b); 91 | z = Z(b); 92 | % $$$ [x,j] = sort(x); 93 | [v,j] = sort(TX(b)); 94 | x = x(j); 95 | z = z(j); 96 | plot(x,z,objColors{i},'LineWidth',4); 97 | hold on; 98 | objName{end+1} = annotation.object(i).name; 99 | range([1 3]) = min(range([1 3]),[min(X) min(Z)]); 100 | range([2 4]) = max(range([2 4]),[max(X) max(Z)]); 101 | % end 102 | end 103 | end 104 | end 105 | range(3) = min(range(3),0); 106 | xlabel('x'); 107 | ylabel('z'); 108 | %axis equal; 109 | axis(range,'equal'); 110 | grid on; 111 | legend(objName,'Location','NorthEastOutside'); 112 | 113 | if nargin > 2 114 | print('-djpeg',fullfile(HOME3D,annotation.folder,[annotation.filename '_footprint.jpg'])); 115 | else 116 | pause; 117 | end 118 | 119 | return; 120 | 121 | % Plot the remaining objects (foldedcards with a single contact point) 122 | clf; 123 | imshow(img); 124 | hold on; 125 | plot([1 size(img,2)],[Hy Hy],'r','LineWidth',4); 126 | for i = 1:Nobjects 127 | if isfield(annotation.object(i),'mesh') && ~isempty(annotation.object(i).mesh) && strcmp(annotation.object(i).mesh.type,'foldedcard') && sum(annotation.object(i).mesh.CptsNdx)==1 128 | TX = annotation.object(i).mesh.v2D(1,:); 129 | TY = annotation.object(i).mesh.v2D(2,:); 130 | b = annotation.object(i).mesh.boundary; 131 | plot([TX(b) TX(b(1))],[TY(b) TY(b(1))],'b','LineWidth',4); 132 | end 133 | end 134 | 135 | if nargin > 2 136 | print('-djpeg',fullfile(HOME3D,annotation.folder,[annotation.filename '_singleContactPoint.jpg'])); 137 | else 138 | pause; 139 | end 140 | -------------------------------------------------------------------------------- /plotPolyEdgeTypes.m: -------------------------------------------------------------------------------- 1 | function seg = plotPolyEdgeTypes(annotation,img,whichPolygons,method) 2 | % Plot polygon and edge types over image. 3 | % 4 | % Polygon types: 5 | % standingplanes => red 6 | % part => yellow 7 | % groundplane => green 8 | % Edge types: 9 | % occlusion => black 10 | % attached => gray 11 | % contact => white 12 | % 13 | % Inputs: 14 | % annotation 15 | % img 16 | % whichPolygons - {valid | all} 17 | % method - {vector | bmap} 18 | % 19 | % Outputs: 20 | % seg - Output bitmap image with polygon and edge types overlayed. 21 | 22 | if nargin < 4 23 | method = 'vector';%'bmap'; 24 | end 25 | if nargout > 0 26 | method = 'bmap'; 27 | end 28 | if nargin < 3 29 | whichPolygons = 'valid'; 30 | % $$$ whichPolygons = 'all'; 31 | end 32 | 33 | % Parameters: 34 | alpha = 0.5; % alpha map between polygon colors and image 35 | cmap = [1 0 0; 0 1 0.7; 0.7 1 0]; 36 | attachedColor = [0.5 0.5 0.5]; 37 | occludedColor = [0 0 0]; 38 | contactColor = [1 1 1]; 39 | 40 | % Set line sizes: 41 | switch method 42 | case 'bmap' 43 | sz = ceil(sqrt(size(img,1)^2+size(img,2)^2)/800); 44 | attachedLineWidth = sz; 45 | occludedLineWidth = 2*sz; 46 | contactLineWidth = 2*sz; 47 | case 'vector' 48 | attachedLineWidth = 1; 49 | occludedLineWidth = 2; 50 | contactLineWidth = 2; 51 | end 52 | 53 | % Remove deleted objects from consideration: 54 | dd = isdeleted(annotation); 55 | annotation.object = annotation.object(~dd); 56 | 57 | % Get masks of different polygon types: 58 | M = zeros(size(img,1),size(img,2),3); 59 | [a,layersNdx] = LMsortlayers(annotation,img); 60 | for i = layersNdx 61 | [X,Y] = getLMpolygon(annotation.object(i).polygon); 62 | m = poly2mask(double(X),double(Y),size(img,1),size(img,2)); 63 | ndx = find(m(:)); 64 | 65 | switch annotation.object(i).world3d.type 66 | case 'standingplanes' 67 | M(ndx) = cmap(1,1); 68 | M(ndx+size(img,1)*size(img,2)) = cmap(1,2); 69 | M(ndx+2*size(img,1)*size(img,2)) = cmap(1,3); 70 | case 'groundplane' 71 | M(ndx) = cmap(2,1); 72 | M(ndx+size(img,1)*size(img,2)) = cmap(2,2); 73 | M(ndx+2*size(img,1)*size(img,2)) = cmap(2,3); 74 | case 'part' 75 | parentNdx = getPartOfParents(annotation,i); 76 | parentNdx = parentNdx(end); 77 | if strcmp(whichPolygons,'all') || strcmp(annotation.object(parentNdx).world3d.type,'standingplanes') 78 | M(ndx) = cmap(3,1); 79 | M(ndx+size(img,1)*size(img,2)) = cmap(3,2); 80 | M(ndx+2*size(img,1)*size(img,2)) = cmap(3,3); 81 | end 82 | otherwise 83 | if strcmp(whichPolygons,'all') 84 | M(ndx) = cmap(1,1); 85 | M(ndx+size(img,1)*size(img,2)) = cmap(1,2); 86 | M(ndx+2*size(img,1)*size(img,2)) = cmap(1,3); 87 | end 88 | end 89 | end 90 | 91 | seg = uint8(alpha*255*M + (1-alpha)*repmat(mean(double(img),3),[1 1 3])); 92 | 93 | if strcmp(method,'vector') 94 | imshow(seg); 95 | hold on; 96 | end 97 | 98 | % Plot edges for attached polygons: 99 | for i = 1:length(annotation.object) 100 | if strcmp(annotation.object(i).world3d.type,'part') 101 | parentNdx = getPartOfParents(annotation,i); 102 | parentNdx = parentNdx(end); 103 | if strcmp(whichPolygons,'all') || strcmp(annotation.object(parentNdx).world3d.type,'standingplanes') 104 | % Get attached edges: 105 | edgesAttached = getEdges(annotation.object(i).polygon); 106 | 107 | switch method 108 | case 'bmap' 109 | seg = plotLineBmap(seg,edgesAttached,attachedColor,attachedLineWidth); 110 | case 'vector' 111 | plot(edgesAttached(:,[1 3])',edgesAttached(:,[2 4])','Color',attachedColor,'LineWidth',2*attachedLineWidth+1); 112 | end 113 | end 114 | end 115 | end 116 | 117 | % Plot edges for standing polygons: 118 | for i = layersNdx 119 | if strcmp(whichPolygons,'all') || strcmp(annotation.object(i).world3d.type,'standingplanes') 120 | edgesOccluded = getEdges(annotation.object(i).polygon); 121 | xc = cellfun(@str2num,{annotation.object(i).world3d.contact(:).x}); 122 | yc = cellfun(@str2num,{annotation.object(i).world3d.contact(:).y}); 123 | edgesContact = [xc(1:end-1); yc(1:end-1); xc(2:end); yc(2:end)]'; 124 | 125 | switch method 126 | case 'bmap' 127 | if ~isempty(edgesOccluded) 128 | seg = plotLineBmap(seg,edgesOccluded,occludedColor,occludedLineWidth); 129 | end 130 | if ~isempty(edgesContact) 131 | seg = plotLineBmap(seg,edgesContact,contactColor,contactLineWidth); 132 | end 133 | case 'vector' 134 | if ~isempty(edgesOccluded) 135 | plot(edgesOccluded(:,[1 3])',edgesOccluded(:,[2 4])','Color',occludedColor,'LineWidth',2*occludedLineWidth+1); 136 | end 137 | if ~isempty(edgesContact) 138 | plot(edgesContact(:,[1 3])',edgesContact(:,[2 4])','Color',0.99*contactColor,'LineWidth',2*contactLineWidth+1); 139 | end 140 | end 141 | end 142 | end 143 | 144 | if strcmp(method,'bmap') && (nargout==0) 145 | clf; 146 | imshow(seg); 147 | end 148 | -------------------------------------------------------------------------------- /partsGraph.m: -------------------------------------------------------------------------------- 1 | function [Ppart, objectClasses] = partsGraph(DB, objectClasses) 2 | % 3 | % Inputs: 4 | % DB - LabelMe DB structure. 5 | % 6 | % Outputs: 7 | % Ppart(i,j) - Probability of object class Oj is a part of Oi. 8 | % objectClasses - Cell array of object classes corresponding to the 9 | % rows/cols of Ppart. 10 | 11 | % Parameters: 12 | Pp = 0.7; % probability of overlaping for between part and object. (it could be part of another object) 13 | thresh_par = 0.90; % parts threshold (to collect local evidences) 14 | Prior_part = 0.25; % prior prob of two objects being part of each other. 15 | 16 | % Extract object classes: 17 | if nargin == 1 18 | objectClasses = LMobjectnames(DB,'name'); 19 | end 20 | objectClasses = lower(objectClasses); 21 | Nobjects = length(objectClasses); 22 | Nimages = length(DB); 23 | 24 | % Compute part probabilities 25 | PC_p = zeros([Nobjects, Nobjects], 'double'); % P(overlap | part) 26 | PC_n = zeros([Nobjects, Nobjects], 'double'); % P(overlap | no part) 27 | Counts = zeros([Nobjects, Nobjects], 'uint16'); % counts of coocurrences 28 | for i = 1:Nimages 29 | display(sprintf('%d out of %d',i,Nimages)); 30 | 31 | if isfield(DB(i).annotation,'object') 32 | [ncols, nrows] = getaproximagesize(DB(i).annotation); 33 | 34 | Nobj = length(DB(i).annotation.object); 35 | % Get object indices for this image 36 | clear ndx X Y area 37 | for j = 1:Nobj 38 | ndx(j) = strmatch(strtrim(lower(DB(i).annotation.object(j).name)), objectClasses, 'exact'); 39 | [X{j},Y{j}] = getLMpolygon(DB(i).annotation.object(j).polygon); 40 | area(j) = polyarea(X{j},Y{j}); 41 | end 42 | 43 | C = zeros(Nobj,Nobj, 'uint8'); 44 | for m = 1:Nobj-1 45 | for n = m+1:Nobj 46 | C(m,n) = detectPart(X{n}, Y{n}, X{m}, Y{m}, thresh_par); 47 | end 48 | end 49 | C = C + C'; 50 | 51 | 52 | % Get Pop for this image 53 | for clm = unique(ndx) 54 | m = find(ndx==clm); 55 | for cln = unique(ndx) % loop on classes in this image (each class counts only once as background) 56 | n = find(ndx==cln); 57 | 58 | if max(area(n)) > max(area(m)) % we expect the part to be the smaller object 59 | % Detect parts 60 | %clear C 61 | %for k = 1:length(n) 62 | % C(k) = detectPart(X{n(k)},Y{n(k)},X{m},Y{m}, thresh_par); 63 | %end 64 | dC = sum(sum(C(n,m)))>0; 65 | 66 | % Probability of overlap between two unrelated objects: 67 | Pn = min(.99,max(0.01, max(sum(area(n)),sum(area(m)))/(eps+ncols*nrows))); % avoid NaN 68 | 69 | % Compute likelihoods for part and no-part 70 | %n = n(1); 71 | PC_p(ndx(n(1)), ndx(m(1))) = PC_p(ndx(n(1)), ndx(m(1))) + dC*log(Pp) + (1-dC)*log(1-Pp); 72 | PC_n(ndx(n(1)), ndx(m(1))) = PC_n(ndx(n(1)), ndx(m(1))) + dC*log(Pn) + (1-dC)*log(1-Pn); 73 | 74 | Counts(ndx(n(1)), ndx(m(1))) = Counts(ndx(n(1)), ndx(m(1))) + 1; 75 | end 76 | end 77 | end 78 | end 79 | end 80 | 81 | % Posterior: 82 | % Ppart = PC_p*Prior_part ./ (PC_p*Prior_part+PC_n*(1-Prior_part)); 83 | ratio = exp(PC_n-PC_p) * (1-Prior_part)/Prior_part; 84 | clear PC_n PC_p 85 | Ppart = 1 ./ (1+ratio); 86 | clear ratio 87 | 88 | % Set to Prior_part all probabilities computed with small counts 89 | Ppart = Ppart.*(Counts>5) + Prior_part.*(Counts<=5); 90 | Ppart = Ppart-diag(diag(Ppart)); 91 | 92 | % Just for visualization: 93 | [i,j] = find(Ppart>.1 & Counts>10); %only show results when there are enough samples 94 | v = Ppart(sub2ind(size(Ppart),i,j)); 95 | [v,n] = sort(v, 'descend'); 96 | for k=n' 97 | disp(sprintf('%s is part of %s (with probability = %1.2f, estimated from %d samples)', objectClasses{j(k)}, objectClasses{i(k)}, Ppart(i(k),j(k)), Counts(i(k),j(k)))) 98 | end 99 | 100 | disp('number of NaN') 101 | M = sum(isnan(Ppart(:))) 102 | 103 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 104 | function C = detectPart(X1,Y1,X2,Y2, thresh_par) 105 | % Return logical indicator. Is O1/O2 part of O2/O1? 106 | 107 | C = 0; 108 | 109 | % Do fast approximation of intersection area: 110 | Rn = [min(X1) min(Y1) max(X1) max(Y1)]; 111 | Rm = [min(X2) min(Y2) max(X2) max(Y2)]; 112 | if Rn(3) thresh_par 118 | [A,ua,area1,area2] = PolyAreas(X1,Y1,X2,Y2); 119 | % detect overlap 120 | C = (A/min(area1,area2)) > thresh_par; 121 | end 122 | -------------------------------------------------------------------------------- /demo.m: -------------------------------------------------------------------------------- 1 | % This script demonstrates the main functionalities of the LabelMe3D 2 | % toolbox. The functionalities depend on the LabelMe toolbox, which can 3 | % be downloaded from here: 4 | % 5 | % http://labelme.csail.mit.edu/LabelMeToolbox/index.html 6 | 7 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 8 | %%% Step 1: Compile mex files in the LabelMe3D toolbox 9 | compile; 10 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 11 | 12 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 13 | %%% Step 2: Add required toolboxes to the Matlab path (change the following 14 | % lines to point to the locations of the LabelMe and LabelMe3D toolboxes) 15 | addpath '~/work/MatlabLibraries/LabelMeToolbox'; % LabelMe toolbox 16 | addpath '~/work/MatlabLibraries/LabelMe3dToolbox'; % LabelMe3D toolbox 17 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 18 | 19 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 20 | % Read a LabelMe annotation and image directly from the LabelMe server 21 | % and compute its 3D information. 22 | 23 | HOMEANNOTATIONS = 'http://labelme.csail.mit.edu/Annotations'; 24 | HOMEIMAGES = 'http://labelme.csail.mit.edu/Images'; 25 | folderlist = {'05june05_static_street_boston'}; 26 | filelist = {'p1010736.jpg'}; 27 | DB = LMdatabase(HOMEANNOTATIONS,HOMEIMAGES,folderlist,filelist); 28 | img = LMimread(DB,1,HOMEIMAGES); 29 | annotation = Recover3DSceneComponents(DB.annotation); 30 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 31 | 32 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 33 | %%% DATA VISUALIZATION. The following illustrates how to visualize the data. 34 | 35 | % Plot 3D annotations: 36 | figure; 37 | LMplot3Dscene(annotation,img); 38 | 39 | % Plot inferred polygon and edge types: 40 | figure; 41 | plotPolyEdgeTypes(annotation,img); 42 | 43 | % Generate textured mesh: 44 | mesh = LM3DgetTexturedMesh(annotation,img); 45 | 46 | % Generate VRML file: 47 | vrmlfile = strrep(annotation.filename,'.jpg','.wrl'); 48 | vrmlfolder = fullfile('cache','vrml'); 49 | if ~exist(vrmlfolder,'dir') 50 | mkdir(vrmlfolder); 51 | end 52 | LM2VRML(annotation,mesh,vrmlfile,vrmlfolder); 53 | display(sprintf('Produced a VRML file here: %s',fullfile(vrmlfolder,vrmlfile))); 54 | 55 | % Sanity check that 3D points project to 2D points: 56 | err = []; 57 | for i = 1:length(annotation.object) 58 | if isfield(annotation.object(i).world3d,'polygon3d') 59 | X = getLMpolygon3D(annotation.object(i).world3d.polygon3d); % Get 3D points 60 | [x,y] = Project3D2D(X,annotation); % Project 3D points to 2D 61 | [xx,yy] = getLMpolygon(annotation.object(i).polygon); % Original 2D polygon 62 | err = [err [abs(x-xx'); abs(y-yy')]]; % Record error 63 | end 64 | end 65 | display(sprintf('Mean error: %f',mean(err(:)))); 66 | 67 | % Plot labeled objects and horizon line: 68 | [xh,yh] = getHorizonLine(annotation); 69 | figure; 70 | LMplot(annotation,img); 71 | hold on; 72 | plot(xh,yh,'r'); 73 | 74 | % Plot depth map: 75 | [Xmap,Ymap,Zmap,Nmap] = getXYZmaps(annotation); 76 | figure; 77 | plotZmap(Zmap); 78 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 79 | 80 | return; 81 | 82 | 83 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 84 | %%% DOWNLOAD DATA, OPTION 1: Download the entire database onto your machine. 85 | 86 | % Change this path to point to the desired location of the LabelMe3D database: 87 | HOMEDATABASE = './LabelMe3D_database'; 88 | 89 | % Define needed paths for the database: 90 | HOMEANNOTATIONS3D = fullfile(HOMEDATABASE,'Annotations3D'); 91 | HOMEIMAGES = fullfile(HOMEDATABASE,'Images'); 92 | 93 | % Download data: 94 | LM3Dinstall(HOMEIMAGES,HOMEANNOTATIONS3D); 95 | 96 | % Read data into Matlab structure. The LabelMe annotation structure for 97 | % the ith image is stored as "DB(i).annotation". 98 | DB = LM3Ddatabase(HOMEANNOTATIONS3D); 99 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 100 | 101 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 102 | %%% DOWNLOAD DATA, OPTION 2: Download only needed images. 103 | 104 | % Read an annotation and image directly from the LabelMe server: 105 | HOMEANNOTATIONS3D = 'http://labelme.csail.mit.edu/Annotations3D'; 106 | HOMEIMAGES = 'http://labelme.csail.mit.edu/Images'; 107 | folderlist = {'05june05_static_street_boston'}; 108 | filelist = {'p1010736.jpg'}; 109 | DB = LM3Ddatabase(HOMEANNOTATIONS3D,HOMEIMAGES,folderlist,filelist); 110 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 111 | 112 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 113 | %%% RUNNING LABELME3D ALGORITHM: Run over all downloaded annotations. 114 | 115 | % WARNING: this will over-write your downloaded XML annotations! 116 | HOMEANNOTATIONS3D = fullfile(HOMEDATABASE,'Annotations3D'); 117 | LM3Dgenerate3D(HOMEANNOTATIONS3D,HOMEIMAGES); 118 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 119 | 120 | -------------------------------------------------------------------------------- /LM2VRMLfill.m: -------------------------------------------------------------------------------- 1 | function LM2VRMLfill(annotation,vrmlfolder,outfile,imfile,img) 2 | % LM2VRML(annotation,outfile,imfile,img) 3 | % 4 | % Converts LabelMe 3D polygons into VRML file for visualization. 5 | % 6 | % Inputs: 7 | % annotation - LabelMe annotation structure 8 | % 9 | % outfile - Output VRML file name (e.g. 'foo.wrl'). 10 | % imfile - Pointer to location of image filename. This is used to 11 | % texturize 3D polygons. 12 | % img - Image. If this is an empty matrix, then the VRML output will not 13 | % have image texture (the polygons will be colored). 14 | 15 | % Get image size 16 | if nargin>=4 17 | [nrows,ncols,dim] = size(img); 18 | else 19 | if ~isfield(annotation,'imagesize') 20 | error('Please pass the image as an input or set the imagesize field.'); 21 | return; 22 | end 23 | nrows = annotation.imagesize.nrows; 24 | ncols = annotation.imagesize.ncols; 25 | img = 1; 26 | end 27 | 28 | % Parameters VRML: 29 | fieldOfView = 0.78; 30 | scaling = 30; 31 | vBB = [-10^5 10^5 -10^5 10^5 -100 10^8]; % volume bounding box (remove objects outside this volume) 32 | 33 | % Get meshes 34 | [X,Y,Z,TX,TY,tri,boundary,textures,objNames] = getTextures(annotation, img); 35 | Nobjects = length(X); 36 | 37 | % Save textures 38 | relativeFolder = strrep(outfile, '.wrl', ''); 39 | mkdir(fullfile(vrmlfolder, relativeFolder)) 40 | for ii = 1:Nobjects 41 | texturefilename{ii} = sprintf('object_%03d.jpg', ii); 42 | imwrite(textures{ii}, fullfile(vrmlfolder, relativeFolder, texturefilename{ii}), 'jpg') 43 | end 44 | 45 | % Get initial camera position: 46 | initialCamera = getInitialViewpoint(X,Y,Z); 47 | 48 | % Dump VRML file: 49 | fp = fopen(fullfile(vrmlfolder,outfile),'w'); 50 | fprintf(fp,'#VRML V2.0 utf8\n\nNavigationInfo {\n headlight TRUE\n type [\"EXAMINE\", \"ANY\"]\n}\n\n'); 51 | fprintf(fp,'Viewpoint {\n position %.2f %.2f %.2f\n orientation 0 0 1 0\n fieldOfView %.2f\n description "Original"\n}\n\n',initialCamera(1), initialCamera(2), initialCamera(3), fieldOfView); 52 | fprintf(fp,'DEF Back1 Background {\n groundColor [1 1 1]\n skyColor [1 1 1]\n}\n\n'); 53 | 54 | for ii = 1:Nobjects 55 | fprintf(fp,'# %s\nShape{\n',objNames{ii}); 56 | if isempty(img) 57 | fprintf(fp,' appearance Appearance {\n material Material {\n diffuseColor %.2f %.2f %.2f\n }\n }\n',colors(ii,1),colors(ii,2),colors(ii,3)); 58 | else 59 | fprintf(fp,' appearance Appearance {\n texture ImageTexture { url \"%s\" }\n }\n', fullfile(relativeFolder, texturefilename{ii})); 60 | end 61 | 62 | fprintf(fp,' geometry IndexedFaceSet {\n coord Coordinate {\n point [\n'); 63 | for jj = 1:length(X{ii}) 64 | fprintf(fp,' %.4f %.4f %.4f,\n',X{ii}(jj)/scaling,Y{ii}(jj)/scaling,-Z{ii}(jj)/scaling); 65 | end 66 | fprintf(fp,' ]\n }\n'); 67 | 68 | %% Get coordIndex: 69 | fprintf(fp,' coordIndex [\n'); 70 | for j = 1:size(tri{ii},1) 71 | fprintf(fp,' %d %d %d -1,\n',tri{ii}(j,:)); 72 | end 73 | fprintf(fp,' ]\n'); 74 | 75 | if ~isempty(img) 76 | %% Get texture coordinates: 77 | fprintf(fp,' texCoord TextureCoordinate {\n point [\n'); 78 | for jj = 1:length(TX{ii}) 79 | fprintf(fp,' %.4f %.4f,\n',TX{ii}(jj),1-TY{ii}(jj)); 80 | end 81 | fprintf(fp,' ]\n'); 82 | fprintf(fp,' }\n'); 83 | 84 | %% Get texture coordinate indices: 85 | fprintf(fp,' texCoordIndex [\n'); 86 | for j = 1:size(tri{ii},1) 87 | fprintf(fp,' %d %d %d -1,\n',tri{ii}(j,:)); 88 | end 89 | fprintf(fp,' ]\n'); 90 | end 91 | 92 | fprintf(fp,' solid FALSE\n'); 93 | fprintf(fp,' }\n'); 94 | fprintf(fp,'}\n\n'); 95 | end 96 | fclose(fp); 97 | 98 | 99 | 100 | 101 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 102 | function initialCamera = getInitialViewpoint(X,Y,Z) 103 | % 104 | % From the object coordinater, get best view point to start viewing the 105 | % scene 106 | 107 | % Get initial camera position: 108 | cube = [Inf -Inf Inf -Inf Inf -Inf 0 0 0]; 109 | for ii = 1:length(X) 110 | %cube = [minX maxX minY maxY minZ maxZ]; 111 | cube(1) = min(cube(1), min(X{ii})); 112 | cube(2) = max(cube(2), max(X{ii})); 113 | cube(3) = min(cube(3), min(Y{ii})); 114 | cube(4) = max(cube(4), max(Y{ii})); 115 | cube(5) = min(cube(5), min(Z{ii})); 116 | cube(6) = max(cube(6), max(Z{ii})); 117 | % mean 118 | cube(7) = cube(7)+mean(X{ii})/length(X); 119 | cube(8) = cube(8)+mean(max(1,Y{ii}))/length(X); 120 | cube(9) = cube(9)+mean(Z{ii})/length(X); 121 | end 122 | %initialCamera = [(cube(1)+cube(2))/2 max(1,(cube(3)+cube(4))/2) cube(5)*0.8]; 123 | initialCamera = [cube(7) cube(8) 0*cube(5)*0.8]; 124 | 125 | initialCamera = [0 5 max(-30,min(cube(5),0))]; 126 | 127 | 128 | -------------------------------------------------------------------------------- /contactPointsNew.m: -------------------------------------------------------------------------------- 1 | function annotation = contactPointsNew(annotation,sup) 2 | % 3 | % This function needs a full rewrite 4 | % - also consider edges: two nearby parallel edges are also a contact 5 | % cue. 6 | 7 | thresh_sup = 50; 8 | largeObjects = getListLargeObjects; 9 | 10 | Nobjects = length(annotation.object); 11 | 12 | for i = 1:Nobjects 13 | [X{i},Y{i}] = getLMpolygon(annotation.object(i).polygon); 14 | end 15 | 16 | for i = 1:Nobjects 17 | % add default 0 contact 18 | for j = 1:length(annotation.object(i).polygon.pt) 19 | annotation.object(i).polygon.pt(j).contact = 0; 20 | end 21 | if sup(i) && ismember(strtrim(lower(annotation.object(i).name)),largeObjects) 22 | % Handle buildings/walls/etc: 23 | % $$$ n = convhull(X{i},Y{i})'; 24 | n = [1:length(X{i})]; 25 | 26 | for j = n 27 | x = X{i}(j); 28 | y = Y{i}(j); 29 | % $$$ if SupportDist(X{sup(i)},Y{sup(i)},x,y)<=thresh_sup && y>annotation.camera.Hy % IMPORTANT: contact points are bellow horizon line 30 | if y>annotation.camera.Hy % IMPORTANT: contact points are bellow horizon line 31 | annotation.object(i).polygon.pt(j).contact = 1; 32 | end 33 | end 34 | else 35 | % Case for all objects 36 | [v,n] = max(Y{i}); 37 | if v>annotation.camera.Hy % IMPORTANT: contact points are bellow horizon line 38 | annotation.object(i).polygon.pt(n).contact = 1; 39 | end 40 | end 41 | end 42 | 43 | % This is a hack but covers an important concept: a contact point has to 44 | % be the lowest in the same veritical line 45 | for i = 1:Nobjects 46 | [x,y] = getLMpolygon(annotation.object(i).polygon); 47 | pb = isThereAnyPointBellow(x,y); 48 | for j = find(pb)' 49 | annotation.object(i).polygon.pt(j).contact = 0; 50 | end 51 | end 52 | 53 | % Another hack: find adjacent points that form lines with high absolute 54 | % slope (i.e. the two poitns form a vertical line). Do not consider the 55 | % point at the top of the line. 56 | for i = 1:Nobjects 57 | [x,y] = getLMpolygon(annotation.object(i).polygon); 58 | vl = isVerticalLine(x,y); 59 | for j = find(vl') 60 | annotation.object(i).polygon.pt(j).contact = 0; 61 | end 62 | end 63 | 64 | % $$$ % Handle occlusion boundaries. Points along occlusion boundaries should 65 | % $$$ % not be contact points. We make sure to leave the lowest point. 66 | % $$$ % TO DO: need to remove points that are close to the *boundary* of an 67 | % $$$ % occluding object. 68 | % $$$ OCC_THRESH = 20; 69 | % $$$ for i = 1:Nobjects 70 | % $$$ [xi,yi] = getLMpolygon(annotation.object(i).polygon); 71 | % $$$ c = find([annotation.object(i).polygon.pt(:).contact]); 72 | % $$$ if ~isempty(c) 73 | % $$$ [v,j] = max(yi(c)); 74 | % $$$ c = c(j); 75 | % $$$ for j = 1:Nobjects 76 | % $$$ if (i~=j) & (j~=sup(i)) 77 | % $$$ [xj,yj] = getLMpolygon(annotation.object(j).polygon); 78 | % $$$ for k = 1:length(xi) 79 | % $$$ if (k~=c) && annotation.object(i).polygon.pt(k).contact && (SupportDist(xj,yj,xi(k),yi(k))x(n)-0.001 & i~=n 122 | % Insert point 123 | m = m+1; 124 | yy(m) = (y(i+1)-y(i))/(x(i+1)-x(i))*(x(n)-x(i))+y(i); 125 | end 126 | end 127 | if length(yy)>0 128 | pb(n) = y(n)SLOPE_THRESH)&(y(1:end-1)SLOPE_THRESH)&(y(1:end-1)<[y(end-1); y(1:end-2)])); 145 | 146 | function d = dist2(x1,y1,x2,y2) 147 | % x1,y1,x2,y2 are column vectors 148 | d = repmat(x1.^2,1,length(x2))-2*x1*x2'+repmat(x2.^2,1,length(x1))' + ... 149 | repmat(y1.^2,1,length(y2))-2*y1*y2'+repmat(y2.^2,1,length(y1))'; 150 | -------------------------------------------------------------------------------- /getPolyEdgeTypes.m: -------------------------------------------------------------------------------- 1 | function varargout = getPolyEdgeTypes(annotation,img,type,whichPolygons) 2 | % Inputs: 3 | % annotation - LabelMe3D annotation structure 4 | % img - image 5 | % type - Any of: 'standingplanes','groundplane','part','attached_edges','contact_edges','occluded_edges' 6 | % whichPolygons (optional) - {'valid' | 'all'} 7 | % 8 | % Outputs: 9 | % seg - Binary masks for desired type 10 | % 11 | % Example: get binary masks for "parts" and "occluded edges" 12 | % 13 | % [partMask,occludedEdgesMask] = getPolyEdgeTypes(annotation,img,{'part','occluded_edges'}); 14 | 15 | if nargin < 4 16 | whichPolygons = 'valid'; 17 | end 18 | 19 | if isstr(type) 20 | type = {type}; 21 | end 22 | 23 | imageSize = size(img); 24 | 25 | % Set line sizes: 26 | sz = ceil(sqrt(imageSize(1)^2+imageSize(2)^2)/800); 27 | attachedLineWidth = sz; 28 | occludedLineWidth = 2*sz; 29 | contactLineWidth = 2*sz; 30 | 31 | % Remove deleted objects from consideration: 32 | dd = isdeleted(annotation); 33 | annotation.object = annotation.object(~dd); 34 | 35 | SegOrder = {'standingplanes','groundplane','part','attached_edges','contact_edges','occluded_edges'}; 36 | 37 | if ~all(ismember(type,SegOrder)) 38 | error('One of the types is invalid.'); 39 | end 40 | 41 | % Initialize outputs: 42 | for i = 1:length(SegOrder) 43 | if ismember(SegOrder{i},type) 44 | seg{i} = logical(zeros(imageSize(1),imageSize(2))); 45 | end 46 | end 47 | 48 | for i = 1:length(annotation.object) 49 | %%% Get masks of different polygon types: 50 | if any(ismember({'standingplanes','groundplane','part'},type)) 51 | [X,Y] = getLMpolygon(annotation.object(i).polygon); 52 | mask = poly2mask(double(X),double(Y),imageSize(1),imageSize(2)); 53 | 54 | switch annotation.object(i).world3d.type 55 | case 'standingplanes' 56 | if ismember('standingplanes',type) 57 | seg{1}(mask) = 1; 58 | end 59 | case 'groundplane' 60 | if ismember('groundplane',type) 61 | seg{2}(mask) = 1; 62 | end 63 | case 'part' 64 | if ismember('part',type) 65 | parentNdx = getPartOfParents(annotation,i); 66 | parentNdx = parentNdx(end); 67 | if strcmp(whichPolygons,'all') || strcmp(annotation.object(parentNdx).world3d.type,'standingplanes') 68 | seg{3}(mask) = 1; 69 | end 70 | end 71 | otherwise 72 | if strcmp(whichPolygons,'all') && ismember('standingplanes',type) 73 | % Render as standing plane: 74 | seg{1}(mask) = 1; 75 | end 76 | end 77 | end 78 | 79 | %%% Plot edges for attached polygons: 80 | if ismember('attached_edges',type) && strcmp(annotation.object(i).world3d.type,'part') 81 | parentNdx = getPartOfParents(annotation,i); 82 | parentNdx = parentNdx(end); 83 | if strcmp(whichPolygons,'all') || strcmp(annotation.object(parentNdx).world3d.type,'standingplanes') 84 | % Get attached edges: 85 | edgesAttached = getEdges(annotation.object(i).polygon); 86 | 87 | seg{4} = seg{4} | plotBinaryLine(edgesAttached,size(seg{4}),attachedLineWidth); 88 | end 89 | end 90 | 91 | %%% Plot edges for standing polygons: 92 | if strcmp(whichPolygons,'all') || strcmp(annotation.object(i).world3d.type,'standingplanes') 93 | edgesOccluded = getEdges(annotation.object(i).polygon); 94 | xc = cellfun(@str2num,{annotation.object(i).world3d.contact(:).x}); 95 | yc = cellfun(@str2num,{annotation.object(i).world3d.contact(:).y}); 96 | edgesContact = [xc(1:end-1); yc(1:end-1); xc(2:end); yc(2:end)]'; 97 | 98 | sc = logical(zeros(imageSize(1),imageSize(2))); 99 | if ~isempty(edgesContact) 100 | sc = plotBinaryLine(edgesContact,size(sc),contactLineWidth); 101 | end 102 | if ~isempty(edgesOccluded) && ismember('occluded_edges',type) 103 | so = plotBinaryLine(edgesOccluded,size(seg{6}),occludedLineWidth)-sc; 104 | 105 | % Remove small drawn components: 106 | ll = bwlabel(so); cc = hist(ll(:),[0:max(ll(:))]); 107 | so = ismember(ll,find(cc(2:end)>2)); 108 | 109 | seg{6} = seg{6} | so; 110 | end 111 | if ismember('contact_edges',type) 112 | seg{5} = seg{5} | sc; 113 | end 114 | end 115 | end 116 | 117 | % Assign outputs: 118 | [junk,n] = ismember(type,SegOrder); 119 | varargout = seg(n); 120 | 121 | return; 122 | 123 | function mask = plotBinaryLine(edges,imageSize,sz) 124 | % Plots lines on a bitmap image. 125 | 126 | N = ceil(max(((edges(:,3)-edges(:,1)).^2+(edges(:,4)-edges(:,2)).^2).^0.5))+10; 127 | alpha = linspace(0,1,N); 128 | sx = zeros(size(edges,1),N); 129 | sy = zeros(size(edges,1),N); 130 | for i = 1:N 131 | sx(:,i) = (1-alpha(i))*edges(:,1)+alpha(i)*edges(:,3); 132 | sy(:,i) = (1-alpha(i))*edges(:,2)+alpha(i)*edges(:,4); 133 | end 134 | s = round([sx(:) sy(:)]); 135 | s = max(s,1); 136 | s(:,1) = min(s(:,1),imageSize(2)); 137 | s(:,2) = min(s(:,2),imageSize(1)); 138 | s = unique(s,'rows'); 139 | ndx = sub2ind(imageSize,s(:,2),s(:,1)); 140 | mask = zeros(imageSize); 141 | mask(ndx) = 1; 142 | cB = strel('square',sz); % This is faster than disk 143 | % $$$ cB = strel('disk',sz); 144 | mask = imdilate(mask,cB); 145 | 146 | return; 147 | 148 | type = 'standingplanes'; 149 | seg = getPolyEdgeTypes(annotation,img,type); 150 | 151 | type = 'part'; 152 | seg = getPolyEdgeTypes(annotation,img,type); 153 | 154 | type = 'groundplane'; 155 | seg = getPolyEdgeTypes(annotation,img,type); 156 | 157 | type = 'attached_edges'; 158 | seg = getPolyEdgeTypes(annotation,img,type); 159 | 160 | type = 'occluded_edges'; 161 | seg = getPolyEdgeTypes(annotation,img,type); 162 | 163 | type = 'contact_edges'; 164 | seg = getPolyEdgeTypes(annotation,img,type); 165 | 166 | type = {'contact_edges','part'}; 167 | [ce,pa] = getPolyEdgeTypes(annotation,img,type); 168 | -------------------------------------------------------------------------------- /InflateSceneXML.m: -------------------------------------------------------------------------------- 1 | function annotation = InflateSceneXML(annotation,notDeleted) 2 | 3 | imageSize = [str2num(annotation.imagesize.nrows) str2num(annotation.imagesize.ncols)]; 4 | 5 | % Add 3D information for groundplane and standingplanes objects: 6 | P = getCameraMatrix(annotation,'RH'); 7 | if isempty(P) 8 | valid = 0; 9 | return; 10 | end 11 | [K,R,C] = decomposeP(P); 12 | for i = notDeleted%1:length(annotation.object) 13 | switch annotation.object(i).world3d.type 14 | case 'groundplane' 15 | % Intersect polygon points with ground plane: 16 | [x,y] = getLMpolygon(annotation.object(i).polygon); 17 | X = projectOntoGroundPlane(x,y,annotation); 18 | annotation.object(i).world3d.polygon3d = setLMpolygon3D(X(1,:),X(2,:),X(3,:)); 19 | 20 | % Get plane parameters: 21 | annotation.object(i).world3d.plane.pix = '0'; 22 | annotation.object(i).world3d.plane.piy = '1'; 23 | annotation.object(i).world3d.plane.piz = '0'; 24 | annotation.object(i).world3d.plane.piw = '0'; 25 | 26 | % Reorder fields: 27 | annotation.object(i).world3d = orderfields(annotation.object(i).world3d,{'type','stale','polygon3d','plane'}); 28 | case 'standingplanes' 29 | % Get planes corresponding to each contact edge by reasoning through 30 | % world contact lines: 31 | [x,y] = getLMpolygon(annotation.object(i).polygon); 32 | [x,y] = LH2RH(x,y,imageSize); 33 | xc = str2num(char({annotation.object(i).world3d.contact(:).x})); 34 | yc = str2num(char({annotation.object(i).world3d.contact(:).y})); 35 | [xc,yc] = LH2RH(xc,yc,imageSize); 36 | 37 | % Get plane parameters: 38 | H = P(:,[1 3 4]); 39 | if length(xc)==1 40 | % Make single plane frontal-parallel: 41 | xx = H\[xc yc 1]'; 42 | PI = [0 0 1 -xx(2)/xx(3)]'; 43 | else 44 | % Handle multiple planes: 45 | PI = []; 46 | for j = 1:length(xc)-1 47 | x1 = [xc(j) yc(j) 1]'; 48 | x2 = [xc(j+1) yc(j+1) 1]'; 49 | li = cross(x1,x2); 50 | lw = H'*li; 51 | lw = lw/sqrt(lw(1)^2+lw(2)^2); 52 | PI(:,j) = [lw(1) 0 lw(2) lw(3)]'; 53 | end 54 | end 55 | 56 | % Project 2D points onto standing planes: 57 | [X,isAdded,nPlane,x,y] = projectOntoStandingPlanes(x,y,P,PI,xc,yc,i); 58 | 59 | if ~isempty(X) 60 | if any(isAdded) 61 | % Store 2D points: 62 | [x,y] = RH2LH(x,y,imageSize); 63 | for j = 1:length(x) 64 | annotation.object(i).polygon.pt(j).x = num2str(x(j)); 65 | annotation.object(i).polygon.pt(j).y = num2str(y(j)); 66 | end 67 | end 68 | 69 | % Store plane parameters: 70 | for j = 1:size(PI,2) 71 | annotation.object(i).world3d.plane(j).pix = num2str(PI(1,j)); 72 | annotation.object(i).world3d.plane(j).piy = num2str(PI(2,j)); 73 | annotation.object(i).world3d.plane(j).piz = num2str(PI(3,j)); 74 | annotation.object(i).world3d.plane(j).piw = num2str(PI(4,j)); 75 | end 76 | 77 | % Store 3D points in annotation structure: 78 | annotation.object(i).world3d.polygon3d = setLMpolygon3D(X(1,:),X(2,:),X(3,:)); 79 | for j = 1:size(X,2) 80 | annotation.object(i).world3d.polygon3d.pt(j).added = num2str(isAdded(j)); 81 | for k = 1:length(nPlane{j}) 82 | annotation.object(i).world3d.polygon3d.pt(j).planeindex(k).index = num2str(nPlane{j}(k)-1); 83 | end 84 | end 85 | 86 | % Reorder fields: 87 | annotation.object(i).world3d = orderfields(annotation.object(i).world3d,{'type','stale','polygon3d','contact','plane'}); 88 | % $$$ else 89 | % $$$ keyboard; 90 | end 91 | case 'part' 92 | % Reorder fields: 93 | annotation.object(i).world3d = orderfields(annotation.object(i).world3d,{'type','stale','parentid'}); 94 | case 'none' 95 | % Reorder fields: 96 | annotation.object(i).world3d = orderfields(annotation.object(i).world3d,{'type','stale'}); 97 | end 98 | end 99 | 100 | % Add 3D information for parts: 101 | for i = notDeleted%1:length(annotation.object) 102 | if strcmp(annotation.object(i).world3d.type,'part') 103 | % Get root object index: 104 | k = getPartOfParents(annotation,i); 105 | nRoot = k(end); 106 | 107 | % Set root ID: 108 | annotation.object(i).world3d.rootid = annotation.object(nRoot).id; 109 | 110 | if isfield(annotation.object(nRoot).world3d,'polygon3d') 111 | % Get polygon: 112 | [x,y] = getLMpolygon(annotation.object(i).polygon); 113 | 114 | switch annotation.object(nRoot).world3d.type 115 | case 'standingplanes' 116 | % Project 2D points onto root object's standing planes: 117 | [x,y] = LH2RH(x,y,imageSize); 118 | [X,isAdded,nPlane,x,y] = projectOntoStandingPlanes(x,y,annotation,nRoot); 119 | case 'groundplane' 120 | [X,isAdded,nPlane,x,y] = projectOntoGroundPlane(x,y,annotation); 121 | otherwise 122 | error('Root object 3D type is invalid'); 123 | end 124 | 125 | if ~isempty(X) 126 | if any(isAdded) 127 | % Store 2D points: 128 | [x,y] = RH2LH(x,y,imageSize); 129 | for j = 1:length(x) 130 | annotation.object(i).polygon.pt(j).x = num2str(x(j)); 131 | annotation.object(i).polygon.pt(j).y = num2str(y(j)); 132 | end 133 | end 134 | 135 | % Store 3D points in annotation structure: 136 | annotation.object(i).world3d.polygon3d = setLMpolygon3D(X(1,:),X(2,:),X(3,:)); 137 | for j = 1:size(X,2) 138 | annotation.object(i).world3d.polygon3d.pt(j).added = num2str(isAdded(j)); 139 | for k = 1:length(nPlane{j}) 140 | annotation.object(i).world3d.polygon3d.pt(j).planeindex(k).index = num2str(nPlane{j}(k)-1); 141 | end 142 | end 143 | end 144 | end 145 | end 146 | end 147 | -------------------------------------------------------------------------------- /getXYZmaps.m: -------------------------------------------------------------------------------- 1 | function [Xmap,Ymap,Zmap,Nmap] = getXYZmaps(annotation,mesh,scaleFactor) 2 | 3 | if nargin < 3 4 | % $$$ maxRes = 360; 5 | scaleFactor = 1; 6 | end 7 | if nargin < 2 8 | mesh = getSceneMesh(annotation); 9 | mesh = subDivideMesh(mesh,2); 10 | end 11 | 12 | P = getCameraMatrix(annotation,'RH'); 13 | 14 | imageSize = [str2num(annotation.imagesize.nrows) str2num(annotation.imagesize.ncols)]; 15 | 16 | if scaleFactor~=1%max(imageSize)>maxRes 17 | % $$$ scaleFactor = maxRes/max(imageSize); 18 | P = [scaleFactor 0 0; 0 scaleFactor 0; 0 0 1]*P; 19 | imageSize = round(scaleFactor*imageSize); 20 | end 21 | 22 | [K,R,C] = decomposeP(P); 23 | 24 | % Ray-cast: 25 | [x,y] = meshgrid(1:imageSize(2),1:imageSize(1)); 26 | [x,y] = LH2RH(x,y,imageSize); 27 | x = [x(:) y(:) ones(prod(size(x)),1)]'; 28 | [X,isValid,faceNdx] = rayTrace(single(P(:,1:3)\x),single(C),mesh.vertices,mesh.faces); 29 | 30 | isValid = reshape(isValid,imageSize); 31 | faceNdx = reshape(faceNdx,imageSize); 32 | Nmap = inf*ones(imageSize,'uint32'); 33 | Nmap(isValid) = mesh.objndx(faceNdx(isValid)); 34 | Xmap = int32(reshape(X(1,:),imageSize)); 35 | Ymap = uint32(reshape(X(2,:),imageSize)); 36 | Zmap = uint32(reshape(X(3,:),imageSize)); 37 | 38 | MAX_DEPTH = 16777215; 39 | MAX_WIDTH = 8388607; 40 | Xmap(~isValid) = MAX_WIDTH; 41 | Ymap(~isValid) = MAX_DEPTH; 42 | Zmap(~isValid) = MAX_DEPTH; 43 | 44 | % $$$ function [Xmap,Ymap,Zmap,Nmap,Xobj,Yobj,Zobj] = getXYZmaps_old(annotation) 45 | % $$$ % [Xmap,Ymap,Zmap,Nmap] = getXYZmaps(annotation) 46 | % $$$ % 47 | % $$$ % Computes depth map given the computed 3D mesh. 48 | % $$$ % Returns the X,Y,Z data for each pixel in the image, along with a 49 | % $$$ % map of object IDs (corresponding to annotation.object(i).id) 50 | % $$$ % indicating which object is closest to the camera center at each x,y 51 | % $$$ % location in the image. Pixels that do not point to any valid object 52 | % $$$ % with 3D information is given the value of 4294967295. 53 | % $$$ % 54 | % $$$ % [Xmap,Ymap,Zmap,Nmap,Xobj,Yobj,Zobj] = getXYZmaps(annotation) 55 | % $$$ % 56 | % $$$ % Computes individual depth maps for the objects. 57 | % $$$ % 58 | % $$$ % Inputs: 59 | % $$$ % annotation - LabelMe annotation structure 60 | % $$$ % 61 | % $$$ % Outputs: 62 | % $$$ % Xmap - Map of X coordinates 63 | % $$$ % Ymap - Map of Y coordinates 64 | % $$$ % Zmap - Map of Z coordinates 65 | % $$$ % Nmap - Map of object IDs for each pixel (ground and standing objects). 66 | % $$$ % Xobj - 3D matrix of X coordinates for each object. 67 | % $$$ % Yobj - 3D matrix of Y coordinates for each object. 68 | % $$$ % Zobj - 3D matrix of Z coordinates for each object. 69 | % $$$ 70 | % $$$ MAX_DEPTH = 16777215; 71 | % $$$ MAX_WIDTH = 8388607; 72 | % $$$ nrows = str2num(annotation.imagesize.nrows); 73 | % $$$ ncols = str2num(annotation.imagesize.ncols); 74 | % $$$ 75 | % $$$ % $$$ % Insert ground plane: 76 | % $$$ % $$$ annotation = insertGroundPlane(annotation); 77 | % $$$ 78 | % $$$ % Get indices of non-deleted objects: 79 | % $$$ notDeleted = find(~isdeleted(annotation))'; 80 | % $$$ 81 | % $$$ Nobjects = length(annotation.object); 82 | % $$$ Xmap = MAX_WIDTH*ones(nrows,ncols,'int32'); 83 | % $$$ Ymap = MAX_DEPTH*ones(nrows,ncols,'uint32'); 84 | % $$$ Zmap = MAX_DEPTH*ones(nrows,ncols,'uint32'); 85 | % $$$ Nmap = inf*ones(nrows,ncols,'uint32'); 86 | % $$$ if nargout>=5 87 | % $$$ Xobj = MAX_WIDTH*ones(nrows,ncols,Nobjects,'int32'); 88 | % $$$ Yobj = MAX_DEPTH*ones(nrows,ncols,Nobjects,'uint32'); 89 | % $$$ Zobj = MAX_DEPTH*ones(nrows,ncols,Nobjects,'uint32'); 90 | % $$$ end 91 | % $$$ 92 | % $$$ % Store depth maps for standing and ground objects: 93 | % $$$ for i = notDeleted%1:Nobjects 94 | % $$$ if ismember(annotation.object(i).polygon.polyType,{'ground','standing'}) 95 | % $$$ [X,Y,Z,mx,my,tri,boundary,CptsNdx,valid] = getObject3D(annotation,i,'grid'); 96 | % $$$ 97 | % $$$ if ~isempty(X) && valid 98 | % $$$ X = int32(X)'; 99 | % $$$ Y = uint32(Y)'; 100 | % $$$ Z = uint32(Z)'; 101 | % $$$ n = sub2ind([nrows ncols],my,mx); 102 | % $$$ nn = ((Z=5 108 | % $$$ Xobj(n+(i-1)*nrows*ncols) = X; 109 | % $$$ Yobj(n+(i-1)*nrows*ncols) = Y; 110 | % $$$ Zobj(n+(i-1)*nrows*ncols) = Z; 111 | % $$$ end 112 | % $$$ end 113 | % $$$ end 114 | % $$$ end 115 | % $$$ 116 | % $$$ % Store depth maps for attached objects: 117 | % $$$ for i = notDeleted%1:Nobjects 118 | % $$$ if strcmp(annotation.object(i).polygon.polyType,'part') 119 | % $$$ [X,Y,Z,mx,my,tri,boundary,CptsNdx,valid] = getObject3D(annotation,i,'grid'); 120 | % $$$ parents = getPartOfParents(annotation,i); 121 | % $$$ 122 | % $$$ if ~isempty(X) && valid && ~isempty(parents) 123 | % $$$ X = int32(X)'; 124 | % $$$ Y = uint32(Y)'; 125 | % $$$ Z = uint32(Z)'; 126 | % $$$ n = sub2ind([nrows ncols],my,mx); 127 | % $$$ nn = ((Z=5 138 | % $$$ Xobj(n+(i-1)*nrows*ncols) = X; 139 | % $$$ Yobj(n+(i-1)*nrows*ncols) = Y; 140 | % $$$ Zobj(n+(i-1)*nrows*ncols) = Z; 141 | % $$$ end 142 | % $$$ end 143 | % $$$ end 144 | % $$$ end 145 | % $$$ 146 | % $$$ if nargout==0 147 | % $$$ plotDepthMap(Zmap); 148 | % $$$ end 149 | -------------------------------------------------------------------------------- /inferEdgeTypesNew.m: -------------------------------------------------------------------------------- 1 | function annotation = inferEdgeTypesNew(annotation,validObjects) 2 | % annotation = inferEdgeTypesNew(annotation) 3 | % 4 | % Infer edge types for all the labeled polygons in the scene. Edges can 5 | % either be (i) attached, (ii) occluded, (iii) contact. 6 | % 7 | % Inputs: 8 | % annotation 9 | % validObjects - Vector of desired object indices to compute. 10 | % 11 | % Outputs: 12 | % annotation.object(i).polygon.edgeType 13 | % annotation.object(i).polygon.pEdgeType 14 | % 15 | % Edge types: 16 | % attached - 1 17 | % contact - 2 18 | % occluded - 3 19 | 20 | % Parameters: 21 | priorContact = 0.05; % Prior of being a contact (or occluded) edge 22 | beta_g = 50; % Beta distribution prior for distance to ground object 23 | beta_l = 1.5; % Beta distribution prior for edge length 24 | beta_o = 2; % Beta distribution prior for edge orientation 25 | rho1 = 0.01; % Trinary term weight 26 | rho2 = 10; % Trinary term weight 27 | minEdgeLen = 0.05; % Minimum contact edge length 28 | insertEdgeTypes = 0; 29 | 30 | if nargin < 2 31 | validObjects = 1:length(annotation.object); 32 | end 33 | 34 | % Collect edge evidence: 35 | [evidenceEdges,annotation] = collectEdgeEvidence(annotation,validObjects); 36 | 37 | Nobjects = length(annotation.object); 38 | ncols = annotation.imagesize.ncols; 39 | for i = validObjects 40 | if strcmp(annotation.object(i).polygon.polyType,'standing') 41 | edges = annotation.object(i).polygon.edges; 42 | Nedges = size(edges,2); 43 | pEdgeType = zeros(3,Nedges); 44 | edgeType = zeros(1,Nedges); 45 | 46 | % Get edge evidence: 47 | pBelow = evidenceEdges.object(i).pBelow; 48 | pOri = evidenceEdges.object(i).pOri; 49 | pLen = evidenceEdges.object(i).pLen; 50 | pGround = evidenceEdges.object(i).pGround; 51 | 52 | % Get attached log-probability: 53 | pAttached = annotation.object(i).polygon.pAttached; 54 | pEdgeType(1,:) = log(pAttached); 55 | 56 | % Look for additional edge constraints: 57 | concaveConstraints = detectShallowConcave(annotation.object(i).polygon,pBelow); 58 | 59 | % Set up factor graph: 60 | % 1 - contact; 2 - occlusion 61 | k = 0; 62 | clear nodes factors; 63 | for j = 1:Nedges 64 | % Local edge evidence: 65 | k = k+1; 66 | p = [priorContact*betapdf(pGround(j),beta_g,1)*betapdf(pOri(j),beta_o,1)*betapdf(pLen(j),beta_l,1)*pBelow(j) 1-priorContact]'; 67 | factors(k).pot = p/sum(p); 68 | factors(k).ndxNodes = j; 69 | factors(k).ndxMsgs = 1; 70 | nodes(j).msgIn = p; 71 | end 72 | 73 | if ~isempty(concaveConstraints) 74 | for j = 1:size(concaveConstraints,1) 75 | k = k+1; 76 | c = concaveConstraints(j,:); 77 | p = ones(2,2,2); 78 | p(2,1,2) = rho1; % contact 79 | p(2,2,2) = rho2; % occlusion 80 | factors(k).pot = p; 81 | factors(k).ndxNodes = c; 82 | factors(k).ndxMsgs = [size(nodes(c(1)).msgIn,2)+1 size(nodes(c(2)).msgIn,2)+1 size(nodes(c(3)).msgIn,2)+1]; 83 | nodes(c(1)).msgIn(:,end+1) = ones(2,1); 84 | nodes(c(2)).msgIn(:,end+1) = ones(2,1); 85 | nodes(c(3)).msgIn(:,end+1) = ones(2,1); 86 | end 87 | end 88 | 89 | % Run BP: 90 | beliefs = bpFactorGraph(nodes,factors); 91 | 92 | % Assign beliefs: 93 | pEdgeType = [beliefs(:).bel]; 94 | 95 | % Assign edge type based on maximal probability: 96 | [v,edgeType] = max(pEdgeType,[],1); 97 | 98 | % Set contact edges: 99 | edges = annotation.object(i).polygon.edges; 100 | Npts = length(getLMpolygon(annotation.object(i).polygon)); 101 | % $$$ Npts = length(annotation.object(i).polygon.pt); 102 | [X,Y] = getLMpolygon(annotation.object(i).polygon); 103 | 104 | % Get contact edges: 105 | c = zeros(1,Npts); 106 | n = find(edgeType==1); 107 | e = edges(:,n); 108 | c(unique(e(:))) = 1; 109 | 110 | % If only one edge segment and it is short, then use horizontal card: 111 | if sum(c)==2 112 | n = find(c); 113 | edgeLen = ((X(n(1))-X(n(2))).^2+(Y(n(1))-Y(n(2))).^2).^0.5/ncols; 114 | if edgeLen < minEdgeLen 115 | [v,j] = min([Y(n(1)) Y(n(2))]); 116 | c(n(j)) = 0; 117 | end 118 | end 119 | 120 | % Get contact points: 121 | n = find(c); 122 | annotation.object(i).polygon = setLMcontact(X(n),Y(n),annotation.object(i).polygon); 123 | end 124 | 125 | % Insert edge types (optional): 126 | if insertEdgeTypes 127 | for j = 1:length(annotation.object(i).polygon.pt) 128 | if strcmp(annotation.object(i).polygon.polyType,'standing') 129 | switch edgeType(j) 130 | case 1 131 | annotation.object(i).polygon.pt(j).edgeType = 'c'; 132 | case 2 133 | annotation.object(i).polygon.pt(j).edgeType = 'o'; 134 | end 135 | else 136 | annotation.object(i).polygon.pt(j).edgeType = 'a'; 137 | end 138 | end 139 | end 140 | 141 | end 142 | 143 | return; 144 | 145 | figure; 146 | x = [0:0.01:1]; 147 | plot(x,betapdf(x,1.5,1)); 148 | axis([0 1 0 1]); 149 | 150 | for i = 1:length(annotation.object) 151 | [X,Y] = getLMpolygon(annotation.object(i).polygon); 152 | edges = annotation.object(i).polygon.edges; 153 | concaveConstraints = detectShallowConcave(annotation.object(i).polygon); 154 | if ~isempty(concaveConstraints) 155 | for j = 1:size(concaveConstraints,1) 156 | clf; 157 | imshow(img); 158 | hold on; 159 | plot([X; X(1)],[Y; Y(1)],'b','LineWidth',4); 160 | c = concaveConstraints(j,:); 161 | plot([X(edges(1,c(1))) X(edges(2,c(1)))],[Y(edges(1,c(1))) Y(edges(2,c(1)))],'r','LineWidth',4); 162 | plot([X(edges(1,c(2))) X(edges(2,c(2)))],[Y(edges(1,c(2))) Y(edges(2,c(2)))],'r','LineWidth',4); 163 | plot([X(edges(1,c(3))) X(edges(2,c(3)))],[Y(edges(1,c(3))) Y(edges(2,c(3)))],'r','LineWidth',4); 164 | pause; 165 | end 166 | end 167 | end 168 | --------------------------------------------------------------------------------