├── Assignment_1 ├── Apple.zip ├── Assignment_1.pdf ├── Elephant.zip └── Pear.zip ├── LICENSE ├── PFMAccess.cpp ├── PFMAccess.h ├── PhotometricStero.cc ├── PhotometricStero.h ├── README.md ├── resultImage ├── appleAlbedo.jpg ├── appleHeight.jpg ├── appleNormal.jpg ├── appleNormalWithAlbedo.jpg ├── elephantAlbedo.jpg ├── elephantHeightMethod1.jpg ├── elephantHeightMethod2Judgement.jpg ├── elephantHeightMethod2NoJudgement.jpg ├── elephantNormal.jpg ├── elephantNormalWithAlbedo.jpg ├── pearAlbedo.jpg ├── pearHeight.jpg ├── pearNormal.jpg └── pearNormalWithAlbedo.jpg └── testPhotoMetricStero.cpp /Assignment_1/Apple.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xyg-coder/Photometric-Stereo/7c1bee1eca417085155a3417956cbbe9114434ac/Assignment_1/Apple.zip -------------------------------------------------------------------------------- /Assignment_1/Assignment_1.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xyg-coder/Photometric-Stereo/7c1bee1eca417085155a3417956cbbe9114434ac/Assignment_1/Assignment_1.pdf -------------------------------------------------------------------------------- /Assignment_1/Elephant.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xyg-coder/Photometric-Stereo/7c1bee1eca417085155a3417956cbbe9114434ac/Assignment_1/Elephant.zip -------------------------------------------------------------------------------- /Assignment_1/Pear.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xyg-coder/Photometric-Stereo/7c1bee1eca417085155a3417956cbbe9114434ac/Assignment_1/Pear.zip -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 XinyuanGui 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /PFMAccess.cpp: -------------------------------------------------------------------------------- 1 | //#include "StdAfx.h" 2 | #include ".\pfmaccess.h" 3 | #include "string.h" 4 | 5 | 6 | CPFMAccess::CPFMAccess(void) 7 | { 8 | m_data = NULL; 9 | m_width = -1; 10 | m_height = -1; 11 | } 12 | 13 | CPFMAccess::~CPFMAccess(void) 14 | { 15 | if(m_data!=NULL) 16 | delete []m_data; 17 | } 18 | 19 | 20 | bool CPFMAccess::LoadFromFile(char* fn) 21 | { 22 | FILE * fp = fopen(fn,"rb"); 23 | if(fp == NULL) 24 | return false; 25 | int dataStartPos = ReadPFMHead(fp, &m_width, &m_height); 26 | if(dataStartPos == 0) 27 | { 28 | fclose(fp); 29 | return false; 30 | } 31 | fseek(fp,dataStartPos,SEEK_SET); 32 | 33 | if(m_data != NULL) 34 | { 35 | delete []m_data; 36 | m_data = NULL; 37 | } 38 | m_data = new float[3*m_width*m_height]; 39 | if(fread(m_data,sizeof(float),3*m_width*m_height,fp) != 3*m_width*m_height) 40 | { 41 | delete []m_data; 42 | m_data =NULL; 43 | fclose(fp); 44 | return false; 45 | } 46 | 47 | fclose(fp); 48 | return true; 49 | } 50 | 51 | bool CPFMAccess::SaveToFile(char* fn) 52 | { 53 | if(m_data == NULL) 54 | return false; 55 | 56 | FILE * fp = fopen(fn,"wb"); 57 | if(fp == NULL) 58 | return false; 59 | 60 | //fprintf(fp ,"PF\x0a%d %d\x0a-1.000000\x0a",m_width, m_height); 61 | fprintf(fp ,"PF\x0a%d\x0a%d\x0a-1.000000\x0a",m_width, m_height); // by zlzhou 62 | 63 | if(fwrite(m_data, sizeof(float), 3*m_width*m_height,fp) != 3*m_width*m_height) 64 | { 65 | fclose(fp); 66 | return false; 67 | } 68 | 69 | fclose(fp); 70 | return true; 71 | } 72 | 73 | 74 | int CPFMAccess::ReadPFMHead(FILE * pFile, int* width,int* height) 75 | { 76 | char Header[513]; 77 | fseek(pFile, 0, SEEK_SET); 78 | int len = fread(Header, sizeof(char), 512, pFile); 79 | Header[len] = 0; 80 | 81 | if(len > 3) 82 | { 83 | Header[len-1] = 0; 84 | if( (Header[0] == 'P' && Header[1] == 'F') || 85 | (Header[0] == 'p' && Header[1] == 'f') ) 86 | { 87 | char* p = strchr(Header,0xa); 88 | if(p) 89 | { 90 | 91 | p++; 92 | 93 | 94 | //for the read of pfm file generated from photoshop 95 | int cx, cy; 96 | char* end; 97 | end = strchr(p,0xa); 98 | end = &(end[1]); 99 | end = strchr(end,0xa); 100 | end[0] = 0; 101 | if(sscanf(p,"%d %d", &cx,&cy)==2) 102 | { 103 | *width = cx; 104 | *height = cy; 105 | p = &end[1]; 106 | end = strchr(p,0xa); 107 | if(end) 108 | { 109 | return (end-Header)+1; 110 | } 111 | 112 | } 113 | 114 | 115 | //for read of other pfm files 116 | // end = strchr(p,0xa); 117 | // if(end) 118 | // { 119 | // end[0] = 0; 120 | // 121 | // int cx,cy; 122 | // if(sscanf(p,"%d %d",&cx,&cy) == 2) 123 | // { 124 | // *width = cx; 125 | // *height = cy; 126 | // p = &end[1]; 127 | // end = strchr(p,0xa); 128 | // if(end) 129 | // { 130 | // return (end-Header)+1; 131 | // } 132 | // } 133 | // } //if (end) 134 | 135 | 136 | } //if(p) 137 | } //if(Header) 138 | }// if (len) 139 | return 0; 140 | } 141 | 142 | bool CPFMAccess::SetSize(int width,int height) 143 | { 144 | if(m_width*m_height == width*height) 145 | return true; 146 | 147 | if(m_data !=NULL) 148 | delete m_data; 149 | m_width = width; 150 | m_height = height; 151 | m_data = new float[3*width*height]; 152 | for(int i=0; i2) 203 | return; 204 | m_data[component_id+3*x+3*y*m_width] = v; 205 | } 206 | 207 | 208 | void CPFMAccess::GetPixelValue(int x,int y,float *value) 209 | { 210 | if(x<0||x>=m_width||y<0||y>=m_height) 211 | { 212 | value[0] = value[1] = value[2] =0.0; 213 | return; 214 | } 215 | for(int i=0;i<3;i++) 216 | value[i] = m_data[i+3*x+3*y*m_width]; 217 | } 218 | 219 | void CPFMAccess::GetPixelValue(int x, int y, float* r, float* g, float* b) 220 | { 221 | if(x<0||x>=m_width||y<0||y>=m_height) 222 | { 223 | *r = *g = *b =0.0; 224 | return; 225 | } 226 | *r = m_data[0+3*x+3*y*m_width]; 227 | *g = m_data[1+3*x+3*y*m_width]; 228 | *b = m_data[2+3*x+3*y*m_width]; 229 | } 230 | 231 | void CPFMAccess::GetPixelRValue(int x, int y, float *value) 232 | { 233 | if(x<0||x>=m_width||y<0||y>=m_height) 234 | { 235 | *value = 0.0; 236 | return; 237 | } 238 | *value = m_data[0+3*x+3*y*m_width]; 239 | } 240 | 241 | void CPFMAccess::GetPixelGValue(int x, int y, float *value) 242 | { 243 | if(x<0||x>=m_width||y<0||y>=m_height) 244 | { 245 | *value = 0.0; 246 | return; 247 | } 248 | *value = m_data[1+3*x+3*y*m_width]; 249 | } 250 | 251 | void CPFMAccess::GetPixelBValue(int x, int y, float *value) 252 | { 253 | if(x<0||x>=m_width||y<0||y>=m_height) 254 | { 255 | *value = 0.0; 256 | return; 257 | } 258 | *value = m_data[2+3*x+3*y*m_width]; 259 | } 260 | 261 | void CPFMAccess::GetPixelComponentValue(int x, int y, int component_id, float *value) 262 | { 263 | if(x<0||x>=m_width||y<0||y>=m_height) 264 | { 265 | *value = 0.0; 266 | return; 267 | } 268 | if(component_id<0 || component_id>2) 269 | { 270 | *value = 0.0; 271 | return; 272 | } 273 | *value = m_data[component_id+3*x+3*y*m_width]; 274 | } 275 | 276 | 277 | 278 | -------------------------------------------------------------------------------- /PFMAccess.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "stdio.h" 3 | #include "stdlib.h" 4 | 5 | 6 | class CPFMAccess 7 | { 8 | public: 9 | CPFMAccess(void); 10 | CPFMAccess(const CPFMAccess&) = delete; 11 | CPFMAccess operator = (const CPFMAccess&) = delete; 12 | ~CPFMAccess(void); 13 | 14 | bool LoadFromFile(char* fn); 15 | bool SaveToFile(char* fn); 16 | float* GetData(){return m_data;} 17 | int GetWidth(){return m_width;} 18 | int GetHeight(){return m_height;} 19 | bool SetSize(int width,int height); 20 | bool SetData(float* data); 21 | 22 | void SetPixelValue(int x,int y,const float * value); 23 | void SetPixelValue(int x,int y,const float r, const float g, const float b); 24 | void SetPixelBValue(int x, int y, const float b); 25 | void SetPixelGValue(int x, int y, const float g); 26 | void SetPixelRValue(int x, int y, const float r); 27 | void SetPixelComponentValue(int x, int y, const int component_id, const float v); 28 | 29 | void GetPixelValue(int x, int y, float* r, float* g, float* b); 30 | void GetPixelValue(int x,int y,float *value); 31 | void GetPixelValue(float x,float y,float *value); 32 | void GetPixelRValue(int x, int y, float* value); 33 | void GetPixelGValue(int x, int y, float* value); 34 | void GetPixelBValue(int x, int y, float* value); 35 | void GetPixelComponentValue(int x, int y, int component_id, float* value); 36 | 37 | int ReadPFMHead(FILE * pFile, int* width,int* height); 38 | public: 39 | float * m_data; 40 | int m_width; 41 | int m_height; 42 | 43 | }; 44 | -------------------------------------------------------------------------------- /PhotometricStero.cc: -------------------------------------------------------------------------------- 1 | #include "PhotometricStero.h" 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | namespace phoSte { 11 | 12 | namespace { 13 | // use iniRSize points to get the initial r 14 | const int iniRSize = 10; 15 | // use 20 points to get the circle information 16 | const int adjustSize = 20; 17 | // iterate iterTimes to get the circle 18 | const int iterTime = 10; 19 | // threshold of stopping the iteration 20 | const double normThresh = 0.01; 21 | 22 | phoSte::circle getCircle(std::vector& contour) { 23 | double circle_x = 0; 24 | double circle_y = 0; 25 | double r = 0; 26 | int pointSize = contour.size(); 27 | if(pointSize < iniRSize) { 28 | std::cout << "the counter is error" << std::endl; 29 | exit(-1); 30 | } 31 | for(int i = 0; i < pointSize; i++) { 32 | circle_x += contour[i].x; 33 | circle_y += contour[i].y; 34 | } 35 | circle_x /= pointSize; 36 | circle_y /= pointSize; 37 | for(int i = 0; i < iniRSize; i++) { 38 | r += sqrt((contour[i].x - circle_x) * (contour[i].x - circle_x) + 39 | (contour[i].y - circle_y) * (contour[i].y - circle_y)); 40 | } 41 | r /= iniRSize; 42 | 43 | // use adjustment calculation to get the circle 44 | if(pointSize < adjustSize) { 45 | std::cout << "the counter is error" << std::endl; 46 | exit(-1); 47 | } 48 | std::vector pointIndex; 49 | pointIndex.reserve(adjustSize); 50 | int distanceOfIndex = pointSize / adjustSize; 51 | pointIndex.push_back(0); 52 | for(int i = 1; i < adjustSize; i++) { 53 | pointIndex.push_back(pointIndex[i - 1] + distanceOfIndex); 54 | } 55 | 56 | cv::Mat B(adjustSize, 3, CV_64F); 57 | cv::Mat L(adjustSize, 1, CV_64F); 58 | double v_norm = INT_MAX; 59 | for(int i = 0; i < iterTime; i++) { 60 | double *pB = (double *)B.data; 61 | double *pL = (double *)L.data; 62 | for(int j = 0; j < adjustSize; j++) { 63 | *pB = contour[pointIndex[j] + i].x - circle_x; 64 | *(pB + 1) = contour[pointIndex[j] + i].y - circle_y; 65 | *(pB + 2) = r; 66 | pB += 3; 67 | *pL = ((contour[pointIndex[j] + i].x - circle_x) * (contour[pointIndex[j] + i].x - circle_x) 68 | + (contour[pointIndex[j] + i].y - circle_y) * (contour[pointIndex[j] + i].y - circle_y) 69 | - r * r) / 2; 70 | pL++; 71 | } 72 | cv::Mat pseudoInverse; 73 | cv::invert(B, pseudoInverse, cv::DECOMP_SVD); 74 | cv::Mat result = pseudoInverse * L; 75 | circle_x += result.at(0, 0); 76 | circle_y += result.at(1, 0); 77 | r += result.at(2, 0); 78 | } 79 | return phoSte::circle(circle_x, circle_y, r); 80 | } 81 | 82 | // get the direction of light according to the metalCircle 83 | phoSte::light getLightDirection(const phoSte::circle& metalCircle, 84 | const cv::Point& maxPoint) { 85 | int maxX = maxPoint.x; 86 | int maxY = maxPoint.y; 87 | double nx = maxX - metalCircle.mCenterX; 88 | double ny = maxY - metalCircle.mCenterY; 89 | double nz = sqrt(metalCircle.mR * metalCircle.mR - nx * nx - ny * ny); 90 | double rootSquareSum = sqrt(nx * nx + ny * ny + nz * nz); 91 | nx /= rootSquareSum; 92 | ny /= rootSquareSum; 93 | nz /= rootSquareSum; 94 | double nDotR = nz; 95 | double lx = 2 * nDotR * nx; 96 | double ly = 2 * nDotR * ny; 97 | double lz = 2 * nDotR * nz - 1; 98 | 99 | return phoSte::light(lx, ly, lz); 100 | } 101 | 102 | void swapNum(double& a, double& b) { 103 | double tmp = a; 104 | a = b; 105 | b = tmp; 106 | 107 | } 108 | 109 | // find the num at the ratio of the nums 110 | // always use the endI as the pivot 111 | // use the random integer because there might be some order of the pixel value 112 | double quickSelect(std::vector& nums, int startI, int endI, int selectI) { 113 | if(startI == endI) 114 | return nums[startI]; 115 | int pivotIndex = rand() % (endI - startI + 1) + startI; 116 | swapNum(nums[endI], nums[pivotIndex]); 117 | double pivot = nums[endI]; 118 | int i = startI - 1; 119 | int j = startI; 120 | while(j != endI) { 121 | if(nums[j] < pivot) { 122 | i++; 123 | swapNum(nums[j], nums[i]); 124 | } 125 | j++; 126 | } 127 | i++; 128 | swapNum(nums[endI], nums[i]); 129 | if(i == selectI) 130 | return nums[i]; 131 | else if(i < selectI) { 132 | return quickSelect(nums, i + 1, endI, selectI); 133 | } else 134 | return quickSelect(nums, startI, i - 1, selectI); 135 | } 136 | 137 | // do the transform to image to output 138 | // maxValue to 255, minValue to 0 139 | int linearTransform(double max, double min, double value) { 140 | return 255 / (max - min) * (value - min); 141 | } 142 | } 143 | 144 | photometryStero::~photometryStero() { 145 | if(!mImageStorage.empty()) { 146 | for(int i = 0; i < mImageStorage.size(); i++) { 147 | delete mImageStorage[i]; 148 | } 149 | } 150 | } 151 | 152 | photometryStero::photometryStero(int n, int startI, int endI, std::string path, 153 | std::string metal1Phere1Name, std::string metal2Phere1Name, 154 | std::string lambertPhereName, std::string objectName, 155 | double discardRatio): mDiscardRatio(discardRatio), imageNum(n) { 156 | int distance = (endI - startI + 1) / n; 157 | for (int i = 0; i < n; i++) { 158 | int index = i * distance + startI; 159 | char imageName[100]; 160 | sprintf(imageName, "image%03d.pbm", index); 161 | std::string imageNameStr = imageName; 162 | imageNameStr = path + imageNameStr; 163 | mImageNames.push_back(imageNameStr); 164 | } 165 | std::string mask1Name = metal1Phere1Name; 166 | std::string mask2Name = metal2Phere1Name; 167 | std::string mask3Name = lambertPhereName; 168 | 169 | mMaskNames.push_back(path + mask1Name); 170 | mMaskNames.push_back(path + mask2Name); 171 | mMaskNames.push_back(path + mask3Name); 172 | mMaskNames.push_back(path + objectName); 173 | } 174 | 175 | // read image and mask image according to the name 176 | // return true if read successfully 177 | // return false if read fail 178 | bool photometryStero::readImage() { 179 | if(mImageNames.empty() || mMaskNames.empty()) 180 | return false; 181 | 182 | // read the image 183 | for(int i = 0; i < mImageNames.size(); i++) { 184 | CPFMAccess* PFMAccessI = new CPFMAccess(); 185 | char *tpChar = new char[mImageNames[i].size() + 1]; 186 | std::strcpy(tpChar, mImageNames[i].c_str()); 187 | if(! (PFMAccessI -> LoadFromFile(tpChar)) ) 188 | return false; 189 | delete[]tpChar; 190 | float *imageData = PFMAccessI -> GetData(); 191 | mImageStorage.push_back(PFMAccessI); 192 | cv::Mat image((PFMAccessI->GetWidth()) * (PFMAccessI->GetHeight()), 3, 193 | CV_32F, imageData); 194 | cv::Mat p2Image (0.2989 * image.col(0) + 0.5870 * image.col(1) 195 | + 0.1140 * image.col(2)); 196 | p2Image = p2Image.reshape(0, PFMAccessI->GetHeight()); 197 | mp2Images.push_back(p2Image); 198 | } 199 | // read the maskImage 200 | for(int i = 0; i < mMaskNames.size(); i++) { 201 | cv::Mat p2Mask(cv::imread(mMaskNames[i], CV_LOAD_IMAGE_GRAYSCALE)); 202 | if(! p2Mask.data ) 203 | return false; 204 | mp2Mask.push_back(p2Mask); 205 | } 206 | return true; 207 | } 208 | 209 | void photometryStero::getLightInformation(const int metalIndex, const int lambIndex) { 210 | // get the metal circle 211 | cv::threshold(mp2Mask[metalIndex], mp2Mask[metalIndex], 255 / 2, 255, cv::THRESH_BINARY); 212 | std::vector> metalContour; 213 | cv::findContours(mp2Mask[metalIndex], metalContour, CV_RETR_EXTERNAL, CV_CHAIN_APPROX_NONE); 214 | // use the contour with the max size to calculate the circle 215 | int mostPoint = 0; 216 | int mostPointIdx = 0; 217 | for(int i = 0; i < metalContour.size(); i++) { 218 | if(metalContour.size() > mostPoint) { 219 | mostPoint = metalContour.size(); 220 | mostPointIdx = i; 221 | } 222 | } 223 | m_metalSphere = getCircle(metalContour[mostPointIdx]); 224 | 225 | // get the lambertian circle 226 | cv::threshold(mp2Mask[lambIndex], mp2Mask[lambIndex], 255 / 2, 255, cv::THRESH_BINARY); 227 | std::vector> lambContour; 228 | cv::findContours(mp2Mask[lambIndex], lambContour, CV_RETR_EXTERNAL, CV_CHAIN_APPROX_NONE); 229 | // use the contour with the max size to calculate the circle 230 | mostPoint = 0; 231 | mostPointIdx = 0; 232 | for(int i = 0; i < lambContour.size(); i++) { 233 | if(lambContour.size() > mostPoint) { 234 | mostPoint = lambContour.size(); 235 | mostPointIdx = i; 236 | } 237 | } 238 | m_lambSpere = getCircle(lambContour[mostPointIdx]); 239 | 240 | // get the direction and the intensity of every image 241 | for(int i = 0; i < mp2Images.size(); i++) { 242 | cv::Point metalMaxPoint; 243 | cv::minMaxLoc(mp2Images[i], NULL, NULL, NULL, &metalMaxPoint, mp2Mask[metalIndex]); 244 | m_light.push_back(getLightDirection(m_metalSphere, metalMaxPoint)); 245 | double lightIntensity; 246 | cv::minMaxLoc(mp2Images[i], NULL, &lightIntensity, NULL, NULL, mp2Mask[lambIndex]); 247 | m_light[i].mIntensity = lightIntensity; 248 | } 249 | } 250 | 251 | // calculate the norm and albedo for every pixel 252 | // here, threshold is for the dark discard 253 | void photometryStero::getPixelNormAndAlbedo(const int objectIndex) { 254 | // calculate the pixel num 255 | std::vector pixelThreshold; 256 | for(int i = 0; i < mp2Mask[objectIndex].rows; i++) { 257 | for(int j = 0; j < mp2Mask[objectIndex].cols; j++) { 258 | if(mp2Mask[objectIndex].at(i, j) >= 255) { 259 | mObjectX.push_back(j); 260 | mObjectY.push_back(i); 261 | } 262 | } 263 | } 264 | std::vector> allPixelValue; 265 | allPixelValue.reserve(imageNum); 266 | int objectPixelNum = mObjectX.size(); 267 | for(int i = 0; i < imageNum; i++) { 268 | std::vector pixelValue; 269 | allPixelValue.push_back(pixelValue); 270 | allPixelValue[i].reserve(objectPixelNum); 271 | for(int j = 0; j < objectPixelNum; j++) { 272 | allPixelValue[i].push_back(mp2Images[i].at(mObjectY[j], mObjectX[j])); 273 | } 274 | } 275 | // threshold for every image; 276 | std::vector allThreshold; 277 | allThreshold.reserve(imageNum); 278 | int thresholdIndex = mDiscardRatio * objectPixelNum; 279 | for(int i = 0; i < imageNum; i++) { 280 | std::vector tmpPixelValue = allPixelValue[i]; 281 | allThreshold.push_back( 282 | quickSelect(tmpPixelValue, 0, objectPixelNum - 1, thresholdIndex)); 283 | } 284 | 285 | cv::Mat I(imageNum, objectPixelNum, CV_64F); 286 | cv::Mat L(imageNum, 3, CV_64F); 287 | mN.create(3, objectPixelNum, CV_64F); 288 | mAlbedo.create(1, objectPixelNum, CV_64F); 289 | double *pL = (double *)L.data; 290 | for(int i = 0; i < imageNum; i++) { 291 | *pL = m_light[i].mx; 292 | *(pL + 1) = m_light[i].my; 293 | *(pL + 2) = m_light[i].mz; 294 | pL += 3; 295 | } 296 | cv::Mat LPseudoInvert; 297 | cv::invert(L, LPseudoInvert, cv::DECOMP_SVD); 298 | double *pI = (double *)I.data; 299 | double *pN = (double *)mN.data; 300 | 301 | for(int i = 0; i < objectPixelNum; i++) { 302 | int inValidNum = 0; 303 | cv::Mat specificL; 304 | for(int j = 0; j < imageNum; j++) { 305 | if(allPixelValue[j][i] < allThreshold[j]) { 306 | inValidNum ++; 307 | if(inValidNum == 1) { 308 | L.copyTo(specificL); 309 | } 310 | double *p2LRow = specificL.ptr(j); 311 | *(p2LRow) = 0; 312 | *(p2LRow + 1) = 0; 313 | *(p2LRow + 2) = 0; 314 | *(pI + j * objectPixelNum + i) = 0; 315 | } else { 316 | *(pI + j * objectPixelNum + i) = allPixelValue[j][i] / m_light[j].mIntensity; 317 | } 318 | if(imageNum - inValidNum < 3) { 319 | break; 320 | } 321 | } 322 | if(inValidNum == 0) { 323 | mN.col(i) = LPseudoInvert * I.col(i); 324 | double nx = mN.col(i).at(0, 0); 325 | double ny = mN.col(i).at(1, 0); 326 | double nz = mN.col(i).at(2, 0); 327 | mAlbedo.at(0, i) = sqrt(nx * nx + ny * ny + nz * nz); 328 | } else if(imageNum - inValidNum >= 3) { 329 | cv::Mat specificLPseudoInvert; 330 | cv::invert(specificL, specificLPseudoInvert, cv::DECOMP_SVD); 331 | mN.col(i) = specificLPseudoInvert * I.col(i); 332 | double nx = mN.col(i).at(0, 0); 333 | double ny = mN.col(i).at(1, 0); 334 | double nz = mN.col(i).at(2, 0); 335 | mAlbedo.at(0, i) = sqrt(nx * nx + ny * ny + nz * nz); 336 | } else { 337 | mN.at(0, i) = 0; 338 | mN.at(1, i) = 0; 339 | mN.at(2, i) = 0; 340 | mAlbedo.at(0, i) = 0; 341 | mInvalidIndex.push_back(i); 342 | } 343 | } 344 | } 345 | 346 | void photometryStero::outputImage() { 347 | double max = INT_MIN, min = INT_MAX; 348 | if (mp2Images.empty()) { 349 | std::cout << "no image to output" << std::endl; 350 | } 351 | cv::Mat out(mp2Images[0].rows, mp2Images[0].cols, CV_8U); 352 | for (int i = 0; i < mp2Images[0].rows; i++) { 353 | for (int j = 0; j < mp2Images[0].cols; j++) { 354 | double pixelValue = mp2Images[0].at(i, j); 355 | max = pixelValue > max ? pixelValue : max; 356 | min = pixelValue < min ? pixelValue : min; 357 | } 358 | } 359 | for (int i = 0; i < mp2Images[0].rows; i++) { 360 | for (int j = 0; j < mp2Images[0].cols; j++) { 361 | double pixelValue = mp2Images[0].at(i, j); 362 | out.at(i, j) = linearTransform(max, min, pixelValue); 363 | } 364 | } 365 | cv::imshow("outputImage", out); 366 | cv::waitKey(0); 367 | } 368 | 369 | cv::Mat photometryStero::outputNormalImage(int objectIndex) { 370 | cv::Mat result = cv::Mat::zeros(mp2Images[0].rows, mp2Images[0].cols, CV_32FC3); 371 | int nextInvalid = mInvalidIndex.empty() ? INT_MAX : mInvalidIndex[0]; 372 | int nextInvalidIndex = 0; 373 | int invalidPixelNum = 0; 374 | for (int i = 0; i < mObjectX.size(); i++) { 375 | double nx = mN.at(0, i); 376 | double ny = mN.at(1, i); 377 | double nz = mN.at(2, i); 378 | if (i == nextInvalid) { 379 | invalidPixelNum++; 380 | result.at(mObjectY[i], mObjectX[i])[0] = 0; 381 | result.at(mObjectY[i], mObjectX[i])[1] = 0; 382 | result.at(mObjectY[i], mObjectX[i])[2] = 0; 383 | if (nextInvalidIndex + 1 == mInvalidIndex.size()) { 384 | nextInvalid = INT_MAX; 385 | } else { 386 | nextInvalidIndex++; 387 | nextInvalid = mInvalidIndex[nextInvalidIndex]; 388 | } 389 | } else { 390 | double rootsquareSum = sqrt(nx * nx + ny * ny + nz * nz); 391 | float nxf = (1 + nx / rootsquareSum) / 2; 392 | float nyf = (1 + ny / rootsquareSum) / 2; 393 | float nzf = (1 + nz / rootsquareSum) / 2; 394 | result.at(mObjectY[i], mObjectX[i])[0] = nxf; 395 | result.at(mObjectY[i], mObjectX[i])[1] = nyf; 396 | result.at(mObjectY[i], mObjectX[i])[2] = nzf; 397 | } 398 | } 399 | std::cout << invalidPixelNum << " pixels are not drawn" << std::endl; 400 | return result; 401 | } 402 | 403 | // this function is only for debug 404 | void photometryStero::addSmallMaskForObject(int size, int midX, int midY) { 405 | cv::Mat smallMask = cv::Mat::zeros(mp2Images[0].rows, mp2Images[0].cols, CV_8U); 406 | int num = 0; 407 | for (int i = midY - size / 2; i <= midY + size / 2; i++) { 408 | for (int j = midX - size / 2; j <= midX + size / 2; j++) { 409 | smallMask.at(i, j) = 255; 410 | num++; 411 | } 412 | } 413 | mp2Mask.push_back(smallMask); 414 | } 415 | 416 | cv::Mat photometryStero::outputAlbedoImage(int objectIndex) { 417 | cv::Mat result = cv::Mat::zeros(mp2Images[0].rows, mp2Images[0].cols, CV_32F); 418 | int nextInvalid = mInvalidIndex.empty() ? INT_MAX : mInvalidIndex[0]; 419 | int nextInvalidIndex = 0; 420 | int invalidPixelNum = 0; 421 | for (int i = 0; i < mObjectX.size(); i++) { 422 | float albedo = mAlbedo.at(0, i); 423 | if (i == nextInvalid) { 424 | invalidPixelNum++; 425 | result.at(mObjectY[i], mObjectX[i]) = 0; 426 | if (nextInvalidIndex + 1 == mInvalidIndex.size()) { 427 | nextInvalid = INT_MAX; 428 | } else { 429 | nextInvalidIndex++; 430 | nextInvalid = mInvalidIndex[nextInvalidIndex]; 431 | } 432 | } else { 433 | result.at(mObjectY[i], mObjectX[i]) = albedo; 434 | } 435 | } 436 | std::cout << invalidPixelNum << " pixels are not drawn" << std::endl; 437 | return result; 438 | } 439 | 440 | // use the normal direction dot the (0, 0, 1) and then times the albedo 441 | cv::Mat photometryStero::outputNormalWithAlbedo(int objectIndex) { 442 | cv::Mat result = cv::Mat::zeros(mp2Images[0].rows, mp2Images[0].cols, CV_32F); 443 | int nextInvalid = mInvalidIndex.empty() ? INT_MAX : mInvalidIndex[0]; 444 | int nextInvalidIndex = 0; 445 | int invalidPixelNum = 0; 446 | for (int i = 0; i < mObjectX.size(); i++) { 447 | float albedo = mAlbedo.at(0, i); 448 | double nx = mN.at(0, i); 449 | double ny = mN.at(1, i); 450 | double nz = mN.at(2, i); 451 | float intensity = nz / sqrt(nx * nx + ny * ny + nz * nz) * albedo; 452 | if (i == nextInvalid) { 453 | invalidPixelNum++; 454 | result.at(mObjectY[i], mObjectX[i]) = 0; 455 | if (nextInvalidIndex + 1 == mInvalidIndex.size()) { 456 | nextInvalid = INT_MAX; 457 | } else { 458 | nextInvalidIndex++; 459 | nextInvalid = mInvalidIndex[nextInvalidIndex]; 460 | } 461 | } else { 462 | result.at(mObjectY[i], mObjectX[i]) = intensity; 463 | } 464 | } 465 | std::cout << invalidPixelNum << " pixels are not drawn" << std::endl; 466 | return result; 467 | } 468 | 469 | // use the normal map to get the height map 470 | // mode 1: use the discard ratio to discard some low lengths 471 | // mode 2: use sigmoid function to map to the depth 472 | // in order to avoid the mistake due to the small nz, I plus 0.05 when nz is too small 473 | cv::Mat photometryStero::getHeightMap(int mode, double parameter) { 474 | // make the map from object pixel to object index 475 | int nextInvalid = mInvalidIndex.empty() ? INT_MAX : mInvalidIndex[0]; 476 | int nextInvalidIndex = 0; 477 | cv::Mat pixel2ObjectIndex = cv::Mat::zeros(mp2Images[0].rows, 478 | mp2Images[0].cols, CV_32S); 479 | for (int i = 0; i < mObjectX.size(); i++) { 480 | if (i == nextInvalid) { 481 | pixel2ObjectIndex.at(mObjectY[i], mObjectX[i]) = -1; 482 | if (nextInvalidIndex + 1 == mInvalidIndex.size()) { 483 | nextInvalid = INT_MAX; 484 | } else { 485 | nextInvalidIndex++; 486 | nextInvalid = mInvalidIndex[nextInvalidIndex]; 487 | } 488 | } else { 489 | // use i + 1 to prevent the collision with 0 490 | pixel2ObjectIndex.at(mObjectY[i], mObjectX[i]) = i + 1; 491 | 492 | } 493 | } 494 | // first of all, iterate until all object points have normal value 495 | // I use the 3*3 window to get average normal 496 | cv::Mat NForHeight; 497 | mN.copyTo(NForHeight); 498 | cv::Mat heightMap = cv::Mat::zeros(mp2Images[0].rows, mp2Images[0].rows, CV_32F); 499 | int nInvalid = mInvalidIndex.size(); 500 | int iterateTime = 0; 501 | while (nInvalid > 0) { 502 | if (iterateTime > 10) { 503 | std::cout << "too many iterations during getting values to the invalid pixel" 504 | << std::endl; 505 | return heightMap; 506 | } 507 | iterateTime++; 508 | for (int i = 0; i < mInvalidIndex.size(); i++) { 509 | int index = mInvalidIndex[i]; 510 | if (pixel2ObjectIndex.at(mObjectY[index], mObjectX[index]) == -1) { 511 | int x = mObjectX[index]; 512 | int y = mObjectY[index]; 513 | double nxSum = 0; 514 | double nySum = 0; 515 | double nzSum = 0; 516 | int nValid = 0; 517 | for (int dx = -1; dx <= 1; dx++) { 518 | for (int dy = -1; dy <= 1; dy++) { 519 | if (pixel2ObjectIndex.at(y + dy, x + dx) > 0) { 520 | nValid++; 521 | nxSum += NForHeight.at(0, pixel2ObjectIndex.at(y + dy, x + dx)); 522 | nySum += NForHeight.at(1, pixel2ObjectIndex.at(y + dy, x + dx)); 523 | nzSum += NForHeight.at(2, pixel2ObjectIndex.at(y + dy, x + dx)); 524 | } 525 | } 526 | } 527 | if (nValid > 0) { 528 | pixel2ObjectIndex.at(y, x) = index + 1; 529 | nInvalid--; 530 | nxSum /= nValid; 531 | nySum /= nValid; 532 | nzSum /= nValid; 533 | double squareSum = sqrt(nxSum * nxSum + nySum * nySum + nzSum * nzSum); 534 | NForHeight.at(0, index) = nxSum / squareSum; 535 | NForHeight.at(1, index) = nySum / squareSum; 536 | NForHeight.at(2, index) = nzSum / squareSum; 537 | } 538 | } 539 | } 540 | } 541 | std::cout << "the iteration time is " << iterateTime; 542 | 543 | // form the sparse matrix and solve the linear equation 544 | // first of all, scan all the points to find the number of equations 545 | int nEquation = 0; 546 | for (int i = 0; i < mObjectX.size(); i++) { 547 | int x = mObjectX[i]; 548 | int y = mObjectY[i]; 549 | if (pixel2ObjectIndex.at(y + 1, x) > 0) { 550 | nEquation++; 551 | } 552 | if (pixel2ObjectIndex.at(y, x + 1) > 0) { 553 | nEquation++; 554 | } 555 | } 556 | // then use the sparse matrix to solve the linear equation 557 | // I set z(allIndex / 2)=0 as the coordinate origin 558 | int midIndex = mObjectX.size() / 2; 559 | int midX = mObjectX[midIndex]; 560 | int midY = mObjectY[midIndex]; 561 | double midNx_midNz = -(NForHeight.at(0, midIndex)) / (NForHeight.at(2, midIndex)); 562 | double midNy_midNz = -(NForHeight.at(1, midIndex)) / (NForHeight.at(2, midIndex)); 563 | Eigen::VectorXd b(nEquation); 564 | typedef Eigen::SparseMatrix SpMat; 565 | // since the sparse matrix of eigen is col main, so use the transpose to store the data 566 | SpMat A(mObjectX.size() - 1, nEquation); 567 | typedef Eigen::Triplet T; 568 | std::vector tripletList; 569 | tripletList.reserve(nEquation * 3); 570 | int currentEquation = 0; 571 | for (int i = 0; i < mObjectX.size(); i++) { 572 | int x = mObjectX[i]; 573 | int y = mObjectY[i]; 574 | if (i != midIndex) { 575 | int thisZIndex = i < midIndex ? i : i - 1; 576 | if (pixel2ObjectIndex.at(y + 1, x) > 0) { 577 | if (y + 1 != midY || x != midX) { 578 | int index = pixel2ObjectIndex.at(y + 1, x) - 1; 579 | int upZIndex = index < midIndex ? index : index - 1; 580 | tripletList.push_back(T(upZIndex, currentEquation, 1)); 581 | tripletList.push_back(T(thisZIndex, currentEquation, -1)); 582 | if (NForHeight.at(2, index) < 0.001) { 583 | b(currentEquation) = -(NForHeight.at(1, index) + 0.05) / (NForHeight.at(2, index) + 0.05); 584 | } else { 585 | b(currentEquation) = -(NForHeight.at(1, index)) / (NForHeight.at(2, index)); 586 | } 587 | currentEquation++; 588 | } else { 589 | int index = pixel2ObjectIndex.at(y + 1, x) - 1; 590 | tripletList.push_back(T(thisZIndex, currentEquation, -1)); 591 | if (NForHeight.at(2, index) < 0.001) { 592 | b(currentEquation) = -(NForHeight.at(1, index) + 0.05) / (NForHeight.at(2, index) + 0.05); 593 | } else { 594 | b(currentEquation) = -(NForHeight.at(1, index)) / (NForHeight.at(2, index)); 595 | } 596 | currentEquation++; 597 | } 598 | } 599 | if (pixel2ObjectIndex.at(y, x + 1) > 0) { 600 | if (y != midY || x + 1 != midX) { 601 | int index = pixel2ObjectIndex.at(y, x + 1) - 1; 602 | int rightZIndex = index < midIndex ? index : index - 1; 603 | tripletList.push_back(T(rightZIndex, currentEquation, 1)); 604 | tripletList.push_back(T(thisZIndex, currentEquation, -1)); 605 | if (NForHeight.at(2, index) < 0.001) { 606 | b(currentEquation) = -(NForHeight.at(0, index) + 0.05) / (NForHeight.at(2, index) + 0.05); 607 | } else { 608 | b(currentEquation) = -(NForHeight.at(0, index)) / (NForHeight.at(2, index)); 609 | } 610 | currentEquation++; 611 | } else { 612 | int index = pixel2ObjectIndex.at(y, x + 1) - 1; 613 | tripletList.push_back(T(thisZIndex, currentEquation, -1)); 614 | if (NForHeight.at(2, index) < 0.001) { 615 | b(currentEquation) = -(NForHeight.at(0, index) + 0.05) / (NForHeight.at(2, index) + 0.05); 616 | } else { 617 | b(currentEquation) = -(NForHeight.at(0, index)) / (NForHeight.at(2, index)); 618 | } 619 | currentEquation++; 620 | } 621 | } 622 | } else { 623 | int thisIndex = midIndex; 624 | if (pixel2ObjectIndex.at(y + 1, x) > 0) { 625 | int index = pixel2ObjectIndex.at(y + 1, x) - 1; 626 | int upZIndex = index < midIndex ? index : index - 1; 627 | tripletList.push_back(T(upZIndex, currentEquation, 1)); 628 | b(currentEquation) = -midNy_midNz; 629 | currentEquation++; 630 | } 631 | if (pixel2ObjectIndex.at(y, x + 1) > 0) { 632 | int index = pixel2ObjectIndex.at(y, x + 1) - 1; 633 | int rightZIndex = index < midIndex ? index : index - 1; 634 | tripletList.push_back(T(rightZIndex, currentEquation, 1)); 635 | b(currentEquation) = -midNx_midNz; 636 | currentEquation++; 637 | } 638 | } 639 | } 640 | A.setFromTriplets(tripletList.begin(), tripletList.end()); 641 | SpMat Atranspose = A.transpose(); 642 | A = A * Atranspose; 643 | A.makeCompressed(); 644 | Eigen::SimplicialLLT > solver; 645 | solver.compute(A); 646 | if (solver.info() != Eigen::Success) { 647 | std::cout << "decomposition fails" << std::endl; 648 | return heightMap; 649 | } 650 | b = Atranspose.transpose() * b; 651 | Eigen::VectorXd z = solver.solve(b); 652 | if (solver.info() != Eigen::Success) { 653 | std::cout << "solving failed" << std::endl; 654 | return heightMap; 655 | } 656 | 657 | // try to discard ratio to show; 658 | switch (mode) { 659 | case 1: { 660 | std::vector zVec; 661 | zVec.reserve(mObjectX.size()); 662 | for (int i = 0; i < mObjectX.size() - 1; i++) { 663 | zVec.push_back(z(i)); 664 | } 665 | zVec.push_back(0); 666 | std::vector zVec2(zVec.begin(), zVec.end()); 667 | double minThreshold = quickSelect(zVec, 0, zVec.size() - 1, zVec.size() * parameter); 668 | double maxZ = z.maxCoeff(); 669 | for (int i = 0; i < mObjectX.size(); i++) { 670 | int x = mObjectX[i]; 671 | int y = mObjectY[i]; 672 | double originz; 673 | if (i < midIndex) { 674 | originz = z(i); 675 | } else if (i > midIndex) { 676 | originz = z(i - 1); 677 | } else { 678 | originz = 0; 679 | } 680 | 681 | if (originz < minThreshold) { 682 | heightMap.at(y, x) = 0; 683 | } else { 684 | float newz = (originz - minThreshold) / (maxZ - minThreshold); 685 | heightMap.at(y, x) = newz; 686 | } 687 | } 688 | break; 689 | } 690 | case 2: { 691 | std::vector zVec; 692 | zVec.reserve(mObjectX.size()); 693 | for (int i = 0; i < mObjectX.size() - 1; i++) { 694 | zVec.push_back(z(i)); 695 | } 696 | zVec.push_back(0); 697 | std::vector zVec2(zVec.begin(), zVec.end()); 698 | for (int i = 0; i < mObjectX.size(); i++) { 699 | int x = mObjectX[i]; 700 | int y = mObjectY[i]; 701 | double originz; 702 | if (i < midIndex) { 703 | originz = z(i); 704 | } else if (i > midIndex) { 705 | originz = z(i - 1); 706 | } else { 707 | originz = 0; 708 | } 709 | heightMap.at(y, x) = 1 / (1 + exp(parameter * originz)); 710 | } 711 | break; 712 | } 713 | default: 714 | break; 715 | } 716 | // output the height map to matrix; 717 | return heightMap; 718 | } 719 | 720 | } 721 | -------------------------------------------------------------------------------- /PhotometricStero.h: -------------------------------------------------------------------------------- 1 | #ifndef PCVASSIGNMENT1_PHOTOMETRICSTERO_H_ 2 | #define PCVASSIGNMENT1_PHOTOMETRICSTERO_H_ 3 | 4 | #include 5 | #include 6 | 7 | #include 8 | #include 9 | #include 10 | 11 | #include "PFMAccess.h" 12 | 13 | namespace phoSte { 14 | struct circle { 15 | circle(double x = 0, double y = 0, double r = 0): mCenterX(x), mCenterY(y), mR(r) { 16 | } 17 | double mCenterX; 18 | double mCenterY; 19 | double mR; 20 | }; 21 | 22 | struct light { 23 | light(double x, double y, double z): mx(x), my(y), mz(z) { 24 | } 25 | double mx; 26 | double my; 27 | double mz; 28 | double mIntensity; 29 | }; 30 | 31 | class photometryStero { 32 | public: 33 | // fill the imageNames and maskNames 34 | // use image with euqal distance 35 | photometryStero(int n, int startI, int endI, std::string path, 36 | std::string metal1Phere1Name, std::string metal2Phere1Name, 37 | std::string lambertPhereName, std::string objectName, double discardRatio = 0.1); 38 | photometryStero(const photometryStero&) = delete; 39 | photometryStero& operator = (const photometryStero&) = delete; 40 | ~photometryStero(); 41 | bool readImage(); // read the images and masks according to the ImageNames 42 | void getLightInformation(const int metalIndex, const int lambIndex); 43 | void getPixelNormAndAlbedo(const int objectIndex); 44 | cv::Mat outputNormalImage(int objectIndex); 45 | cv::Mat outputAlbedoImage(int objectIndex); 46 | cv::Mat outputNormalWithAlbedo(int objectIndex); 47 | cv::Mat getHeightMap(int mode, double parameter); 48 | // below are functions for test 49 | // 50 | void outputImage(); 51 | // build one mask with size*size in the middle to test the N; 52 | void addSmallMaskForObject(int size, int midX, int midY); 53 | private: 54 | const int imageNum; // the num of images used to calculate; 55 | std::vector mImageNames; // the name of images 56 | std::vector mMaskNames; // the name of mask 57 | std::vector mp2Images; 58 | std::vector mp2Mask; 59 | // when the PFMACCess destructs, the data will be destroyed 60 | // so it's very dangerous, it should not be copied 61 | std::vector mImageStorage; 62 | // tht circle data for the metalSphere 63 | phoSte::circle m_metalSphere; 64 | phoSte::circle m_lambSpere; 65 | std::vector m_light; 66 | cv::Mat mN; 67 | cv::Mat mAlbedo; 68 | std::vector mObjectX; 69 | std::vector mObjectY; 70 | std::vector mInvalidIndex; 71 | const double mDiscardRatio; 72 | }; 73 | } 74 | 75 | #endif 76 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Photometric Stero 2 | ===== 3 | 4 | The math base of "Photometric Stero" can be easily got from Internet. You can experiment with the code and the materials here. And be careful to change the path in the main function of this code.
5 | I write `PhotometricStero` class and `test` and do some change to the PFMAccess from Dr.Qin.
6 | Below is the record of my idea.
7 | 8 | ## Calculate the Normal map and Albedo map 9 | * I use the `PFMAccess` to access the data in the `.pbm` file. Notice that the destructor of PFMAccess class will destroy the data in the float pointer, which is very dangerous. Thus I prohibit the copy constructor of the class. So, the vector can only store its pointer. Also remember to free the PFMAccess later.
10 | * Then I use the float data to construct the opencv Mat `Mat (int rows, int cols, int type, void *data, size_t step=AUTO_STEP)`, it can use the float data from the PFMAccess directly.
11 | * Also remember that the copy constructor of Mat is like `std::shared_ptr`. So you don't need to fear that redundant data copy.
12 | * I choose one mask of metal phere to get the max position of the phere to get the light direction.
13 | * Here, the `void cv::minMaxLoc (InputArray src, double *minVal, double *maxVal=0, Point *minLoc=0, Point *maxLoc=0, InputArray mask=noArray())` is very useful since I can directly set the mask as one parameter.
14 | * Here, to get the circle of metal phere, I use `cv::findContours` to get the contour of the sphere. In order to avoid the potential noise, I adopt the contours with most points. Then, I initialize the position of circle center with the average position of circle points, and radius with the average distance of some points to the initial center. And I use some points to iterate to get the precise circle center and radius.
15 | * Then I use the brightest point on the lambertian phere to get the light intensity.
16 | * With the light direction and intensity, I get the normal and albedo.
17 | * Because the high error of the dark point on the object. Dr.Qin recommends us to omit 10% darkest points in every image. So I use the quickSelect algorithm to every image to get the 10% threshold. And these dark points won't attend later calculation.
18 | * And since the calculation needs at least 3 images to get the normal direction and albedo. I count the invalid image for every pixel. And only if valid image of one pixel is more than 3, this pixel will be calculated. Otherwise its value will be zero.
19 | * Then I use the matrix operation to get normal and albedo. We should notice here that we should normalize every pixel value with the source intensity of image. Moreover, since most pixels on the object is valid for all images, the pseudo-inverse of the whole L is usually used. So, I calculate it before the iteration.
20 | 21 | ### effect: 22 | When I map the normal to picture, I use the method mentioned in the `./Assignment_1/Assignment_1.pdf`. 23 | 24 | | image type |apple | elephant | pear | 25 | | ---------------|-------- | ---------- | ------------| 26 | | normal map |![Image failed](./resultImage/appleNormal.jpg "apple normal map") | ![Image failed](./resultImage/elephantNormal.jpg "elephant normal map") | ![Image failed](./resultImage/pearNormal.jpg "pear normal map")| 27 | | albedo map | ![Image failed](./resultImage/appleAlbedo.jpg "apple albedo map") | ![Image failed](./resultImage/elephantAlbedo.jpg "elephant albedo map") | ![Image failed](./resultImage/pearAlbedo.jpg "pear albedo map")| 28 | | normal with albedo map | ![Image failed](./resultImage/appleNormalWithAlbedo.jpg "apple normal with albedo map") | ![Image failed](./resultImage/elephantNormalWithAlbedo.jpg "elephant normal with albedo map") | ![Image failed](./resultImage/pearNormalWithAlbedo.jpg "pear normal with albedo map")| 29 | 30 | ## Calculate the depth of every pixel 31 | * For these invalid point which doesn't have normal, I take one `3*3` window and assign the average normal direction to the invalid point.
32 | * With every normal direction (nx, ny, nz) of every pixel, we have `(nx, ny, nz).dot((z(x+1, y)) - (z(x, y))) = 0, (nx, ny, nz).dot(z(x, y+1) - z(x, y)) = 0`. Thus we have `number_of_points * 2` equations.
33 | * When I try to solve the equation `Pz = b`, the rank of P is `number_of_points - 1`. Thus, I set the depth of the point with median index to be zero and move it to constant matrix `b`.
34 | * The parameter matrix `P` is of size `number_of_points * 2, number_of_points - 1`. So we have to create sparse matrix to solve the linear equation. I use `Eigen` library to solve it.
35 | * In the equation, if `nz` of one point is near to 0, the equation will be unstable. Thus, I make one judgement: `when nz is smaller than one constant, than add one small number to it`. I don't know if it is mathematically right. If you have better idea, please contact me at `xinyuan.gui95@gmail.com`.
36 | 37 | ### Depth effect 38 | It's kind of hard to map the depth to 2d-image because the depth of the object varies a lot. I try to use linear function and sigmoid function to map.
39 | 40 | 41 | #### linear transform without judgement of nz 42 | | apple | elephant | pear | 43 | | ---------------|-------- | ---------- | 44 | | ![Image failed](./resultImage/appleHeight.jpg "apple height map") | ![Image failed](./resultImage/elephantHeightMethod1.jpg "elephant depth map") | ![Image failed](./resultImage/pearHeight.jpg "pear depth map")| 45 | 46 | 47 | 48 | #### sigmoid transform with and without judgement of nz 49 | |elephant with judgement of nz | elephant without judgement| 50 | |------------------------------|---------------------------| 51 | | ![Image failed](./resultImage/elephantHeightMethod2NoJudgement.jpg "elephant depth map without judgement") | ![Image failed](./resultImage/elephantHeightMethod2Judgement.jpg "elephant depth map with judgement")| 52 | 53 | From the picture, we can see that the judgement decreases the outlier to some extent.
54 | 55 | ### How to use 56 | * In order to use it, you should have correctrly installed the `openCV` and `Eigen` library.
57 | * The main class is in `PhotometricStero.h` and `PhotometricStero.cc`.
58 | * `PFMAccess` class is used to access `.pbm` file, but you should never copy that. Use its pointer is a good idea.
59 | * The `testPhotometricStero.cpp` contains the `main` function and some examples. You should be careful to change the path.
60 | * project guide and materials is in the `./Assignment_1/` 61 | 62 | 63 | If you have any question, contact me at `xinyuan.gui95@gmail.com` 64 | -------------------------------------------------------------------------------- /resultImage/appleAlbedo.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xyg-coder/Photometric-Stereo/7c1bee1eca417085155a3417956cbbe9114434ac/resultImage/appleAlbedo.jpg -------------------------------------------------------------------------------- /resultImage/appleHeight.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xyg-coder/Photometric-Stereo/7c1bee1eca417085155a3417956cbbe9114434ac/resultImage/appleHeight.jpg -------------------------------------------------------------------------------- /resultImage/appleNormal.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xyg-coder/Photometric-Stereo/7c1bee1eca417085155a3417956cbbe9114434ac/resultImage/appleNormal.jpg -------------------------------------------------------------------------------- /resultImage/appleNormalWithAlbedo.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xyg-coder/Photometric-Stereo/7c1bee1eca417085155a3417956cbbe9114434ac/resultImage/appleNormalWithAlbedo.jpg -------------------------------------------------------------------------------- /resultImage/elephantAlbedo.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xyg-coder/Photometric-Stereo/7c1bee1eca417085155a3417956cbbe9114434ac/resultImage/elephantAlbedo.jpg -------------------------------------------------------------------------------- /resultImage/elephantHeightMethod1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xyg-coder/Photometric-Stereo/7c1bee1eca417085155a3417956cbbe9114434ac/resultImage/elephantHeightMethod1.jpg -------------------------------------------------------------------------------- /resultImage/elephantHeightMethod2Judgement.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xyg-coder/Photometric-Stereo/7c1bee1eca417085155a3417956cbbe9114434ac/resultImage/elephantHeightMethod2Judgement.jpg -------------------------------------------------------------------------------- /resultImage/elephantHeightMethod2NoJudgement.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xyg-coder/Photometric-Stereo/7c1bee1eca417085155a3417956cbbe9114434ac/resultImage/elephantHeightMethod2NoJudgement.jpg -------------------------------------------------------------------------------- /resultImage/elephantNormal.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xyg-coder/Photometric-Stereo/7c1bee1eca417085155a3417956cbbe9114434ac/resultImage/elephantNormal.jpg -------------------------------------------------------------------------------- /resultImage/elephantNormalWithAlbedo.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xyg-coder/Photometric-Stereo/7c1bee1eca417085155a3417956cbbe9114434ac/resultImage/elephantNormalWithAlbedo.jpg -------------------------------------------------------------------------------- /resultImage/pearAlbedo.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xyg-coder/Photometric-Stereo/7c1bee1eca417085155a3417956cbbe9114434ac/resultImage/pearAlbedo.jpg -------------------------------------------------------------------------------- /resultImage/pearHeight.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xyg-coder/Photometric-Stereo/7c1bee1eca417085155a3417956cbbe9114434ac/resultImage/pearHeight.jpg -------------------------------------------------------------------------------- /resultImage/pearNormal.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xyg-coder/Photometric-Stereo/7c1bee1eca417085155a3417956cbbe9114434ac/resultImage/pearNormal.jpg -------------------------------------------------------------------------------- /resultImage/pearNormalWithAlbedo.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xyg-coder/Photometric-Stereo/7c1bee1eca417085155a3417956cbbe9114434ac/resultImage/pearNormalWithAlbedo.jpg -------------------------------------------------------------------------------- /testPhotoMetricStero.cpp: -------------------------------------------------------------------------------- 1 | #include "PhotometricStero.h" 2 | 3 | void drawAppleNormal(); 4 | void drawAppleAlbedo(); 5 | void drawElephantNormal(); 6 | void drawElephantAlbedo(); 7 | void drawPearNormal(); 8 | void drawPearAlbedo(); 9 | void drawAppleNormalWithAlbedo(); 10 | void drawElephantNormalwithAlbedo(); 11 | void drawPearNormalWithAlbedo(); 12 | void getAppleHeight(); 13 | void getElephantHeight(); 14 | void getPearHeight(); 15 | 16 | int main() { 17 | //drawAppleNormal(); 18 | //drawElephantNormal(); 19 | //drawPearNormal(); 20 | //drawAppleAlbedo(); 21 | //drawElephantAlbedo(); 22 | //drawPearAlbedo(); 23 | //drawAppleNormalWithAlbedo(); 24 | //drawElephantNormalwithAlbedo(); 25 | //drawPearNormalWithAlbedo(); 26 | //getAppleHeight(); 27 | getElephantHeight(); 28 | //getPearHeight(); 29 | } 30 | 31 | void drawAppleNormal() { 32 | phoSte::photometryStero A(21, 2, 22, 33 | "C:\\xinyuan Gui\\pcv_Assignment_1\\Assignment_1\\Apple\\", 34 | "mask_dir_1.png", "mask_dir_2.png", "mask_I.png", "applemask.png"); 35 | A.readImage(); 36 | A.getLightInformation(1, 2); 37 | A.getPixelNormAndAlbedo(3); 38 | cv::Mat result = A.outputNormalImage(3); 39 | cv::imshow("apple normal image", result); 40 | cv::imwrite("C:\\xinyuan Gui\\pcv_Assignment_1\\Assignment_1\\Apple\\normal.jpg", result); 41 | cv::waitKey(0); 42 | } 43 | 44 | void drawAppleAlbedo() { 45 | phoSte::photometryStero A(21, 2, 22, 46 | "C:\\xinyuan Gui\\pcv_Assignment_1\\Assignment_1\\Apple\\", 47 | "mask_dir_1.png", "mask_dir_2.png", "mask_I.png", "applemask.png"); 48 | A.readImage(); 49 | A.getLightInformation(1, 2); 50 | A.getPixelNormAndAlbedo(3); 51 | cv::Mat result = A.outputAlbedoImage(3); 52 | cv::imshow("apple albedo image", result); 53 | cv::waitKey(0); 54 | } 55 | 56 | void drawAppleNormalWithAlbedo() { 57 | phoSte::photometryStero A(21, 2, 22, 58 | "C:\\xinyuan Gui\\pcv_Assignment_1\\Assignment_1\\Apple\\", 59 | "mask_dir_1.png", "mask_dir_2.png", "mask_I.png", "applemask.png"); 60 | A.readImage(); 61 | A.getLightInformation(1, 2); 62 | A.getPixelNormAndAlbedo(3); 63 | cv::Mat result = A.outputNormalWithAlbedo(3); 64 | cv::imshow("apple albedo image", result); 65 | cv::waitKey(0); 66 | } 67 | 68 | void drawElephantNormal() { 69 | phoSte::photometryStero A(21, 1, 21, 70 | "C:\\xinyuan Gui\\pcv_Assignment_1\\Assignment_1\\Elephant\\", 71 | "mask_dir_1.png", "mask_dir_2.png", "mask_I.png", "mask.png"); 72 | A.readImage(); 73 | A.getLightInformation(1, 2); 74 | A.getPixelNormAndAlbedo(3); 75 | cv::Mat result = A.outputNormalImage(3); 76 | cv::imshow("elephant normal image", result); 77 | cv::imwrite("C:\\xinyuan Gui\\pcv_Assignment_1\\Assignment_1\\Elephant\\elephantNormal.jpg", result); 78 | cv::waitKey(0); 79 | } 80 | 81 | void drawElephantNormalwithAlbedo() { 82 | phoSte::photometryStero A(21, 1, 21, 83 | "C:\\xinyuan Gui\\pcv_Assignment_1\\Assignment_1\\Elephant\\", 84 | "mask_dir_1.png", "mask_dir_2.png", "mask_I.png", "mask.png"); 85 | A.readImage(); 86 | A.getLightInformation(1, 2); 87 | A.getPixelNormAndAlbedo(3); 88 | cv::Mat result = A.outputNormalWithAlbedo(3); 89 | cv::imshow("elephant normal image", result); 90 | cv::waitKey(0); 91 | } 92 | 93 | void drawElephantAlbedo() { 94 | phoSte::photometryStero A(21, 1, 21, 95 | "C:\\xinyuan Gui\\pcv_Assignment_1\\Assignment_1\\Elephant\\", 96 | "mask_dir_1.png", "mask_dir_2.png", "mask_I.png", "mask.png"); 97 | A.readImage(); 98 | A.getLightInformation(1, 2); 99 | A.getPixelNormAndAlbedo(3); 100 | cv::Mat result = A.outputAlbedoImage(3); 101 | cv::imshow("elephant albedo image", result); 102 | cv::waitKey(0); 103 | } 104 | 105 | void drawPearNormal() { 106 | phoSte::photometryStero A(21, 1, 21, 107 | "C:\\xinyuan Gui\\pcv_Assignment_1\\Assignment_1\\Pear\\", 108 | "mask_dir_1.png", "mask_dir_2.png", "mask_I.png", "pearmask.png"); 109 | A.readImage(); 110 | A.getLightInformation(1, 2); 111 | A.getPixelNormAndAlbedo(3); 112 | cv::Mat result = A.outputNormalImage(3); 113 | cv::imshow("pear normal image", result); 114 | cv::imwrite("C:\\xinyuan Gui\\pcv_Assignment_1\\Assignment_1\\Pear\\pearNormal.jpg", result); 115 | cv::waitKey(0); 116 | } 117 | 118 | void drawPearAlbedo() { 119 | phoSte::photometryStero A(21, 1, 21, 120 | "C:\\xinyuan Gui\\pcv_Assignment_1\\Assignment_1\\Pear\\", 121 | "mask_dir_1.png", "mask_dir_2.png", "mask_I.png", "pearmask.png"); 122 | A.readImage(); 123 | A.getLightInformation(1, 2); 124 | A.getPixelNormAndAlbedo(3); 125 | cv::Mat result = A.outputAlbedoImage(3); 126 | cv::imshow("pear albedo image", result); 127 | cv::waitKey(0); 128 | } 129 | 130 | void drawPearNormalWithAlbedo() { 131 | phoSte::photometryStero A(21, 1, 21, 132 | "C:\\xinyuan Gui\\pcv_Assignment_1\\Assignment_1\\Pear\\", 133 | "mask_dir_1.png", "mask_dir_2.png", "mask_I.png", "pearmask.png"); 134 | A.readImage(); 135 | A.getLightInformation(1, 2); 136 | A.getPixelNormAndAlbedo(3); 137 | cv::Mat result = A.outputNormalWithAlbedo(3); 138 | cv::imshow("pear albedo image", result); 139 | cv::waitKey(0); 140 | } 141 | 142 | void getAppleHeight() { 143 | std::string path = "C:\\xinyuan Gui\\pcv_Assignment_1\\Assignment_1\\Apple\\"; 144 | phoSte::photometryStero A(21, 2, 22, 145 | path, "mask_dir_1.png", "mask_dir_2.png", "mask_I.png", "applemask.png"); 146 | A.readImage(); 147 | A.getLightInformation(1, 2); 148 | A.getPixelNormAndAlbedo(3); 149 | cv::Mat heightMap = A.getHeightMap(2, -0.1); 150 | std::string fileName = "appleDepth.png"; 151 | cv::imwrite(path + fileName, heightMap); 152 | cv::imshow(fileName, heightMap); 153 | cv::waitKey(0); 154 | } 155 | 156 | void getElephantHeight() { 157 | std::string path = "C:\\xinyuan Gui\\pcv_Assignment_1\\Assignment_1\\Elephant\\"; 158 | phoSte::photometryStero A(21, 1, 21, 159 | path, "mask_dir_1.png", "mask_dir_2.png", "mask_I.png", "mask.png"); 160 | A.readImage(); 161 | A.getLightInformation(1, 2); 162 | A.getPixelNormAndAlbedo(3); 163 | cv::Mat heightMap = A.getHeightMap(2, -0.1); 164 | std::string fileName = "elephantHeight.png"; 165 | cv::imwrite(path + fileName, heightMap); 166 | cv::imshow(fileName, heightMap); 167 | cv::waitKey(0); 168 | } 169 | 170 | void getPearHeight() { 171 | std::string path = "C:\\xinyuan Gui\\pcv_Assignment_1\\Assignment_1\\Pear\\"; 172 | phoSte::photometryStero A(21, 1, 21, 173 | path, "mask_dir_1.png", "mask_dir_2.png", "mask_I.png", "pearmask.png"); 174 | A.readImage(); 175 | A.getLightInformation(1, 2); 176 | A.getPixelNormAndAlbedo(3); 177 | cv::Mat heightMap = A.getHeightMap(2, -0.1); 178 | std::string fileName = "pearHeight.png"; 179 | cv::imwrite(path + fileName, heightMap); 180 | cv::imshow(fileName, heightMap); 181 | cv::waitKey(0); 182 | } 183 | --------------------------------------------------------------------------------