├── README.md ├── SIFT ├── __pycache__ │ ├── harris.cpython-36.pyc │ ├── hessian.cpython-36.pyc │ ├── process.cpython-36.pyc │ ├── contrast.cpython-36.pyc │ └── findextrema.cpython-36.pyc ├── README.txt ├── contrast.py ├── factor.py ├── hessian.py ├── process.py ├── harris.py ├── descriptor.py ├── findextrema.py └── sift.py ├── .gitignore ├── project_jaffe.py └── project_ck.py /README.md: -------------------------------------------------------------------------------- 1 | # Face-Recognizaiton-CNN-SIFT 2 | Using SIFT to find keypoint, CNN to extract feature and classify ck and jaffe image 3 | -------------------------------------------------------------------------------- /SIFT/__pycache__/harris.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Leonardyao/Face-Recognizaiton-CNN-SIFT/HEAD/SIFT/__pycache__/harris.cpython-36.pyc -------------------------------------------------------------------------------- /SIFT/__pycache__/hessian.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Leonardyao/Face-Recognizaiton-CNN-SIFT/HEAD/SIFT/__pycache__/hessian.cpython-36.pyc -------------------------------------------------------------------------------- /SIFT/__pycache__/process.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Leonardyao/Face-Recognizaiton-CNN-SIFT/HEAD/SIFT/__pycache__/process.cpython-36.pyc -------------------------------------------------------------------------------- /SIFT/__pycache__/contrast.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Leonardyao/Face-Recognizaiton-CNN-SIFT/HEAD/SIFT/__pycache__/contrast.cpython-36.pyc -------------------------------------------------------------------------------- /SIFT/__pycache__/findextrema.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Leonardyao/Face-Recognizaiton-CNN-SIFT/HEAD/SIFT/__pycache__/findextrema.cpython-36.pyc -------------------------------------------------------------------------------- /SIFT/README.txt: -------------------------------------------------------------------------------- 1 | 1.Run the sift: 2 | Sift_fd.py is the main file, and the function: feature_detect will return the coordinates of feature points detected by the algorithm 3 | 2.Run the pixel compare: 4 | testcompare.py is the main file, it usese pixels generated by harris corner detection method -------------------------------------------------------------------------------- /SIFT/contrast.py: -------------------------------------------------------------------------------- 1 | import numpy 2 | import Image 3 | 4 | 5 | class contrast(object): 6 | 7 | def lowcontrast(self, arr): 8 | 9 | H, W = arr.shape 10 | re = [] 11 | 12 | for i in range(1,H-1): 13 | for j in range(1,W-1): 14 | temp = arr[i-1:i+2,j-1:j+2] 15 | winStd = temp.std() 16 | winMean = temp.mean() 17 | if winStd == 0: 18 | continue 19 | elif abs((arr[i][j] - winMean)/winStd) > 0.3: 20 | re.append((i,j)) 21 | 22 | return re 23 | 24 | 25 | # con = contrast() 26 | # im = Image.open('f:\\5500\\tt\\s.jpg').convert('L') 27 | # ima = numpy.array(im) 28 | # re = con.lowcontrast(ima) 29 | # print len(re) 30 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | MANIFEST 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | .pytest_cache/ 49 | 50 | # Translations 51 | *.mo 52 | *.pot 53 | 54 | # Django stuff: 55 | *.log 56 | local_settings.py 57 | db.sqlite3 58 | 59 | # Flask stuff: 60 | instance/ 61 | .webassets-cache 62 | 63 | # Scrapy stuff: 64 | .scrapy 65 | 66 | # Sphinx documentation 67 | docs/_build/ 68 | 69 | # PyBuilder 70 | target/ 71 | 72 | # Jupyter Notebook 73 | .ipynb_checkpoints 74 | 75 | # pyenv 76 | .python-version 77 | 78 | # celery beat schedule file 79 | celerybeat-schedule 80 | 81 | # SageMath parsed files 82 | *.sage.py 83 | 84 | # Environments 85 | .env 86 | .venv 87 | env/ 88 | venv/ 89 | ENV/ 90 | env.bak/ 91 | venv.bak/ 92 | 93 | # Spyder project settings 94 | .spyderproject 95 | .spyproject 96 | 97 | # Rope project settings 98 | .ropeproject 99 | 100 | # mkdocs documentation 101 | /site 102 | 103 | # mypy 104 | .mypy_cache/ 105 | -------------------------------------------------------------------------------- /SIFT/factor.py: -------------------------------------------------------------------------------- 1 | import math 2 | import numpy 3 | 4 | class factor(object): 5 | 6 | def calFactor(self,pair0,pair1,pair2): 7 | """ 8 | given three pixel pairs and divide the two area of the triangle 9 | whose vertices are the three pixels in those three pairs. 10 | """ 11 | abPat,acPat,bcPat,ABSrc,ACSrc,BCSrc = self.dist(pair0,pair1,pair2) 12 | 13 | # divide the two triangles' area 14 | factor = self.calArea(ABSrc,ACSrc,BCSrc)/self.calArea(abPat,acPat,bcPat) 15 | 16 | return factor 17 | 18 | 19 | def dist(self,pair0,pair1,pair2): 20 | """ 21 | return the distance bewteen 3 match pixels 22 | """ 23 | 24 | ax = pair0[0][0] 25 | ay = pair0[0][1] 26 | Ax = pair0[1][0] 27 | Ay = pair0[1][1] 28 | 29 | bx = pair1[0][0] 30 | by = pair1[0][1] 31 | Bx = pair1[1][0] 32 | By = pair1[1][1] 33 | 34 | cx = pair2[0][0] 35 | cy = pair2[0][1] 36 | Cx = pair2[1][0] 37 | Cy = pair2[1][1] 38 | 39 | dista_b = math.sqrt((ax-bx)**2+(ay-by)**2) 40 | dista_c = math.sqrt((ax-cx)**2+(ay-cy)**2) 41 | distb_c = math.sqrt((bx-cx)**2+(by-cy)**2) 42 | 43 | distA_B = math.sqrt((Ax-Bx)**2+(Ay-By)**2) 44 | distA_C = math.sqrt((Ax-Cx)**2+(Ay-Cy)**2) 45 | distB_C = math.sqrt((Bx-Cx)**2+(By-Cy)**2) 46 | 47 | return dista_b,dista_c,distb_c,distA_B,distA_C,distB_C 48 | 49 | 50 | def cal_area(self,x,y,z): 51 | """ 52 | compute the area of a triangle whose side lengths are the 3 input 53 | """ 54 | p = (x+y+z)/2 55 | 56 | # Hellen formula 57 | return math.sqrt(p*(p-x)*(p-y)*(p-z)) 58 | 59 | -------------------------------------------------------------------------------- /SIFT/hessian.py: -------------------------------------------------------------------------------- 1 | from PIL import Image 2 | import numpy 3 | import scipy 4 | from scipy.ndimage import filters 5 | 6 | class hessian(object): 7 | 8 | def __init__(self): 9 | self.eps = 0.000001 10 | self.patternEdgeThreshold = 4.1 11 | self.sourceEdgeThreshold = 4.1 12 | 13 | def patEdgeDetect(self,arr): 14 | """ 15 | takes an image array as input, return the pixels on the edges of the 16 | pattern image 17 | """ 18 | 19 | imx = numpy.zeros(arr.shape) 20 | filters.gaussian_filter(arr, (3,3), (0,1), imx) 21 | imy = numpy.zeros(arr.shape) 22 | filters.gaussian_filter(arr, (3,3), (1,0), imy) 23 | 24 | Wxx = filters.gaussian_filter(imx*imx,3) 25 | Wxy = filters.gaussian_filter(imx*imy,3) 26 | Wyy = filters.gaussian_filter(imy*imy,3) 27 | 28 | # compute the determint and trace of the array 29 | Wdet = Wxx*Wyy - Wxy**2 30 | Wtr = Wxx + Wyy 31 | 32 | # This threshold value is set by (r+1)**2/r and experiments 33 | Thres = self.patternEdgeThreshold 34 | coor = [] 35 | 36 | Hess = Wtr**2/(Wdet+self.eps) 37 | re = numpy.where(Hess>Thres) 38 | Num = len(re[0]) 39 | 40 | for i in range(Num): 41 | coor.append((re[0][i],re[1][i])) 42 | 43 | return tuple(coor) 44 | 45 | 46 | def Srcedgedetect(self,arr): 47 | """ 48 | takes an image array as input, return the pixels on the edges of the 49 | source image 50 | """ 51 | 52 | imx = numpy.zeros(arr.shape) 53 | filters.gaussian_filter(arr, (3,3), (0,1), imx) 54 | imy = numpy.zeros(arr.shape) 55 | filters.gaussian_filter(arr, (3,3), (1,0), imy) 56 | 57 | Wxx = filters.gaussian_filter(imx*imx,3) 58 | Wxy = filters.gaussian_filter(imx*imy,3) 59 | Wyy = filters.gaussian_filter(imy*imy,3) 60 | 61 | # compute the determint and trace of the array 62 | Wdet = Wxx*Wyy - Wxy**2 63 | Wtr = Wxx + Wyy 64 | 65 | # This threshold value is set by (r+1)**2/r and experiments 66 | Thres = self.sourceEdgeThreshold 67 | coor = [] 68 | 69 | Hess = Wtr**2/(Wdet+self.eps) 70 | re = numpy.where(Hess>Thres) 71 | Num = len(re[0]) 72 | 73 | for i in range(Num): 74 | coor.append((re[0][i],re[1][i])) 75 | 76 | return tuple(coor) 77 | -------------------------------------------------------------------------------- /SIFT/process.py: -------------------------------------------------------------------------------- 1 | from PIL import Image 2 | import scipy 3 | import numpy 4 | import scipy.ndimage 5 | 6 | 7 | class process(object): 8 | def __init__(self): 9 | self.sigma = 1.6 10 | 11 | def createdog(self,imagearr): 12 | """ 13 | return a list of image arrays containning four octaves, 14 | each ovtives has four dog image arrays 15 | """ 16 | re = [0,1,2,3] 17 | re[0] = self.diff(self.gs_blur(self.sigma,imagearr)) 18 | for i in range(1,4): 19 | base = self.sampling(re[i-1][2]) 20 | re[i] = self.diff(self.gs_blur(self.sigma, base)) 21 | return re 22 | 23 | 24 | def diff(self,images): 25 | """ 26 | generate 4 difference gaussian images 27 | input: a list of images in array form, the number of the list is five 28 | return: a list contains four images in image form, which are generated 29 | by the gaussian difference operation. 30 | """ 31 | diffArray = [0,1,2,3] 32 | 33 | # compute the difference bewteen two adjacent images in the same ovtave 34 | for i in range(1,5): 35 | diffArray[i-1] = images[i]-images[i-1] 36 | 37 | return numpy.array(diffArray) 38 | 39 | 40 | def gs_blur(self,k,img): 41 | """ 42 | use gaussina blur to generate five images in different sigma value 43 | input: a k as constant, and an image in array form 44 | return: a list contains five images in image form which are blurred 45 | """ 46 | SIG = self.sigma 47 | sig = [SIG,k*SIG,k*k*SIG,k*k*k*SIG,k*k*k*k*SIG] 48 | gsArray = [0,1,2,3,4] 49 | scaleImages = [0,1,2,3,4] 50 | 51 | for i in range(5): 52 | gsArray[i] = scipy.ndimage.filters.gaussian_filter(img,sig[i]) 53 | 54 | return gsArray 55 | 56 | 57 | def normalize(self,arr): 58 | """ 59 | normalize the pixel intensity 60 | """ 61 | arr = arr/(arr.max()/255.0) 62 | return arr 63 | 64 | 65 | def sampling(self,arr): 66 | """ 67 | do the equal-distance sampling to resize the oringal 68 | image to its 1/4 69 | input: an image in image form 70 | return: a shrinked image in image form 71 | """ 72 | H=0 73 | W=0 74 | if arr.shape[0]%2 == 0: 75 | H = arr.shape[0]/2 76 | else: 77 | H = 1+arr.shape[0]/2 78 | 79 | if arr.shape[1]%2 == 0: 80 | W = arr.shape[1]/2 81 | else: 82 | W = 1+arr.shape[1]/2 83 | 84 | new_arr = numpy.zeros((H,W),dtype = numpy.int) 85 | for i in range(H): 86 | for j in range(W): 87 | new_arr[i][j] = arr[2*i][2*j] 88 | return new_arr 89 | -------------------------------------------------------------------------------- /SIFT/harris.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from PIL import Image 5 | import numpy 6 | import scipy 7 | from scipy.ndimage import filters 8 | 9 | # threshold = 0.0001 10 | # eps = 0.000001 11 | 12 | class harris(object): 13 | def __init__(self): 14 | self.threshold = 0.0001 15 | self.eps = 0.000001 16 | 17 | def corner(self,arr): 18 | """ 19 | takes an image array as input and return the pixels 20 | which are corners of the image 21 | """ 22 | harrisim = self.compute_harris_response(arr) 23 | filtered_coords = self.get_harris_points(harrisim, 3, self.threshold) 24 | return filtered_coords 25 | 26 | def compute_harris_response(self,im,sigma=1.5): 27 | """ Compute the Harris corner detector response function 28 | for each pixel in a graylevel image. """ 29 | 30 | # derivatives 31 | imx = numpy.zeros(im.shape) 32 | filters.gaussian_filter(im, (sigma,sigma), (0,1), imx) 33 | imy = numpy.zeros(im.shape) 34 | filters.gaussian_filter(im, (sigma,sigma), (1,0), imy) 35 | 36 | # compute components of the Harris matrix 37 | Wxx = filters.gaussian_filter(imx*imx,sigma) 38 | Wxy = filters.gaussian_filter(imx*imy,sigma) 39 | Wyy = filters.gaussian_filter(imy*imy,sigma) 40 | 41 | # determinant and trace 42 | Wdet = Wxx*Wyy - Wxy**2 43 | Wtr = Wxx + Wyy+self.eps 44 | 45 | return Wdet / Wtr 46 | 47 | def get_harris_points(self,harrisim,min_dist,threshold): 48 | """ 49 | Return corners from a Harris response image 50 | min_dist is the minimum number of pixels separating 51 | corners and image boundary. 52 | """ 53 | 54 | # find top corner candidates above a threshold 55 | corner_threshold = harrisim.max() * self.threshold 56 | 57 | harrisim_t = (harrisim > corner_threshold) * 1 58 | 59 | # get coordinates of candidates 60 | coords = numpy.array(harrisim_t.nonzero()).T 61 | 62 | # ...and their values 63 | candidate_values = [harrisim[c[0],c[1]] for c in coords] 64 | 65 | # sort candidates 66 | index = numpy.argsort(candidate_values) 67 | 68 | # store allowed point locations in array 69 | allowed_locations = numpy.zeros(harrisim.shape) 70 | 71 | allowed_locations[min_dist:-min_dist,min_dist:-min_dist] = 1 72 | 73 | # select the best points taking min_distance into account 74 | filtered_coords = [] 75 | for i in index: 76 | if allowed_locations[coords[i,0],coords[i,1]] == 1: 77 | filtered_coords.append(tuple(coords[i])) 78 | 79 | allowed_locations[(coords[i,0]-min_dist):(coords[i,0]+min_dist), 80 | (coords[i,1]-min_dist):(coords[i,1]+min_dist)] = 0 81 | 82 | return tuple(filtered_coords) 83 | -------------------------------------------------------------------------------- /SIFT/descriptor.py: -------------------------------------------------------------------------------- 1 | from PIL import Image 2 | import scipy 3 | import numpy 4 | import scipy.ndimage 5 | import math 6 | 7 | 8 | class descriptor(object): 9 | def __init__(self): 10 | self.eps = 0.00001 11 | 12 | def creatDes(self, features, imarr): 13 | """ 14 | return pixels and their descriptor together as a dictionary 15 | the coordinates of pixels are the keys, the descriptors are 16 | the value 17 | """ 18 | desDict = {} 19 | arr = imarr.astype(numpy.float64) 20 | desNum = len(features) 21 | 22 | for i in range(desNum): 23 | desDict[(features[i][0],features[i][1])] = 24 | self.allocate(features[i][0],features[i][1],arr) 25 | 26 | return desDict 27 | 28 | 29 | def direction(self,i,j,imarr): 30 | """ 31 | computes each pixel's gradient magnitude and orientation 32 | """ 33 | mij = math.sqrt((imarr[i+1,j]-imarr[i-1,j])**2 34 | +(imarr[i,j+1]-imarr[i,j-1])**2) 35 | theta = math.atan((imarr[i,j+1]-imarr[i,j-1]) 36 | /(imarr[i+1,j]-imarr[i-1,j]+self.eps)) 37 | 38 | return mij,theta 39 | 40 | 41 | def allocate(self,i,j,imarr): 42 | """ 43 | computes the 16 local area's gradient magnitude and 44 | orientation around the current pixel, 45 | each local area contains 8 pixels 46 | """ 47 | vec = [0]*16 48 | vec[0] = self.localdir(i-8,j-8,imarr) 49 | vec[1] = self.localdir(i-8,j,imarr) 50 | vec[2] = self.localdir(i-8,j+8,imarr) 51 | vec[3] = self.localdir(i-8,j+16,imarr) 52 | 53 | vec[4] = self.localdir(i,j-8,imarr) 54 | vec[5] = self.localdir(i,j,imarr) 55 | vec[6] = self.localdir(i,j+8,imarr) 56 | vec[7] = self.localdir(i,j+16,imarr) 57 | 58 | vec[8] = self.localdir(i+8,j-8,imarr) 59 | vec[9] = self.localdir(i+8,j,imarr) 60 | vec[10] = self.localdir(i+8,j+8,imarr) 61 | vec[11] = self.localdir(i+8,j+16,imarr) 62 | 63 | vec[12] = self.localdir(i+16,j-8,imarr) 64 | vec[13] = self.localdir(i+16,j,imarr) 65 | vec[14] = self.localdir(i+16,j+8,imarr) 66 | vec[15] = self.localdir(i+16,j+16,imarr) 67 | 68 | return [val for subl in vec for val in subl] 69 | 70 | def localdir(self,i,j,imarr): 71 | """ 72 | return singal pixel's direction histogram 73 | the histogram has 18 region 74 | """ 75 | P = math.pi 76 | localDir = [0]*18 77 | 78 | for b in range(i-8,i): 79 | for c in range(j-8,j): 80 | m,t = self.direction(b,c,imarr) 81 | if t>=P*-9/18 and t<=P*-8/18: 82 | localDir[0]+=m 83 | if t>P*-8/18 and t<=P*-7/18: 84 | localDir[1]+=m 85 | if t>P*-7/18 and t<=P*-6/18: 86 | localDir[2]+=m 87 | if t>P*-6/18 and t<=P*-5/18: 88 | localDir[3]+=m 89 | if t>P*-5/18 and t<=P*-4/18: 90 | localDir[4]+=m 91 | if t>P*-4/18 and t<=P*-3/18: 92 | localDir[5]+=m 93 | if t>P*-3/18 and t<=P*-2/18: 94 | localDir[6]+=m 95 | if t>P*-2/18and t<=P*-1/18: 96 | localDir[7]+=m 97 | if t>P*-1/18 and t<=0: 98 | localDir[8]+=m 99 | if t>0 and t<=P*1/18: 100 | localDir[9]+=m 101 | if t>P*1/18 and t<=P*2/18: 102 | localDir[10]+=m 103 | if t>P*2/18 and t<=P*3/18: 104 | localDir[11]+=m 105 | if t>P*3/18 and t<=P*4/18: 106 | localDir[12]+=m 107 | if t>P*4/18 and t<=P*5/18: 108 | localDir[13]+=m 109 | if t>P*5/18 and t<=P*6/18: 110 | localDir[14]+=m 111 | if t>P*6/18 and t<=P*7/18: 112 | localDir[15]+=m 113 | if t>P*7/18 and t<=P*8/18: 114 | localDir[16]+=m 115 | if t>P*8/18 and t<=P*9/18: 116 | localDir[17]+=m 117 | 118 | return localDir 119 | 120 | -------------------------------------------------------------------------------- /SIFT/findextrema.py: -------------------------------------------------------------------------------- 1 | from PIL import Image 2 | import scipy 3 | import numpy 4 | import scipy.ndimage 5 | import math 6 | import itertools 7 | 8 | import harris 9 | import hessian 10 | import contrast 11 | 12 | 13 | class findextrema(object): 14 | 15 | def getPatextremes(self,ims,pa): 16 | """ 17 | find local extremas on pattern image 18 | """ 19 | 20 | # instantiate funtional class 21 | hs = harris.harris() 22 | hess = hessian.hessian() 23 | cont = contrast.contrast() 24 | 25 | coordinates = [] 26 | temp = {} 27 | 28 | H = [0,1,2,3] 29 | W = [0,1,2,3] 30 | 31 | for i in range(4): 32 | H[i] = len(ims[i][0]) 33 | W[i] = len(ims[i][0][0]) 34 | 35 | localArea = [0,1,2] 36 | 37 | # get the unstable and low contrast pixel 38 | hs_points = hs.corner(pa) 39 | hess_points = hess.Patedgedetect(pa) 40 | low_contrast = cont.lowcontrast(pa) 41 | 42 | # compute the pixels which are not situable for pixel matching 43 | bad_points = list(set(hs_points) | set(hess_points) | set(low_contrast)) 44 | bad = dict.fromkeys(bad_points, 0) 45 | 46 | for m in range(4): 47 | for n in range(1,3): 48 | for i in range(16,H[m]-16): 49 | for j in range(16,W[m]-16): 50 | if bad.has_key((i*2**m,j*2**m))==False : 51 | 52 | # compare local pixel with its 26 neighbour 53 | currentPixel = ims[m][n][i][j] 54 | localArea[0] = ims[m][n-1][i-1:i+2,j-1:j+2] 55 | localArea[1] = ims[m][n][i-1:i+2,j-1:j+2] 56 | localArea[2] = ims[m][n+1][i-1:i+2,j-1:j+2] 57 | 58 | Area = numpy.array(localArea) 59 | 60 | maxLocal = numpy.array(Area).max() 61 | minLocal = numpy.array(Area).min() 62 | 63 | if (currentPixel == maxLocal) or (currentPixel == minLocal): 64 | if temp.has_key((i*2**m,j*2**m)) == False: 65 | coordinates.append([int(i*2**m),int(j*2**m)]) 66 | temp[(i*2**m,j*2**m)] = [i*2**m,j*2**m] 67 | return coordinates 68 | 69 | 70 | def get_Srcextremes(self,ims,sa): 71 | """ 72 | find local extremas on pattern image 73 | """ 74 | 75 | # instantiate funtional class 76 | hs = harris.harris() 77 | hess = hessian.hessian() 78 | cont = contrast.contrast() 79 | 80 | coordinates = [] 81 | temp = {} 82 | 83 | H = [0,1,2,3] 84 | W = [0,1,2,3] 85 | 86 | for i in range(4): 87 | H[i] = len(ims[i][0]) 88 | W[i] = len(ims[i][0][0]) 89 | 90 | localArea = [0,1,2] 91 | 92 | # get the unstable and low contrast pixel 93 | hs_points = hs.corner(sa) 94 | hess_points = hess.Srcedgedetect(sa) 95 | low_contrast = cont.lowcontrast(sa) 96 | 97 | # compute the pixels which are not situable for pixel matching 98 | bad_points = list(set(hs_points) | set(hess_points) | set(low_contrast)) 99 | bad = dict.fromkeys(bad_points, 0) 100 | 101 | 102 | for m in range(4): 103 | for n in range(1,3): 104 | for i in range(16,H[m]-16): 105 | for j in range(16,W[m]-16): 106 | if bad.has_key((i*2**m,j*2**m))==False : 107 | 108 | # compare local pixel with its 26 neighbour 109 | currentPixel = ims[m][n][i][j] 110 | localArea[0] = ims[m][n-1][i-1:i+2,j-1:j+2] 111 | localArea[1] = ims[m][n][i-1:i+2,j-1:j+2] 112 | localArea[2] = ims[m][n+1][i-1:i+2,j-1:j+2] 113 | 114 | Area = numpy.array(localArea) 115 | 116 | maxLocal = numpy.array(Area).max() 117 | minLocal = numpy.array(Area).min() 118 | 119 | if (currentPixel == maxLocal) or (currentPixel == minLocal): 120 | if temp.has_key((i*2**m,j*2**m)) == False: 121 | coordinates.append([int(i*2**m),int(j*2**m)]) 122 | temp[(i*2**m,j*2**m)] = [i*2**m,j*2**m] 123 | return coordinates 124 | 125 | 126 | -------------------------------------------------------------------------------- /SIFT/sift.py: -------------------------------------------------------------------------------- 1 | """ 2 | author: alan 3 | 4 | zhougf01@gmail.com 5 | 6 | """ 7 | from PIL import Image 8 | import numpy 9 | import scipy 10 | import scipy.ndimage 11 | import scipy.spatial 12 | from scipy.ndimage import filters 13 | from scipy.spatial import cKDTree 14 | import math 15 | import itertools 16 | 17 | import process 18 | import findextrema 19 | import descriptor 20 | import harris 21 | import hessian 22 | import factor 23 | 24 | class sift(object): 25 | def __init__(self): 26 | self.distanceThresh = 0.00000000001 27 | self.similarityThresh = 0.8 28 | 29 | def match(self,patname,srcname): 30 | """ 31 | given a pattern image and a source image, return the match result and 32 | the scaling factor 33 | """ 34 | 35 | p = Image.open(patname).convert('L') 36 | pa = numpy.array(p) 37 | pa *= 255.0/pa.max() 38 | 39 | s = Image.open(srcname).convert('L') 40 | sa = numpy.array(s) 41 | sa *= 255.0/sa.max() 42 | 43 | pre = process.process() 44 | ex= findextrema.findextrema() 45 | des = descriptor.descriptor() 46 | scale = factor.factor() 47 | 48 | 49 | pdata = pre.creatdog(pa) 50 | sdata = pre.creatdog(sa) 51 | 52 | pDes = [] 53 | sDes = [] 54 | 55 | # dict to store all the feature matching result 56 | result = {} 57 | 58 | pFeatures = ex.get_Patextremes(pdata,pa) 59 | sFeatures = ex.get_Srcextremes(sdata,sa) 60 | 61 | # assign decriptors for each feature points 62 | pDes = des.creatDes(pFeatures,pa) 63 | sDes = des.creatDes(sFeatures,sa) 64 | 65 | tree = [] 66 | 67 | if sDes=={} or pDes=={}: 68 | return False 69 | else: 70 | # use cKD tree struture to compute the two similar pixels 71 | tree = scipy.spatial.cKDTree(sDes.values()) 72 | slocList = sDes.keys() 73 | pDict = {} 74 | sDict = {} 75 | for p in pDes.keys(): 76 | x = pDes[p] 77 | re = tree.query(x,k=2,eps=self.distanceThresh,p=2, 78 | distance_upper_bound=numpy.inf) 79 | 80 | if re[0][1]!=0 and re[0][0]/re[0][1] < self.similarityThresh: 81 | pLoc = p 82 | sLoc = slocList[re[1][0]] 83 | distance = re[0][0] 84 | 85 | # did not been compared before 86 | if sDict.has_key(sLoc)==False: 87 | 88 | # add the result and compared pattern pixel 89 | # and source pixel 90 | result[(pLoc,sLoc)] = distance 91 | pDict[pLoc] = sLoc 92 | sDict[sLoc] = pLoc 93 | 94 | elif distance < result.get((sDict[sLoc],sLoc)): 95 | 96 | # updates the result and compared pattern pixel 97 | # and source pixel 98 | del result[(sDict[sLoc],sLoc)] 99 | result[(pLoc,sLoc)] = distance 100 | del pDict[sDict[sLoc]] 101 | pDict[pLoc] = sLoc 102 | sDict[sLoc] = pLoc 103 | 104 | elif re[0][1]==0: 105 | pLoc = p 106 | sLoc = slocList[re[1][0]] 107 | distance = re[0][0] 108 | 109 | # did not been compared before 110 | if sDict.has_key(sLoc)==False: 111 | 112 | # add the result and compared pattern pixel 113 | # and source pixel 114 | result[(pLoc,sLoc)] = distance 115 | pDict[pLoc] = sLoc 116 | sDict[sLoc] = pLoc 117 | 118 | elif distance < result.get((sDict[sLoc],sLoc)): 119 | 120 | # updates the result and compared pattern pixel 121 | # and source pixel 122 | del result[(sDict[sLoc],sLoc)] 123 | result[(pLoc,sLoc)] = distance 124 | del pDict[sDict[sLoc]] 125 | pDict[pLoc] = sLoc 126 | sDict[sLoc] = pLoc 127 | 128 | # the list of matched pixels, sorted by the distance 129 | finResult = sorted(result.items(), reverse=False, key=lambda d: d[1]) 130 | 131 | match1 = finResult[0][0] 132 | match2 = finResult[1][0] 133 | match3 = finResult[2][0] 134 | 135 | scalingFactor = scale.cal_factor(match1,match2,match3) 136 | 137 | return finResult,scalingFactor 138 | -------------------------------------------------------------------------------- /project_jaffe.py: -------------------------------------------------------------------------------- 1 | 2 | # coding: utf-8 3 | 4 | 5 | import cv2 # opencv 6 | import tensorflow as tf #tensorflow 7 | import glob # to read files 8 | import numpy as np 9 | import os 10 | from random import shuffle 11 | 12 | 13 | dirImage = './jaffe/' 14 | filenames = [] 15 | labels = [] 16 | filenames += glob.glob(dirImage+"/*"+".tiff") 17 | for i in filenames: 18 | labels.append(''.join(i.strip().split('.')[2:3])[:-1]) 19 | #print(labels) 20 | #print("Processing " + str(len(filenames)) + " images") 21 | #print(filenames) 22 | 23 | 24 | def get_sift(img): 25 | #img = denormalize_image(img) 26 | img = cv2.imread(img) 27 | #img = np.array(img*255, dtype="uint8") 28 | gray= cv2.cvtColor(img, cv2.COLOR_RGB2GRAY) 29 | sift = cv2.xfeatures2d.SIFT_create() 30 | kp = sift.detect(gray,None) 31 | img=cv2.drawKeypoints(gray,kp,None) 32 | #cv2.imshow('result', img) 33 | #cv2.waitKey(0) 34 | #cv2.destroyWindow("result") 35 | return img 36 | 37 | 38 | 39 | images = [] 40 | for file in filenames: 41 | #img = np.asarray(cv2.imread(file, 0)) 42 | img = get_sift(file) 43 | images.append(img) 44 | #images = np.asarray(images) 45 | #print(images) 46 | 47 | 48 | emotion = ['NE', 'HA', 'SA', 'SU', 'AN', 'DI', 'FE' ] 49 | for i in range(7): 50 | for j in range(len(labels)): 51 | if labels[j] == emotion[i]: 52 | labels[j] = i 53 | labels_count = len(emotion) 54 | #print(labels) 55 | TRAINING_SIZE = 170 56 | VALIDATION_SIZE = len(images) - TRAINING_SIZE 57 | temp = np.array([images, labels]) 58 | temp = temp.transpose() 59 | shuffle(temp) 60 | images = list(temp[:, 0]) 61 | images = np.stack(images, axis = 0) 62 | labels = list(temp[:, 1]) 63 | labels = [int(i) for i in labels] 64 | train_images = images[:TRAINING_SIZE, :, : ] # 170, 256, 256 65 | test_images = images[TRAINING_SIZE:, :, : ] 66 | print(type(train_images)) 67 | 68 | 69 | #label_file = open('temp_labels.txt'); 70 | #label_data = label_file.read() 71 | #labels = [] 72 | #for i in xrange(len(label_data)): 73 | # if (label_data[i]!='\n'): 74 | # labels.append(float(label_data[i])) 75 | train_labels = np.asarray(labels[:TRAINING_SIZE], dtype=int) 76 | test_labels = np.asarray(labels[170:], dtype=int) 77 | #print(train_labels.shape[0]) 78 | 79 | 80 | def dense_to_one_hot(labels_dense, num_classes): 81 | num_labels = len(labels_dense) 82 | index_offset = np.arange(num_labels) * num_classes 83 | labels_one_hot = np.zeros((num_labels, num_classes)) 84 | # print(type(np.int_(index_offset + labels_dense.ravel()))) 85 | labels_one_hot.flat[np.int_(index_offset + labels_dense.ravel())] = 1 86 | return labels_one_hot 87 | 88 | 89 | train_labels = dense_to_one_hot(train_labels.ravel(), labels_count) 90 | # labels = labels.astype(np.uint8) 91 | test_labels = dense_to_one_hot(test_labels.ravel(), labels_count) 92 | #print(train_labels.shape) 93 | 94 | 95 | train_images = train_images.reshape(TRAINING_SIZE, -1) 96 | test_images = test_images.reshape(VALIDATION_SIZE, -1) 97 | print(train_images.shape) 98 | 99 | 100 | train_image_pixels = train_images.shape[1] 101 | #print('Flat pixel values is %d'%(train_image_pixels)) 102 | 103 | 104 | train_image_width = np.int_(np.ceil(np.sqrt(train_image_pixels/3.0))) 105 | train_image_height = np.int_(np.ceil(np.sqrt(train_image_pixels/3.0))) 106 | 107 | 108 | 109 | 110 | # # Build TF CNN model 111 | 112 | 113 | def new_conv_layer(input_data, num_input_channels, num_filters, filter_shape, pool_shape, name): 114 | # setup input filter state 115 | conv_filt_shape = [filter_shape[0], filter_shape[1], num_input_channels, num_filters] 116 | 117 | # initialise weights and bias 118 | weights = tf.Variable(tf.truncated_normal(conv_filt_shape, stddev=0.03), name=name+'_W') 119 | bias = tf.Variable(tf.truncated_normal([num_filters]), name=name+'_b') 120 | 121 | # setup convolutional layer 122 | out_layer = tf.nn.conv2d(input_data, weights, [1, 1, 1, 1], padding='SAME') 123 | out_layer += bias 124 | 125 | # applying relu non-linear activation 126 | out_layer = tf.nn.relu(out_layer) 127 | 128 | # performing max pooling 129 | ksize = [1, 2, 2, 1] 130 | strides = [1, 2, 2, 1] 131 | out_layer = tf.nn.max_pool(out_layer, ksize=ksize, strides=strides, padding='SAME') 132 | 133 | return out_layer 134 | 135 | # input & output of NN 136 | 137 | # images 138 | x = tf.placeholder(tf.float32, [None, train_image_width * train_image_height*3.0]) 139 | # dynamically reshaping input 140 | x_shaped = tf.reshape(x, [-1, train_image_width, train_image_height, 3]) 141 | # labels 142 | y = tf.placeholder(tf.float32, [None, labels_count]) 143 | keep_prob = tf.placeholder(tf.float32) 144 | 145 | # creating sparse layers of CNN 146 | conv1 = tf.layers.conv2d(x_shaped, filters=32, kernel_size=[3, 3], strides=[1, 1], padding='same',activation=tf.nn.relu,kernel_initializer=tf.truncated_normal_initializer(stddev=0.1),name='conv1') 147 | bn1 = tf.layers.batch_normalization(conv1, training=True, name='bn1') 148 | pool1 = tf.layers.max_pooling2d(bn1, pool_size=[2, 2], strides=[2, 2], padding='same', name='pool1') 149 | conv2 = tf.layers.conv2d(pool1, filters=128, kernel_size=[3, 3], strides=[1, 1], padding='same',activation=tf.nn.relu,kernel_initializer=tf.truncated_normal_initializer(stddev=0.1),name='conv2') 150 | bn2 = tf.layers.batch_normalization(conv2, training=True, name='bn2') 151 | pool2 = tf.layers.max_pooling2d(bn2, pool_size=[2, 2], strides=[2, 2], padding='same', name='pool2') 152 | conv3 = tf.layers.conv2d(pool1, filters=256, kernel_size=[3, 3], strides=[1, 1], padding='same',activation=tf.nn.relu,kernel_initializer=tf.truncated_normal_initializer(stddev=0.1),name='conv3') 153 | bn3 = tf.layers.batch_normalization(conv3, training=True, name='bn3') 154 | pool3 = tf.layers.max_pooling2d(bn3, pool_size=[2, 2], strides=[2, 2], padding='same', name='pool3') 155 | conv4 = tf.layers.conv2d(pool1, filters=128, kernel_size=[3, 3], strides=[1, 1], padding='same',activation=tf.nn.relu,kernel_initializer=tf.truncated_normal_initializer(stddev=0.1),name='conv4') 156 | bn4 = tf.layers.batch_normalization(conv4, training=True, name='bn4') 157 | pool4 = tf.layers.max_pooling2d(bn4, pool_size=[2, 2], strides=[2, 2], padding='same', name='pool4') 158 | 159 | flatten_layer = tf.contrib.layers.flatten(pool4, 'flatten_layer') 160 | weights = tf.get_variable(shape=[flatten_layer.shape[-1], labels_count], dtype=tf.float32, 161 | initializer=tf.truncated_normal_initializer(stddev=0.1), name='fc_weights') 162 | biases = tf.get_variable(shape=[labels_count], dtype=tf.float32,initializer=tf.constant_initializer(0.0), name='fc_biases') 163 | 164 | logit_output = tf.nn.bias_add(tf.matmul(flatten_layer, weights), biases, name='logit_output') 165 | cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y, logits=logit_output)) 166 | y_ = tf.nn.softmax(logit_output) 167 | pred_label = tf.argmax(logit_output, 1) 168 | label = tf.argmax(y, 1) 169 | accuracy = tf.reduce_mean(tf.cast(tf.equal(pred_label, label), tf.float32)) 170 | ''' 171 | layer1 = new_conv_layer(x_shaped, 3, 32, [5, 5], [2, 2], name='layer1') 172 | bn1 = tf.layers.batch_normalization(layer1, training=True, name='bn1') 173 | pool1 = tf.layers.max_pooling2d(bn1, pool_size=[2, 2], strides=[2, 2], padding='same', name='pool1') 174 | layer2 = new_conv_layer(layer1, 32, 64, [5, 5], [2, 2], name='layer2') 175 | bn2 = tf.layers.batch_normalization(conv2, training=True, name='bn2') 176 | layer3 = new_conv_layer(layer2, 64, 128, [5, 5], [2, 2], name='layer3') 177 | layer4 = new_conv_layer(layer3, 128, 256, [5, 5], [2, 2], name='layer4') 178 | flattened = tf.reshape(layer4, [-1, 16 * 16 * 256]) 179 | #flattened = tf.reshape(layer1, [-1, 32 * 32 * 512]) 180 | 181 | 182 | # calculating dense layers of CNN 183 | wd1 = tf.Variable(tf.truncated_normal([16 * 16 * 256, 500], stddev=0.03), name='wd1') 184 | bd1 = tf.Variable(tf.truncated_normal([500], stddev=0.01), name='bd1') 185 | dense_layer1 = tf.matmul(flattened, wd1) + bd1 186 | dense_layer1 = tf.nn.relu(dense_layer1) 187 | 188 | # dropout for reducing overfitting 189 | keep_prob = tf.placeholder(tf.float32) 190 | h_fc1_drop = tf.nn.dropout(dense_layer1, keep_prob) 191 | 192 | # another layer for softmax calculation and readout 193 | wd2 = tf.Variable(tf.truncated_normal([500, labels_count], stddev=0.03), name='wd2') 194 | bd2 = tf.Variable(tf.truncated_normal([labels_count], stddev=0.01), name='bd2') 195 | dense_layer2 = tf.matmul(dense_layer1, wd2) + bd2 196 | y_ = tf.nn.softmax(dense_layer2) 197 | 198 | 199 | keep_prob = tf.placeholder(tf.float32) 200 | wd2 = tf.Variable(tf.truncated_normal([32 * 32 * 512, labels_count], stddev=0.03), name='wd2') 201 | bd2 = tf.Variable(tf.truncated_normal([labels_count], stddev=0.01), name='bd2') 202 | dense_layer2 = tf.matmul(flattened, wd2) + bd2 203 | y_ = tf.nn.softmax(dense_layer2) 204 | ''' 205 | # cross entropy cost function 206 | #cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=dense_layer2, labels=y)) 207 | 208 | 209 | # # Training CNN 210 | 211 | 212 | # set to 3000 iterations 213 | epochs = 100 214 | DROPOUT = 0.5 215 | batch_size = 16 216 | 217 | # settings 218 | learning_rate = 1e-3 219 | 220 | os.environ["CUDA_VISIBLE_DEVICES"] = "5" 221 | # adding optimiser 222 | optimiser = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cross_entropy) 223 | 224 | # defining accuracy assessment operation 225 | correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1)) 226 | accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) 227 | 228 | # setting up the initialisation operator 229 | init_op = tf.global_variables_initializer() 230 | 231 | # recording variable to store accuracy 232 | tf.summary.scalar('accuracy', accuracy) 233 | saver = tf.train.Saver() 234 | 235 | merged = tf.summary.merge_all() 236 | writer = tf.summary.FileWriter('logs') 237 | 238 | with tf.Session() as sess: 239 | # initialising variables 240 | sess.run(init_op) 241 | total_batch = int(len(train_labels) / batch_size) 242 | for epoch in range(epochs): 243 | avg_cost = 0 244 | batch_index = 0 245 | for j in range(total_batch): 246 | train_batch_images = train_images[batch_index:(batch_index+batch_size), :] 247 | train_batch_labels = train_labels[batch_index:(batch_index+batch_size), :] 248 | _, c = sess.run([optimiser, cross_entropy], feed_dict={x:train_batch_images, y:train_batch_labels, keep_prob:DROPOUT}) 249 | avg_cost += c 250 | batch_index += batch_size 251 | train_acc = sess.run(accuracy, feed_dict={x:train_images, y:train_labels}) 252 | test_acc = sess.run(accuracy, feed_dict={x: test_images, y: test_labels}) 253 | print("Epoch:", (epoch + 1), "cost =", "{:.3f}".format(avg_cost), "train accuracy: {:.3f}".format(train_acc), "test accuracy: {:.3f}".format(test_acc)) 254 | save_path = './model/jaffe-model.ckpt' 255 | saver.save(sess, save_path) 256 | summary = sess.run(merged, feed_dict={x: test_images, y: test_labels}) 257 | writer.add_summary(summary, epoch) 258 | print("\nTraining complete!") 259 | writer.add_graph(sess.graph) 260 | print(sess.run(accuracy, feed_dict={x: test_images, y: test_labels})) 261 | 262 | -------------------------------------------------------------------------------- /project_ck.py: -------------------------------------------------------------------------------- 1 | 2 | # coding: utf-8 3 | 4 | 5 | import cv2 # opencv 6 | import tensorflow as tf #tensorflow 7 | import glob # to read files 8 | import numpy as np 9 | import os 10 | from random import shuffle 11 | 12 | 13 | dirImage = './ck/' 14 | filenames = [] 15 | labels = [] 16 | filedir = os.listdir(dirImage) 17 | #print(filedir) 18 | for file in filedir: 19 | labeldir = os.listdir(dirImage + file) 20 | for label in labeldir: 21 | imgfile = os.listdir(dirImage + file + "/"+ label) 22 | for img in imgfile: 23 | filenames.append(str(dirImage + file + "/" + label + "/" + img)) 24 | labels.append(label[-1]) 25 | #print(filenames) 26 | 27 | ''' 28 | def get_sift(img): 29 | #img = denormalize_image(img) 30 | img = cv2.imread(img) 31 | img = cv2.resize(img, (245, 320), interpolation=cv2.INTER_CUBIC) 32 | #img = np.array(img*255, dtype="uint8") 33 | gray= cv2.cvtColor(img, cv2.COLOR_RGB2GRAY) 34 | sift = cv2.xfeatures2d.SIFT_create() 35 | kp = sift.detect(gray,None) 36 | img=cv2.drawKeypoints(gray,kp,None) 37 | #cv2.imshow('result', img) 38 | #cv2.waitKey(0) 39 | #cv2.destroyWindow("result") 40 | return img 41 | 42 | 43 | images = [] 44 | for file in filenames: 45 | #img = np.asarray(cv2.imread(file, 0)) 46 | img = get_sift(file) 47 | images.append(img) 48 | #images = np.asarray(images) 49 | np.save('sift_image.npy', images) 50 | #print(images) 51 | ''' 52 | images = np.load('sift_image.npy') 53 | 54 | temp = np.array([list(images), labels]) 55 | temp = temp.transpose() 56 | shuffle(temp) 57 | images = list(temp[:, 0]) 58 | images = np.stack(images, axis = 0) 59 | labels = list(temp[:, 1]) 60 | labels = [int(i) for i in labels] 61 | 62 | emotion = ['1', '2', '3', '4', '5', '6', '7', '8'] #8 class 63 | labels_count = len(emotion) 64 | #print(labels) 65 | TRAINING_SIZE = int(len(images) * 0.8) 66 | VALIDATION_SIZE = len(images) - TRAINING_SIZE 67 | train_images = images[:TRAINING_SIZE] 68 | test_images = images[TRAINING_SIZE:] 69 | #print(train_images.shape) 70 | 71 | 72 | #label_file = open('temp_labels.txt'); 73 | #label_data = label_file.read() 74 | #labels = [] 75 | #for i in xrange(len(label_data)): 76 | # if (label_data[i]!='\n'): 77 | # labels.append(float(label_data[i])) 78 | train_labels = np.asarray(labels[:TRAINING_SIZE], dtype=int) 79 | test_labels = np.asarray(labels[TRAINING_SIZE:], dtype=int) 80 | #print(train_labels.shape[0]) 81 | 82 | 83 | def dense_to_one_hot(labels_dense, num_classes): 84 | num_labels = len(labels_dense) 85 | index_offset = np.arange(num_labels) * num_classes 86 | labels_one_hot = np.zeros((num_labels, num_classes)) 87 | # print(type(np.int_(index_offset + labels_dense.ravel()))) 88 | labels_one_hot.flat[np.int_(index_offset + labels_dense.ravel())] = 1 89 | return labels_one_hot 90 | 91 | train_image_width = 245 92 | train_image_height = 320 93 | train_labels = dense_to_one_hot(train_labels.ravel(), labels_count) 94 | # labels = labels.astype(np.uint8) 95 | test_labels = dense_to_one_hot(test_labels.ravel(), labels_count) 96 | #print(train_labels.shape) 97 | 98 | 99 | train_images = train_images.reshape(TRAINING_SIZE, -1) 100 | test_images = test_images.reshape(VALIDATION_SIZE, -1) 101 | #print(len(train_images), train_images.shape) 102 | 103 | 104 | train_image_pixels = train_images.shape[1] 105 | #print('Flat pixel values is %d'%(train_image_pixels)) 106 | 107 | 108 | 109 | #print(train_image_width) 110 | #print(train_image_height) 111 | 112 | 113 | # # Build TF CNN model 114 | 115 | 116 | def new_conv_layer(input_data, num_input_channels, num_filters, filter_shape, pool_shape, name): 117 | # setup input filter state 118 | conv_filt_shape = [filter_shape[0], filter_shape[1], num_input_channels, num_filters] 119 | 120 | # initialise weights and bias 121 | weights = tf.Variable(tf.truncated_normal(conv_filt_shape, stddev=0.03), name=name+'_W') 122 | bias = tf.Variable(tf.truncated_normal([num_filters]), name=name+'_b') 123 | 124 | # setup convolutional layer 125 | out_layer = tf.nn.conv2d(input_data, weights, [1, 1, 1, 1], padding='SAME') 126 | out_layer += bias 127 | 128 | # applying relu non-linear activation 129 | out_layer = tf.nn.relu(out_layer) 130 | 131 | # performing max pooling 132 | ksize = [1, 2, 2, 1] 133 | strides = [1, 2, 2, 1] 134 | out_layer = tf.nn.max_pool(out_layer, ksize=ksize, strides=strides, padding='SAME') 135 | 136 | return out_layer 137 | 138 | 139 | # input & output of NN 140 | 141 | # images 142 | x = tf.placeholder(tf.float32, [None, train_image_width * train_image_height*3.0]) 143 | # dynamically reshaping input 144 | x_shaped = tf.reshape(x, [-1, train_image_width, train_image_height, 3]) 145 | # labels 146 | y = tf.placeholder(tf.float32, [None, labels_count]) 147 | keep_prob = tf.placeholder(tf.float32) 148 | 149 | # creating sparse layers of CNN 150 | conv1 = tf.layers.conv2d(x_shaped, filters=32, kernel_size=[3, 3], strides=[1, 1], padding='same',activation=tf.nn.relu,kernel_initializer=tf.truncated_normal_initializer(stddev=0.1),name='conv1') 151 | bn1 = tf.layers.batch_normalization(conv1, training=True, name='bn1') 152 | pool1 = tf.layers.max_pooling2d(bn1, pool_size=[2, 2], strides=[2, 2], padding='same', name='pool1') 153 | conv2 = tf.layers.conv2d(pool1, filters=64, kernel_size=[3, 3], strides=[1, 1], padding='same',activation=tf.nn.relu,kernel_initializer=tf.truncated_normal_initializer(stddev=0.1),name='conv2') 154 | bn2 = tf.layers.batch_normalization(conv2, training=True, name='bn2') 155 | pool2 = tf.layers.max_pooling2d(bn2, pool_size=[2, 2], strides=[2, 2], padding='same', name='pool2') 156 | conv3 = tf.layers.conv2d(pool1, filters=128, kernel_size=[3, 3], strides=[1, 1], padding='same',activation=tf.nn.relu,kernel_initializer=tf.truncated_normal_initializer(stddev=0.1),name='conv3') 157 | bn3 = tf.layers.batch_normalization(conv3, training=True, name='bn3') 158 | pool3 = tf.layers.max_pooling2d(bn3, pool_size=[2, 2], strides=[2, 2], padding='same', name='pool3') 159 | 160 | conv4 = tf.layers.conv2d(pool1, filters=64, kernel_size=[3, 3], strides=[1, 1], padding='same',activation=tf.nn.relu,kernel_initializer=tf.truncated_normal_initializer(stddev=0.1),name='conv4') 161 | bn4 = tf.layers.batch_normalization(conv4, training=True, name='bn4') 162 | pool4 = tf.layers.max_pooling2d(bn4, pool_size=[2, 2], strides=[2, 2], padding='same', name='pool4') 163 | 164 | flatten_layer = tf.contrib.layers.flatten(pool4, 'flatten_layer') 165 | weights = tf.get_variable(shape=[flatten_layer.shape[-1], labels_count], dtype=tf.float32, 166 | initializer=tf.truncated_normal_initializer(stddev=0.1), name='fc_weights') 167 | biases = tf.get_variable(shape=[labels_count], dtype=tf.float32,initializer=tf.constant_initializer(0.0), name='fc_biases') 168 | 169 | logit_output = tf.nn.bias_add(tf.matmul(flatten_layer, weights), biases, name='logit_output') 170 | cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y, logits=logit_output)) 171 | y_ = tf.nn.softmax(logit_output) 172 | pred_label = tf.argmax(logit_output, 1) 173 | label = tf.argmax(y, 1) 174 | accuracy = tf.reduce_mean(tf.cast(tf.equal(pred_label, label), tf.float32)) 175 | ''' 176 | layer1 = new_conv_layer(x_shaped, 3, 32, [5, 5], [2, 2], name='layer1') 177 | layer2 = new_conv_layer(layer1, 32, 64, [5, 5], [2, 2], name='layer2') 178 | layer3 = new_conv_layer(layer2, 64, 128, [5, 5], [2, 2], name='layer3') 179 | layer4 = new_conv_layer(layer3, 128, 256, [5, 5], [2, 2], name='layer4') 180 | flattened = tf.reshape(layer4, [-1, 16 * 16 * 256]) 181 | #flattened = tf.reshape(layer3, [-1, 32 * 32 * 128]) 182 | 183 | 184 | # calculating dense layers of CNN 185 | wd1 = tf.Variable(tf.truncated_normal([16 * 16 * 256, 1000], stddev=0.03), name='wd1') 186 | bd1 = tf.Variable(tf.truncated_normal([1000], stddev=0.01), name='bd1') 187 | dense_layer1 = tf.matmul(flattened, wd1) + bd1 188 | dense_layer1 = tf.nn.relu(dense_layer1) 189 | 190 | # dropout for reducing overfitting 191 | keep_prob = tf.placeholder(tf.float32) 192 | h_fc1_drop = tf.nn.dropout(dense_layer1, keep_prob) 193 | 194 | # another layer for softmax calculation and readout 195 | wd2 = tf.Variable(tf.truncated_normal([1000, labels_count], stddev=0.03), name='wd2') 196 | bd2 = tf.Variable(tf.truncated_normal([labels_count], stddev=0.01), name='bd2') 197 | dense_layer2 = tf.matmul(dense_layer1, wd2) + bd2 198 | y_ = tf.nn.softmax(dense_layer2) 199 | 200 | 201 | # cross entropy cost function 202 | cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=dense_layer2, labels=y)) 203 | ''' 204 | 205 | # # Training CNN 206 | 207 | 208 | # set to 3000 iterations 209 | epochs = 1000 210 | DROPOUT = 0.5 211 | batch_size = 2 212 | 213 | # settings 214 | learning_rate = 1e-3 215 | 216 | os.environ["CUDA_VISIBLE_DEVICES"] = "1, 2" 217 | # adding optimiser 218 | optimiser = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cross_entropy) 219 | 220 | # defining accuracy assessment operation 221 | correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1)) 222 | accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) 223 | 224 | # setting up the initialisation operator 225 | init_op = tf.global_variables_initializer() 226 | 227 | # recording variable to store accuracy 228 | tf.summary.scalar('accuracy', accuracy) 229 | saver = tf.train.Saver() 230 | 231 | merged = tf.summary.merge_all() 232 | writer = tf.summary.FileWriter('logs') 233 | 234 | tf_config = tf.ConfigProto() 235 | tf_config.gpu_options.allow_growth = True 236 | tf_config.allow_soft_placement = True 237 | 238 | with tf.Session() as sess: 239 | # initialising variables 240 | sess.run(init_op) 241 | total_batch = int(len(train_labels) / batch_size) 242 | total_batch_test = int(len(test_labels) / batch_size) 243 | for epoch in range(epochs): 244 | avg_cost = 0 245 | batch_index = 0 246 | for j in range(total_batch): 247 | train_batch_images = train_images[batch_index:(batch_index+batch_size), :] 248 | train_batch_labels = train_labels[batch_index:(batch_index+batch_size), :] 249 | _, c, train_acc = sess.run([optimiser, cross_entropy, accuracy], feed_dict={x:train_batch_images, y:train_batch_labels, keep_prob:DROPOUT}) 250 | avg_cost += c 251 | batch_index += batch_size 252 | #train_acc = sess.run(accuracy, feed_dict={x:train_images, y:train_labels}) 253 | for j in range(total_batch_test): 254 | test_batch_images = test_images[batch_index:(batch_index+batch_size), :] 255 | test_batch_labels = test_labels[batch_index:(batch_index+batch_size), :] 256 | test_acc = sess.run(accuracy, feed_dict={x: test_batch_images, y: test_batch_labels}) 257 | print("Epoch:", (epoch + 1), "cost =", "{:.3f}".format(avg_cost), "train accuracy: {:.3f}".format(train_acc), "test accuracy: {:.3f}".format(test_acc)) 258 | save_path = './model/ck-model.ckpt' 259 | saver.save(sess, save_path) 260 | summary = sess.run(merged, feed_dict={x: test_images, y: test_labels}) 261 | writer.add_summary(summary, epoch) 262 | print("\nTraining complete!") 263 | writer.add_graph(sess.graph) 264 | print(sess.run(accuracy, feed_dict={x: test_images, y: test_labels})) 265 | 266 | --------------------------------------------------------------------------------