├── .gitignore
├── README.md
├── __init__.py
├── cache
├── confusion_matrix.npy
├── fc2_mean.npy
├── fc2_std.npy
├── image_mean.npy
├── image_std.npy
├── mvcnn_test_accuracy_shuffled.npy
├── mvcnn_test_accuracy_unshuffled.npy
├── svm_top_k_1.pkl
├── svm_top_k_10.pkl
├── svm_top_k_10_confusion.npy
├── svm_top_k_10_random.pkl
├── svm_top_k_10_random_confusion.npy
├── svm_top_k_15.pkl
├── svm_top_k_15_confusion.npy
├── svm_top_k_15_random.pkl
├── svm_top_k_15_random_confusion.npy
├── svm_top_k_1_confusion.npy
├── svm_top_k_1_random.pkl
├── svm_top_k_1_random_confusion.npy
├── svm_top_k_20.pkl
├── svm_top_k_20_confusion.npy
├── svm_top_k_20_random.pkl
├── svm_top_k_20_random_confusion.npy
├── svm_top_k_25.pkl
├── svm_top_k_25_confusion.npy
├── svm_top_k_25_random.pkl
├── svm_top_k_25_random_confusion.npy
├── svm_top_k_3.pkl
├── svm_top_k_3_confusion.npy
├── svm_top_k_3_random.pkl
├── svm_top_k_3_random_confusion.npy
├── svm_top_k_5.pkl
├── svm_top_k_5_confusion.npy
├── svm_top_k_5_random.pkl
├── svm_top_k_5_random_confusion.npy
├── svm_top_k_7.pkl
├── svm_top_k_7_confusion.npy
├── svm_top_k_7_random.pkl
└── svm_top_k_7_random_confusion.npy
├── classification
├── __init__.py
└── mvcnn_FPS.py
├── data
├── saliency_labels_train.txt
└── saliency_labels_valid.txt
├── dependencies.txt
├── generate_views
├── CMakeLists.txt
├── README.md
├── cmake
│ └── FindLIBIGL.cmake
├── include
│ ├── helpers
│ │ └── RandomUtils.h
│ └── render
│ │ ├── OpenGLStuff.h
│ │ ├── Phong.h
│ │ └── Program.h
└── src
│ ├── helpers
│ └── RandomUtils.cpp
│ ├── main.cpp
│ └── render
│ ├── OpenGLStuff.cpp
│ ├── Phong.cpp
│ └── shaders
│ ├── phong.frag
│ ├── phong.geom
│ └── phong.vert
├── globals.py
├── logs
├── 10_15_20_25.log
├── 1_3_5_7.log
├── saliency.log
├── saliency_log_4_27.log
├── top_1.log
├── top_10.log
├── top_15.log
├── top_20.log
├── top_25.log
├── top_3.log
├── top_5.log
├── top_7.log
├── training_04_13_08_08.log
└── training_04_19_02_01.log
├── models
├── __init__.py
├── multiview_cnn.py
├── multiview_svm.py
└── saliency.py
├── poster.pdf
├── requirements.txt
├── screenshots
├── confusion_matrix.png
└── sample_data.png
├── scripts
├── generate_confusion_matrix.py
├── generate_saliency_data.py
├── plot_logs.py
├── precompute_fc_mean_std.py
├── precompute_image_mean_std.py
├── test_mvcnn.py
├── test_saliency.py
├── train_saliency.py
├── train_svm.py
├── view_confusion_matrix.py
└── view_salient.py
└── utils
├── __init__.py
├── custom_datagen.py
└── helpers.py
/.gitignore:
--------------------------------------------------------------------------------
1 | *.swo
2 | *.swp
3 | *.pyc
4 | __pycache__
5 | .DS_Store
6 | **/build
7 | *.h5
8 |
9 | train_cnn/model_weights.*
10 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | ## Setup
2 |
3 | Clone the repo, add the package to the python path, download python dependencies.
4 |
5 | ```
6 | PYTHONPATH=$(pwd):$PYTHONPATH
7 | git clone https://github.com/bradyz/geometry_processing.git
8 |
9 | pip install -r requirements.txt
10 | or
11 | pip install --user -r requirements.txt
12 | ```
13 |
14 | ## Data Dependencies
15 |
16 | Modelnet with 25 viewpoints each - https://drive.google.com/open?id=0B0d9M5p2RxBqN0IzOXpudjMyTDQ
17 |
18 | Our model's weights - https://drive.google.com/open?id=0B0d9M5p2RxBqMlNZOFg1YmlYR3c
19 |
20 | ## Contents
21 |
22 | 1. [View Generator](#view_gen) - take 2D projections of mesh files.
23 | 2. [Train CNN](#train_cnn) - fine tune a VGG-16 CNN on the new images.
24 | 3. [Classifier](#classify) - train a SVM on the CNN features.
25 | 3. [References](#references) - papers and resources used.
26 |
27 | ## View Generator
28 |
29 | Given a model and a list of viewpoints - .png image files that correspond to 2D projection will be generated.
30 |
31 | Preprocessing consists of centering the mesh, uniformly scaling the bounding box to a unit cube, and taking viewpoints that are centered at the centroid.
32 |
33 | Currently there are 25 viewpoints being generated that fall around the unit sphere from 5 different phis and 5 different thetas (spherical coordinates).
34 |
35 |
36 |
37 | ## Train CNN
38 |
39 | The model used in this project is a VGG-16 with pretrained weights (ImageNet), with two additional layers fc1 (2048), fc2 (1024).
40 |
41 | Training was done for 10 epochs on 100k training images (4000 meshes) over 10 labels of ModelNet10. The images were 224 x 224 rgb. Cross entropy loss was used in combination with a SGD optimizer with a batch size of 64. Training took approximately 5 hours a NVIDIA K40 gpu.
42 |
43 | After training, classification accuracy, given a single pose, is at 80% on a test set of 20k images.
44 |
45 |
46 |
47 | ## Classifier
48 |
49 | The question asked is - given a mesh and several viewpoints, does it help to use all of the viewpoints (MVCNN), or does a selected subset of size k give better accuracy?
50 |
51 | We use a one-vs-rest linear SVM, similar to MVCNN, to classify activation values of the final fc layer.
52 |
53 | The current methods consist of the using the following (currently unimplemented) -
54 |
55 | * Sort by minimized entropy
56 | * Random K
57 | * FPS (farthest point selection) on sorted
58 |
59 | ## References
60 |
61 | Multi-view Convolutional Neural Networks for 3D Shape Recognition - https://arxiv.org/pdf/1505.00880.pdf
62 | Princeton ModelNet - http://modelnet.cs.princeton.edu/
63 |
--------------------------------------------------------------------------------
/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/bradyz/geometry_processing/946729f379ad6d70d6937aec459c46e6d840a3cb/__init__.py
--------------------------------------------------------------------------------
/cache/confusion_matrix.npy:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/bradyz/geometry_processing/946729f379ad6d70d6937aec459c46e6d840a3cb/cache/confusion_matrix.npy
--------------------------------------------------------------------------------
/cache/fc2_mean.npy:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/bradyz/geometry_processing/946729f379ad6d70d6937aec459c46e6d840a3cb/cache/fc2_mean.npy
--------------------------------------------------------------------------------
/cache/fc2_std.npy:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/bradyz/geometry_processing/946729f379ad6d70d6937aec459c46e6d840a3cb/cache/fc2_std.npy
--------------------------------------------------------------------------------
/cache/image_mean.npy:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/bradyz/geometry_processing/946729f379ad6d70d6937aec459c46e6d840a3cb/cache/image_mean.npy
--------------------------------------------------------------------------------
/cache/image_std.npy:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/bradyz/geometry_processing/946729f379ad6d70d6937aec459c46e6d840a3cb/cache/image_std.npy
--------------------------------------------------------------------------------
/cache/mvcnn_test_accuracy_shuffled.npy:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/bradyz/geometry_processing/946729f379ad6d70d6937aec459c46e6d840a3cb/cache/mvcnn_test_accuracy_shuffled.npy
--------------------------------------------------------------------------------
/cache/mvcnn_test_accuracy_unshuffled.npy:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/bradyz/geometry_processing/946729f379ad6d70d6937aec459c46e6d840a3cb/cache/mvcnn_test_accuracy_unshuffled.npy
--------------------------------------------------------------------------------
/cache/svm_top_k_1.pkl:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/bradyz/geometry_processing/946729f379ad6d70d6937aec459c46e6d840a3cb/cache/svm_top_k_1.pkl
--------------------------------------------------------------------------------
/cache/svm_top_k_10.pkl:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/bradyz/geometry_processing/946729f379ad6d70d6937aec459c46e6d840a3cb/cache/svm_top_k_10.pkl
--------------------------------------------------------------------------------
/cache/svm_top_k_10_confusion.npy:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/bradyz/geometry_processing/946729f379ad6d70d6937aec459c46e6d840a3cb/cache/svm_top_k_10_confusion.npy
--------------------------------------------------------------------------------
/cache/svm_top_k_10_random.pkl:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/bradyz/geometry_processing/946729f379ad6d70d6937aec459c46e6d840a3cb/cache/svm_top_k_10_random.pkl
--------------------------------------------------------------------------------
/cache/svm_top_k_10_random_confusion.npy:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/bradyz/geometry_processing/946729f379ad6d70d6937aec459c46e6d840a3cb/cache/svm_top_k_10_random_confusion.npy
--------------------------------------------------------------------------------
/cache/svm_top_k_15.pkl:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/bradyz/geometry_processing/946729f379ad6d70d6937aec459c46e6d840a3cb/cache/svm_top_k_15.pkl
--------------------------------------------------------------------------------
/cache/svm_top_k_15_confusion.npy:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/bradyz/geometry_processing/946729f379ad6d70d6937aec459c46e6d840a3cb/cache/svm_top_k_15_confusion.npy
--------------------------------------------------------------------------------
/cache/svm_top_k_15_random.pkl:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/bradyz/geometry_processing/946729f379ad6d70d6937aec459c46e6d840a3cb/cache/svm_top_k_15_random.pkl
--------------------------------------------------------------------------------
/cache/svm_top_k_15_random_confusion.npy:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/bradyz/geometry_processing/946729f379ad6d70d6937aec459c46e6d840a3cb/cache/svm_top_k_15_random_confusion.npy
--------------------------------------------------------------------------------
/cache/svm_top_k_1_confusion.npy:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/bradyz/geometry_processing/946729f379ad6d70d6937aec459c46e6d840a3cb/cache/svm_top_k_1_confusion.npy
--------------------------------------------------------------------------------
/cache/svm_top_k_1_random.pkl:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/bradyz/geometry_processing/946729f379ad6d70d6937aec459c46e6d840a3cb/cache/svm_top_k_1_random.pkl
--------------------------------------------------------------------------------
/cache/svm_top_k_1_random_confusion.npy:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/bradyz/geometry_processing/946729f379ad6d70d6937aec459c46e6d840a3cb/cache/svm_top_k_1_random_confusion.npy
--------------------------------------------------------------------------------
/cache/svm_top_k_20.pkl:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/bradyz/geometry_processing/946729f379ad6d70d6937aec459c46e6d840a3cb/cache/svm_top_k_20.pkl
--------------------------------------------------------------------------------
/cache/svm_top_k_20_confusion.npy:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/bradyz/geometry_processing/946729f379ad6d70d6937aec459c46e6d840a3cb/cache/svm_top_k_20_confusion.npy
--------------------------------------------------------------------------------
/cache/svm_top_k_20_random.pkl:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/bradyz/geometry_processing/946729f379ad6d70d6937aec459c46e6d840a3cb/cache/svm_top_k_20_random.pkl
--------------------------------------------------------------------------------
/cache/svm_top_k_20_random_confusion.npy:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/bradyz/geometry_processing/946729f379ad6d70d6937aec459c46e6d840a3cb/cache/svm_top_k_20_random_confusion.npy
--------------------------------------------------------------------------------
/cache/svm_top_k_25.pkl:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/bradyz/geometry_processing/946729f379ad6d70d6937aec459c46e6d840a3cb/cache/svm_top_k_25.pkl
--------------------------------------------------------------------------------
/cache/svm_top_k_25_confusion.npy:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/bradyz/geometry_processing/946729f379ad6d70d6937aec459c46e6d840a3cb/cache/svm_top_k_25_confusion.npy
--------------------------------------------------------------------------------
/cache/svm_top_k_25_random.pkl:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/bradyz/geometry_processing/946729f379ad6d70d6937aec459c46e6d840a3cb/cache/svm_top_k_25_random.pkl
--------------------------------------------------------------------------------
/cache/svm_top_k_25_random_confusion.npy:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/bradyz/geometry_processing/946729f379ad6d70d6937aec459c46e6d840a3cb/cache/svm_top_k_25_random_confusion.npy
--------------------------------------------------------------------------------
/cache/svm_top_k_3.pkl:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/bradyz/geometry_processing/946729f379ad6d70d6937aec459c46e6d840a3cb/cache/svm_top_k_3.pkl
--------------------------------------------------------------------------------
/cache/svm_top_k_3_confusion.npy:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/bradyz/geometry_processing/946729f379ad6d70d6937aec459c46e6d840a3cb/cache/svm_top_k_3_confusion.npy
--------------------------------------------------------------------------------
/cache/svm_top_k_3_random.pkl:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/bradyz/geometry_processing/946729f379ad6d70d6937aec459c46e6d840a3cb/cache/svm_top_k_3_random.pkl
--------------------------------------------------------------------------------
/cache/svm_top_k_3_random_confusion.npy:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/bradyz/geometry_processing/946729f379ad6d70d6937aec459c46e6d840a3cb/cache/svm_top_k_3_random_confusion.npy
--------------------------------------------------------------------------------
/cache/svm_top_k_5.pkl:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/bradyz/geometry_processing/946729f379ad6d70d6937aec459c46e6d840a3cb/cache/svm_top_k_5.pkl
--------------------------------------------------------------------------------
/cache/svm_top_k_5_confusion.npy:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/bradyz/geometry_processing/946729f379ad6d70d6937aec459c46e6d840a3cb/cache/svm_top_k_5_confusion.npy
--------------------------------------------------------------------------------
/cache/svm_top_k_5_random.pkl:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/bradyz/geometry_processing/946729f379ad6d70d6937aec459c46e6d840a3cb/cache/svm_top_k_5_random.pkl
--------------------------------------------------------------------------------
/cache/svm_top_k_5_random_confusion.npy:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/bradyz/geometry_processing/946729f379ad6d70d6937aec459c46e6d840a3cb/cache/svm_top_k_5_random_confusion.npy
--------------------------------------------------------------------------------
/cache/svm_top_k_7.pkl:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/bradyz/geometry_processing/946729f379ad6d70d6937aec459c46e6d840a3cb/cache/svm_top_k_7.pkl
--------------------------------------------------------------------------------
/cache/svm_top_k_7_confusion.npy:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/bradyz/geometry_processing/946729f379ad6d70d6937aec459c46e6d840a3cb/cache/svm_top_k_7_confusion.npy
--------------------------------------------------------------------------------
/cache/svm_top_k_7_random.pkl:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/bradyz/geometry_processing/946729f379ad6d70d6937aec459c46e6d840a3cb/cache/svm_top_k_7_random.pkl
--------------------------------------------------------------------------------
/cache/svm_top_k_7_random_confusion.npy:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/bradyz/geometry_processing/946729f379ad6d70d6937aec459c46e6d840a3cb/cache/svm_top_k_7_random_confusion.npy
--------------------------------------------------------------------------------
/classification/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/bradyz/geometry_processing/946729f379ad6d70d6937aec459c46e6d840a3cb/classification/__init__.py
--------------------------------------------------------------------------------
/classification/mvcnn_FPS.py:
--------------------------------------------------------------------------------
1 | import pickle
2 | import os
3 | import cv2
4 | import sys
5 | import Image
6 | import numpy as np
7 | import random as rd
8 | from fnmatch import fnmatch
9 | from sklearn import linear_model
10 | from matplotlib import pyplot as plt
11 | %matplotlib inline
12 |
13 | import keras.layers
14 | from keras.models import Model, Sequential
15 | from keras.applications.vgg16 import VGG16
16 | from keras.layers import Dense, Flatten, Input
17 | from tensorflow.python.platform import gfile
18 | from keras.preprocessing.image import ImageDataGenerator
19 | from keras.callbacks import CSVLogger, ModelCheckpoint, ReduceLROnPlateau
20 |
21 | SAVE_WEIGHTS_FILE = '/your/weights/path/model_weights.h5'
22 | VALID_DIR = "/your/testimages/path/ModelNetViewpoints/test/"
23 | IMAGE_SIZE = 224
24 | NUM_CLASSES = 10
25 |
26 | def load_model_vgg():
27 | img_input = Input(tensor=Input(shape=(IMAGE_SIZE, IMAGE_SIZE, 3)))
28 | base_model = VGG16(include_top=False, input_tensor=img_input)
29 |
30 | for layer in base_model.layers:
31 | layer.trainable = False
32 |
33 | x = base_model.output
34 | x = Flatten(name='flatten')(x)
35 | x = Dense(2048, activation='relu', name='fc1')(x)
36 | x = Dense(1024, activation='relu', name='fc2')(x)
37 | x = Dense(NUM_CLASSES, activation='softmax', name='predictions')(x)
38 | model = Model(input=img_input, output=x)
39 | model.load_weights(SAVE_WEIGHTS_FILE, by_name=True)
40 | #print('Model loaded with weights from %s.' % SAVE_WEIGHTS_FILE)
41 |
42 | return model
43 |
44 | class mvcnnclass:
45 |
46 | def __init__(self,
47 | model,
48 | featurelayer='fc1',
49 | numallview=25,
50 | numviewselection=7,
51 | data_path=VALID_DIR
52 | ):
53 | self.model = model
54 | self.featurelayer = featurelayer
55 | self.numallview = numallview
56 | self.numviewselection = numviewselection
57 | self.data_path = data_path
58 |
59 | def get_data(self, objname):
60 | viewimgfilepaths, objIDs = [],[]
61 | objlist = np.sort(os.listdir(self.data_path+objname+'/'))
62 | modelnum = objlist[::self.numallview]
63 | for i,name in enumerate(modelnum):
64 | name = name.replace('.off_1_1.png','')
65 | files = np.sort(gfile.Glob(self.data_path+objname+'/'+name+'*'))
66 | viewimgfilepaths.append(files)
67 | objIDs.append(name)
68 | print '%s views are loaded!' % objname
69 | return viewimgfilepaths, objIDs
70 |
71 | def image_from_path(self, image_path):
72 | image = cv2.imread(image_path, cv2.IMREAD_COLOR)
73 | try:
74 | image = np.reshape(image, (1, IMAGE_SIZE, IMAGE_SIZE, 3))
75 | except:
76 | print('IMAGE LOADING FAILDED!!!')
77 | print('->path: %s' % image_path)
78 | raise
79 | return image
80 |
81 | def singleview_classification(self, image_path):
82 | image = self.image_from_path(image_path)
83 | prediction = self.model.predict(image)
84 | return prediction[0], np.argsort(prediction[0])[:-6:-1]
85 |
86 | def feature_extraction(self, imagepath, featurelayer):
87 | image = self.image_from_path(imagepath)
88 | intermediate_layer_model = Model(input=self.model.input,
89 | output=self.model.get_layer(featurelayer).output)
90 | return intermediate_layer_model.predict(image)
91 |
92 | def feat_distance(self, feat1, feat2):
93 | sim = spatial.distance.cosine(feat1, feat2)
94 | return 1-sim
95 |
96 | def output_entropy(self, prediction, eps=10e-4):
97 | return np.sum(-(prediction+eps)*np.log2(prediction+eps))
98 |
99 | def view_score(self, feat_test, feat_ref, prediction, portion=1.0):
100 | entropy_score = self.output_entropy(prediction)
101 | fps_score = self.feat_distance(feat_test, feat_ref)
102 | return entropy_score+portion*fps_score
103 |
104 | def entropy_selection(self, viewimgfilepaths):
105 | filenames = []
106 | entropies = []
107 | for file in viewimgfilepaths:
108 | sys.stdout.write('.')
109 | prediction, _ = self.singleview_classification(file)
110 | entropies = np.append(entropies, self.output_entropy(prediction))
111 | filenames = np.append(filenames, file)
112 | solution_feats, solution_filepath = [],[]
113 | argminentropy = np.argsort(entropies)[:self.numviewselection]
114 | solution_filepath = filenames[argminentropy]
115 | for file in solution_filepath:
116 | solution_feats.append(self.feature_extraction(file, self.featurelayer))
117 | solution_feats = np.asarray(solution_feats)
118 | solution_feats = np.reshape(solution_feats,
119 | (solution_filepath.shape[0],
120 | self.model.get_layer(self.featurelayer).output.shape[1]))
121 | solution_filepath = np.asarray(solution_filepath)
122 | return solution_feats, solution_filepath
123 |
124 | def fps_selection(self, viewimgfilepaths):
125 | feats, filenames = [],[]
126 | for file in viewimgfilepaths:
127 | sys.stdout.write('.')
128 | feat = self.feature_extraction(file, self.featurelayer)
129 | filenames.append(file)
130 | feats.append(feat)
131 | solution_feats, solution_filepath = [],[]
132 | initindx = rd.randint(0, len(filenames)-1)
133 |
134 | solution_feats.append(feats.pop(initindx))
135 | solution_filepath.append(filenames.pop(initindx))
136 |
137 | for i in range(self.numviewselection-1):
138 | distances = [self.feat_distance(f, solution_feats[0]) for f in feats]
139 | for i, f in enumerate(feats):
140 | for j, s in enumerate(solution_feats):
141 | distances[i] = min(distances[i], self.feat_distance(f, s))
142 | solution_feats.append(feats.pop(distances.index(max(distances))))
143 | solution_filepath.append(filenames.pop(distances.index(max(distances))))
144 | solution_feats = np.asarray(solution_feats)
145 | solution_feats = np.reshape(solution_feats,
146 | (len(solution_filepath),
147 | self.model.get_layer(self.featurelayer).output.shape[1]))
148 | solution_filepath = np.asarray(solution_filepath)
149 | sys.stdout.write('!\n')
150 | print "FPS selection done."
151 | return solution_feats, solution_filepath
152 |
153 | def feature_pooling(self, selected_feats):
154 | return np.amax(selected_feats, axis=0)
155 |
156 | def mvcnn_classification(self, objname):
157 | predictions, class_names = [],[]
158 | viewimgfilepaths, objID =mvcnn.get_data(objname)
159 | for i, objIDpaths in enumerate(viewimgfilepaths):
160 | print "Object ID-> %s" % objID[i]
161 | bestfeats, bestfilepath = self.entropy_selection(objIDpaths)
162 | agg_feat = self.feature_pooling(bestfeats)
163 |
164 | feat_input = Input(tensor=Input(shape=(agg_feat.shape)))
165 | if self.featurelayer=='fc1':
166 | x = self.model.get_layer('fc2')(feat_input)
167 | x = self.model.get_layer('predictions')(x)
168 | else:
169 | x = self.model.get_layer('predictions')(feat_input)
170 | cnn2_model = Model(input=feat_input, output=x)
171 |
172 | prediction = cnn2_model.predict(np.array([agg_feat]))
173 | class_name = np.sort(os.listdir(VALID_DIR))[np.argmax(prediction, axis=1)]
174 | sys.stdout.write('!\n')
175 | predictions.append(prediction)
176 | class_names.append(class_name)
177 | print "Classification Done."
178 | return prediction, class_name
179 |
180 | def singleview_analysis(self, savetopath):
181 | objlist = np.sort(os.listdir(self.data_path))
182 | print objlist
183 | for _, objname in enumerate(objlist):
184 | viewimgfilepaths, objID =mvcnn.get_data(objname)
185 | for i, file in enumerate(viewimgfilepaths):
186 | print "Model ID : %s" % objID[i]
187 | fc1_feats, fc2_feats, fc1_fps, fc2_fps = [],[],[],[]
188 | filename, entropies, classindxs = [],[],[]
189 | recordlist = []
190 | savefile = savetopath+objname+"/"+objID[i]+"_result.npy"
191 | if os.path.isfile(savefile):
192 | print "%s_result.npy file already exists!" % objID[i]
193 | else:
194 | for _, viewimg in enumerate(file):
195 | filename.append(viewimg.replace(self.data_path+objname+'/',''))
196 | prediction, classindx = self.singleview_classification(viewimg)
197 | entropies.append(self.output_entropy(prediction))
198 | classindxs.append(classindx)
199 | fc1_feats.append(self.feature_extraction(viewimg, 'fc1'))
200 | fc2_feats.append(self.feature_extraction(viewimg, 'fc2'))
201 | agg_fc1_feat = np.asarray(fc1_feats)
202 | agg_fc1_feat = np.reshape(agg_fc1_feat,(len(file), fc1_feats[0].size))
203 | agg_fc1_feat = self.feature_pooling(agg_fc1_feat)
204 | agg_fc2_feat = np.asarray(fc2_feats)
205 | agg_fc2_feat = np.reshape(agg_fc2_feat,(len(file), fc2_feats[0].size))
206 | agg_fc2_feat = self.feature_pooling(agg_fc2_feat)
207 | for j, fc1_feat in enumerate(fc1_feats):
208 | fc1_fps.append(self.feat_distance(fc1_feat, agg_fc1_feat))
209 | fc2_fps.append(self.feat_distance(fc2_feats[j], agg_fc2_feat))
210 |
211 | record={"imgids": filename,
212 | "labels": classindxs,
213 | "entropy": entropies,
214 | "fc1_fps": fc1_fps,
215 | "fc1": fc1_feats,
216 | "fc1_global": agg_fc1_feat,
217 | "fc1_fps": fc2_fps,
218 | "fc2": fc2_feats,
219 | "fc2_global": agg_fc2_feat,
220 | }
221 | recordlist.append(record)
222 | np.save(savefile, recordlist)
223 |
224 | if __name__ == '__main__':
225 | vggmodel = load_model_vgg()
226 | mvcnn = mvcnnclass(vggmodel, featurelayer='fc1')
227 | pred, classname = mvcnn.mvcnn_classification('bed')
228 | mvcnn.singleview_analysis(ANALYSIS_DIR)
229 |
230 |
--------------------------------------------------------------------------------
/dependencies.txt:
--------------------------------------------------------------------------------
1 | sudo apt-get install python3-tk
2 |
--------------------------------------------------------------------------------
/generate_views/CMakeLists.txt:
--------------------------------------------------------------------------------
1 | cmake_minimum_required(VERSION 2.8)
2 | project(GLSL)
3 |
4 | set(EXECUTABLE_OUTPUT_PATH ${PROJECT_BINARY_DIR}/bin)
5 |
6 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -g -lpng -O3" )
7 | set(CMAKE_FIND_LIBRARY_SUFFIXES "${CMAKE_FIND_LIBRARY_SUFFIXES}")
8 |
9 | link_directories(
10 | "/usr/local/lib"
11 | )
12 |
13 | include_directories(
14 | "/usr/local/include"
15 | "${GLSL_SOURCE_DIR}"
16 | "${PROJECT_SOURCE_DIR}/include"
17 | )
18 |
19 | set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${CMAKE_CURRENT_SOURCE_DIR}/cmake)
20 |
21 | find_package(LIBIGL QUIET)
22 |
23 | if (NOT LIBIGL_FOUND)
24 | message(FATAL_ERROR "libigl not found")
25 | endif()
26 |
27 | option(LIBIGL_USE_STATIC_LIBRARY "Use LibIGL as static library" OFF)
28 |
29 | # add a customizable menu bar
30 | option(LIBIGL_WITH_NANOGUI "Use Nanogui menu" OFF)
31 |
32 | # libigl options: choose your dependencies (by default everything is OFF except opengl)
33 | option(LIBIGL_WITH_VIEWER "Use OpenGL viewer" ON)
34 | option(LIBIGL_WITH_OPENGL "Use OpenGL" ON)
35 | option(LIBIGL_WITH_OPENGL_GLFW "Use GLFW" ON)
36 | option(LIBIGL_WITH_BBW "Use BBW" OFF)
37 | option(LIBIGL_WITH_EMBREE "Use Embree" OFF)
38 | option(LIBIGL_WITH_PNG "Use PNG" OFF)
39 | option(LIBIGL_WITH_TETGEN "Use Tetgen" OFF)
40 | option(LIBIGL_WITH_TRIANGLE "Use Triangle" OFF)
41 | option(LIBIGL_WITH_XML "Use XML" OFF)
42 | option(LIBIGL_WITH_LIM "Use LIM" OFF)
43 | option(LIBIGL_WITH_COMISO "Use CoMiso" OFF)
44 | option(LIBIGL_WITH_MATLAB "Use Matlab" OFF) # This option is not supported yet
45 | option(LIBIGL_WITH_MOSEK "Use MOSEK" OFF) # This option is not supported yet
46 | option(LIBIGL_WITH_CGAL "Use CGAL" OFF)
47 | if(LIBIGL_WITH_CGAL) # Do not remove or move this block, the cgal build system fails without it
48 | find_package(CGAL REQUIRED)
49 | set(CGAL_DONT_OVERRIDE_CMAKE_FLAGS TRUE CACHE BOOL "CGAL's CMAKE Setup is super annoying ")
50 | include(${CGAL_USE_FILE})
51 | endif()
52 |
53 | add_subdirectory("${LIBIGL_INCLUDE_DIR}/../shared/cmake" "libigl")
54 |
55 | # libigl information
56 | message("libigl includes: ${LIBIGL_INCLUDE_DIRS}")
57 | message("libigl libraries: ${LIBIGL_LIBRARIES}")
58 | message("libigl extra sources: ${LIBIGL_EXTRA_SOURCES}")
59 | message("libigl extra libraries: ${LIBIGL_EXTRA_LIBRARIES}")
60 | message("libigl definitions: ${LIBIGL_DEFINITIONS}")
61 |
62 | # Prepare the build environment
63 | include_directories(${LIBIGL_INCLUDE_DIRS})
64 | add_definitions(${LIBIGL_DEFINITIONS})
65 |
66 | find_package(OpenGL REQUIRED)
67 | include_directories(${OpenGL_INCLUDE_DIRS})
68 | link_directories(${OpenGL_LIBRARY_DIRS})
69 | add_definitions(${OpenGL_DEFINITIONS})
70 |
71 | find_package(GLEW REQUIRED)
72 | include_directories(${GLEW_INCLUDE_DIRS})
73 | link_libraries(${GLEW_LIBRARIES})
74 |
75 | find_package(PkgConfig REQUIRED)
76 | pkg_search_module(GLFW REQUIRED glfw3)
77 | include_directories(${GLFW_INCLUDE_DIRS})
78 |
79 | find_package(JPEG REQUIRED)
80 | include_directories(${JPEG_INCLUDE_DIR})
81 |
82 | find_package(Eigen3 REQUIRED)
83 | include_directories(${EIGEN3_INCLUDE_DIR})
84 |
85 | # Subdivision Algorithms Demo.
86 | FILE(GLOB SRC ./src/main.cpp ./src/**/*.cpp)
87 | add_executable(generate_views ${SRC} ${LIBIGL_EXTRA_SOURCES})
88 | target_link_libraries(generate_views
89 | ${OPENGL_gl_LIBRARY}
90 | ${GLFW_LIBRARIES}
91 | ${GLEW_LIBRARY}
92 | ${JPEG_LIBRARIES}
93 | ${LDFLAGS}
94 | ${LIBIGL_LIBRARIES}
95 | ${LIBIGL_EXTRA_LIBRARIES})
96 |
--------------------------------------------------------------------------------
/generate_views/README.md:
--------------------------------------------------------------------------------
1 | ## Dependencies
2 |
3 | Libigl
4 | Png++
5 | Eigen
6 | Glm
7 | Glfw
8 | OpenGL 4.1
9 |
10 |
11 | ## Usage
12 |
13 | ```bash
14 | mkdir build
15 | cd build
16 | cmake ..
17 | ```
18 |
19 | At this point you should add the libigl include path (use ccmake).
20 |
21 | In the build directory -
22 |
23 | ```bash
24 | make -j4
25 | ```
26 |
27 | In the main directory, supply the text file with all the paths to the data, and
28 | the output directory (must include the slash on the end).
29 |
30 | All the nested paths must be created, so if you have /path/to/model/mesh.off,
31 | $OUTPUT_DIR/path/to/model must be created before the script runs.
32 |
33 | ```bash
34 | ./build/bin/generate_views $INPUT_TXT $OUTPUT_DIR
35 | ```
36 |
--------------------------------------------------------------------------------
/generate_views/cmake/FindLIBIGL.cmake:
--------------------------------------------------------------------------------
1 | # - Try to find the LIBIGL library
2 | # Once done this will define
3 | #
4 | # LIBIGL_FOUND - system has LIBIGL
5 | # LIBIGL_INCLUDE_DIR - **the** LIBIGL include directory
6 | # LIBIGL_INCLUDE_DIRS - LIBIGL include directories
7 | # LIBIGL_SOURCES - the LIBIGL source files
8 | if(NOT LIBIGL_FOUND)
9 |
10 | FIND_PATH(LIBIGL_INCLUDE_DIR igl/readOBJ.h
11 | ${PROJECT_SOURCE_DIR}/../../include
12 | ${PROJECT_SOURCE_DIR}/../include
13 | ${PROJECT_SOURCE_DIR}/include
14 | ${PROJECT_SOURCE_DIR}/../libigl/include
15 | ${PROJECT_SOURCE_DIR}/../../libigl/include
16 | $ENV{LIBIGL}/include
17 | $ENV{LIBIGLROOT}/include
18 | $ENV{LIBIGL_ROOT}/include
19 | $ENV{LIBIGL_DIR}/include
20 | $ENV{LIBIGL_DIR}/inc
21 | /usr/include
22 | /usr/local/include
23 | /usr/local/igl/libigl/include
24 | )
25 |
26 |
27 | if(LIBIGL_INCLUDE_DIR)
28 | set(LIBIGL_FOUND TRUE)
29 | set(LIBIGL_INCLUDE_DIRS ${LIBIGL_INCLUDE_DIR} ${LIBIGL_INCLUDE_DIR}/../external/Singular_Value_Decomposition)
30 | #set(LIBIGL_SOURCES
31 | # ${LIBIGL_INCLUDE_DIR}/igl/viewer/Viewer.cpp
32 | #)
33 | endif()
34 |
35 | endif()
36 |
--------------------------------------------------------------------------------
/generate_views/include/helpers/RandomUtils.h:
--------------------------------------------------------------------------------
1 | #ifndef RANDOMUTILS_H
2 | #define RANDOMUTILS_H
3 |
4 | #include
5 | #include
6 | #include
7 |
8 | #include
9 |
10 | const glm::vec4 GREEN = glm::vec4(0.0f, 1.0f, 0.0f, 1.0f);
11 | const glm::vec4 BLUE = glm::vec4(0.0f, 0.0f, 1.0f, 1.0f);
12 | const glm::vec4 WHITE = glm::vec4(1.0f, 1.0f, 1.0f, 1.0f);
13 | const glm::vec4 CYAN = glm::vec4(0.0f, 1.0f, 1.0f, 1.0f);
14 | const glm::vec4 RED = glm::vec4(1.0f, 0.0f, 0.0f, 1.0f);
15 | const glm::vec4 BROWN = glm::vec4(0.72f, 0.60f, 0.41f, 1.0f);
16 | const glm::vec4 BLACK = glm::vec4(0.0f, 0.0f, 0.0f, 1.0f);
17 |
18 | void LoadOBJ(const std::string& file,
19 | std::vector& vertices,
20 | std::vector& faces,
21 | std::vector& normals);
22 |
23 | void LoadOBJWithNormals(const std::string& file,
24 | std::vector& vertices,
25 | std::vector& faces,
26 | std::vector& normals);
27 |
28 | std::string loadShader(const std::string& file);
29 |
30 | std::vector getVertexNormals (const std::vector& vertices,
31 | const std::vector& faces);
32 |
33 | void fixDuplicateVertices (std::vector& v,
34 | std::vector& f);
35 |
36 | namespace glm {
37 | std::ostream& operator<<(std::ostream& os, const glm::vec2& v);
38 | std::ostream& operator<<(std::ostream& os, const glm::vec3& v);
39 | std::ostream& operator<<(std::ostream& os, const glm::vec4& v);
40 | std::ostream& operator<<(std::ostream& os, const glm::mat3& v);
41 | std::ostream& operator<<(std::ostream& os, const glm::mat4& v);
42 | }
43 |
44 | #endif
45 |
--------------------------------------------------------------------------------
/generate_views/include/render/OpenGLStuff.h:
--------------------------------------------------------------------------------
1 | #ifndef OPENGLSTUFF_H
2 | #define OPENGLSTUFF_H
3 |
4 | #include
5 | #include
6 |
7 | #include
8 | #include
9 |
10 | #include
11 | #include
12 |
13 | extern glm::vec4 LIGHT_POSITION;
14 |
15 | extern const float kNear;
16 | extern const float kFar;
17 | extern const float kFov;
18 |
19 | extern float camera_distance;
20 |
21 | extern glm::vec3 up;
22 | extern glm::vec3 look;
23 | extern glm::vec3 tangent;
24 | extern glm::mat3 orientation;
25 |
26 | extern glm::vec3 eye;
27 | extern glm::vec3 center;
28 |
29 | extern glm::mat4 view_matrix;
30 | extern glm::mat4 projection_matrix;
31 |
32 | extern const std::string window_title;
33 |
34 | extern int window_width;
35 | extern int window_height;
36 |
37 | extern const int WIDTH;
38 | extern const int HEIGHT;
39 |
40 | extern bool do_action;
41 |
42 | const std::string PHONG_VERT = "./src/render/shaders/phong.vert";
43 | const std::string PHONG_GEOM = "./src/render/shaders/phong.geom";
44 | const std::string PHONG_FRAG = "./src/render/shaders/phong.frag";
45 |
46 | enum {
47 | kVertexBuffer,
48 | kIndexBuffer,
49 | kVertexNormalBuffer,
50 | kNumVbos
51 | };
52 |
53 | enum {
54 | kPhongVao,
55 | kNumVaos
56 | };
57 |
58 | extern GLuint array_objects[kNumVaos];
59 | extern GLuint buffer_objects[kNumVaos][kNumVbos];
60 |
61 | void initOpenGL();
62 | bool keepLoopingOpenGL(glm::vec3 eye_pos);
63 | void cleanupOpenGL();
64 | void endLoopOpenGL();
65 |
66 | GLuint setupShader (const char* shaderName, GLenum shaderType);
67 |
68 | extern const char* OpenGlErrorToString(GLenum error);
69 |
70 | #define CHECK_SUCCESS(x) \
71 | if (!(x)) { \
72 | glfwTerminate(); \
73 | exit(EXIT_FAILURE); \
74 | }
75 |
76 | #define CHECK_GL_SHADER_ERROR(id) \
77 | { \
78 | GLint status = 0; \
79 | GLint length = 0; \
80 | glGetShaderiv(id, GL_COMPILE_STATUS, &status); \
81 | glGetShaderiv(id, GL_INFO_LOG_LENGTH, &length); \
82 | if (!status) { \
83 | string log(length, 0); \
84 | glGetShaderInfoLog(id, length, nullptr, &log[0]); \
85 | cerr << "Line :" << __LINE__ << " OpenGL Shader Error: Log = \n" \
86 | << &log[0]; \
87 | glfwTerminate(); \
88 | exit(EXIT_FAILURE); \
89 | } \
90 | }
91 |
92 | #define CHECK_GL_PROGRAM_ERROR(id) \
93 | { \
94 | GLint status = 0; \
95 | GLint length = 0; \
96 | glGetProgramiv(id, GL_LINK_STATUS, &status); \
97 | glGetProgramiv(id, GL_INFO_LOG_LENGTH, &length); \
98 | if (!status) { \
99 | string log(length, 0); \
100 | glGetProgramInfoLog(id, length, nullptr, &log[0]); \
101 | cerr << "Line :" << __LINE__ << " OpenGL Program Error: Log = \n" \
102 | << &log[0]; \
103 | glfwTerminate(); \
104 | exit(EXIT_FAILURE); \
105 | } \
106 | }
107 |
108 | #define CHECK_GL_ERROR(statement) \
109 | { \
110 | { statement; } \
111 | GLenum error = GL_NO_ERROR; \
112 | if ((error = glGetError()) != GL_NO_ERROR) { \
113 | cerr << "Line :" << __LINE__ << " OpenGL Error: code = " << error \
114 | << " description = " << OpenGlErrorToString(error); \
115 | glfwTerminate(); \
116 | exit(EXIT_FAILURE); \
117 | } \
118 | }
119 |
120 | #endif
121 |
--------------------------------------------------------------------------------
/generate_views/include/render/Phong.h:
--------------------------------------------------------------------------------
1 | #ifndef PHONG_H
2 | #define PHONG_H
3 |
4 | #include
5 |
6 | #include
7 |
8 | #include "helpers/RandomUtils.h"
9 | #include "render/Program.h"
10 |
11 | struct PhongProgram: public Program {
12 | GLint obj_color_location;
13 | GLint eye_location;
14 |
15 | PhongProgram (glm::mat4* view_p, glm::mat4* proj_p) :
16 | Program(view_p, proj_p) {
17 | }
18 |
19 | virtual void setup();
20 | void draw (const std::vector& vertices,
21 | const std::vector& faces,
22 | const std::vector& normals,
23 | const glm::mat4& model, const glm::vec4& color,
24 | const glm::vec4& eye);
25 | };
26 |
27 | #endif
28 |
--------------------------------------------------------------------------------
/generate_views/include/render/Program.h:
--------------------------------------------------------------------------------
1 | #ifndef PROGRAM_H
2 | #define PROGRAM_H
3 |
4 | #include
5 |
6 | #include
7 |
8 | struct Program {
9 | int vaoIndex;
10 |
11 | GLuint programId;
12 |
13 | GLint projection_matrix_location;
14 | GLint model_matrix_location;
15 | GLint view_matrix_location;
16 |
17 | const glm::mat4& view;
18 | const glm::mat4& proj;
19 |
20 | GLint light_position_location;
21 |
22 | Program (glm::mat4* view_p, glm::mat4* proj_p) :
23 | view(*view_p), proj(*proj_p) { }
24 |
25 | virtual void setup() = 0;
26 | };
27 |
28 | #endif
29 |
--------------------------------------------------------------------------------
/generate_views/src/helpers/RandomUtils.cpp:
--------------------------------------------------------------------------------
1 | #include
2 | #include
3 | #include
4 | #include
5 | #include
6 | #include
7 | #include