├── .gitignore ├── Makefile ├── PySCNutils ├── __init__.py ├── datasets │ ├── ADNI.py │ ├── __init__.py │ ├── classificationdataset.py │ ├── modelnet.py │ ├── off3dfile.py │ ├── retrievaldataset.py │ ├── shapenet55.py │ ├── shrec09.py │ └── shrec15.py ├── metrics │ ├── __init__.py │ └── base.py ├── misc │ └── __init__.py ├── networks.py ├── plotting │ ├── __init__.py │ └── curves.py └── training │ ├── __init__.py │ ├── softmax.py │ └── triplet.py ├── README.md ├── SparseConvNet ├── BatchProducer.cu ├── BatchProducer.h ├── ConvolutionalLayer.cu ├── ConvolutionalLayer.h ├── ConvolutionalTriangularLayer.cu ├── ConvolutionalTriangularLayer.h ├── IndexLearnerLayer.cu ├── IndexLearnerLayer.h ├── Makefile ├── MaxPoolingLayer.cu ├── MaxPoolingLayer.h ├── MaxPoolingTriangularLayer.cu ├── MaxPoolingTriangularLayer.h ├── NetworkInNetworkLayer.cu ├── NetworkInNetworkLayer.h ├── NetworkInNetworkPReLULayer.cu ├── NetworkInNetworkPReLULayer.h ├── Off3DFormatPicture.cpp ├── Off3DFormatPicture.h ├── Off3DFormatTriangularPicture.cpp ├── OnlineHandwritingPicture.cpp ├── OnlineHandwritingPicture.h ├── OnlineHandwritingTriangularPicture.cpp ├── Picture.cpp ├── Picture.h ├── README.md ├── ReallyConvolutionalLayer.cu ├── ReallyConvolutionalLayer.h ├── Regions.cu ├── Regions.h ├── Rng.cpp ├── Rng.h ├── SigmoidLayer.cu ├── SigmoidLayer.h ├── SoftmaxClassifier.cu ├── SoftmaxClassifier.h ├── SparseConvNet.cu ├── SparseConvNet.h ├── SparseConvNetCUDA.cu ├── SparseConvNetCUDA.h ├── SparseGrid.h ├── SpatiallySparseBatch.cu ├── SpatiallySparseBatch.h ├── SpatiallySparseBatchInterface.cu ├── SpatiallySparseBatchInterface.h ├── SpatiallySparseDataset.cpp ├── SpatiallySparseDataset.h ├── SpatiallySparseDatasetModelNet.cpp ├── SpatiallySparseDatasetModelNet.h ├── SpatiallySparseDatasetSHREC2015.cpp ├── SpatiallySparseDatasetSHREC2015.h ├── SpatiallySparseLayer.cu ├── SpatiallySparseLayer.h ├── TerminalPoolingLayer.cu ├── TerminalPoolingLayer.h ├── VoxelPicture.cpp ├── VoxelPicture.h ├── mnist.cpp ├── modelnet.cpp ├── shrec2015.cpp ├── shrec2015triangular.cpp ├── signature.h ├── types.cpp ├── types.h ├── utilities.cu ├── utilities.h ├── vectorCUDA.cu ├── vectorCUDA.h ├── vectorHash.cpp └── vectorHash.h ├── _SparseConvNet.pxd ├── requirements.txt ├── setup.py ├── sparseNetwork.pyx └── tests ├── __init__.py ├── test_scn.py └── voxel_picture_test.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.o 2 | shrec2015 3 | modelnet 4 | *.features 5 | 6 | 7 | .DS_Store 8 | Data/ 9 | SparseConvNet/Data 10 | 11 | SparseConvNet/weights/ 12 | 13 | *.so 14 | *.pyc 15 | 16 | 17 | build/ 18 | *.log 19 | 20 | 21 | sparseNetwork.cpp 22 | notebooks/.ipynb_checkpoints/ 23 | 24 | *.swp 25 | 26 | CMakeLists.txt 27 | 28 | old/ -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | NVCC=nvcc 2 | LIBS=-lopencv_core -lopencv_highgui -lopencv_imgproc -lcublas -larmadillo -lpython2.7 3 | CYTHON_BUILD_DIR=build/temp.macosx-10.10-x86_64-2.7 4 | ifeq ($(shell uname -s),Linux) 5 | LIBS += -lrt 6 | CYTHON_BUILD_DIR=build/temp.linux-x86_64-2.7 7 | endif 8 | 9 | clean: 10 | rm -r build/ *.pyc *.cpp *.so || true 11 | fullclean: clean 12 | @$(MAKE) -C SparseConvNet $(MAKECMDGOALS) 13 | 14 | build: clean 15 | python setup.py build_ext -if || true 16 | 17 | test: 18 | python -m unittest tests/ 19 | 20 | full: build 21 | @$(MAKE) -C SparseConvNet $(MAKECMDGOALS) 22 | $(NVCC) --shared -o PySparseConvNet.so $(CYTHON_BUILD_DIR)/sparseNetwork.o SparseConvNet/*.o $(LIBS) 23 | 24 | update: build 25 | $(NVCC) --shared -o PySparseConvNet.so $(CYTHON_BUILD_DIR)/sparseNetwork.o SparseConvNet/*.o $(LIBS) 26 | -------------------------------------------------------------------------------- /PySCNutils/__init__.py: -------------------------------------------------------------------------------- 1 | # import sys 2 | # 3 | # WORK_DIR = '/var/workplace/PySparseConvNet/' 4 | # if WORK_DIR not in sys.path: 5 | # sys.path.insert(0, WORK_DIR) 6 | # 7 | # try: 8 | # from PySparseConvNet import Off3DPicture 9 | # from PySparseConvNet import SparseDataset 10 | # except ImportError: 11 | # print("PySparseConvNet doesn't imports") 12 | # # raise 13 | -------------------------------------------------------------------------------- /PySCNutils/datasets/ADNI.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | 3 | import pandas as pd 4 | import nibabel as nib 5 | import numpy as np 6 | from tqdm import tqdm 7 | import matplotlib.pyplot as plt 8 | import os 9 | import gc 10 | 11 | import PySparseConvNet as pyscn 12 | 13 | 14 | def do_sampling(tensors, n_samples): 15 | if n_samples <= tensors.shape[0]: 16 | ids = np.arange(0, n_samples, dtype=np.int) 17 | else: 18 | ids = np.r_[ 19 | np.arange(0, tensors.shape[0], dtype=np.int), 20 | np.random.permutation(tensors.shape[0])[ 21 | :(n_samples - tensors.shape[0])] 22 | ] 23 | return tensors[ids] 24 | 25 | 26 | class ADNIDataset: 27 | """ 28 | ls /media/toshiba/shared_storage/libfun/ADNI2_masked 29 | wc -l /media/toshiba/shared_storage/libfun/clean_meta_full.csv 30 | 31 | in csv change `/home/mount/neuro-x01-hdd/` to 32 | `/media/toshiba/shared_storage/libfun/` 33 | 34 | http://ida.loni.usc.edu/collaboration/access/appLicense.jsp 35 | 36 | The Image Data Archive at the Laboratory of Neuro Imaging (IDA) provides a 37 | safe repository for medical imaging data. LONI seeks to improve the 38 | understanding of the brain in health and disease through the development of 39 | algorithms and approaches for the comprehensive and quantitative mapping of 40 | its structure and function. 41 | """ 42 | sparse_train_dataset = None 43 | sparse_test_dataset = None 44 | name = 'ADNI MRI Alzheimer detection dataset' 45 | BASEDIR = '/media/toshiba/shared_storage/libfun/' 46 | number_of_features = 1 47 | sampling_methods = { 48 | 'upsampling': max, 49 | 'downsampling': min, 50 | 'meansampling': lambda x: int(np.round(np.mean(x))) 51 | } 52 | 53 | def __init__(self, labels=None, threshold=0.01, sampling=None): 54 | self._validate_sampling(sampling) 55 | self.sampling = sampling 56 | self.threshold = threshold 57 | path_to_csv = os.path.join(self.BASEDIR, 'clean_meta_full.csv') 58 | self.metadata = pd.read_csv(path_to_csv) 59 | self.metadata.Path = self.metadata.Path.str.replace( 60 | '/home/mount/neuro-x01-hdd/', 61 | '/media/toshiba/shared_storage/libfun/') 62 | if labels is not None: 63 | self.metadata = self.metadata[self.metadata.Label.isin(labels)] 64 | self.labels = sorted(self.metadata.Label.unique().tolist()) 65 | self.number_of_classes = len(self.labels) 66 | self._validate_labels(labels) 67 | self.classes = [] 68 | 69 | def summary(self): 70 | print("{} dataset wrapper object:".format(self.name)) 71 | print("Number of classes {}".format(self.number_of_classes)) 72 | print("Number of files {}".format(sum(self.get_count_of_classes()))) 73 | 74 | def get_count_of_classes(self): 75 | return self.metadata.Label.value_counts() 76 | 77 | def _validate_labels(self, classes): 78 | assert (set(classes).issubset(set(self.metadata.Label.unique())) 79 | or classes is None), ("classes should be None to take all" 80 | " classes or subset of all labels of" 81 | " dataset") 82 | 83 | def _validate_sampling(self, sampling): 84 | assert sampling in self.sampling_methods.keys() or sampling is None 85 | 86 | def _create_voxel_picture_from_tensor(self, tensor, label, spatial_size): 87 | x, y, z = np.where(tensor > self.threshold) 88 | c = tensor[x, y, z] 89 | indices = np.c_[x, y, z] 90 | features = c[:, np.newaxis].astype(np.float) 91 | 92 | return pyscn.PyVoxelPicture(indices, features, 93 | spatial_size, label=label) 94 | 95 | def generate_train_test_dataset(self, proportion=.3, do_shuffle=True, 96 | spatial_size=126): 97 | 98 | for label in self.labels: 99 | smc_mask = (self.metadata.Label == label).values 100 | 101 | data_for_label = np.zeros((smc_mask.sum(), 102 | 110, 110, 110), dtype='float32') 103 | 104 | for it, im in tqdm(enumerate(self.metadata[smc_mask].Path.values), 105 | total=smc_mask.sum(), 106 | desc='Reading MRI to memory', 107 | leave=False): 108 | mx = nib.load(im).get_data()\ 109 | .max(axis=0).max(axis=0).max(axis=0) 110 | data_for_label[it, :, :, :] = np.array( 111 | nib.load(im).get_data()) / mx 112 | self.classes.append(data_for_label) 113 | 114 | if self.sparse_test_dataset is not None: 115 | del self.sparse_test_dataset 116 | if self.sparse_train_dataset is not None: 117 | del self.sparse_train_dataset 118 | gc.collect() 119 | 120 | self.sparse_train_dataset = pyscn.SparseDataset( 121 | "ADNI (Train subset)", 'TRAINBATCH', 122 | self.number_of_features, self.number_of_classes) 123 | self.sparse_test_dataset = pyscn.SparseDataset( 124 | "ADNI (Test subset)", 'TESTBATCH', self.number_of_features, 125 | self.number_of_classes) 126 | 127 | if self.sampling in self.sampling_methods.keys(): 128 | n_samples_in_train = self.sampling_methods[self.sampling]( 129 | int(np.ceil((1-proportion) * _class.shape[0])) 130 | for _class in self.classes) 131 | else: 132 | n_samples_in_train = None 133 | 134 | for _class_id, _class_data in enumerate(self.classes): 135 | if do_shuffle: 136 | np.random.shuffle(_class_data) 137 | n_samples_test = int(np.ceil(proportion * _class_data.shape[0])) 138 | test_samples = _class_data[:n_samples_test] 139 | if n_samples_in_train is not None: 140 | train_samples = do_sampling(_class_data[n_samples_test:], 141 | n_samples_in_train) 142 | else: 143 | train_samples = _class_data[n_samples_test:] 144 | 145 | for _sample in train_samples: 146 | self.sparse_train_dataset.add_voxel_picture( 147 | self._create_voxel_picture_from_tensor(_sample, _class_id, 148 | spatial_size)) 149 | for _sample in test_samples: 150 | self.sparse_test_dataset.add_picture( 151 | self._create_voxel_picture_from_tensor(_sample, _class_id, 152 | spatial_size)) 153 | 154 | return self.sparse_test_dataset, self.sparse_train_dataset, self.labels 155 | 156 | @staticmethod 157 | def plot_slices(tensor): 158 | fig, ax = plt.subplots(3, 3, figsize=(13, 13)) 159 | ids = np.asarray(np.ceil(np.linspace(10, 80, 9)), dtype=int) 160 | 161 | for _id, _ax in zip(ids, ax.ravel()): 162 | _ax.imshow(tensor[_id, :, :]) 163 | _ax.set_title('slice #{}'.format(_id)) 164 | -------------------------------------------------------------------------------- /PySCNutils/datasets/__init__.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | if os.path.exists('/media/toshiba'): 4 | DEFAULT_DATASET_FOLDER = '/media/toshiba/shape_retrieval_datasets/' 5 | else: 6 | DEFAULT_DATASET_FOLDER = '/media/hdd/shape_retrieval_datasets/' 7 | 8 | DATASET_FOLDER = os.environ.get('DATASET_FOLDER', DEFAULT_DATASET_FOLDER) 9 | -------------------------------------------------------------------------------- /PySCNutils/datasets/modelnet.py: -------------------------------------------------------------------------------- 1 | from .retrievaldataset import RetrievalDataset 2 | from .classificationdataset import ClassificationDataset 3 | from . import DATASET_FOLDER 4 | import os 5 | from glob import glob 6 | 7 | 8 | class ModelNet40(ClassificationDataset, RetrievalDataset): 9 | name = 'ModelNet40' 10 | BASEDIR = os.path.join(DATASET_FOLDER, 'ModelNet/ModelNet40/') 11 | num_of_tests_per_class = {} 12 | 13 | def __init__(self, max_num_of_tests_per_class=20): 14 | self.search_map_to_class = {} 15 | self.class_labels = sorted(os.listdir(self.BASEDIR)) 16 | self.class_for_sample = {} 17 | for _label, _class_name in enumerate(self.class_labels): 18 | samples = sorted( 19 | os.listdir(os.path.join( 20 | self.BASEDIR, _class_name, 'test' 21 | )))[:max_num_of_tests_per_class] 22 | self.num_of_tests_per_class[_label] = len(samples) 23 | for fname in samples: 24 | self.search_map_to_class[fname] = _label 25 | self.class_for_sample[fname] = _class_name 26 | 27 | self.search_to_label = self.search_map_to_class.__getitem__ 28 | self.query_to_label = self.search_map_to_class.__getitem__ 29 | self.search_to_file = self.query_to_file 30 | 31 | self.classes = [ 32 | glob(os.path.join(self.BASEDIR, _class_folder, 'train/*.off')) 33 | for _class_folder in self.class_labels 34 | ] 35 | 36 | def query_to_file(self, query, **kwargs): 37 | return os.path.join(self.BASEDIR, self.class_for_sample[query], 38 | 'test', query) 39 | 40 | def get_all_queries(self): 41 | return list(self.search_map_to_class.keys()) 42 | 43 | def get_search_set_for_query_sample(self, query_sample): 44 | search_set = list(self.search_map_to_class.keys()) 45 | search_set.remove(query_sample) 46 | return search_set 47 | 48 | def num_relevant_samples_for_query(self, query): 49 | return self.num_of_tests_per_class[self.query_to_label(query)] - 1 50 | -------------------------------------------------------------------------------- /PySCNutils/datasets/off3dfile.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import os 3 | os.environ["ETS_TOOLKIT"] = "qt4" 4 | from mayavi import mlab 5 | 6 | from PySparseConvNet import Off3DPicture 7 | 8 | class Off3DFile(object): 9 | """Add some helpful methods to Off files 10 | """ 11 | file_path = None 12 | 13 | def __init__(self, file_path): 14 | if os.path.exists(file_path): 15 | self.file_path = file_path 16 | self.vertices, self.faces = self.read_off() 17 | else: 18 | raise IOError("File {} doesn't exists!".format(file_path)) 19 | 20 | def read_off(self): 21 | with open(self.file_path, 'r') as _file: 22 | if 'OFF' != _file.readline().strip(): 23 | raise ValueError('Not a valid OFF header') 24 | n_verts, n_faces, n_dontknow = tuple( 25 | [int(s) for s in _file.readline().strip().split(' ')]) 26 | verts = [] 27 | for i_vert in range(n_verts): 28 | verts.append( 29 | [float(s) for s in _file.readline().strip().split(' ')]) 30 | faces = [] 31 | for i_face in range(n_faces): 32 | faces.append( 33 | [int(s) for s in _file.readline().strip().split(' ')][1:]) 34 | return np.array(verts), np.array(faces) 35 | 36 | def plot_triangles(self, ax=None, title=''): 37 | import matplotlib.pyplot as plt 38 | if not title: 39 | title = self.file_path.split('/')[-1] 40 | if ax is None: 41 | fig = plt.figure(figsize=(6, 5)) 42 | ax = fig.add_subplot(111, projection='3d') 43 | verts_np, patches = self.read_off() 44 | # ax.scatter(x, y, z, zdir='z', c= 'red') 45 | ax.plot_trisurf(verts_np[:, 0], verts_np[:, 1], verts_np[:, 2], 46 | triangles=patches, lw=0) 47 | ax.set_title(title) 48 | 49 | def voxelize(self, ss, rs): 50 | pic = Off3DPicture(self.file_path, rs) 51 | pairs, _ = pic.codifyInputData(ss) 52 | list_of_coordinates = [] 53 | for key_id, feature_idx in pairs: 54 | list_of_coordinates.append(( 55 | (key_id / ss / ss) % ss, 56 | (key_id / ss) % ss, 57 | key_id % ss 58 | )) 59 | del pic 60 | return zip(*list_of_coordinates) 61 | 62 | def plot_shape(self, kind, spatialSize=None, renderSize=None, 63 | features=None): 64 | """ outline=True - box edges 65 | title=True - print kind of picture 66 | **kwargs # one of: vertA, faceA, features, x, y, z, spatialSize 67 | """ 68 | f = mlab.figure(bgcolor=(1, 1, 1)) 69 | if kind == 'solid': 70 | vertA, faceA = self.vertices, self.faces 71 | mlab.triangular_mesh(vertA[:, 0], vertA[:, 1], vertA[:, 2], faceA) 72 | elif kind == 'transparent': 73 | vertA, faceA = self.vertices, self.faces 74 | mlab.triangular_mesh(vertA[:,0], vertA[:, 1], vertA[:, 2], faceA, 75 | opacity=0.1) 76 | elif kind == 'wireframe': 77 | vertA, faceA = self.vertices, self.faces 78 | mlab.triangular_mesh(vertA[:, 0], vertA[:, 1], vertA[:, 2], faceA, 79 | representation='wireframe') 80 | elif kind == "Bool": 81 | x, y, z = map(np.array, self.voxelize(spatialSize, renderSize)) 82 | assert len(x) == len(y) == len(z) 83 | N = len(x) 84 | scalars = np.arange(N) # Key point: set an integer for each point 85 | colors = np.zeros((N, 4), dtype=np.uint8) 86 | colors[:, -1] = 255 # No transparency 87 | if features is not None: 88 | features = features.ravel() 89 | colors[:, 0] = 255 90 | colors[:, 1] = ( 91 | 255 * (1 - features / np.max(features))).astype(np.uint8) 92 | colors[:, 2] = ( 93 | 255 * (1 - features / np.max(features))).astype(np.uint8) 94 | else: 95 | colors[:, 0] = 0 96 | colors[:, 1] = 255 97 | colors[:, 2] = 0 98 | 99 | pts = mlab.quiver3d( 100 | x-spatialSize/2, 101 | y-spatialSize/2+0.5, 102 | z-spatialSize/2+0.5, 103 | np.ones(N), np.zeros(N), np.zeros(N), 104 | scalars=scalars, mode='cube', scale_factor=0.7, line_width=10) 105 | pts.glyph.color_mode = 'color_by_scalar' 106 | try: 107 | pts.module_manager.scalar_lut_manager.lut.table = colors 108 | except: 109 | pass 110 | mlab.draw() 111 | 112 | return f 113 | -------------------------------------------------------------------------------- /PySCNutils/datasets/retrievaldataset.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from functools import partial 3 | from tqdm import tqdm 4 | 5 | 6 | def normilize_matrix_of_vectors(_all_feature_vectors): 7 | sam, veclen = _all_feature_vectors.shape 8 | return _all_feature_vectors / ( 9 | np.sqrt( 10 | (_all_feature_vectors**2).sum(axis=1) 11 | ).reshape(sam, 1).dot( 12 | np.ones((1, veclen))) 13 | ) 14 | 15 | 16 | def eval_query(query_class, retrieved_class, num_of_relevant): 17 | G = (query_class == retrieved_class) 18 | G_sum = G.cumsum(axis=0).astype(float) 19 | R_points = np.zeros((num_of_relevant,), dtype=int) 20 | for rec in range(1, num_of_relevant + 1): 21 | R_points[rec - 1] = (G_sum == rec).nonzero()[0][0] 22 | P_points = G_sum[R_points] / (R_points + 1).astype(float) 23 | Av_Precision = P_points.mean() 24 | NN = int(G[0]) 25 | FT = G_sum[num_of_relevant - 1] / num_of_relevant 26 | ST = G_sum[2 * num_of_relevant - 1] / num_of_relevant 27 | P_32 = G_sum[32 - 1] / 32 28 | R_32 = G_sum[32 - 1] / num_of_relevant 29 | 30 | if P_32 == 0 and R_32 == 0: 31 | E = 0 32 | else: 33 | E = 2 * P_32 * R_32 / (P_32 + R_32) 34 | 35 | NORM_VALUE = 1 + sum(1.0 / np.log2(np.arange(2, num_of_relevant + 1))) 36 | dcg_i = (1.0 / np.log2(np.arange(2, len(retrieved_class) + 1))) * G[1:] 37 | dcg_i = np.hstack((float(G[0]), dcg_i[:])) 38 | dcg = dcg_i.sum() / NORM_VALUE 39 | return P_points, NN, FT, ST, dcg, E, Av_Precision 40 | 41 | 42 | class RetrievalDataset(object): 43 | # lists or sets of unique strings 44 | 45 | def query_to_file(self, query, **kwargs): 46 | raise NotImplementedError() 47 | 48 | def get_search_set_for_query_sample(self, query_sample): 49 | raise NotImplementedError() 50 | 51 | def search_to_file(self, search): 52 | raise NotImplementedError() 53 | 54 | def search_to_label(self, search): 55 | raise NotImplementedError() 56 | 57 | def query_to_label(self, query): 58 | raise NotImplementedError() 59 | 60 | def num_relevant_samples_for_query(self, query): 61 | raise NotImplementedError() 62 | 63 | def evaluate_ranking_for_query(self, _query, metric, **kwargs): 64 | """ 65 | 66 | :param _query: 67 | :param metric: 68 | :param n_relevant_samples: Number of relevant samples for query 69 | :param kwargs: 70 | :return: 71 | """ 72 | # for _query in self.query_set: 73 | potential = partial(metric, self.query_to_file(_query, **kwargs)) 74 | search_set = self.get_search_set_for_query_sample(_query) 75 | n_relevant_samples = self.num_relevant_samples_for_query(_query) 76 | 77 | resulting_ranking = np.array( 78 | map(potential, map(self.search_to_file, search_set)) 79 | ).argsort() 80 | return eval_query( 81 | self.query_to_label(_query), 82 | np.array(map( 83 | self.search_to_label, 84 | map(search_set.__getitem__, resulting_ranking))), 85 | n_relevant_samples) 86 | 87 | def evaluate_ranking_for_query_set(self, query_set, metric, **kwargs): 88 | number_of_queries = len(query_set) 89 | 90 | # P_points=np.zeros((number_of_queries, self.num_of_relevant_samples)) 91 | p_points = [0] * number_of_queries 92 | 93 | av_precision = np.zeros((number_of_queries,)) 94 | nearest_neighbour = np.zeros((number_of_queries,)) 95 | first_tier = np.zeros((number_of_queries,)) 96 | second_tier = np.zeros((number_of_queries,)) 97 | dcg = np.zeros((number_of_queries,)) 98 | e_measure = np.zeros((number_of_queries,)) 99 | 100 | for qqq, _query in enumerate(tqdm( 101 | query_set, leave=False, unit='sample', 102 | desc='Iter val. queries')): 103 | (p_points[qqq], nearest_neighbour[qqq], first_tier[qqq], 104 | second_tier[qqq], dcg[qqq], e_measure[qqq], 105 | av_precision[qqq]) = self.evaluate_ranking_for_query( 106 | _query, metric, **kwargs) 107 | 108 | nearest_neighbour_av = nearest_neighbour.mean() 109 | first_tier_av = first_tier.mean() 110 | second_tier_av = second_tier.mean() 111 | dcg_av = dcg.mean() 112 | e_measure_av = e_measure.mean() 113 | mean_av_precision = av_precision.mean() 114 | 115 | def recall_fun(n): 116 | return np.arange(1, n + 1, dtype=float) / n 117 | 118 | lengths_of_ppoints = map(len, p_points) 119 | 120 | if kwargs.get('average_pr', True): 121 | max_num_of_relevant = max(lengths_of_ppoints) 122 | interp_p_points = np.zeros( 123 | (number_of_queries, max_num_of_relevant)) 124 | 125 | for _i, arr in enumerate(p_points): 126 | if len(arr) < max_num_of_relevant: 127 | interp_p_points[_i] = np.interp( 128 | recall_fun(max_num_of_relevant), 129 | recall_fun(len(arr)), arr) 130 | else: 131 | interp_p_points[_i] = arr 132 | precision = interp_p_points.mean(axis=0) 133 | recall = recall_fun(max_num_of_relevant) 134 | else: 135 | precision = p_points 136 | recall = map(recall_fun, lengths_of_ppoints) 137 | 138 | return (precision, recall, nearest_neighbour_av, first_tier_av, 139 | second_tier_av, dcg_av, e_measure_av, mean_av_precision) 140 | -------------------------------------------------------------------------------- /PySCNutils/datasets/shapenet55.py: -------------------------------------------------------------------------------- 1 | from .classificationdataset import ClassificationDataset 2 | from .retrievaldataset import RetrievalDataset 3 | from . import DATASET_FOLDER 4 | import os 5 | from glob import glob 6 | 7 | 8 | class SHREC16_dataset(ClassificationDataset, RetrievalDataset): 9 | name = 'ShapeNet55' 10 | BASEDIR = os.path.join(DATASET_FOLDER, 11 | 'SHREC16_3D_SHAPE_RETRIEVAL_SHAPENET55') 12 | train_dir = os.path.join(BASEDIR, 'train') 13 | test_dir = os.path.join(BASEDIR, 'val') 14 | is_pre_split = True 15 | 16 | def __init__(self, validation_limit=None): 17 | """ 18 | """ 19 | self.name = 'ShapeNet55' 20 | self.class_labels = os.listdir(self.train_dir) 21 | self.class_labels.sort() 22 | self.classes = [ 23 | glob(os.path.join(self.train_dir, _class_label, "*.off")) 24 | for _class_label in self.class_labels 25 | ] 26 | 27 | self.search_map_to_class = {} 28 | self._num_of_samples_per_class = {} 29 | for _class_folder in os.listdir(self.test_dir): 30 | samples_for_class = glob(os.path.join( 31 | self.test_dir, _class_folder, '*.off'))[:validation_limit] 32 | self._num_of_samples_per_class[_class_folder] = len( 33 | samples_for_class) 34 | self.search_map_to_class.update( 35 | (fpath, self.class_labels.index(_class_folder)) 36 | for fpath in samples_for_class) 37 | 38 | self.query_to_label = self.search_map_to_class.__getitem__ 39 | self.search_to_label = self.search_map_to_class.__getitem__ 40 | 41 | def get_all_queries(self): 42 | return list(self.search_map_to_class.keys()) 43 | 44 | def query_to_file(self, x, **kwargs): 45 | return x 46 | 47 | def search_to_file(self, x, **kwargs): 48 | return x 49 | 50 | def get_search_set_for_query_sample(self, query_sample): 51 | search_set = list(self.search_map_to_class.keys()) 52 | search_set.remove(query_sample) 53 | return search_set 54 | 55 | def num_relevant_samples_for_query(self, query): 56 | return self._num_of_samples_per_class[ 57 | query.split('/')[-2] 58 | ] - 1 59 | -------------------------------------------------------------------------------- /PySCNutils/datasets/shrec09.py: -------------------------------------------------------------------------------- 1 | from .retrievaldataset import RetrievalDataset 2 | from .classificationdataset import ClassificationDataset 3 | from . import DATASET_FOLDER 4 | from scipy.io import loadmat 5 | import os 6 | from glob import glob 7 | 8 | 9 | class SHREC09(ClassificationDataset, RetrievalDataset): 10 | BASEDIR = os.path.join( 11 | DATASET_FOLDER, 12 | 'SHREC_2009_Shape_Retrieval_Contest_of_Partial_3D_Models/') 13 | 14 | samples_per_class = 18 15 | name = 'SHREC_2009' 16 | class_labels = [ 17 | 'Bird', 'Fish', 'NonFlyingInsect', 'FlyingInsect', 'Biped', 18 | 'Quadruped', 'ApartmentHouse', 'Skyscraper', 'SingleHouse', 19 | 'Bottle', 'Cup', 'Glasses', 'HandGun', 'SubmachineGun', 20 | 'MusicalInstrument', 'Mug', 'FloorLamp', 'DeskLamp', 'Sword', 21 | 'Cellphone', 'DeskPhone', 'Monitor', 'Bed', 'NonWheelChair', 22 | 'WheelChair', 'Sofa', 'RectangleTable', 'RoundTable', 'Bookshelf', 23 | 'HomePlant', 'Tree', 'Biplane', 'Helicopter', 'Monoplane', 'Rocket', 24 | 'Ship', 'Motorcycle', 'Car', 'MilitaryVehicle', 'Bicycle' 25 | ] 26 | 27 | def __init__(self): 28 | def convert_filepath_to_num(pth): 29 | filename = pth.split('/')[-1] 30 | x = int(filename.lstrip('D0').rstrip('.of')) 31 | return x 32 | _all_samples = sorted( 33 | glob(os.path.join(self.BASEDIR, 'TargetModels/*.off')), 34 | key=convert_filepath_to_num) 35 | self.classes = list(zip(*[iter(_all_samples)]*18)) 36 | 37 | self.num_of_relevant_samples = self.samples_per_class 38 | 39 | self.search_map_to_class = { 40 | __sample.split('/')[-1].rstrip('.of'): _num 41 | for _num, _list_of_samples in enumerate(self.classes) 42 | for __sample in _list_of_samples 43 | } 44 | self.search_to_label = self.search_map_to_class.__getitem__ 45 | self.query_labels = {} 46 | for fname in ['range_query_labels', 'parts_query_labels']: 47 | rql = loadmat(os.path.join(self.BASEDIR, 48 | 'evaluate_rank_lists_code/{}.mat'.format(fname))) 49 | _query_labels = rql[fname] 50 | self.query_labels.update({ 51 | _query_labels[i, 0][0]: int(_query_labels[i, 1][0]) - 1 52 | for i in xrange(_query_labels.shape[0]) 53 | }) 54 | self.query_to_label = self.query_labels.__getitem__ 55 | 56 | def query_to_file(self, query, _type=None): 57 | if _type is None: 58 | _type = { 59 | 'R': 'scan_hr', 60 | 'P': 'parts' 61 | }[query[0]] 62 | return os.path.join( 63 | self.BASEDIR, 64 | { 65 | 'parts': 'partial_query_models/{}.off', 66 | 'scan_hr': 'range_query_models_high_resolution/{}.off', 67 | 'scan_lr': 'range_query_models_low_resolution/{}.off' 68 | }[_type].format(query) 69 | ) 70 | 71 | def search_to_file(self, search): 72 | return os.path.join( 73 | self.BASEDIR, 74 | 'TargetModels/{}.off'.format(search) 75 | ) 76 | 77 | def get_search_set_for_query_sample(self, query_sample): 78 | return list(self.search_map_to_class.keys()) 79 | 80 | -------------------------------------------------------------------------------- /PySCNutils/datasets/shrec15.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | from .classificationdataset import ClassificationDataset 3 | from . import DATASET_FOLDER 4 | import collections 5 | import os 6 | 7 | 8 | def parse_cla(cla_file): 9 | classifier = collections.OrderedDict() 10 | with open(cla_file) as f: 11 | if "PSB" not in f.readline(): 12 | raise IOError("file must be \"cla\" format file.") 13 | # n_class, n_data = map(int, f.readline().split(' ')) 14 | while True: 15 | line = f.readline() 16 | if line == '': 17 | break 18 | split_line = line.split(' ') 19 | if len(split_line) == 3: 20 | name, parent_name, n = split_line 21 | if int(n) > 0: 22 | ids = [int(f.readline()) for i in xrange(int(n))] 23 | classifier.setdefault(name, ids) 24 | return classifier 25 | 26 | 27 | class SHREC2015_dataset(ClassificationDataset): 28 | BASEDIR = os.path.join(DATASET_FOLDER, 'shrec15-non-rigid') 29 | 30 | def __init__(self): 31 | self.name = "SHREC2015" 32 | parsed_cla = parse_cla(os.path.join( 33 | self.BASEDIR, 'SHREC15_Non-rigid_ToolKit/test.cla' 34 | )) 35 | self.class_labels = list(parsed_cla.keys()) 36 | self.class_labels.sort() 37 | self.classes = [ 38 | map( 39 | os.path.join(self.BASEDIR, "SHREC15NonRigidTestDB/T{}.off" 40 | ).format, parsed_cla[_class_name]) 41 | for _class_name in self.class_labels] 42 | -------------------------------------------------------------------------------- /PySCNutils/metrics/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gangiman/PySparseConvNet/a07b4ade74b049022cf56a533ceac7caea4efe12/PySCNutils/metrics/__init__.py -------------------------------------------------------------------------------- /PySCNutils/metrics/base.py: -------------------------------------------------------------------------------- 1 | from collections import OrderedDict 2 | from scipy.spatial.distance import cosine 3 | import numpy as np 4 | import shelve 5 | import os 6 | 7 | try: 8 | from PySparseConvNet import Off3DPicture 9 | from PySparseConvNet import SparseDataset 10 | except ImportError: 11 | print("PySparseConvNet doesn't imports") 12 | raise 13 | 14 | 15 | def get_feature_extractor(network, weights_path, _layer=17, 16 | renderSize=40): 17 | if weights_path is not None: 18 | weights_folder = os.path.dirname(weights_path) 19 | weights_file = os.path.basename(weights_path) 20 | prefix, epoch = weights_file.split('_epoch') 21 | epoch = epoch.strip('-.cnn') 22 | network.loadWeights( 23 | os.path.join(weights_folder, prefix), int(epoch)) 24 | 25 | def wraped(pic_path): 26 | pic = Off3DPicture(pic_path, renderSize) 27 | loi = network.layer_activations(pic) 28 | return np.array(loi[_layer]['features']) 29 | 30 | return wraped 31 | 32 | 33 | class Metric(object): 34 | 35 | name = None 36 | caching = True 37 | 38 | def __call__(self, *args): 39 | x, y = args 40 | try: 41 | assert all(os.path.exists(_arg) for _arg in args) 42 | except AssertionError: 43 | print(args) 44 | raise 45 | if x == y: 46 | return 0 47 | else: 48 | return self.compute_metric(x, y) 49 | 50 | def compute_metric(self, x, y): 51 | raise NotImplementedError() 52 | 53 | 54 | class NNMetric(Metric): 55 | 56 | dist = OrderedDict(( 57 | ('L2', lambda x, y: np.linalg.norm(x-y)), 58 | ('cos', cosine) 59 | )) 60 | 61 | def __init__(self, network, weights_path=None, norm='cos', _layer=17, 62 | render_size=40, cache_dir='/tmp', net_hash=None): 63 | if weights_path is None: 64 | self.name = net_hash 65 | else: 66 | self.name = os.path.basename(weights_path).rstrip('.cnn') 67 | self.metric = self.dist[norm] 68 | self.unique_hash = "{}_{}_{}_{}".format(self.name, norm, _layer, 69 | render_size) 70 | 71 | self.extractor = get_feature_extractor(network, weights_path, 72 | _layer=_layer, 73 | renderSize=render_size) 74 | if not os.path.isdir(cache_dir): 75 | raise IOError("No such directory '{}'".format(cache_dir)) 76 | self.cache_file = os.path.join(cache_dir, 77 | self.unique_hash + '.db') 78 | self.cache = shelve.open(self.cache_file, writeback=True) 79 | 80 | def __del__(self): 81 | self.cache.close() 82 | 83 | def get_vector(self, off_path): 84 | if off_path not in self.cache: 85 | feature_vector = self.extractor(off_path) 86 | feature_vector /= np.sqrt((feature_vector**2).sum()) 87 | self.cache[off_path] = feature_vector 88 | return self.cache[off_path] 89 | 90 | def compute_metric(self, *args): 91 | return self.metric(*map(self.get_vector, args)) 92 | 93 | 94 | class RandomMetric(NNMetric): 95 | 96 | centroids = None 97 | 98 | def __init__(self, vl=192, norm='cos', ds=None, contraction=10): 99 | self.vector_length = vl 100 | self.contraction_factor = contraction 101 | self.ds = ds 102 | if ds is not None: 103 | c = self.ds.class_count 104 | self.centroids = np.hstack(( 105 | np.eye(c), 106 | np.zeros((c, self.vector_length - c)) 107 | )) 108 | 109 | from uuid import uuid1 110 | super(RandomMetric, self).__init__(None, _layer=0, norm=norm, 111 | net_hash=str(uuid1())[:8]) 112 | self.extractor = self._extractor 113 | 114 | def _extractor(self, off_path): 115 | if self.ds is None: 116 | return np.random.randn(self.vector_length) 117 | else: 118 | off_class_label = off_path.split('/')[-3] 119 | class_id = self.ds.class_labels.index(off_class_label) 120 | return self.centroids[class_id] + np.random.randn( 121 | self.vector_length) / self.contraction_factor 122 | -------------------------------------------------------------------------------- /PySCNutils/misc/__init__.py: -------------------------------------------------------------------------------- 1 | from random import sample 2 | from string import ascii_letters 3 | 4 | 5 | def get_random_hash(length=11): 6 | return ''.join(sample(ascii_letters + 7 | ''.join(str(i) for i in range(10)), length)) 8 | -------------------------------------------------------------------------------- /PySCNutils/networks.py: -------------------------------------------------------------------------------- 1 | import PySparseConvNet as pyscn 2 | 3 | 4 | def generate_network(dimension=3, l=5, k=32, fn='VLEAKYRELU', nInputFeatures=1, 5 | nClasses=50, p=0.0, cudaDevice=-1): 6 | network = pyscn.SparseNetwork( 7 | dimension, nInputFeatures, nClasses, cudaDevice=cudaDevice) 8 | for i in range(l + 1): 9 | network.addLeNetLayerMP( 10 | (i + 1) * k, 2, 1, 3 if (i < l) else 1, 2 if (i < l) else 1, fn, 11 | p * i * 1.0 / l) 12 | return network 13 | 14 | 15 | def generate_wide_network(dimension=3, l=5, filter_mult=None, fn='VLEAKYRELU', 16 | nInputFeatures=1, nClasses=50, p=0.0, cudaDevice=-1): 17 | assert len(filter_mult) == l + 1 18 | network = pyscn.SparseNetwork( 19 | dimension, nInputFeatures, nClasses, cudaDevice=cudaDevice) 20 | """ for l = 5, k = 32 21 | sparse_net.addLeNetLayerMP(32, 2, 1, 3, 2, 'VLEAKYRELU', 0.0) 22 | sparse_net.addLeNetLayerMP(64, 2, 1, 3, 2, 'VLEAKYRELU', 0.0) 23 | sparse_net.addLeNetLayerMP(96, 2, 1, 3, 2, 'VLEAKYRELU', 0.0) 24 | sparse_net.addLeNetLayerMP(128, 2, 1, 3, 2, 'VLEAKYRELU', 0.0) 25 | sparse_net.addLeNetLayerMP(160, 2, 1, 3, 2, 'VLEAKYRELU', 0.0) 26 | sparse_net.addLeNetLayerMP(192, 2, 1, 1, 1, 'VLEAKYRELU', 0.0) 27 | """ 28 | for i, fm in enumerate(filter_mult): 29 | network.addLeNetLayerMP( 30 | fm, 31 | 2, 1, 3 if (i < l) else 1, 2 if (i < l) else 1, fn, 32 | p * i * 1.0 / l) 33 | return network 34 | 35 | 36 | def create_DeepC2Network(dimension, n_layers, n_filters_multiplier, fn, nInputFeatures, 37 | nClasses, dropout, nThreads=1): 38 | sparse_net = pyscn.SparseNetwork(dimension, nInputFeatures, nClasses, nThreads=nThreads) 39 | for i in range(n_layers + 1): 40 | sparse_net.addLeNetLayerMP( 41 | (i + 1) * n_filters_multiplier, 42 | 2, 43 | 1, 44 | 3 if (i < n_layers) else 1, 45 | 2 if (i < n_layers) else 1, 46 | fn, 47 | dropout * i * 1.0 / n_layers) 48 | sparse_net.addSoftmaxLayer() 49 | return sparse_net 50 | 51 | 52 | def custom_DeepC2Network(dimension, n_filters_multiplier, 53 | fsizes, fstrides, pool_sizes, pool_strides, 54 | fn, nInputFeatures, 55 | nClasses, dropout, nThreads=1): 56 | sparse_net = pyscn.SparseNetwork(dimension, nInputFeatures, nClasses, nThreads=nThreads) 57 | n_layers = len(fsizes) 58 | assert(len(fsizes) == len(fstrides)) 59 | assert(len(fsizes) == len(pool_sizes)) 60 | assert(len(fsizes) == len(pool_strides)) 61 | for i in xrange(n_layers + 1): 62 | sparse_net.addLeNetLayerMP( 63 | (i + 1) * n_filters_multiplier, 64 | fsizes[i] if i < n_layers else 2, 65 | fstrides[i] if i < n_layers else 1, 66 | pool_sizes[i] if i < n_layers else 1, 67 | pool_strides[i] if i < n_layers else 1, 68 | fn, 69 | dropout * i * 1.0 / n_layers) 70 | sparse_net.addSoftmaxLayer() 71 | return sparse_net 72 | 73 | 74 | 75 | def create_dC2(): 76 | nFeatures = 1 77 | nClasses = 40 78 | return create_DeepC2Network(3, 5, 32, 'VLEAKYRELU', nFeatures, nClasses, 0.0) 79 | -------------------------------------------------------------------------------- /PySCNutils/plotting/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gangiman/PySparseConvNet/a07b4ade74b049022cf56a533ceac7caea4efe12/PySCNutils/plotting/__init__.py -------------------------------------------------------------------------------- /PySCNutils/plotting/curves.py: -------------------------------------------------------------------------------- 1 | import matplotlib.pyplot as plt 2 | import numpy as np 3 | from operator import itemgetter 4 | 5 | 6 | def PR_per_class(P_points,labels=None, C=18): 7 | if labels is None: 8 | labels = map(str, range(len(P_points))) 9 | assert len(labels) == len(P_points) 10 | Recall=np.arange(1, C + 1, dtype=float) / C 11 | fig, ax = plt.subplots(1, 1, figsize=(11, 7)) 12 | plt.grid(True) 13 | plt.hold(True) 14 | ax.xthick = np.arange(0.0, 1.1, 0.1) 15 | reco = Recall 16 | for _idx, pres in enumerate(P_points): 17 | plt.plot(reco, pres, label=labels[_idx]) 18 | plt.hold(False) 19 | plt.xlabel('Recall') 20 | plt.ylabel('Prescion') 21 | plt.legend() 22 | 23 | # @interact(**{__name: Checkbox(description=__name, value=True) for __name in strings_with_arrays}) 24 | # def learning_curve_for_starting_point(**kwargs): 25 | # fig, ax = plt.subplots(1, 1, figsize=(11,7)) 26 | # plt.grid(True) 27 | # plt.hold(True) 28 | # # ax.xthick = np.arange(0.0, 1.1, 0.1) 29 | # for _name, (reco, pres) in strings_with_arrays.items(): 30 | # if kwargs[_name]: 31 | # plt.plot(reco, pres, label=_name) 32 | # plt.hold(False) 33 | # plt.xlabel('Recall') 34 | # plt.ylabel('Prescion') 35 | # plt.legend() 36 | 37 | 38 | def plot_class_distribution(freq_dict): 39 | pairs = freq_dict.items() 40 | fig = plt.figure(figsize=(9, 4)) 41 | ax = fig.add_subplot(111) 42 | N = len(pairs) 43 | ind = np.arange(N) # the x locations for the groups 44 | width = 0.35 # the width of the bars 45 | ax.bar(ind, map(itemgetter(1), pairs), width) 46 | # axes and labels 47 | ax.set_xlim(-width, len(ind) + width) 48 | # ax.set_ylim(0,45) 49 | ax.set_ylabel('number of samples') 50 | ax.set_title('number of samples by class') 51 | # xTickMarks = [fashionista_tags[int(i)] for _, i in sorted_pairs] 52 | xTickMarks = map(itemgetter(0), pairs) 53 | ax.set_xticks(ind + width) 54 | xtickNames = ax.set_xticklabels(xTickMarks) 55 | plt.setp(xtickNames, rotation=90, fontsize=10) 56 | # plt.show() 57 | return ax 58 | -------------------------------------------------------------------------------- /PySCNutils/training/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gangiman/PySparseConvNet/a07b4ade74b049022cf56a533ceac7caea4efe12/PySCNutils/training/__init__.py -------------------------------------------------------------------------------- /PySCNutils/training/softmax.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | from tqdm import tqdm 3 | import os 4 | 5 | 6 | def nop(*args, **kwargs): 7 | pass 8 | 9 | 10 | def train(ds, network, experiment_hash, 11 | batch_size=150, test_every_n_epochs=5, 12 | lr_policy=nop, 13 | momentum_policy=nop, 14 | dataset_gen_args=(), 15 | weights_dir='./weights', 16 | epoch=0, epoch_limit=100, 17 | train_iteration_hook=nop, 18 | test_iteration_hook=nop): 19 | ds.summary() 20 | weights_temp = os.path.join(weights_dir, experiment_hash) 21 | 22 | testSet, trainSet, labels = ds.generate_train_test_dataset( 23 | **dict(dataset_gen_args)) 24 | 25 | for _ in tqdm(xrange(epoch_limit), 26 | total=epoch_limit, unit="epoch"): 27 | learning_rate = lr_policy(epoch) 28 | momentum = momentum_policy(epoch) 29 | train_report = network.processDataset(trainSet, batch_size, 30 | learningRate=learning_rate, 31 | momentum=momentum) 32 | train_iteration_hook(train_report=train_report) 33 | if epoch > 0 and epoch % test_every_n_epochs == 0: 34 | network.saveWeights(weights_temp, epoch) 35 | test_reports = network.processDatasetRepeatTest(testSet, 36 | batch_size, 3) 37 | test_iteration_hook(_network=network, 38 | learning_rate=learning_rate, 39 | momentum=momentum, 40 | epoch=epoch, 41 | weights_path="{}_epoch-{}.cnn".format( 42 | weights_temp, epoch), 43 | test_reports=test_reports) 44 | epoch += 1 45 | -------------------------------------------------------------------------------- /PySCNutils/training/triplet.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | import numpy as np 3 | from tqdm import tqdm 4 | from itertools import islice 5 | from itertools import izip 6 | from functools import partial 7 | import os 8 | 9 | try: 10 | from PySparseConvNet import Off3DPicture 11 | from PySparseConvNet import SparseDataset 12 | from PySparseConvNet import SparseNetwork 13 | except ImportError: 14 | print("PySparseConvNet doesn't imports") 15 | raise 16 | 17 | 18 | def get_functions(norm_type='cosine', margin=0.1, norm_for_l2=True): 19 | """ 20 | function linear: triplet_loss = ReLU(L + norm(g,p) - norm(g,n)) 21 | function logarithmic: triplet_loss = ln(1 + ReLU(norm(g,p)/(norm(g,n) + L)) 22 | g (ground) - some sample, p (positive) - same class sample, 23 | n - (negative) not same class sample. 24 | 25 | :param norm_type: - 'L2' or 'cosine' 26 | :param margin: float 27 | :param norm_for_l2: 28 | :return: 29 | """ 30 | from autograd import grad 31 | import autograd.numpy as _np 32 | 33 | def cos_norm(x, y): 34 | return 1 - _np.dot(x, y)/_np.sqrt(_np.dot(x, x) * _np.dot(y, y)) 35 | 36 | def l2_norm(x, y): 37 | if norm_for_l2: 38 | xn = x/_np.sqrt((x * x).sum()) 39 | yn = y/_np.sqrt((y * y).sum()) 40 | else: 41 | xn, yn = x, y 42 | return ((xn - yn) ** 2).sum() 43 | 44 | if norm_type == 'cosine': 45 | norm = cos_norm 46 | elif norm_type == 'L2': 47 | norm = l2_norm 48 | 49 | def relu(x): 50 | return _np.maximum(0, x) 51 | 52 | def _linear_triplet_loss(trip): 53 | g, p, n = trip 54 | return relu(margin + norm(g, p) - norm(g, n)) 55 | 56 | return _linear_triplet_loss, grad(_linear_triplet_loss), norm 57 | 58 | 59 | def nop(*args, **kwargs): 60 | pass 61 | 62 | 63 | def get_decreasing_weights(l, fall='line'): 64 | if l == 1: 65 | return np.array([1.]) 66 | line_range = np.arange(1, l - 1, dtype=np.float)[::-1] 67 | if fall == 'log': 68 | log_range = np.log(line_range + 1) 69 | elif fall == 'line': 70 | log_range = line_range 71 | else: 72 | raise Exception("fuuu") 73 | log_range /= log_range[0] if log_range.any() else 1 74 | full_weights = np.hstack(( 75 | np.array((1., 1.)), 76 | log_range 77 | )) 78 | return full_weights / full_weights.sum() 79 | 80 | 81 | def weighted_sampling_of_best(arr, best=None): 82 | l = len(arr) 83 | if l == 1: 84 | return 0 85 | if best == 'min': 86 | sorted_arr_ids = arr.argsort() # from smallest to biggest 87 | elif best == 'max': 88 | sorted_arr_ids = arr.argsort()[::-1] # from smallest to biggest 89 | else: 90 | raise AssertionError('mode is not min/max.') 91 | return np.random.choice(sorted_arr_ids, 1, p=get_decreasing_weights(l))[0] 92 | 93 | 94 | def train(ds, network, experiment_hash, 95 | batch_size=150, test_every_n_batches=100, 96 | unique_classes_in_batch=5, 97 | lr_policy=nop, 98 | momentum_policy=nop, 99 | pair_taking_method=0, 100 | render_size=40, weights_dir='./weights', 101 | in_batch_sample_selection=False, norm_type='cosine', L=1, 102 | epoch=0, epoch_limit=None, 103 | batch_iteration_hook=nop, epoch_iteration_hook=nop, 104 | pairs_limit=None): 105 | linear_triplet_loss, ltl_grad, norm = get_functions(norm_type=norm_type, 106 | margin=L) 107 | ds.summary() 108 | gen = ds.generate_triplets(batch_size=batch_size, 109 | unique_classes_in_batch=unique_classes_in_batch, 110 | method=pair_taking_method, limit=pairs_limit) 111 | 112 | weights_temp = os.path.join(weights_dir, experiment_hash) 113 | print('Taking {} batches in to dataset'.format(test_every_n_batches)) 114 | if epoch_limit is None: 115 | if pairs_limit is None: 116 | total_pairs_num = sum(map(lambda l: len(l) * (len(l) - 1), 117 | ds.classes)) 118 | else: 119 | total_pairs_num = ds.get_limit(pairs_limit) * ds.class_count 120 | total_number_of_epochs = int(np.ceil( 121 | total_pairs_num 122 | / (batch_size / 3.0) / test_every_n_batches)) 123 | else: 124 | total_number_of_epochs = epoch_limit 125 | for _ in tqdm(xrange(total_number_of_epochs), 126 | total=total_number_of_epochs, unit="epoch"): 127 | train_ds = SparseDataset( 128 | ds.name + " train", 'TRAINBATCH', 1, ds.class_count, shuffle=False) 129 | ranges_for_all = [] 130 | for batch_samples, ranges in tqdm(islice(gen, test_every_n_batches), 131 | leave=False, 132 | desc="Creating dataset"): 133 | ranges_for_all.append(ranges) 134 | for _sample in batch_samples: 135 | train_ds.add_picture(Off3DPicture(_sample, render_size)) 136 | if not ranges_for_all: 137 | break 138 | batch_gen = network.batch_generator(train_ds, batch_size) 139 | learning_rate = lr_policy(epoch) 140 | momentum = momentum_policy(epoch) 141 | for bid, (batch, _ranges) in tqdm( 142 | enumerate(izip(batch_gen, ranges_for_all)), 143 | leave=False, unit='batch', total=test_every_n_batches): 144 | activation = network.processBatchForward(batch) 145 | feature_vectors = activation['features'] 146 | delta = np.zeros_like(feature_vectors) 147 | batch_loss = [] 148 | for _offset, _range in zip(3 * np.cumsum([0] + _ranges)[:-1], _ranges): 149 | if in_batch_sample_selection: 150 | one_class_ids = np.arange(2 * _range) + _offset 151 | other_class_ids = np.arange(2 * _range, 3 * _range) + _offset 152 | while one_class_ids.any(): 153 | anchor = one_class_ids[0] 154 | positive_id = weighted_sampling_of_best( 155 | np.apply_along_axis( 156 | partial(norm, feature_vectors[anchor]), 1, 157 | feature_vectors[one_class_ids[1:]]), 158 | best='max') 159 | positive_id += 1 160 | negative_id = weighted_sampling_of_best( 161 | np.apply_along_axis( 162 | partial(norm, feature_vectors[anchor]), 1, 163 | feature_vectors[other_class_ids]), best='min') 164 | triplet_slice = [anchor, 165 | one_class_ids[positive_id], 166 | other_class_ids[negative_id]] 167 | one_class_ids = np.delete(one_class_ids, [0, positive_id]) 168 | other_class_ids = np.delete(other_class_ids, negative_id) 169 | delta[triplet_slice] = ltl_grad( 170 | feature_vectors[triplet_slice]) 171 | batch_loss.append(linear_triplet_loss( 172 | feature_vectors[triplet_slice])) 173 | else: 174 | for _i in range(_range): 175 | triplet_slice = _offset + (np.arange(3) * _range) + _i 176 | delta[triplet_slice] = ltl_grad(feature_vectors[triplet_slice]) 177 | batch_loss.append(linear_triplet_loss(feature_vectors[triplet_slice])) 178 | batch_iteration_hook( 179 | batch_loss=batch_loss, 180 | epoch=epoch, 181 | bid=bid 182 | ) 183 | network.processBatchBackward(batch, delta, 184 | learningRate=learning_rate, 185 | momentum=momentum) 186 | network.saveWeights(weights_temp, epoch) 187 | epoch_iteration_hook(_network=network, 188 | learning_rate=learning_rate, 189 | momentum=momentum, 190 | epoch=epoch, 191 | weights_path="{}_epoch-{}.cnn".format( 192 | weights_temp, epoch)) 193 | epoch += 1 194 | del train_ds 195 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # PySparseConvNet Overview 2 | 3 | Fork of [Ben Graham's](http://www2.warwick.ac.uk/fac/sci/statistics/staff/academic-research/graham/)[SparseConvNet](https://github.com/btgraham/SparseConvNet) project but with a wrapper written in Cython. Optimized to process 3D mesh objects. 4 | 5 | 6 | The SparseConvNet Library is written in C++ programming language, and utilizes a lot of CUDA capabilities for speed and efficiency. 7 | But it is very limited when it comes to 8 | * extending functionality — class structure and CUDA kernels are very complex, and requires re-compilation on every modification. 9 | * changing loss functions — only learning configuration was SoftMax with log-likelihood loss function. 10 | * Fine grained access to layer activations — there were no way to extract activations and therefore features from hidden layers. 11 | * interactivity for exploration of models — every experiment was a compiled binary with now way to perform operations step by step, 12 | to explore properties of models. 13 | 14 | Because of all these problems we developed PySparseConvNet to solves them. On implementation level it’s a python compiled module that 15 | can be used by Python interpreter, and harness all of it’s powerful features. Most of modern Deep Learning tools, such as (Theano, Chainer, Tensorflow) using Python as a way to perofrm interactive computing. 16 | 17 | 18 | Interface of PySparseConvNet is much simpler, and consist’s of 4 classes: 19 | * SparseNetwork — Network object class, it has all the methods to changing it’s structure, manipulate weights and activations. 20 | * SparseDataset — Container class for sparse samples and their labels. 21 | * SparseBatch — Gives access to data in dataset when brocessing separate mini-batches. 22 | * Off3DPicture — Wrapper class for 3D models in OFF (Object File Format), used to voxelize samples to be processed by SparseNetwork. 23 | 24 | ## Using PySparseConvNet 25 | 26 | We developed and used this library on Linux servers running: 27 | * Ubuntu Server 14.04-16.04 28 | * CPython 2.7.3-2.7.11 29 | * CUDA 7.0-8.0 30 | * GNU Make 4.0 31 | * libarmadillo (version 4.450.4) 32 | * libsparsehash (version 2.0.2-1) 33 | 34 | ## Python Requirements 35 | 36 | Besides environment described above you'll need following python modules: 37 | 38 | * Cython >= 0.24 39 | * numpy >= 1.8 40 | 41 | and some stuff that is imported in jupyter notebooks. 42 | 43 | ## Installation 44 | 45 | If you have ubuntu and properly installed CUDA you can do: 46 | 47 | ```bash 48 | git clone git@github.com:gangiman/PySparseConvNet.git 49 | cd PySparseConvNet 50 | # to install cython, needed for compilation 51 | pip install -r requirements.txt 52 | # install original SparseConvNet C++ dependancies 53 | sudo apt-get install -y libsparsehash-dev libarmadillo-dev 54 | make full 55 | make test 56 | # if you want to install as system module or in your virtual environement 57 | python setup.py install 58 | ``` 59 | 60 | 61 | ************************************************************************** 62 | PySparseConvNet is free software: you can redistribute it and/or modify 63 | it under the terms of the GNU General Public License as published by 64 | the Free Software Foundation, either version 3 of the License, or 65 | (at your option) any later version. 66 | 67 | SparseConvNet is distributed in the hope that it will be useful, 68 | but WITHOUT ANY WARRANTY; without even the implied warranty of 69 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 70 | [GNU General Public License](http://www.gnu.org/licenses/) for more details. 71 | ************************************************************************** 72 | -------------------------------------------------------------------------------- /SparseConvNet/BatchProducer.cu: -------------------------------------------------------------------------------- 1 | // don't delete the returned threads when done with them; ownership is with 2 | // cnn.batchPool 3 | 4 | #define MULTITHREAD_BATCH_PRODUCTION 5 | 6 | #include "BatchProducer.h" 7 | #include "SpatiallySparseBatch.h" 8 | #include "SpatiallySparseBatchInterface.h" 9 | #include "utilities.h" 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | #include 17 | 18 | BatchProducer::BatchProducer(SparseConvNetCUDA &cnn, 19 | SpatiallySparseDataset &dataset, int spatialSize, 20 | int batchSize) 21 | : cnn(cnn), batchCounter(-1), dataset(dataset), spatialSize(spatialSize), 22 | batchSize(batchSize) { 23 | assert(batchSize > 0); 24 | nBatches = (dataset.pictures.size() + batchSize - 1) / batchSize; 25 | // std::cout << "n_batches: " << nBatches << std::endl; 26 | permutation = range(dataset.pictures.size()); 27 | if (dataset.type == TRAINBATCH && dataset.do_shuffle) { 28 | RNG rng; 29 | rng.vectorShuffle(permutation); 30 | } 31 | while (cnn.batchPool[0].interfaces.size() <= cnn.layers.size()) { 32 | cnn.sharedSubInterfaces.push_back(new SpatiallySparseBatchSubInterface()); 33 | for (int c = 0; c < cnn.nBatchProducerThreads; c++) { 34 | cnn.batchPool[c].interfaces.emplace_back(cnn.sharedSubInterfaces.back()); 35 | } 36 | } 37 | 38 | #ifdef MULTITHREAD_BATCH_PRODUCTION 39 | for (int nThread = 0; nThread < cnn.nBatchProducerThreads; ++nThread) 40 | workers.emplace_back(&BatchProducer::batchProducerThread, this, nThread); 41 | #endif 42 | } 43 | 44 | void BatchProducer::preprocessBatch(int c, int cc, RNG &rng) { 45 | cnn.batchPool[cc].reset(); 46 | cnn.batchPool[cc].type = dataset.type; 47 | cnn.batchPool[cc].interfaces[0].nFeatures = dataset.nFeatures; 48 | cnn.batchPool[cc].interfaces[0].spatialSize = spatialSize; 49 | cnn.batchPool[cc].interfaces[0].featuresPresent.hVector() = 50 | range(dataset.nFeatures); 51 | int n_pictures = dataset.pictures.size(); 52 | for (int i = c * batchSize; i < min((c + 1) * batchSize, n_pictures); i++) { 53 | // check whether picture is already loaded 54 | if (!dataset.pictures[permutation[i]]->is_loaded) { 55 | dataset.pictures[permutation[i]]->loadPicture(); 56 | } 57 | 58 | std::shared_ptr pic(dataset.pictures[permutation[i]]->distort(rng, dataset.type)); 59 | cnn.batchPool[cc].sampleNumbers.push_back(permutation[i]); 60 | cnn.batchPool[cc].batchSize++; 61 | cnn.batchPool[cc].interfaces[0].grids.push_back(SparseGrid()); 62 | cnn.batchPool[cc].labels.hVector().push_back(pic->label); 63 | pic->codifyInputData( 64 | cnn.batchPool[cc].interfaces[0].grids.back(), 65 | cnn.batchPool[cc].interfaces[0].sub->features.hVector(), 66 | cnn.batchPool[cc].interfaces[0].nSpatialSites, 67 | cnn.batchPool[cc].interfaces[0].spatialSize); 68 | 69 | // Force clean 70 | pic.reset(); 71 | dataset.pictures[permutation[i]]->unloadPicture(); 72 | } 73 | // std::cout << cnn.batchPool[cc].interfaces[0].sub->features.size() << std::endl; 74 | // std::cout << cnn.batchPool[cc].interfaces[0].nFeatures * cnn.batchPool[cc].interfaces[0].nSpatialSites << std::endl; 75 | // std::cout << cnn.batchPool[cc].interfaces[0].nSpatialSites << std::endl; 76 | assert(cnn.batchPool[cc].interfaces[0].sub->features.size() == 77 | cnn.batchPool[cc].interfaces[0].nFeatures * 78 | cnn.batchPool[cc].interfaces[0].nSpatialSites); 79 | if (cnn.inputNormalizingConstants.size() > 0) { 80 | std::vector &features = 81 | cnn.batchPool[cc].interfaces[0].sub->features.hVector(); 82 | for (int i = 0; i < features.size(); ++i) 83 | features[i] *= cnn.inputNormalizingConstants 84 | [i % (cnn.batchPool[cc].interfaces[0].nFeatures)]; 85 | } 86 | for (int i = 0; i < cnn.layers.size(); i++) 87 | cnn.layers[i]->preprocess(cnn.batchPool[cc], 88 | cnn.batchPool[cc].interfaces[i], 89 | cnn.batchPool[cc].interfaces[i + 1]); 90 | // Shifted to line 126 !!!!!! 91 | // cnn.batchPool[cc].interfaces[0].sub->features.copyToGPUAsync( 92 | // cnn.batchMemStreams[cc]); 93 | // cnn.batchPool[cc].labels.copyToGPUAsync(cnn.batchMemStreams[cc]); 94 | // for (int i = 0; i <= cnn.layers.size(); ++i) { 95 | // cnn.batchPool[cc].interfaces[i].featuresPresent.copyToGPUAsync( 96 | // cnn.batchMemStreams[cc]); 97 | // cnn.batchPool[cc].interfaces[i].rules.copyToGPUAsync( 98 | // cnn.batchMemStreams[cc]); 99 | // } 100 | // cudaStreamSynchronize(cnn.batchMemStreams[cc].stream); 101 | } 102 | 103 | #ifdef MULTITHREAD_BATCH_PRODUCTION 104 | void BatchProducer::batchProducerThread(int nThread) { 105 | cudaSetDevice(cnn.deviceID); 106 | // cudaSafeCall(cudaStreamDestroy(cnn.batchMemStreams[nThread].stream)); 107 | // cudaSafeCall(cudaStreamCreate(&cnn.batchMemStreams[nThread].stream)); 108 | RNG rng; 109 | for (int c = nThread; c < nBatches; c += cnn.nBatchProducerThreads) { 110 | cnn.batchLock[nThread].lock(); 111 | while (cnn.batchPool[nThread].batchSize > 112 | 0) { // Don't overwrite unused batches 113 | cnn.batchLock[nThread].unlock(); 114 | std::this_thread::sleep_for(std::chrono::milliseconds(10)); 115 | cnn.batchLock[nThread].lock(); 116 | } 117 | preprocessBatch(c, nThread, rng); 118 | cnn.batchLock[nThread].unlock(); 119 | } 120 | } 121 | 122 | SpatiallySparseBatch *BatchProducer::nextBatch() { 123 | if (batchCounter >= 0) { 124 | int cc = batchCounter % cnn.nBatchProducerThreads; 125 | cnn.batchLock[cc].unlock(); 126 | cnn.batchPool[cc].batchSize = 0; 127 | } 128 | batchCounter++; 129 | 130 | if (batchCounter < nBatches) { 131 | int cc = batchCounter % cnn.nBatchProducerThreads; 132 | for (bool ready = false; not ready;) { 133 | bool accessible = cnn.batchLock[cc].try_lock(); 134 | if (accessible) 135 | if (cnn.batchPool[cc].batchSize == 0) 136 | cnn.batchLock[cc].unlock(); 137 | else 138 | ready = true; 139 | } 140 | ///////////////////////////////////////////////////////// 141 | cnn.batchPool[cc].interfaces[0].sub->features.copyToGPUAsync(cnn.memStream); 142 | cnn.batchPool[cc].labels.copyToGPUAsync(cnn.memStream); 143 | for (int i = 0; i <= cnn.layers.size(); ++i) { 144 | cnn.batchPool[cc].interfaces[i].featuresPresent.copyToGPUAsync( 145 | cnn.memStream); 146 | cnn.batchPool[cc].interfaces[i].rules.copyToGPUAsync(cnn.memStream); 147 | } 148 | cudaStreamSynchronize(cnn.memStream.stream); 149 | ///////////////////////////////////////////////////////// 150 | return &cnn.batchPool[cc]; 151 | } else { 152 | for (int i = 0; i < cnn.nBatchProducerThreads; i++) 153 | workers[i].join(); 154 | return NULL; 155 | } 156 | } 157 | 158 | #else 159 | 160 | void BatchProducer::batchProducerThread(int nThread) {} 161 | 162 | SpatiallySparseBatch *BatchProducer::nextBatch() { 163 | if (batchCounter >= 0) { 164 | int cc = batchCounter % cnn.nBatchProducerThreads; 165 | cnn.batchPool[cc].batchSize = 0; 166 | } 167 | batchCounter++; 168 | if (batchCounter == nBatches) { 169 | return NULL; 170 | } else { 171 | RNG rng; 172 | int cc = batchCounter % cnn.nBatchProducerThreads; 173 | preprocessBatch(batchCounter, cc, rng); 174 | ///////////////////////////////////////////////////////// 175 | cnn.batchPool[cc].interfaces[0].sub->features.copyToGPUAsync(cnn.memStream); 176 | cnn.batchPool[cc].labels.copyToGPUAsync(cnn.memStream); 177 | for (int i = 0; i <= cnn.layers.size(); ++i) { 178 | cnn.batchPool[cc].interfaces[i].featuresPresent.copyToGPUAsync( 179 | cnn.memStream); 180 | cnn.batchPool[cc].interfaces[i].rules.copyToGPUAsync(cnn.memStream); 181 | } 182 | cudaStreamSynchronize(cnn.memStream.stream); 183 | ///////////////////////////////////////////////////////// 184 | return &cnn.batchPool[cc]; 185 | } 186 | } 187 | #endif 188 | 189 | BatchProducer::~BatchProducer() { 190 | if (batchCounter < nBatches) { 191 | SpatiallySparseBatch *batch = nextBatch(); 192 | while (batch) { 193 | batch = nextBatch(); 194 | } 195 | } 196 | } 197 | -------------------------------------------------------------------------------- /SparseConvNet/BatchProducer.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | #include "SpatiallySparseBatch.h" 5 | #include "SpatiallySparseDataset.h" 6 | #include "SparseConvNetCUDA.h" 7 | 8 | class BatchProducer { 9 | public: 10 | SparseConvNetCUDA &cnn; 11 | int batchCounter; 12 | int nBatches; 13 | std::vector workers; 14 | SpatiallySparseDataset &dataset; 15 | int batchSize; 16 | int spatialSize; 17 | std::vector permutation; 18 | BatchProducer(SparseConvNetCUDA &cnn, SpatiallySparseDataset &dataset, 19 | int spatialSize, int batchSize); 20 | ~BatchProducer(); 21 | SpatiallySparseBatch *nextBatch(); 22 | void batchProducerThread(int nThread); 23 | void preprocessBatch(int c, int cc, RNG &rng); 24 | }; 25 | -------------------------------------------------------------------------------- /SparseConvNet/ConvolutionalLayer.cu: -------------------------------------------------------------------------------- 1 | // This does not really implement a convolution. It just gathers data together 2 | // in prepartion for matrix muliplictation. "Proper convolution" = 3 | // "ConvolutionalLayer" + "NetworkInNetworkLayer" 4 | 5 | #include "ConvolutionalLayer.h" 6 | #include 7 | #include 8 | #include 9 | #include "utilities.h" 10 | #include "Regions.h" 11 | 12 | __global__ void dPropForwardToMatrixMultiplyInput(float *d_features, 13 | float *d_convolved, 14 | int *rules, int count, 15 | int nIn) { 16 | __shared__ int r[KERNELBLOCKSIZE]; 17 | for (int c = blockIdx.x * KERNELBLOCKSIZE; c < count; 18 | c += (1 << 12) * KERNELBLOCKSIZE) { 19 | int n = min(KERNELBLOCKSIZE, count - c); 20 | r[threadIdx.x] = (threadIdx.x < n) ? rules[c + threadIdx.x] * nIn : 0; 21 | __syncthreads(); 22 | for (int q = 0; q < n; q++) { 23 | int rq = r[q]; 24 | int i = (c + q) * nIn; 25 | for (int j = threadIdx.x; j < nIn; j += KERNELBLOCKSIZE) { 26 | d_convolved[i + j] = d_features[rq + j]; 27 | } 28 | } 29 | __syncthreads(); 30 | } 31 | } 32 | void propForwardToMatrixMultiply(float *inFeatures, float *outFeatures, 33 | int *rules, int count, int nIn, 34 | cudaMemStream &memStream) { 35 | assert(count > 0); 36 | int batch = min(1 << 12, (count + KERNELBLOCKSIZE - 1) / KERNELBLOCKSIZE); 37 | dPropForwardToMatrixMultiplyInput 38 | << >> 39 | (inFeatures, outFeatures, rules, count, nIn); 40 | cudaCheckError(); 41 | } 42 | __global__ void dPropBackwardFromMatrixMultiplyOutput(float *d_deltaGrid, 43 | float *d_deltaConvolved, 44 | int *rules, int count, 45 | int nIn) { 46 | __shared__ int r[KERNELBLOCKSIZE]; 47 | for (int c = blockIdx.x * KERNELBLOCKSIZE; c < count; 48 | c += (1 << 12) * KERNELBLOCKSIZE) { 49 | int n = min(KERNELBLOCKSIZE, count - c); 50 | r[threadIdx.x] = (threadIdx.x < n) ? rules[c + threadIdx.x] * nIn : 0; 51 | __syncthreads(); 52 | for (int q = 0; q < n; q++) { 53 | int rq = r[q]; 54 | int i = (c + q) * nIn; 55 | for (int j = threadIdx.x; j < nIn; j += KERNELBLOCKSIZE) { 56 | if (rq >= 0) 57 | atomicAdd(&d_deltaGrid[rq + j], d_deltaConvolved[i + j]); 58 | } 59 | } 60 | __syncthreads(); 61 | } 62 | } 63 | void propBackwardFromMatrixMultiply(float *inDFeatures, float *outDFeatures, 64 | int *rules, int count, int nIn, 65 | cudaMemStream &memStream) { 66 | assert(count > 0); 67 | int batch = min(1 << 12, (count + KERNELBLOCKSIZE - 1) / KERNELBLOCKSIZE); 68 | dPropBackwardFromMatrixMultiplyOutput 69 | << >> 70 | (inDFeatures, outDFeatures, rules, count, nIn); 71 | cudaCheckError(); 72 | } 73 | 74 | template 75 | void convolutionFeaturesPresent(std::vector &d_src, std::vector &d_dest, 76 | int nf, int nfp, int nCopies) { 77 | for (int i = 0; i < nfp * nCopies; ++i) { 78 | d_dest[i] = d_src[i % nfp] + nf * (i / nfp); 79 | } 80 | } 81 | template void convolutionFeaturesPresent(std::vector &d_src, 82 | std::vector &d_dest, int nf, 83 | int nfp, int nCopies); 84 | 85 | ConvolutionalLayer::ConvolutionalLayer(cudaMemStream &memStream, int filterSize, 86 | int filterStride, int dimension, 87 | int nFeaturesIn, int minActiveInputs) 88 | : SpatiallySparseLayer(memStream), filterSize(filterSize), 89 | filterStride(filterStride), dimension(dimension), 90 | nFeaturesIn(nFeaturesIn), minActiveInputs(minActiveInputs) { 91 | fs = ipow(filterSize, dimension); 92 | nFeaturesOut = fs * nFeaturesIn; 93 | // std::cout << "Convolution " << filterSize << "^" << dimension << "x" 94 | // << nFeaturesIn << "->" << nFeaturesOut; 95 | // if (filterStride > 1) 96 | // std::cout << " stride:" << filterStride; 97 | // if (minActiveInputs > 1) 98 | // std::cout << " minActiveInputs:" << minActiveInputs; 99 | // std::cout << std::endl; 100 | } 101 | void ConvolutionalLayer::preprocess(SpatiallySparseBatch &batch, 102 | SpatiallySparseBatchInterface &input, 103 | SpatiallySparseBatchInterface &output) { 104 | output.nFeatures = nFeaturesOut; 105 | assert(input.nFeatures == nFeaturesIn); 106 | assert(input.spatialSize >= filterSize); 107 | assert((input.spatialSize - filterSize) % filterStride == 0); 108 | output.spatialSize = (input.spatialSize - filterSize) / filterStride + 1; 109 | output.nSpatialSites = 0; 110 | output.grids.resize(batch.batchSize); 111 | output.backpropErrors = input.backpropErrors; 112 | RegularSquareRegions regions(inSpatialSize, outSpatialSize, dimension, 113 | filterSize, filterStride); 114 | for (int item = 0; item < batch.batchSize; item++) { 115 | gridRules(input.grids[item], output.grids[item], regions, 116 | output.nSpatialSites, output.rules.hVector(), true, 117 | minActiveInputs); 118 | } 119 | output.featuresPresent.resize(input.featuresPresent.size() * fs); 120 | convolutionFeaturesPresent(input.featuresPresent.hVector(), 121 | output.featuresPresent.hVector(), input.nFeatures, 122 | input.featuresPresent.size(), fs); 123 | } 124 | void ConvolutionalLayer::forwards(SpatiallySparseBatch &batch, 125 | SpatiallySparseBatchInterface &input, 126 | SpatiallySparseBatchInterface &output) { 127 | output.sub->features.resize(output.nSpatialSites * 128 | output.featuresPresent.size()); 129 | propForwardToMatrixMultiply(input.sub->features.dPtr(), 130 | output.sub->features.dPtr(), output.rules.dPtr(), 131 | output.nSpatialSites * fs, 132 | input.featuresPresent.size(), memStream); 133 | } 134 | void ConvolutionalLayer::backwards(SpatiallySparseBatch &batch, 135 | SpatiallySparseBatchInterface &input, 136 | SpatiallySparseBatchInterface &output, 137 | float learningRate, float momentum) { 138 | if (input.backpropErrors) { 139 | input.sub->dfeatures.resize(input.nSpatialSites * 140 | input.featuresPresent.size()); 141 | input.sub->dfeatures.setZero(memStream); 142 | propBackwardFromMatrixMultiply( 143 | input.sub->dfeatures.dPtr(), output.sub->dfeatures.dPtr(), 144 | output.rules.dPtr(), output.nSpatialSites * fs, 145 | input.featuresPresent.size(), memStream); 146 | } 147 | } 148 | int ConvolutionalLayer::calculateInputSpatialSize(int outputSpatialSize) { 149 | outSpatialSize = outputSpatialSize; 150 | inSpatialSize = filterSize + (outputSpatialSize - 1) * filterStride; 151 | return inSpatialSize; 152 | } 153 | -------------------------------------------------------------------------------- /SparseConvNet/ConvolutionalLayer.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "SpatiallySparseLayer.h" 3 | 4 | class ConvolutionalLayer : public SpatiallySparseLayer { 5 | private: 6 | int fs; 7 | 8 | public: 9 | int inSpatialSize; 10 | int outSpatialSize; 11 | int filterSize; 12 | int filterStride; 13 | int dimension; 14 | int nFeaturesIn; 15 | int nFeaturesOut; 16 | int minActiveInputs; 17 | ConvolutionalLayer( 18 | cudaMemStream &memStream, int filterSize, int filterStride, int dimension, 19 | int nFeaturesIn, 20 | int minActiveInputs = 1); // normally 1, <=ipow(filterSize,dimension) 21 | void preprocess(SpatiallySparseBatch &batch, 22 | SpatiallySparseBatchInterface &input, 23 | SpatiallySparseBatchInterface &output); 24 | void forwards(SpatiallySparseBatch &batch, 25 | SpatiallySparseBatchInterface &input, 26 | SpatiallySparseBatchInterface &output); 27 | void backwards(SpatiallySparseBatch &batch, 28 | SpatiallySparseBatchInterface &input, 29 | SpatiallySparseBatchInterface &output, float learningRate, 30 | float momentum); 31 | int calculateInputSpatialSize(int outputSpatialSize); 32 | }; 33 | 34 | template 35 | void convolutionFeaturesPresent(std::vector &d_src, std::vector &d_dest, 36 | int nf, int nfp, int nCopies); 37 | void propForwardToMatrixMultiply(float *inFeatures, float *outFeatures, 38 | int *rules, int count, int nIn, 39 | cudaMemStream &memStream); 40 | void propBackwardFromMatrixMultiply(float *inDFeatures, float *outDFeatures, 41 | int *rules, int count, int nIn, 42 | cudaMemStream &memStream); 43 | -------------------------------------------------------------------------------- /SparseConvNet/ConvolutionalTriangularLayer.cu: -------------------------------------------------------------------------------- 1 | #include "ConvolutionalTriangularLayer.h" 2 | #include "ConvolutionalLayer.h" 3 | #include 4 | #include 5 | #include "utilities.h" 6 | #include "Regions.h" 7 | 8 | ConvolutionalTriangularLayer::ConvolutionalTriangularLayer( 9 | cudaMemStream &memStream, 10 | int filterSize, int filterStride, int dimension, int nFeaturesIn, 11 | int minActiveInputs) 12 | : SpatiallySparseLayer(memStream), filterSize(filterSize), 13 | filterStride(filterStride), dimension(dimension), 14 | nFeaturesIn(nFeaturesIn), minActiveInputs(minActiveInputs) { 15 | fs = triangleSize(filterSize, dimension); 16 | nFeaturesOut = fs * nFeaturesIn; 17 | std::cout << dimension 18 | << "D ConvolutionalTriangularLayer side-length=" << filterSize 19 | << " " << nFeaturesIn << "x" << fs << "->" << nFeaturesOut; 20 | if (filterStride > 1) 21 | std::cout << ", stride " << filterStride; 22 | std::cout << std::endl; 23 | } 24 | void ConvolutionalTriangularLayer::preprocess( 25 | SpatiallySparseBatch &batch, SpatiallySparseBatchInterface &input, 26 | SpatiallySparseBatchInterface &output) { 27 | assert(input.nFeatures == nFeaturesIn); 28 | assert(input.spatialSize >= filterSize); 29 | assert((input.spatialSize - filterSize) % filterStride == 0); 30 | output.nFeatures = nFeaturesOut; 31 | output.spatialSize = (input.spatialSize - filterSize) / filterStride + 1; 32 | output.nSpatialSites = 0; 33 | output.grids.resize(batch.batchSize); 34 | output.backpropErrors = input.backpropErrors; 35 | RegularTriangularRegions regions(inSpatialSize, outSpatialSize, dimension, 36 | filterSize, filterStride); 37 | for (int item = 0; item < batch.batchSize; item++) 38 | gridRules(input.grids[item], output.grids[item], regions, 39 | output.nSpatialSites, output.rules.hVector(), minActiveInputs); 40 | output.featuresPresent.resize(input.featuresPresent.size() * fs); 41 | convolutionFeaturesPresent(input.featuresPresent.hVector(), 42 | output.featuresPresent.hVector(), input.nFeatures, 43 | input.featuresPresent.size(), fs); 44 | } 45 | void ConvolutionalTriangularLayer::forwards( 46 | SpatiallySparseBatch &batch, SpatiallySparseBatchInterface &input, 47 | SpatiallySparseBatchInterface &output) { 48 | output.sub->features.resize(output.nSpatialSites * 49 | output.featuresPresent.size()); 50 | propForwardToMatrixMultiply(input.sub->features.dPtr(), 51 | output.sub->features.dPtr(), output.rules.dPtr(), 52 | output.nSpatialSites * fs, 53 | input.featuresPresent.size(), memStream); 54 | } 55 | void ConvolutionalTriangularLayer::backwards( 56 | SpatiallySparseBatch &batch, SpatiallySparseBatchInterface &input, 57 | SpatiallySparseBatchInterface &output, float learningRate, float momentum) { 58 | if (input.backpropErrors) { 59 | input.sub->dfeatures.resize(input.nSpatialSites * 60 | input.featuresPresent.size()); 61 | input.sub->dfeatures.setZero(memStream); 62 | propBackwardFromMatrixMultiply( 63 | input.sub->dfeatures.dPtr(), output.sub->dfeatures.dPtr(), 64 | output.rules.dPtr(), output.nSpatialSites * fs, 65 | input.featuresPresent.size(), memStream); 66 | } 67 | } 68 | int ConvolutionalTriangularLayer::calculateInputSpatialSize( 69 | int outputSpatialSize) { 70 | outSpatialSize = outputSpatialSize; 71 | inSpatialSize = filterSize + (outputSpatialSize - 1) * filterStride; 72 | std::cout << "(" << outSpatialSize << "C" << inSpatialSize << ") "; 73 | return inSpatialSize; 74 | } 75 | -------------------------------------------------------------------------------- /SparseConvNet/ConvolutionalTriangularLayer.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "SpatiallySparseLayer.h" 3 | 4 | class ConvolutionalTriangularLayer : public SpatiallySparseLayer { 5 | private: 6 | int fs; 7 | 8 | public: 9 | int inSpatialSize; 10 | int outSpatialSize; 11 | int filterSize; 12 | int filterStride; 13 | int dimension; 14 | int nFeaturesIn; 15 | int nFeaturesOut; 16 | int minActiveInputs; 17 | ConvolutionalTriangularLayer(cudaMemStream &memStream, int filterSize, 18 | int filterStride, int dimension, int nFeaturesIn, 19 | int minActiveInputs); 20 | void preprocess(SpatiallySparseBatch &batch, 21 | SpatiallySparseBatchInterface &input, 22 | SpatiallySparseBatchInterface &output); 23 | void forwards(SpatiallySparseBatch &batch, 24 | SpatiallySparseBatchInterface &input, 25 | SpatiallySparseBatchInterface &output); 26 | void backwards(SpatiallySparseBatch &batch, 27 | SpatiallySparseBatchInterface &input, 28 | SpatiallySparseBatchInterface &output, float learningRate, 29 | float momentum); 30 | int calculateInputSpatialSize(int outputSpatialSize); 31 | }; 32 | -------------------------------------------------------------------------------- /SparseConvNet/IndexLearnerLayer.cu: -------------------------------------------------------------------------------- 1 | #include "IndexLearnerLayer.h" 2 | #include "SigmoidLayer.h" 3 | #include 4 | #include 5 | #include "SoftmaxClassifier.h" 6 | #include "NetworkInNetworkLayer.h" 7 | #include "utilities.h" 8 | 9 | __global__ void dGradientDescentShrunkMatrixNoMomentum( 10 | float *d_delta, float *d_weights, int nOut, int nOutDropout, 11 | int *inFeaturesPresent, int *outFeaturesPresent, float learningRate) { 12 | int i = blockIdx.x * nOutDropout; 13 | int ii = inFeaturesPresent[blockIdx.x] * nOut; 14 | for (int j = threadIdx.x; j < nOutDropout; j += KERNELBLOCKSIZE) { 15 | int jj = outFeaturesPresent[j]; 16 | // no momentum, weight updated infrequently if the dataset is much larger 17 | // than each minibatch 18 | d_weights[ii + jj] -= learningRate * d_delta[i + j]; 19 | } 20 | } 21 | 22 | IndexLearnerLayer::IndexLearnerLayer(cudaMemStream &memStream, 23 | cublasHandle_t &cublasHandle, 24 | int nFeaturesIn, int nFeaturesOut) 25 | : SpatiallySparseLayer(memStream), cublasHandle(cublasHandle), 26 | nFeaturesIn(nFeaturesIn), nFeaturesOut(nFeaturesOut) { 27 | std::cout << "IndexLearnerLayer" << std::endl; 28 | float scale = pow(6.0f / (nFeaturesIn + nFeaturesOut), 0.5f); 29 | W.resize(nFeaturesIn * nFeaturesOut); 30 | W.setZero(); // Uniform(-scale,scale); 31 | MW.resize(nFeaturesIn * nFeaturesOut); 32 | MW.setZero(); 33 | } 34 | void IndexLearnerLayer::preprocess(SpatiallySparseBatch &batch, 35 | SpatiallySparseBatchInterface &input, 36 | SpatiallySparseBatchInterface &output) { 37 | if (batch.type == TRAINBATCH) { 38 | assert(input.nFeatures == nFeaturesIn); 39 | output.nFeatures = nFeaturesOut; 40 | output.spatialSize = input.spatialSize; 41 | output.nSpatialSites = input.nSpatialSites; 42 | output.grids = input.grids; 43 | output.backpropErrors = true; 44 | } 45 | } 46 | 47 | void IndexLearnerLayer::forwards(SpatiallySparseBatch &batch, 48 | SpatiallySparseBatchInterface &input, 49 | SpatiallySparseBatchInterface &output) { 50 | output.featuresPresent.hVector() = indexLearnerIndices; 51 | output.sub->features.resize(output.nSpatialSites * 52 | output.featuresPresent.size()); 53 | output.sub->dfeatures.resize(output.nSpatialSites * 54 | output.featuresPresent.size()); 55 | w.resize(input.featuresPresent.size() * output.featuresPresent.size()); 56 | dShrinkMatrixForDropout << >> 58 | (W.dPtr(), w.dPtr(), input.featuresPresent.dPtr(), 59 | output.featuresPresent.dPtr(), output.nFeatures, 60 | output.featuresPresent.size()); 61 | cudaCheckError(); 62 | d_rowMajorSGEMM_alphaAB_betaC( 63 | cublasHandle, input.sub->features.dPtr(), w.dPtr(), 64 | output.sub->features.dPtr(), output.nSpatialSites, 65 | input.featuresPresent.size(), output.featuresPresent.size(), 1.0f, 0.0f, 66 | __FILE__, __LINE__); 67 | applySigmoid(output, output, SOFTMAX, memStream); 68 | cudaCheckError(); 69 | } 70 | void IndexLearnerLayer::backwards(SpatiallySparseBatch &batch, 71 | SpatiallySparseBatchInterface &input, 72 | SpatiallySparseBatchInterface &output, 73 | float learningRate, float momentum) { 74 | applySigmoidBackProp(output, output, SOFTMAX, memStream); 75 | input.sub->dfeatures.resize(input.nSpatialSites * 76 | input.featuresPresent.size()); 77 | dw.resize(input.featuresPresent.size() * output.featuresPresent.size()); 78 | d_rowMajorSGEMM_alphaAtB_betaC( 79 | cublasHandle, input.sub->features.dPtr(), output.sub->dfeatures.dPtr(), 80 | dw.dPtr(), input.featuresPresent.size(), output.nSpatialSites, 81 | output.featuresPresent.size(), 1.0, 0.0); 82 | cudaCheckError(); 83 | 84 | if (input.backpropErrors) { 85 | d_rowMajorSGEMM_alphaABt_betaC( 86 | cublasHandle, output.sub->dfeatures.dPtr(), w.dPtr(), 87 | input.sub->dfeatures.dPtr(), output.nSpatialSites, 88 | output.featuresPresent.size(), input.featuresPresent.size(), 1.0, 0.0); 89 | cudaCheckError(); 90 | } 91 | dGradientDescentShrunkMatrixNoMomentum 92 | << >> 94 | (dw.dPtr(), W.dPtr(), output.nFeatures, output.featuresPresent.size(), 95 | input.featuresPresent.dPtr(), output.featuresPresent.dPtr(), 96 | learningRate); 97 | cudaCheckError(); 98 | } 99 | void IndexLearnerLayer::loadWeightsFromStream(std::ifstream &f, bool momentum) { 100 | f.read((char *)&W.hVector()[0], sizeof(float) * W.size()); 101 | }; 102 | void IndexLearnerLayer::putWeightsToStream(std::ofstream &f, bool momentum) { 103 | f.write((char *)&W.hVector()[0], sizeof(float) * W.size()); 104 | }; 105 | int IndexLearnerLayer::calculateInputSpatialSize(int outputSpatialSize) { 106 | return outputSpatialSize; 107 | } 108 | 109 | void IndexLearner(SpatiallySparseBatchInterface &input, 110 | SpatiallySparseBatch &batch, int nTop, 111 | cudaMemStream &memStream) { 112 | assert(batch.batchSize == input.nSpatialSites); 113 | assert(ipow(batch.batchSize, 2) == input.sub->features.size()); 114 | assert(batch.type == TRAINBATCH); 115 | 116 | float *probs = &input.sub->features.hVector()[0]; 117 | for (int i = 0; i < batch.batchSize; ++i) 118 | batch.probabilities.push_back(std::vector( 119 | probs + i * batch.batchSize, probs + (i + 1) * batch.batchSize)); 120 | for (int i = 0; i < batch.batchSize; i++) 121 | batch.predictions.push_back(vectorTopIndices(batch.probabilities[i], nTop)); 122 | 123 | batch.mistakes += batch.batchSize; 124 | for (int i = 0; i < batch.batchSize; i++) { 125 | batch.negativeLogLikelihood -= log(max(batch.probabilities[i][i], 1.0e-15)); 126 | for (int j = 0; j < nTop; j++) { 127 | if (batch.predictions[i][j] == i) { 128 | batch.mistakes--; 129 | } 130 | } 131 | } 132 | // Begin backprop. Top layer: d Cost / d SoftmaxInput 133 | vectorCUDA labels; 134 | labels.hVector() = range(batch.batchSize); 135 | input.sub->dfeatures.resize(input.nSpatialSites * 136 | input.featuresPresent.size()); 137 | dDerivativeOfCostWRTpreSoftmaxTopLevelWeights 138 | << <1, NTHREADS, 0, memStream.stream>>> 139 | (batch.batchSize, input.sub->dfeatures.dPtr(), input.sub->features.dPtr(), 140 | labels.dPtr(), batch.batchSize); 141 | cudaCheckError(); 142 | } 143 | -------------------------------------------------------------------------------- /SparseConvNet/IndexLearnerLayer.h: -------------------------------------------------------------------------------- 1 | // See Ben Graham 2 | // http://www2.warwick.ac.uk/fac/sci/statistics/staff/academic-research/graham/indexlearning.pdf 3 | // and 4 | // http://papers.nips.cc/paper/5548-discriminative-unsupervised-feature-learning-with-convolutional-neural-networks.pdf 5 | 6 | #pragma once 7 | #include 8 | #include "SpatiallySparseLayer.h" 9 | #include "Rng.h" 10 | #include "vectorCUDA.h" 11 | #include 12 | #include 13 | 14 | class IndexLearnerLayer : public SpatiallySparseLayer { 15 | private: 16 | RNG rng; 17 | cublasHandle_t &cublasHandle; 18 | vectorCUDA W; // Weights 19 | vectorCUDA MW; // momentum 20 | vectorCUDA w; // shrunk versions 21 | vectorCUDA dw; // For backprop 22 | public: 23 | std::vector indexLearnerIndices; // Variable to deliver indices in use 24 | // for "double minibatch gradient 25 | // descent" 26 | int nFeaturesIn; 27 | int nFeaturesOut; 28 | float dropout; 29 | IndexLearnerLayer(cudaMemStream &memStream, cublasHandle_t &cublasHandle, 30 | int nFeaturesIn, int nFeaturesOut); 31 | void preprocess(SpatiallySparseBatch &batch, 32 | SpatiallySparseBatchInterface &input, 33 | SpatiallySparseBatchInterface &output); 34 | void forwards(SpatiallySparseBatch &batch, 35 | SpatiallySparseBatchInterface &input, 36 | SpatiallySparseBatchInterface &output); 37 | void backwards(SpatiallySparseBatch &batch, 38 | SpatiallySparseBatchInterface &input, 39 | SpatiallySparseBatchInterface &output, float learningRate, 40 | float momentum); 41 | void loadWeightsFromStream(std::ifstream &f, bool momentum); 42 | void putWeightsToStream(std::ofstream &f, bool momentum); 43 | int calculateInputSpatialSize(int outputSpatialSize); 44 | }; 45 | 46 | void IndexLearner(SpatiallySparseBatchInterface &input, 47 | SpatiallySparseBatch &batch, int nTop, 48 | cudaMemStream &memStream); 49 | -------------------------------------------------------------------------------- /SparseConvNet/Makefile: -------------------------------------------------------------------------------- 1 | CC=g++ 2 | DEBUG ?= 0 3 | ifeq ($(DEBUG), 1) 4 | CFLAGS=--std=c++11 -g -fPIC 5 | NVCCFLAGS=--std=c++11 -arch sm_50 -g -Xcompiler -fPIC 6 | else 7 | CFLAGS=--std=c++11 -O3 -fPIC 8 | NVCCFLAGS=--std=c++11 -arch sm_50 -O3 -Xcompiler -fPIC 9 | endif 10 | NVCC=nvcc 11 | OBJ=BatchProducer.o ConvolutionalLayer.o ConvolutionalTriangularLayer.o IndexLearnerLayer.o MaxPoolingLayer.o MaxPoolingTriangularLayer.o NetworkInNetworkLayer.o NetworkInNetworkPReLULayer.o Picture.o Regions.o Rng.o SigmoidLayer.o SoftmaxClassifier.o SparseConvNet.o SparseConvNetCUDA.o SpatiallySparseBatch.o SpatiallySparseBatchInterface.o SpatiallySparseDataset.o SpatiallySparseLayer.o TerminalPoolingLayer.o types.o utilities.o vectorCUDA.o ReallyConvolutionalLayer.o vectorHash.o Off3DFormatPicture.o VoxelPicture.o 12 | LIBS=-lopencv_core -lopencv_highgui -lopencv_imgproc -lcublas -larmadillo 13 | ifeq ($(shell uname -s),Linux) 14 | LIBS += -lrt 15 | endif 16 | 17 | %.o: %.cpp $(DEPS) 18 | $(CC) -c -o $@ $< $(CFLAGS) 19 | %.o: %.cu $(DEPS) 20 | $(NVCC) -c -o $@ $< $(NVCCFLAGS) 21 | 22 | clean: 23 | rm *.o 24 | 25 | shrec2015: $(OBJ) Off3DFormatPicture.o SpatiallySparseDatasetSHREC2015.o shrec2015.o 26 | $(NVCC) -o shrec2015 $(OBJ) Off3DFormatPicture.o SpatiallySparseDatasetSHREC2015.o shrec2015.o $(LIBS) $(NVCCFLAGS) 27 | 28 | modelnet: $(OBJ) Off3DFormatPicture.o SpatiallySparseDatasetModelNet.o modelnet.o 29 | $(NVCC) -o modelnet $(OBJ) Off3DFormatPicture.o SpatiallySparseDatasetModelNet.o modelnet.o $(LIBS) $(NVCCFLAGS) 30 | 31 | shrec2015triangular: $(OBJ) Off3DFormatTriangularPicture.o SpatiallySparseDatasetSHREC2015.o shrec2015triangular.o 32 | $(NVCC) -o shrec2015triangular $(OBJ) Off3DFormatTriangularPicture.o SpatiallySparseDatasetSHREC2015.o shrec2015triangular.o $(LIBS) $(NVCCFLAGS) 33 | 34 | full: $(OBJ) 35 | fullclean: clean 36 | -------------------------------------------------------------------------------- /SparseConvNet/MaxPoolingLayer.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include "SpatiallySparseLayer.h" 4 | #include "Rng.h" 5 | 6 | void maxPool(float *g1, float *g2, int *rules, int count, int sd, int nOut, 7 | int *d_choice, cudaMemStream &memStream); 8 | void maxPoolBackProp(float *d1, float *d2, int count, int nOut, int *d_choice, 9 | cudaMemStream &memStream); 10 | 11 | class MaxPoolingLayer : public SpatiallySparseLayer { 12 | public: 13 | int inSpatialSize; 14 | int outSpatialSize; 15 | int poolSize; 16 | int poolStride; 17 | int dimension; 18 | int sd; 19 | MaxPoolingLayer(cudaMemStream &memStream, int poolSize, int poolStride, 20 | int dimension); 21 | void preprocess(SpatiallySparseBatch &batch, 22 | SpatiallySparseBatchInterface &input, 23 | SpatiallySparseBatchInterface &output); 24 | void forwards(SpatiallySparseBatch &batch, 25 | SpatiallySparseBatchInterface &input, 26 | SpatiallySparseBatchInterface &output); 27 | void backwards(SpatiallySparseBatch &batch, 28 | SpatiallySparseBatchInterface &input, 29 | SpatiallySparseBatchInterface &output, float learningRate, 30 | float momentum); 31 | int calculateInputSpatialSize(int outputSpatialSize); 32 | }; 33 | 34 | class PseudorandomOverlappingFractionalMaxPoolingLayer 35 | : public SpatiallySparseLayer { 36 | int sd; 37 | 38 | public: 39 | int inSpatialSize; 40 | int outSpatialSize; 41 | float fmpShrink; 42 | int poolSize; 43 | int dimension; 44 | RNG rng; 45 | PseudorandomOverlappingFractionalMaxPoolingLayer(cudaMemStream &memStream, 46 | int poolSize, 47 | float fmpShrink, 48 | int dimension); 49 | void preprocess(SpatiallySparseBatch &batch, 50 | SpatiallySparseBatchInterface &input, 51 | SpatiallySparseBatchInterface &output); 52 | void forwards(SpatiallySparseBatch &batch, 53 | SpatiallySparseBatchInterface &input, 54 | SpatiallySparseBatchInterface &output); 55 | void backwards(SpatiallySparseBatch &batch, 56 | SpatiallySparseBatchInterface &input, 57 | SpatiallySparseBatchInterface &output, float learningRate, 58 | float momentum); 59 | int calculateInputSpatialSize(int outputSpatialSize); 60 | }; 61 | 62 | class PseudorandomNonOverlappingFractionalMaxPoolingLayer 63 | : public SpatiallySparseLayer { 64 | int sd; 65 | 66 | public: 67 | int inSpatialSize; 68 | int outSpatialSize; 69 | float fmpShrink; 70 | int poolSize; 71 | int dimension; 72 | RNG rng; 73 | PseudorandomNonOverlappingFractionalMaxPoolingLayer(cudaMemStream &memStream, 74 | int poolSize, 75 | float fmpShrink, 76 | int dimension); 77 | void preprocess(SpatiallySparseBatch &batch, 78 | SpatiallySparseBatchInterface &input, 79 | SpatiallySparseBatchInterface &output); 80 | void forwards(SpatiallySparseBatch &batch, 81 | SpatiallySparseBatchInterface &input, 82 | SpatiallySparseBatchInterface &output); 83 | void backwards(SpatiallySparseBatch &batch, 84 | SpatiallySparseBatchInterface &input, 85 | SpatiallySparseBatchInterface &output, float learningRate, 86 | float momentum); 87 | int calculateInputSpatialSize(int outputSpatialSize); 88 | }; 89 | 90 | class RandomOverlappingFractionalMaxPoolingLayer : public SpatiallySparseLayer { 91 | int sd; 92 | 93 | public: 94 | int inSpatialSize; 95 | int outSpatialSize; 96 | float fmpShrink; 97 | int poolSize; 98 | int dimension; 99 | RNG rng; 100 | RandomOverlappingFractionalMaxPoolingLayer(cudaMemStream &memStream, 101 | int poolSize, float fmpShrink, 102 | int dimension); 103 | void preprocess(SpatiallySparseBatch &batch, 104 | SpatiallySparseBatchInterface &input, 105 | SpatiallySparseBatchInterface &output); 106 | void forwards(SpatiallySparseBatch &batch, 107 | SpatiallySparseBatchInterface &input, 108 | SpatiallySparseBatchInterface &output); 109 | void backwards(SpatiallySparseBatch &batch, 110 | SpatiallySparseBatchInterface &input, 111 | SpatiallySparseBatchInterface &output, float learningRate, 112 | float momentum); 113 | int calculateInputSpatialSize(int outputSpatialSize); 114 | }; 115 | 116 | class RandomNonOverlappingFractionalMaxPoolingLayer 117 | : public SpatiallySparseLayer { 118 | int sd; 119 | 120 | public: 121 | int inSpatialSize; 122 | int outSpatialSize; 123 | float fmpShrink; 124 | int poolSize; 125 | int dimension; 126 | RNG rng; 127 | RandomNonOverlappingFractionalMaxPoolingLayer(cudaMemStream &memStream, 128 | int poolSize, float fmpShrink, 129 | int dimension); 130 | void preprocess(SpatiallySparseBatch &batch, 131 | SpatiallySparseBatchInterface &input, 132 | SpatiallySparseBatchInterface &output); 133 | void forwards(SpatiallySparseBatch &batch, 134 | SpatiallySparseBatchInterface &input, 135 | SpatiallySparseBatchInterface &output); 136 | void backwards(SpatiallySparseBatch &batch, 137 | SpatiallySparseBatchInterface &input, 138 | SpatiallySparseBatchInterface &output, float learningRate, 139 | float momentum); 140 | int calculateInputSpatialSize(int outputSpatialSize); 141 | }; 142 | -------------------------------------------------------------------------------- /SparseConvNet/MaxPoolingTriangularLayer.cu: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include "utilities.h" 4 | #include "MaxPoolingLayer.h" 5 | #include "MaxPoolingTriangularLayer.h" 6 | #include "Regions.h" 7 | 8 | MaxPoolingTriangularLayer::MaxPoolingTriangularLayer(cudaMemStream &memStream, 9 | int poolSize, 10 | int poolStride, 11 | int dimension) 12 | : SpatiallySparseLayer(memStream), poolSize(poolSize), 13 | poolStride(poolStride), dimension(dimension) { 14 | S = triangleSize(poolSize, dimension); 15 | std::cout << dimension 16 | << "D MaxPoolingTriangularLayer side-length=" << poolSize 17 | << " volume=" << S << " stride " << poolStride << std::endl; 18 | } 19 | void MaxPoolingTriangularLayer::preprocess( 20 | SpatiallySparseBatch &batch, SpatiallySparseBatchInterface &input, 21 | SpatiallySparseBatchInterface &output) { 22 | assert(input.spatialSize == inSpatialSize); 23 | output.nFeatures = input.nFeatures; 24 | output.featuresPresent.hVector() = input.featuresPresent.hVector(); 25 | output.spatialSize = outSpatialSize; 26 | output.nSpatialSites = 0; 27 | output.grids.resize(batch.batchSize); 28 | output.backpropErrors = input.backpropErrors; 29 | RegularTriangularRegions regions(inSpatialSize, outSpatialSize, dimension, 30 | poolSize, poolStride); 31 | for (int item = 0; item < batch.batchSize; item++) 32 | gridRules(input.grids[item], output.grids[item], regions, 33 | output.nSpatialSites, output.rules.hVector()); 34 | } 35 | void MaxPoolingTriangularLayer::forwards( 36 | SpatiallySparseBatch &batch, SpatiallySparseBatchInterface &input, 37 | SpatiallySparseBatchInterface &output) { 38 | output.sub->poolingChoices.resize(output.nSpatialSites * 39 | output.featuresPresent.size()); 40 | output.sub->features.resize(output.nSpatialSites * 41 | output.featuresPresent.size()); 42 | cudaCheckError(); 43 | maxPool(input.sub->features.dPtr(), output.sub->features.dPtr(), 44 | output.rules.dPtr(), output.nSpatialSites, S, 45 | output.featuresPresent.size(), output.sub->poolingChoices.dPtr(), 46 | memStream); 47 | cudaCheckError(); 48 | } 49 | void MaxPoolingTriangularLayer::backwards(SpatiallySparseBatch &batch, 50 | SpatiallySparseBatchInterface &input, 51 | SpatiallySparseBatchInterface &output, 52 | float learningRate, float momentum) { 53 | if (input.backpropErrors) { 54 | input.sub->dfeatures.resize(input.nSpatialSites * 55 | input.featuresPresent.size()); 56 | input.sub->dfeatures.setZero(memStream); 57 | maxPoolBackProp(input.sub->dfeatures.dPtr(), output.sub->dfeatures.dPtr(), 58 | output.nSpatialSites, output.featuresPresent.size(), 59 | output.sub->poolingChoices.dPtr(), memStream); 60 | } 61 | } 62 | int MaxPoolingTriangularLayer::calculateInputSpatialSize( 63 | int outputSpatialSize) { 64 | outSpatialSize = outputSpatialSize; 65 | inSpatialSize = poolSize + (outputSpatialSize - 1) * poolStride; 66 | std::cout << "(" << outSpatialSize << "MP" << inSpatialSize << ") "; 67 | return inSpatialSize; 68 | } 69 | -------------------------------------------------------------------------------- /SparseConvNet/MaxPoolingTriangularLayer.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | #include "SpatiallySparseLayer.h" 5 | #include "Rng.h" 6 | 7 | class MaxPoolingTriangularLayer : public SpatiallySparseLayer { 8 | public: 9 | int inSpatialSize; 10 | int outSpatialSize; 11 | int poolSize; 12 | int poolStride; 13 | int dimension; 14 | int S; 15 | MaxPoolingTriangularLayer(cudaMemStream &memStream, int poolSize, 16 | int poolStride, int dimension); 17 | void preprocess(SpatiallySparseBatch &batch, 18 | SpatiallySparseBatchInterface &input, 19 | SpatiallySparseBatchInterface &output); 20 | void forwards(SpatiallySparseBatch &batch, 21 | SpatiallySparseBatchInterface &input, 22 | SpatiallySparseBatchInterface &output); 23 | void backwards(SpatiallySparseBatch &batch, 24 | SpatiallySparseBatchInterface &input, 25 | SpatiallySparseBatchInterface &output, float learningRate, 26 | float momentum); 27 | int calculateInputSpatialSize(int outputSpatialSize); 28 | }; 29 | -------------------------------------------------------------------------------- /SparseConvNet/NetworkInNetworkLayer.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include "SpatiallySparseLayer.h" 4 | #include "Rng.h" 5 | 6 | class NetworkInNetworkLayer : public SpatiallySparseLayer { 7 | private: 8 | RNG rng; 9 | cublasHandle_t &cublasHandle; 10 | 11 | public: 12 | vectorCUDA W; // Weights 13 | vectorCUDA MW; // momentum 14 | vectorCUDA w; // shrunk versions 15 | vectorCUDA dw; // For backprop 16 | vectorCUDA B; // Weights 17 | vectorCUDA MB; // momentum 18 | vectorCUDA b; // shrunk versions 19 | vectorCUDA db; // For backprop 20 | ActivationFunction fn; 21 | int nFeaturesIn; 22 | int nFeaturesOut; 23 | float dropout; 24 | NetworkInNetworkLayer( 25 | cudaMemStream &memStream, cublasHandle_t &cublasHandle, int nFeaturesIn, 26 | int nFeaturesOut, float dropout = 0, ActivationFunction fn = NOSIGMOID, 27 | float alpha = 1 // used to determine intialization weights only 28 | ); 29 | void preprocess(SpatiallySparseBatch &batch, 30 | SpatiallySparseBatchInterface &input, 31 | SpatiallySparseBatchInterface &output); 32 | void forwards(SpatiallySparseBatch &batch, 33 | SpatiallySparseBatchInterface &input, 34 | SpatiallySparseBatchInterface &output); 35 | void scaleWeights(SpatiallySparseBatchInterface &input, 36 | SpatiallySparseBatchInterface &output, 37 | float &scalingUnderneath, bool topLayer); 38 | void backwards(SpatiallySparseBatch &batch, 39 | SpatiallySparseBatchInterface &input, 40 | SpatiallySparseBatchInterface &output, float learningRate, 41 | float momentum); 42 | void loadWeightsFromStream(std::ifstream &f, bool momentum); 43 | void putWeightsToStream(std::ofstream &f, bool momentum); 44 | int calculateInputSpatialSize(int outputSpatialSize); 45 | }; 46 | 47 | __global__ void dShrinkMatrixForDropout(float *m, float *md, 48 | int *inFeaturesPresent, 49 | int *outFeaturesPresent, int nOut, 50 | int nOutDropout); 51 | __global__ void dShrinkVectorForDropout(float *m, float *md, 52 | int *outFeaturesPresent, int nOut, 53 | int nOutDropout); 54 | __global__ void dGradientDescent(float *d_delta, float *d_momentum, 55 | float *d_weights, int nOut, float learningRate, 56 | float momentum); 57 | __global__ void 58 | dGradientDescentShrunkMatrix(float *d_delta, float *d_momentum, 59 | float *d_weights, int nOut, int nOutDropout, 60 | int *inFeaturesPresent, int *outFeaturesPresent, 61 | float learningRate, float momentum); 62 | __global__ void dGradientDescentShrunkVector(float *d_delta, float *d_momentum, 63 | float *d_weights, int nOut, 64 | int nOutDropout, 65 | int *outFeaturesPresent, 66 | float learningRate, 67 | float momentum); 68 | void replicateArray(float *src, float *dst, int nRows, int nColumns, 69 | cudaMemStream &memStream); 70 | void columnSum(float *matrix, float *target, int nRows, int nColumns, 71 | cudaMemStream &memStream); 72 | -------------------------------------------------------------------------------- /SparseConvNet/NetworkInNetworkPReLULayer.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include "SpatiallySparseLayer.h" 4 | #include "NetworkInNetworkLayer.h" 5 | #include "Rng.h" 6 | 7 | class NetworkInNetworkPReLULayer : public SpatiallySparseLayer { 8 | private: 9 | RNG rng; 10 | cublasHandle_t &cublasHandle; 11 | 12 | public: 13 | vectorCUDA W; // Weights 14 | vectorCUDA MW; // momentum 15 | vectorCUDA w; // shrunk versions 16 | vectorCUDA dw; // For backprop 17 | vectorCUDA B; // Weights 18 | vectorCUDA MB; // momentum 19 | vectorCUDA b; // shrunk versions 20 | vectorCUDA db; // For backprop 21 | vectorCUDA PReLU; // negative slopes 22 | vectorCUDA MPReLU; // momentum 23 | vectorCUDA prelu; // shrunk versions 24 | vectorCUDA dprelu; // For backprop 25 | 26 | int nFeaturesIn; 27 | int nFeaturesOut; 28 | float dropout; 29 | NetworkInNetworkPReLULayer( 30 | cudaMemStream &memStream, cublasHandle_t &cublasHandle, int nFeaturesIn, 31 | int nFeaturesOut, float dropout = 0, 32 | float alpha = 1 // used to determine intialization weights only 33 | ); 34 | void preprocess(SpatiallySparseBatch &batch, 35 | SpatiallySparseBatchInterface &input, 36 | SpatiallySparseBatchInterface &output); 37 | void forwards(SpatiallySparseBatch &batch, 38 | SpatiallySparseBatchInterface &input, 39 | SpatiallySparseBatchInterface &output); 40 | void scaleWeights(SpatiallySparseBatchInterface &input, 41 | SpatiallySparseBatchInterface &output, 42 | float &scalingUnderneath, bool topLayer); 43 | void backwards(SpatiallySparseBatch &batch, 44 | SpatiallySparseBatchInterface &input, 45 | SpatiallySparseBatchInterface &output, float learningRate, 46 | float momentum); 47 | void loadWeightsFromStream(std::ifstream &f, bool momentum); 48 | void putWeightsToStream(std::ofstream &f, bool momentum); 49 | int calculateInputSpatialSize(int outputSpatialSize); 50 | }; 51 | -------------------------------------------------------------------------------- /SparseConvNet/Off3DFormatPicture.cpp: -------------------------------------------------------------------------------- 1 | #include "Off3DFormatPicture.h" 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | int mapToGridOFF(float coord, int inputFieldSize) { 10 | return std::max( 11 | 0, std::min(inputFieldSize - 1, (int)(coord + 0.5 * inputFieldSize))); 12 | } 13 | // Draw triangle with vertices {a, a+u, a+v} on grid 14 | // For every entry in grid (except the null vector -1) add a +1 to features 15 | // vector. 16 | void drawTriangleOFF(SparseGrid &grid, int inputFieldSize, 17 | std::vector &features, int &nSpatialSites, float a0, 18 | float a1, float a2, float u0, float u1, float u2, float v0, 19 | float v1, float v2) { 20 | float base = powf(u0 * u0 + u1 * u1 + u2 * u2, 0.5); 21 | u0 /= base; 22 | u1 /= base; 23 | u2 /= base; // scale u to a unit vector 24 | float offset = u0 * v0 + u1 * v1 + u2 * v2; // u dot v 25 | v0 -= offset * u0; 26 | v1 -= offset * u1; 27 | v2 -= offset * u2; // make v orthogonal to u 28 | float height = powf(v0 * v0 + v1 * v1 + v2 * v2, 0.5); 29 | v0 /= height; 30 | v1 /= height; 31 | v2 /= height; // scale v to be a unit vector 32 | // u and v are now orthogonal 33 | // The triangle now has points {a, a+base*u, a+offset*u+height*v} 34 | 35 | for (float h = 0; h <= height; h = std::min(h + 1, height) + (h == height)) { 36 | float l = base * (1 - h / height); 37 | for (float b = 0; b <= l; b = std::min(b + 1, l) + (b == l)) { 38 | float p0 = a0 + (b + offset * h / height) * u0 + h * v0, 39 | p1 = a1 + (b + offset * h / height) * u1 + h * v1, 40 | p2 = a2 + (b + offset * h / height) * u2 + h * v2; 41 | int n = 42 | mapToGridOFF(p0, inputFieldSize) * inputFieldSize * inputFieldSize + 43 | mapToGridOFF(p1, inputFieldSize) * inputFieldSize + 44 | mapToGridOFF(p2, inputFieldSize); 45 | if (grid.mp.find(n) == grid.mp.end()) { 46 | grid.mp[n] = nSpatialSites++; 47 | features.push_back(1); // NOTE: here we can push back vector of features 48 | } 49 | } 50 | } 51 | } 52 | 53 | OffSurfaceModelPicture::OffSurfaceModelPicture(std::string filename, 54 | int renderSize, int label) 55 | : Picture(label), renderSize(renderSize) { 56 | picture_path = filename; 57 | is_loaded = false; 58 | } 59 | 60 | void OffSurfaceModelPicture::loadPicture() { 61 | std::ifstream file(picture_path.c_str()); 62 | std::string off; 63 | getline(file, off); 64 | 65 | int nPoints, nTriangles, nLines; 66 | 67 | // In some files the first line is concatenated with the second line like this: 68 | // OFF348 256 0 69 | // So, we need to parse this situation 70 | if (off.compare("OFF") != 0) { 71 | std::istringstream off_stream(off); 72 | std::vector words{std::istream_iterator{off_stream}, 73 | std::istream_iterator{}}; 74 | nPoints = std::stoi(words[0].substr(3)); 75 | nTriangles = std::stoi(words[1]); 76 | nLines = std::stoi(words[2]); 77 | } else { 78 | file >> nPoints >> nTriangles >> nLines; 79 | } 80 | points.set_size(nPoints, 3); 81 | surfaces.resize(nTriangles); 82 | for (int i = 0; i < nPoints; i++) 83 | file >> points(i, 0) >> points(i, 1) >> points(i, 2); 84 | for (int i = 0; i < nTriangles; i++) { 85 | surfaces[i].resize(3); 86 | int three; 87 | file >> three >> surfaces[i][0] >> surfaces[i][1] >> surfaces[i][2]; 88 | } 89 | is_loaded = true; 90 | } 91 | 92 | void OffSurfaceModelPicture::unloadPicture() { 93 | points.reset(); 94 | std::vector>().swap(surfaces); // force vector reallocation 95 | is_loaded = false; 96 | } 97 | 98 | void OffSurfaceModelPicture::normalize() { // Fit centrally in the cube 99 | // [-renderSize/2,renderSize/2]^3 100 | arma::mat pointsm = arma::min(points, 0); 101 | arma::mat pointsM = arma::max(points, 0); 102 | float scale = arma::mat(pointsM - pointsm).max(); 103 | assert(scale > 0); 104 | points = points - arma::repmat(0.5 * (pointsm + pointsM), points.n_rows, 1); 105 | points *= renderSize / scale; 106 | } 107 | 108 | void OffSurfaceModelPicture::random_rotation(RNG &rng) { 109 | arma::mat L, Q, R; 110 | L.set_size(3, 3); 111 | for (int i = 0; i < 3; i++) 112 | for (int j = 0; j < 3; j++) 113 | L(i, j) = rng.uniform(); 114 | arma::qr(Q, R, L); 115 | points = points * Q; 116 | } 117 | 118 | void OffSurfaceModelPicture::jiggle(RNG &rng, float alpha) { 119 | for (int i = 0; i < 3; i++) 120 | points.col(i) += renderSize * rng.uniform(-alpha, alpha); 121 | } 122 | 123 | void OffSurfaceModelPicture::affineTransform(RNG &rng, float alpha) { 124 | arma::mat L = arma::eye(3, 3); 125 | for (int i = 0; i < 3; i++) 126 | for (int j = 0; j < 3; j++) 127 | L(i, j) += rng.uniform(-alpha, alpha); 128 | points = points * L; 129 | } 130 | 131 | void OffSurfaceModelPicture::codifyInputData(SparseGrid &grid, 132 | std::vector &features, 133 | int &nSpatialSites, 134 | int spatialSize) { 135 | features.push_back(0); // Background feature 136 | grid.backgroundCol = nSpatialSites++; 137 | for (int i = 0; i < surfaces.size(); ++i) { 138 | // assume triangles 139 | drawTriangleOFF(grid, spatialSize, features, nSpatialSites, 140 | points(surfaces[i][0], 0), points(surfaces[i][0], 1), 141 | points(surfaces[i][0], 2), 142 | points(surfaces[i][1], 0) - points(surfaces[i][0], 0), 143 | points(surfaces[i][1], 1) - points(surfaces[i][0], 1), 144 | points(surfaces[i][1], 2) - points(surfaces[i][0], 2), 145 | points(surfaces[i][2], 0) - points(surfaces[i][0], 0), 146 | points(surfaces[i][2], 1) - points(surfaces[i][0], 1), 147 | points(surfaces[i][2], 2) - points(surfaces[i][0], 2)); 148 | } 149 | } 150 | 151 | Picture *OffSurfaceModelPicture::distort(RNG &rng, batchType type) { 152 | OffSurfaceModelPicture *pic = new OffSurfaceModelPicture(*this); 153 | if (type != UNLABELEDBATCH) 154 | { 155 | pic->random_rotation(rng); 156 | pic->normalize(); 157 | if (type == TRAINBATCH) { 158 | pic->affineTransform(rng, 0.2); 159 | pic->jiggle(rng, 0.2); 160 | } 161 | }else{ 162 | pic->normalize(); 163 | } 164 | return pic; 165 | } 166 | -------------------------------------------------------------------------------- /SparseConvNet/Off3DFormatPicture.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "Picture.h" 3 | #include "Rng.h" 4 | #include 5 | #include 6 | #include 7 | 8 | class OffSurfaceModelPicture : public Picture { 9 | private: 10 | arma::mat points; 11 | std::vector> 12 | surfaces; // Will assume all surfaces are triangles for now 13 | public: 14 | int renderSize; 15 | std::string picture_path; 16 | OffSurfaceModelPicture(std::string filename, int renderSize, int label_ = -1); 17 | virtual ~OffSurfaceModelPicture() { 18 | points.reset(); 19 | } 20 | void loadPicture(); 21 | void unloadPicture() override; 22 | void normalize(); // Fit centrally in the cube [-scale_n/2,scale_n/2]^3 23 | void random_rotation(RNG &rng); 24 | void jiggle(RNG &rng, float alpha); 25 | void affineTransform(RNG &rng, float alpha); 26 | void codifyInputData(SparseGrid &grid, std::vector &features, int &nSpatialSites, int spatialSize); 27 | Picture *distort(RNG &rng, batchType type = TRAINBATCH); 28 | }; 29 | -------------------------------------------------------------------------------- /SparseConvNet/Off3DFormatTriangularPicture.cpp: -------------------------------------------------------------------------------- 1 | #if 0 2 | #include "Off3DFormatPicture.h" 3 | #include 4 | #include 5 | // Draw triangle with vertices {a, a+u, a+v} on grid 6 | // For every entry in grid (except the null vector -1) add a +1 to features 7 | // vector. 8 | void drawTriangleOFF(SparseGrid &grid, int inputFieldSize, 9 | std::vector &features, int &nSpatialSites, float a0, 10 | float a1, float a2, float u0, float u1, float u2, float v0, 11 | float v1, float v2) { 12 | float base = powf(u0 * u0 + u1 * u1 + u2 * u2, 0.5); 13 | u0 /= base; 14 | u1 /= base; 15 | u2 /= base; // scale u to a unit vector 16 | float offset = u0 * v0 + u1 * v1 + u2 * v2; // u dot v 17 | v0 -= offset * u0; 18 | v1 -= offset * u1; 19 | v2 -= offset * u2; // make v orthogonal to u 20 | float height = powf(v0 * v0 + v1 * v1 + v2 * v2, 0.5); 21 | v0 /= height; 22 | v1 /= height; 23 | v2 /= height; // scale v to be a unit vector 24 | // u and v are now orthogonal 25 | // The triangle now has points {a, a+base*u, a+offset*u+height*v} 26 | 27 | for (float h = 0; h <= height; h = std::min(h + 1, height) + (h == height)) { 28 | float l = base * (1 - h / height); 29 | for (float b = 0; b <= l; b = std::min(b + 1, l) + (b == l)) { 30 | float p0 = a0 + (b + offset * h / height) * u0 + h * v0, 31 | p1 = a1 + (b + offset * h / height) * u1 + h * v1, 32 | p2 = a2 + (b + offset * h / height) * u2 + h * v2; 33 | int i0 = p0 + 0.25 * inputFieldSize, //(0.25,0.25,0.25) for the pyramid 34 | // center in cubic coord system, 35 | // corresponds to 36 | //(1/2,1/2/sqrt(3),1/sqrt(24)) 37 | i1 = p1 + 0.25 * inputFieldSize, i2 = p2 + 0.25 * inputFieldSize; 38 | if (i0 >= 0 && i1 >= 0 && i2 >= 0 && i0 + i1 + i2 < inputFieldSize) { 39 | int n = i0 * inputFieldSize * inputFieldSize + i1 * inputFieldSize + i2; 40 | if (grid.mp.find(n) == grid.mp.end()) { 41 | grid.mp[n] = nSpatialSites++; 42 | features.push_back(1); 43 | } 44 | } 45 | // else 46 | // std::cout << "!"<> nPoints >> nTriangles >> nLines; 59 | points.set_size(nPoints, 3); 60 | surfaces.resize(nTriangles); 61 | for (int i = 0; i < nPoints; i++) 62 | file >> points(i, 0) >> points(i, 1) >> points(i, 2); 63 | for (int i = 0; i < nTriangles; i++) { 64 | surfaces[i].resize(3); 65 | int three; 66 | file >> three >> surfaces[i][0] >> surfaces[i][1] >> surfaces[i][2]; 67 | } 68 | } 69 | 70 | OffSurfaceModelPicture::~OffSurfaceModelPicture() {} 71 | void OffSurfaceModelPicture::normalize() { // Fit centrally in the cube 72 | // [-renderSize/2,renderSize/2]^3 73 | arma::mat pointsm = arma::min(points, 0); 74 | arma::mat pointsM = arma::max(points, 0); 75 | float scale = arma::mat(pointsM - pointsm).max(); 76 | assert(scale > 0); 77 | points = points - arma::repmat(0.5 * (pointsm + pointsM), points.n_rows, 1); 78 | points *= renderSize / scale; 79 | } 80 | 81 | void OffSurfaceModelPicture::random_rotation(RNG &rng) { 82 | arma::mat L, Q, R; 83 | L.set_size(3, 3); 84 | for (int i = 0; i < 3; i++) 85 | for (int j = 0; j < 3; j++) 86 | L(i, j) = rng.uniform(); 87 | arma::qr(Q, R, L); 88 | points = points * Q; 89 | } 90 | 91 | void OffSurfaceModelPicture::jiggle(RNG &rng, float alpha) { 92 | for (int i = 0; i < 3; i++) 93 | points.col(i) += renderSize * rng.uniform(-alpha, alpha); 94 | } 95 | 96 | void OffSurfaceModelPicture::affineTransform(RNG &rng, float alpha) { 97 | static const arma::mat L = arma::eye(3, 3); 98 | for (int i = 0; i < 3; i++) 99 | for (int j = 0; j < 3; j++) 100 | L(i, j) += rng.uniform(-alpha, alpha); 101 | points = points * L; 102 | } 103 | 104 | void OffSurfaceModelPicture::codifyInputData(SparseGrid &grid, 105 | std::vector &features, 106 | int &nSpatialSites, 107 | int spatialSize) { 108 | features.push_back(0); // Background feature 109 | grid.backgroundCol = nSpatialSites++; 110 | for (int i = 0; i < surfaces.size(); ++i) { 111 | // assume triangles 112 | drawTriangleOFF(grid, spatialSize, features, nSpatialSites, 113 | points(surfaces[i][0], 0), points(surfaces[i][0], 1), 114 | points(surfaces[i][0], 2), 115 | points(surfaces[i][1], 0) - points(surfaces[i][0], 0), 116 | points(surfaces[i][1], 1) - points(surfaces[i][0], 1), 117 | points(surfaces[i][1], 2) - points(surfaces[i][0], 2), 118 | points(surfaces[i][2], 0) - points(surfaces[i][0], 0), 119 | points(surfaces[i][2], 1) - points(surfaces[i][0], 1), 120 | points(surfaces[i][2], 2) - points(surfaces[i][0], 2)); 121 | } 122 | } 123 | 124 | // Map tetrahedron to simplex 125 | arma::mat convPyrCub = 126 | "1 0 0; -0.57735 1.1547 0; -0.40825 -0.40825 1.2247"; 127 | // (0,0,0) -> (0.0.0) 128 | // (1,0,0) -> (1,0,0) 129 | // (0.5,0.866,0) -> (0,1,0) 130 | // (0.5,0.289,0.82) -> (0,0,10 131 | 132 | Picture *OffSurfaceModelPicture::distort(RNG &rng, batchType type) { 133 | OffSurfaceModelPicture *pic = new OffSurfaceModelPicture(*this); 134 | 135 | pic->random_rotation(rng); 136 | pic->normalize(); 137 | if (type == TRAINBATCH) { 138 | pic->affineTransform(rng, 0.2); 139 | pic->jiggle(rng, 0.2); 140 | } 141 | pic->points = pic->points * convPyrCub; 142 | return pic; 143 | } 144 | #endif -------------------------------------------------------------------------------- /SparseConvNet/OnlineHandwritingPicture.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "Picture.h" 4 | #include "Rng.h" 5 | #include 6 | #include 7 | 8 | class OnlineHandwritingPicture : public Picture { 9 | float penSpeed3d; 10 | float offset3d; 11 | 12 | public: 13 | std::vector ops; 14 | int renderSize; 15 | OnlineHandwritingEncoding enc; 16 | OnlineHandwritingPicture(int renderSize, OnlineHandwritingEncoding enc, 17 | int label, float penSpeed3d = 0.02); 18 | ~OnlineHandwritingPicture(); 19 | void normalize(); // Fit centrally in the square 20 | // [-renderSize/2,renderSize/2]^2 21 | Picture *distort(RNG &rng, batchType type); 22 | void codifyInputData(SparseGrid &grid, std::vector &features, 23 | int &nSpatialSites, int spatialSize); 24 | void jiggle(RNG &rng, float alpha); 25 | void draw(int spatialSize); 26 | }; 27 | -------------------------------------------------------------------------------- /SparseConvNet/Picture.cpp: -------------------------------------------------------------------------------- 1 | #include "Picture.h" 2 | 3 | #include 4 | 5 | std::string Picture::identify() { return std::string(); } 6 | Picture::Picture(int label) : label(label) {} 7 | Picture::~Picture() { 8 | //std::cout << "Picture destroy" << std::endl; 9 | } 10 | -------------------------------------------------------------------------------- /SparseConvNet/Picture.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | #include "SparseGrid.h" 5 | #include "Rng.h" 6 | #include "types.h" 7 | 8 | class Picture { 9 | public: 10 | virtual void codifyInputData(SparseGrid &grid, std::vector &features, 11 | int &nSpatialSites, int spatialSize) = 0; 12 | virtual Picture *distort(RNG &rng, batchType type) { return this; } 13 | virtual std::string identify(); 14 | virtual void loadPicture() = 0; 15 | virtual void unloadPicture() {} 16 | bool is_loaded; 17 | int label; //-1 for unknown 18 | Picture(int label = -1); 19 | virtual ~Picture(); 20 | }; 21 | -------------------------------------------------------------------------------- /SparseConvNet/README.md: -------------------------------------------------------------------------------- 1 | # SparseConvNet 2 | ## A spatially-sparse convolutional neural network 3 | ### [Ben Graham](http://www2.warwick.ac.uk/fac/sci/statistics/staff/academic-research/graham/), [University of Warwick](http://www2.warwick.ac.uk/fac/sci/statistics/), 2013-2015, GPLv3 4 | 5 | SparseConvNet is a convolutional neural network for processing sparse data on a variety of lattices, i.e. 6 | (i) the square lattice, 7 | (ii) the triangular lattice, 8 | (iii) the cubic lattice, 9 | (iv) the tetrahedral lattice, ... 10 | ![lattice](/figures/lattices.png) 11 | ... and of course the hyper-cubic and hyper-tetrahedral 4D lattices as well. 12 | 13 | Data is sparse if most sites take the value zero. For example, if a loop of string has a knot in it, and you trace the shape of the string in a 3D lattice, most sites will not form part of the knot (left). Applying a 2x2x2 convolution (middle), and a pooling operation (right), the set of non-zero sites stays fairly small: 14 | ![lattice](/figures/trefoil.png) 15 | 16 | This can be used to analyse 3D models, or space-time paths. 17 | Here are some examples from a [3D object dataset](http://www.itl.nist.gov/iad/vug/sharp/contest/2014/Generic3D/index.html). The insides are hollow, so the data is fairly sparse. The computational complexity of processing the models is related to the [fractal dimension](http://en.wikipedia.org/wiki/Fractal_dimension) of the underlying objects. 18 | 19 | ![lattice](/figures/shrec.png) 20 | Top row: four exemplars of snakes. Bottom row: an ant, an elephant, a robot and a tortoise. 21 | 22 | ## [Wiki](https://github.com/btgraham/SparseConvNet/wiki) 23 | ## [Dependencies and Installation](https://github.com/btgraham/SparseConvNet/wiki/Installation) 24 | 25 | ************************************************************************** 26 | SparseConvNet is free software: you can redistribute it and/or modify 27 | it under the terms of the GNU General Public License as published by 28 | the Free Software Foundation, either version 3 of the License, or 29 | (at your option) any later version. 30 | 31 | SparseConvNet is distributed in the hope that it will be useful, 32 | but WITHOUT ANY WARRANTY; without even the implied warranty of 33 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 34 | [GNU General Public License](http://www.gnu.org/licenses/) for more details. 35 | ************************************************************************** 36 | -------------------------------------------------------------------------------- /SparseConvNet/ReallyConvolutionalLayer.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "SpatiallySparseLayer.h" 3 | 4 | class ReallyConvolutionalLayer : public SpatiallySparseLayer { 5 | private: 6 | int fs; 7 | RNG rng; 8 | float leaky; 9 | 10 | public: 11 | int inSpatialSize; 12 | int outSpatialSize; 13 | int filterSize; 14 | int filterStride; 15 | int dimension; 16 | ActivationFunction fn; 17 | int nFeaturesIn; 18 | int nFeaturesOut; 19 | float dropout; 20 | int minActiveInputs; 21 | vectorCUDA W; // Weights 22 | vectorCUDA MW; // momentum 23 | vectorCUDA w; // shrunk versions 24 | vectorCUDA dw; // For backprop 25 | vectorCUDA B; // Weights 26 | vectorCUDA MB; // momentum 27 | vectorCUDA b; // shrunk versions 28 | vectorCUDA db; // For backprop 29 | ReallyConvolutionalLayer(cudaMemStream &memStream, int nFeaturesIn, 30 | int nFeaturesOut, int filterSize, int filterStride, 31 | int dimension, ActivationFunction fn, float dropout, 32 | int minActiveInputs = 1, float poolingToFollow = 1); 33 | void preprocess(SpatiallySparseBatch &batch, 34 | SpatiallySparseBatchInterface &input, 35 | SpatiallySparseBatchInterface &output); 36 | void forwards(SpatiallySparseBatch &batch, 37 | SpatiallySparseBatchInterface &input, 38 | SpatiallySparseBatchInterface &output); 39 | void backwards(SpatiallySparseBatch &batch, 40 | SpatiallySparseBatchInterface &input, 41 | SpatiallySparseBatchInterface &output, float learningRate, 42 | float momentum); 43 | void scaleWeights(SpatiallySparseBatchInterface &input, 44 | SpatiallySparseBatchInterface &output, 45 | float &scalingUnderneath, bool topLayer); 46 | int calculateInputSpatialSize(int outputSpatialSize); 47 | }; 48 | -------------------------------------------------------------------------------- /SparseConvNet/Regions.h: -------------------------------------------------------------------------------- 1 | // Supports 1,2,3 and 4 dimensions. Could extend to 5d, etc 2 | // Supports convolutions and max-pooling with fixed size and stride 3 | // Supports fractional max-pooling 4 | // Supports triangular/tetrahedral lattices 5 | #pragma once 6 | #include 7 | #include "Rng.h" 8 | #include "SparseGrid.h" 9 | 10 | class RectangularRegions { 11 | public: 12 | int nIn; 13 | int nOut; 14 | int dimension; 15 | int s; 16 | int sd; 17 | RectangularRegions(int nIn, int nOut, int dimension, int s); 18 | ~RectangularRegions(); 19 | // up to 4d - could extend to 5d ... 20 | // given a point in the output layer, location of "top-left" and 21 | // "bottom-right" 22 | // corner of the corresponding pooling region in the input layer 23 | // (left<=x inputL; 48 | std::vector inputR; 49 | std::vector outputL; 50 | std::vector outputR; 51 | }; 52 | 53 | class PseudorandomOverlappingFmpTicks : public FractionalMaxPoolingTicks { 54 | public: 55 | PseudorandomOverlappingFmpTicks(int nIn, int nOut, int poolSize, RNG &rng); 56 | }; 57 | class PseudorandomNonOverlappingFmpTicks : public FractionalMaxPoolingTicks { 58 | public: 59 | PseudorandomNonOverlappingFmpTicks(int nIn, int nOut, int poolSize, RNG &rng); 60 | }; 61 | class RandomOverlappingFmpTicks : public FractionalMaxPoolingTicks { 62 | public: 63 | RandomOverlappingFmpTicks(int nIn, int nOut, int poolSize, RNG &rng); 64 | }; 65 | class RandomNonOverlappingFmpTicks : public FractionalMaxPoolingTicks { 66 | public: 67 | RandomNonOverlappingFmpTicks(int nIn, int nOut, int poolSize, RNG &rng); 68 | }; 69 | 70 | template 71 | class FractionalPoolingRegions : public RectangularRegions { 72 | std::vector pb; 73 | 74 | public: 75 | FractionalPoolingRegions(int nIn, int nOut, int dimension, int poolSize, 76 | RNG &rng); 77 | ~FractionalPoolingRegions(); 78 | int inputL(int axis, int j); 79 | int inputR(int axis, int j); 80 | int outputL(int axis, int i); 81 | int outputR(int axis, int i); 82 | }; 83 | 84 | void gridRules(SparseGrid &inputGrid, SparseGrid &outputGrid, 85 | RectangularRegions ®ions, int &nOutputSpatialSites, 86 | std::vector &rules, bool uniformSizeRegions, 87 | int minActiveInputs = 1); 88 | 89 | class RegularTriangularRegions { 90 | public: 91 | int poolSize; 92 | int poolStride; 93 | int nIn; // nIn==1: o nIn==2: oo nIn==3: ooo nIn==4: oooo etc 94 | // (o=site, x=nothing) 95 | int nOut; // ox oox ooox 96 | // oxx ooxx 97 | // oxxx 98 | // Similarly for nOut. 99 | int dimension; // dimension==2 for triangles, 3 for tetrahedron/pyramids, 4 100 | // for tetrahedral hyperpyramids 101 | int s; // 1d pooling region size (same as poolSize in 102 | // RegularTriangularRegions) 103 | int S; // total pooling region size (dimension==1, S==s; dimension==2, 104 | // S=s*(s+1)/2, ....) 105 | std::vector ord; // order of the S points chosen from the 106 | // ipow(s,dimension) points in making up the filter 107 | // shape 108 | RegularTriangularRegions(int nIn, int nOut, int dimension, int poolSize, 109 | int poolStride); 110 | // up to 4d 111 | // given a point in the output layer, location of "top-left" corner of the 112 | // corresponding pooling region in the input layer 113 | int inputL(int j); 114 | // given an input-layer point, what is the range of points in the output layer 115 | // to which it can be pooled ((left<=x &rules, int minActiveInputs = 1); 128 | -------------------------------------------------------------------------------- /SparseConvNet/Rng.cpp: -------------------------------------------------------------------------------- 1 | #define RNG_CPP 2 | #include "Rng.h" 3 | #include 4 | #include 5 | #ifdef TARGET_OS_MAC 6 | #include 7 | #include 8 | #endif 9 | #include 10 | #include 11 | #include 12 | 13 | RNG::RNG() : stdNormal(0, 1), uniform01(0, 1) { 14 | // auto seed = 15 | // std::chrono::high_resolution_clock::now().time_since_epoch().count(); 16 | // std::mt19937 gen(seed); 17 | timespec ts; 18 | #ifdef TARGET_OS_MAC 19 | clock_serv_t cclock; 20 | mach_timespec_t mts; 21 | host_get_clock_service(mach_host_self(), CALENDAR_CLOCK, &cclock); 22 | clock_get_time(cclock, &mts); 23 | mach_port_deallocate(mach_task_self(), cclock); 24 | ts.tv_sec = mts.tv_sec; 25 | ts.tv_nsec = mts.tv_nsec; 26 | #elif defined __linux__ 27 | clock_gettime(CLOCK_REALTIME, &ts); 28 | #endif 29 | 30 | RNGseedGeneratorMutex.lock(); 31 | gen.seed(RNGseedGenerator() + ts.tv_nsec); 32 | RNGseedGeneratorMutex.unlock(); 33 | } 34 | 35 | int RNG::randint(int n) { 36 | if (n == 0) 37 | return 0; 38 | else 39 | return gen() % n; 40 | } 41 | float RNG::uniform(float a, float b) { return a + (b - a) * uniform01(gen); } 42 | float RNG::normal(float mean, float sd) { return mean + sd * stdNormal(gen); } 43 | int RNG::bernoulli(float p) { 44 | if (uniform01(gen) < p) 45 | return 1; 46 | else 47 | return 0; 48 | } 49 | template int RNG::index(std::vector &v) { 50 | if (v.size() == 0) 51 | std::cout << "RNG::index called for empty std::vector!\n"; 52 | return gen() % v.size(); 53 | } 54 | std::vector RNG::NchooseM(int n, int m) { 55 | std::vector ret(m, 100); 56 | int ctr = m; 57 | for (int i = 0; i < n; i++) 58 | if (uniform01(gen) < ctr * 1.0 / (n - i)) 59 | ret[m - ctr--] = i; 60 | return ret; 61 | } 62 | std::vector RNG::permutation(int n) { 63 | std::vector ret; 64 | for (int i = 0; i < n; i++) 65 | ret.push_back(i); 66 | std::shuffle(ret.begin(), ret.end(), gen); 67 | return ret; 68 | } 69 | template void RNG::vectorShuffle(std::vector &v) { 70 | std::shuffle(v.begin(), v.end(), gen); 71 | } 72 | 73 | template void RNG::vectorShuffle(std::vector &v); 74 | -------------------------------------------------------------------------------- /SparseConvNet/Rng.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | 5 | #ifdef RNG_CPP 6 | std::mutex RNGseedGeneratorMutex; 7 | std::mt19937 RNGseedGenerator; 8 | #else 9 | extern std::mutex RNGseedGeneratorMutex; 10 | extern std::mt19937 RNGseedGenerator; 11 | #endif 12 | 13 | class RNG { 14 | std::normal_distribution<> stdNormal; 15 | std::uniform_real_distribution<> uniform01; 16 | 17 | public: 18 | std::mt19937 gen; 19 | 20 | RNG(); 21 | int randint(int n); 22 | float uniform(float a = 0, float b = 1); 23 | float normal(float mean = 0, float sd = 1); 24 | int bernoulli(float p); 25 | template int index(std::vector &v); 26 | std::vector NchooseM(int n, int m); 27 | std::vector permutation(int n); 28 | template void vectorShuffle(std::vector &v); 29 | }; 30 | -------------------------------------------------------------------------------- /SparseConvNet/SigmoidLayer.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "SpatiallySparseLayer.h" 3 | 4 | void applySigmoid(SpatiallySparseBatchInterface &input, 5 | SpatiallySparseBatchInterface &output, ActivationFunction fn, 6 | cudaMemStream &memStream); 7 | void applySigmoidBackProp(SpatiallySparseBatchInterface &input, 8 | SpatiallySparseBatchInterface &output, 9 | ActivationFunction fn, cudaMemStream &memStream); 10 | 11 | class SigmoidLayer : public SpatiallySparseLayer { 12 | public: 13 | ActivationFunction fn; 14 | SigmoidLayer(cudaMemStream &memStream, ActivationFunction fn); 15 | void preprocess(SpatiallySparseBatch &batch, 16 | SpatiallySparseBatchInterface &input, 17 | SpatiallySparseBatchInterface &output); 18 | void forwards(SpatiallySparseBatch &batch, 19 | SpatiallySparseBatchInterface &input, 20 | SpatiallySparseBatchInterface &output); 21 | void backwards(SpatiallySparseBatch &batch, 22 | SpatiallySparseBatchInterface &input, 23 | SpatiallySparseBatchInterface &output, float learningRate, 24 | float momentum); 25 | int calculateInputSpatialSize(int outputSpatialSize); 26 | }; 27 | -------------------------------------------------------------------------------- /SparseConvNet/SoftmaxClassifier.cu: -------------------------------------------------------------------------------- 1 | #include "SoftmaxClassifier.h" 2 | #include 3 | #include 4 | #include 5 | #include "utilities.h" 6 | 7 | __global__ void dDerivativeOfCostWRTpreSoftmaxTopLevelWeights( 8 | int batchSize, float *topDelta, float *topGrid, int *labels, int N) { 9 | for (int k = 0; k < batchSize; k++) { 10 | for (int i = threadIdx.x; i < N; i += NTHREADS) { 11 | topDelta[k * N + i] = topGrid[k * N + i] - (i == labels[k]); 12 | } 13 | } 14 | } 15 | 16 | void SoftmaxClassifier(SpatiallySparseBatchInterface &input, 17 | SpatiallySparseBatch &batch, int nTop, 18 | cudaMemStream &memStream) { 19 | // Assume no dropout in the output layer! nClasses:=input.nFeatures. 20 | assert(batch.batchSize == input.nSpatialSites); 21 | assert(input.nFeatures == input.featuresPresent.size()); 22 | 23 | if (batch.type == 24 | TRAINBATCH) { // Begin backprop. Top layer: d Cost / d SoftmaxInput 25 | input.sub->dfeatures.resize(input.nSpatialSites * 26 | input.featuresPresent.size()); 27 | dDerivativeOfCostWRTpreSoftmaxTopLevelWeights 28 | << <1, NTHREADS, 0, memStream.stream>>> 29 | (batch.batchSize, input.sub->dfeatures.dPtr(), 30 | input.sub->features.dPtr(), batch.labels.dPtr(), input.nFeatures); 31 | } 32 | 33 | input.sub->features.copyToCPUAsync(memStream); 34 | batch.labels.copyToCPUAsync(memStream); 35 | 36 | float *probs = &input.sub->features.hVector()[0]; 37 | for (int i = 0; i < batch.batchSize; ++i) 38 | batch.probabilities.push_back(std::vector( 39 | probs + i * input.nFeatures, probs + (i + 1) * input.nFeatures)); 40 | for (int i = 0; i < batch.batchSize; i++) 41 | batch.predictions.push_back(vectorTopIndices(batch.probabilities[i], nTop)); 42 | 43 | if (batch.type != UNLABELEDBATCH) { 44 | batch.mistakes += batch.batchSize; 45 | for (int i = 0; i < batch.batchSize; i++) { 46 | batch.negativeLogLikelihood -= 47 | log(max(batch.probabilities[i][batch.labels.hVector()[i]], 1.0e-15)); 48 | for (int j = 0; j < nTop; j++) { 49 | if (batch.predictions[i][j] == batch.labels.hVector()[i]) { 50 | batch.mistakes--; 51 | } 52 | } 53 | } 54 | } 55 | // std::cout << (int)batch.negativeLogLikelihood << " " << std::flush; 56 | input.sub->features.copyToGPUAsync(memStream); 57 | cudaCheckError(); 58 | } 59 | -------------------------------------------------------------------------------- /SparseConvNet/SoftmaxClassifier.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "SpatiallySparseBatchInterface.h" 3 | #include "SpatiallySparseBatch.h" 4 | 5 | //////////////////////////////////////////////////////////////////////////////////////////////// 6 | // Calculate softmaxProbability(i) - indicator(i=label) 7 | // for i=0,1,...N-1 with N the number of character classes. 8 | __global__ void dDerivativeOfCostWRTpreSoftmaxTopLevelWeights( 9 | int batchSize, float *topDelta, float *topGrid, int *labels, int N); 10 | void SoftmaxClassifier(SpatiallySparseBatchInterface &input, 11 | SpatiallySparseBatch &batch, int nTop, 12 | cudaMemStream &memStream); 13 | -------------------------------------------------------------------------------- /SparseConvNet/SparseConvNet.h: -------------------------------------------------------------------------------- 1 | // Ben Graham, University of Warwick, 2015, b.graham@warwick.ac.uk 2 | // SparseConvNet is free software: you can redistribute it and/or modify 3 | // it under the terms of the GNU General Public License as published by 4 | // the Free Software Foundation, either version 3 of the License, or 5 | //(at your option) any later version. 6 | 7 | // SparseConvNet is distributed in the hope that it will be useful, 8 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | // GNU General Public License for more details. 11 | 12 | #pragma once 13 | #include "SpatiallySparseDataset.h" 14 | #include 15 | #include 16 | #include 17 | #include "SparseConvNetCUDA.h" 18 | 19 | class SparseConvNetCUDA; 20 | 21 | class SparseConvNet { 22 | private: 23 | public: 24 | std::unique_ptr cnn; 25 | 26 | SparseConvNet(int dimension, int nInputFeatures, int nClasses, 27 | int pciBusID = -1, int nTop = 1, int nThreads = 1); 28 | ~SparseConvNet(); 29 | void addLeNetLayerMP(int nFeatures, int filterSize, int filterStride, 30 | int poolSize, int poolStride, 31 | ActivationFunction activationFn = RELU, 32 | float dropout = 0.0f, int minActiveInputs = 1); 33 | void addLeNetLayerPOFMP(int nFeatures, int filterSize, int filterStride, 34 | int poolSize, float fmpShrink, 35 | ActivationFunction activationFn = RELU, 36 | float dropout = 0.0f, int minActiveInputs = 1); 37 | void addLeNetLayerROFMP(int nFeatures, int filterSize, int filterStride, 38 | int poolSize, float fmpShrink, 39 | ActivationFunction activationFn = RELU, 40 | float dropout = 0.0f, int minActiveInputs = 1); 41 | void addLeNetLayerPDFMP(int nFeatures, int filterSize, int filterStride, 42 | int poolSize, float fmpShrink, 43 | ActivationFunction activationFn = RELU, 44 | float dropout = 0.0f, int minActiveInputs = 1); 45 | void addLeNetLayerRDFMP(int nFeatures, int filterSize, int filterStride, 46 | int poolSize, float fmpShrink, 47 | ActivationFunction activationFn = RELU, 48 | float dropout = 0.0f, int minActiveInputs = 1); 49 | void addTerminalPoolingLayer(int poolSize); 50 | void addSoftmaxLayer(); 51 | void addIndexLearnerLayer(); 52 | pd_report processDataset(SpatiallySparseDataset &dataset, int batchSize = 100, 53 | float learningRate = 0, float momentum = 0.99); 54 | std::vector processDatasetRepeatTest(SpatiallySparseDataset &dataset, 55 | int batchSize = 100, int nReps = 12, 56 | std::string predictionsFilename = "", 57 | std::string confusionMatrixFilename = ""); 58 | float processIndexLearnerDataset(SpatiallySparseDataset &dataset, 59 | int batchSize = 100, 60 | float learningRate = 0.0, 61 | float momentum = 0.99); 62 | void processDatasetDumpTopLevelFeatures(SpatiallySparseDataset &dataset, 63 | int batchSize, int reps = 1); 64 | void loadWeights(std::string baseName, int epoch, bool momentum = false, 65 | int firstNlayers = 1000000); 66 | void saveWeights(std::string baseName, int epoch, bool momentum = false); 67 | void calculateInputRegularizingConstants(SpatiallySparseDataset dataset); 68 | }; 69 | 70 | class SparseConvTriangLeNet { 71 | private: 72 | std::auto_ptr cnn; 73 | 74 | public: 75 | SparseConvTriangLeNet(int dimension, int nInputFeatures, int nClasses, 76 | int pciBusID = -1, int nTop = 1); 77 | ~SparseConvTriangLeNet(); 78 | void addLeNetLayerMP(int nFeatures, int filterSize, int filterStride, 79 | int poolSize, int poolStride, 80 | ActivationFunction activationFn = RELU, 81 | float dropout = 0.0f, int minActiveInputs = 1); 82 | void addTerminalPoolingLayer(int poolSize); 83 | void addSoftmaxLayer(); 84 | void addIndexLearnerLayer(); 85 | pd_report processDataset(SpatiallySparseDataset &dataset, int batchSize = 100, 86 | float learningRate = 0, float momentum = 0.99); 87 | void processDatasetRepeatTest(SpatiallySparseDataset &dataset, 88 | int batchSize = 100, int nReps = 12, 89 | std::string predictionsFilename = "", 90 | std::string confusionMatrixFilename = ""); 91 | float processIndexLearnerDataset(SpatiallySparseDataset &dataset, 92 | int batchSize = 100, float learningRate = 0, 93 | float momentum = 0.99); 94 | void processDatasetDumpTopLevelFeatures(SpatiallySparseDataset &dataset, 95 | int batchSize, int reps = 1); 96 | void loadWeights(std::string baseName, int epoch, bool momentum = false, 97 | int firstNlayers = 1000000); 98 | void saveWeights(std::string baseName, int epoch, bool momentum = false); 99 | void calculateInputRegularizingConstants(SpatiallySparseDataset dataset); 100 | }; 101 | -------------------------------------------------------------------------------- /SparseConvNet/SparseConvNetCUDA.h: -------------------------------------------------------------------------------- 1 | // Ben Graham, University of Warwick, 2015 b.graham@warwick.ac.uk 2 | // SparseConvNet is free software: you can redistribute it and/or modify 3 | // it under the terms of the GNU General Public License as published by 4 | // the Free Software Foundation, either version 3 of the License, or 5 | //(at your option) any later version. 6 | 7 | // SparseConvNet is distributed in the hope that it will be useful, 8 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | // GNU General Public License for more details. 11 | 12 | // You should have received a copy of the GNU General Public License 13 | // along with SparseConvNet. If not, see . 14 | 15 | #pragma once 16 | #include "SpatiallySparseBatch.h" 17 | #include "SpatiallySparseLayer.h" 18 | #include "SpatiallySparseDataset.h" 19 | #include 20 | #include 21 | #include 22 | #include 23 | #define N_MAX_BATCH_PRODUCER_THREADS 20 24 | 25 | struct pd_report { 26 | float errorRate; 27 | float nll; 28 | int MegaMultiplyAdds_per_sample; 29 | long time; 30 | int GigaMultiplyAdds_per_s; 31 | int rate; 32 | }; 33 | 34 | struct activation { 35 | long grid_size; 36 | int feature_size; 37 | int nSpatialSites; 38 | int spatialSize; 39 | int nFeatures; 40 | std::vector features; 41 | std::vector> sparse_grid; 42 | }; 43 | 44 | 45 | class SparseConvNetCUDA { 46 | public: 47 | std::vector layers; 48 | std::vector inputNormalizingConstants; 49 | int dimension; 50 | int nClasses; 51 | int nTop; 52 | int inputSpatialSize; 53 | int nInputFeatures; 54 | int nOutputFeatures; 55 | int deviceID; 56 | int nBatchProducerThreads; 57 | cudaMemStream memStream; 58 | cudaMemStream batchMemStreams[N_MAX_BATCH_PRODUCER_THREADS]; 59 | std::mutex batchLock[N_MAX_BATCH_PRODUCER_THREADS]; 60 | std::vector batchPool; 61 | std::vector initialSubInterfaces; 62 | std::vector sharedSubInterfaces; 63 | cublasHandle_t cublasHandle; 64 | 65 | SparseConvNetCUDA(int dimension, int nInputFeatures, int nClasses, 66 | int pciBusID = -1, int nTop = 1, 67 | int nBatchProducerThreads = 1); 68 | ~SparseConvNetCUDA(); 69 | void processBatch(SpatiallySparseBatch &batch, float learningRate, 70 | float momentum, std::ofstream &f, std::ofstream &g); 71 | activation processBatchForward(SpatiallySparseBatch &batch); 72 | void processBatchBackward(SpatiallySparseBatch &batch, 73 | float learningRate, float momentum, 74 | const std::vector &dfeatures); 75 | void processIndexLearnerBatch(SpatiallySparseBatch &batch, float learningRate, 76 | float momentum, std::ofstream &f); 77 | 78 | void addLearntLayer(int nFeatures, ActivationFunction activationFn = RELU, 79 | float dropout = 0.0f, float alpha = 1.0f); 80 | void addNetworkInNetworkLayer(int nFeatures, 81 | ActivationFunction activationFn = RELU, 82 | float dropout = 0.0f); 83 | void addConvolutionalLayer(int nFeatures, int filterSize, int filterStride, 84 | ActivationFunction activationFn = RELU, 85 | float dropout = 0.0f, int minActiveInputs = 1, 86 | float poolingToFollow = 1.0f); 87 | void addLeNetLayerMP(int nFeatures, int filterSize, int filterStride, 88 | int poolSize, int poolStride, 89 | ActivationFunction activationFn = RELU, 90 | float dropout = 0.0f, int minActiveInputs = 1); 91 | void addLeNetLayerROFMP(int nFeatures, int filterSize, int filterStride, 92 | int poolSize, float fmpShrink, 93 | ActivationFunction activationFn = RELU, 94 | float dropout = 0.0f, int minActiveInputs = 1); 95 | void addLeNetLayerPOFMP(int nFeatures, int filterSize, int filterStride, 96 | int poolSize, float fmpShrink, 97 | ActivationFunction activationFn = RELU, 98 | float dropout = 0.0f, int minActiveInputs = 1); 99 | void addLeNetLayerRDFMP(int nFeatures, int filterSize, int filterStride, 100 | int poolSize, float fmpShrink, 101 | ActivationFunction activationFn = RELU, 102 | float dropout = 0.0f, int minActiveInputs = 1); 103 | void addLeNetLayerPDFMP(int nFeatures, int filterSize, int filterStride, 104 | int poolSize, float fmpShrink, 105 | ActivationFunction activationFn = RELU, 106 | float dropout = 0.0f, int minActiveInputs = 1); 107 | void addTriangularConvolutionalLayer(int nFeatures, int filterSize, 108 | int filterStride, 109 | ActivationFunction activationFn = RELU, 110 | float dropout = 0.0f, 111 | int minActiveInputs = 1, 112 | float poolingToFollow = 1.0f); 113 | void addTriangularLeNetLayerMP(int nFeatures, int filterSize, 114 | int filterStride, int poolSize, int poolStride, 115 | ActivationFunction activationFn = RELU, 116 | float dropout = 0.0f, int minActiveInputs = 1); 117 | int computeInputSpatialSize(int outputSpatialSize = 1); 118 | void addSoftmaxLayer(); 119 | void addTerminalPoolingLayer(int poolSize, int S); 120 | void addIndexLearnerLayer(); 121 | pd_report processDataset(SpatiallySparseDataset &dataset, int batchSize = 100, 122 | float learningRate = 0, float momentum = 0.99); 123 | std::vector layer_activations( 124 | SpatiallySparseDataset &dataset); 125 | std::vector> predict(SpatiallySparseDataset &dataset); 126 | std::vector processDatasetRepeatTest(SpatiallySparseDataset &dataset, 127 | int batchSize = 100, int nReps = 12, 128 | std::string predictionsFilename = "", 129 | std::string confusionMatrixFilename = ""); 130 | float processIndexLearnerDataset(SpatiallySparseDataset &dataset, 131 | int batchSize = 100, 132 | float learningRate = 0.0, 133 | float momentum = 0.99); 134 | void processBatchDumpTopLevelFeaturess(SpatiallySparseBatch &batch, 135 | std::ofstream &f); 136 | void processDatasetDumpTopLevelFeatures(SpatiallySparseDataset &dataset, 137 | int batchSize, int reps = 1); 138 | void loadWeights(std::string baseName, int epoch, bool momentum, 139 | int firstNlayers = 1000000); 140 | void saveWeights(std::string baseName, int epoch, bool momentum); 141 | void calculateInputRegularizingConstants(SpatiallySparseDataset dataset); 142 | }; 143 | -------------------------------------------------------------------------------- /SparseConvNet/SparseGrid.h: -------------------------------------------------------------------------------- 1 | // We need a sparse n-dimensional table 2 | // We can either use 3 | 4 | // a) Google's sparsehash dense_hash_map, or 5 | #define USE_GOOGLE_SPARSEHASH 6 | 7 | // b) the C++11 std::unordered_map 8 | // #define USE_UNORDERED_MAP 9 | 10 | // c) the C++11 std::unordered_map with tbb::scalable_alloc (or similar) to 11 | // prevent threads getting in each others way as they access memory to grow the 12 | // maps, 13 | // #define USE_UNORDERED_MAP 14 | 15 | // d) vectors disguised as a hash map (ok in 2 dimensions) 16 | // #define USE_VECTOR_HASH 17 | 18 | #pragma once 19 | #include 20 | 21 | #ifdef USE_GOOGLE_SPARSEHASH 22 | #include 23 | #include 24 | typedef google::dense_hash_map, 25 | std::equal_to> SparseGridMap; 26 | class SparseGrid { 27 | public: 28 | int backgroundCol; 29 | SparseGridMap mp; 30 | SparseGrid() { 31 | backgroundCol = -1; // Indicate that no "null vector" is needed 32 | mp.set_empty_key(-99); // dense_hash_map needs an empty key that will not be 33 | // used as a real key 34 | mp.set_deleted_key(-98); // and another for deleting 35 | mp.min_load_factor(0.0f); 36 | } 37 | }; 38 | #endif 39 | 40 | #ifdef USE_UNORDERED_MAP 41 | #include 42 | typedef std::unordered_map< 43 | int64_t, int, std::hash, std::equal_to, 44 | std::allocator>> SparseGridMap; 45 | class SparseGrid { 46 | public: 47 | int backgroundCol; // Set to -1 when no "null vector" is needed 48 | SparseGridMap mp; 49 | SparseGrid() : backgroundCol(-1) {} 50 | }; 51 | #endif 52 | 53 | #ifdef USE_UNORDERED_MAP_TBB 54 | // Libraries -ltbb -ltbbmalloc 55 | #include 56 | #include 57 | typedef std::unordered_map< 58 | int64_t, int, std::hash, std::equal_to, 59 | tbb::scalable_allocator>> SparseGridMap; 60 | class SparseGrid { 61 | public: 62 | int backgroundCol; // Set to -1 when no "null vector" is needed 63 | SparseGridMap mp; 64 | SparseGrid() : backgroundCol(-1) {} 65 | }; 66 | #endif 67 | 68 | #ifdef USE_VECTOR_HASH 69 | #include "vectorHash.h" 70 | typedef vectorHash SparseGridMap; 71 | class SparseGrid { 72 | public: 73 | int backgroundCol; // Set to -1 when no "null vector" is needed 74 | SparseGridMap mp; 75 | SparseGrid() : backgroundCol(-1) {} 76 | }; 77 | #endif 78 | -------------------------------------------------------------------------------- /SparseConvNet/SpatiallySparseBatch.cu: -------------------------------------------------------------------------------- 1 | #include "SpatiallySparseBatch.h" 2 | #include "utilities.h" 3 | 4 | SpatiallySparseBatch::SpatiallySparseBatch( 5 | SpatiallySparseBatchSubInterface *inputSub) { 6 | interfaces.emplace_back(inputSub); 7 | reset(); 8 | } 9 | void SpatiallySparseBatch::reset() { 10 | batchSize = 0; 11 | sampleNumbers.resize(0); 12 | for (int i = 0; i < interfaces.size(); ++i) 13 | interfaces[i].reset(); 14 | interfaces[0].sub->reset(); 15 | labels.resize(0); 16 | predictions.resize(0); 17 | probabilities.resize(0); 18 | negativeLogLikelihood = 0; 19 | mistakes = 0; 20 | } 21 | -------------------------------------------------------------------------------- /SparseConvNet/SpatiallySparseBatch.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include "SpatiallySparseBatchInterface.h" 4 | #include "vectorCUDA.h" 5 | #include "types.h" 6 | 7 | class SpatiallySparseBatch { 8 | SpatiallySparseBatchSubInterface *inputSub; 9 | 10 | public: 11 | SpatiallySparseBatch(SpatiallySparseBatchSubInterface *inputSub); 12 | batchType type; 13 | int batchSize; 14 | std::vector sampleNumbers; 15 | std::vector interfaces; 16 | vectorCUDA labels; 17 | std::vector> predictions; 18 | std::vector> probabilities; 19 | float negativeLogLikelihood; 20 | int mistakes; 21 | void reset(); 22 | }; 23 | -------------------------------------------------------------------------------- /SparseConvNet/SpatiallySparseBatchInterface.cu: -------------------------------------------------------------------------------- 1 | #include 2 | #include "SpatiallySparseBatchInterface.h" 3 | 4 | SpatiallySparseBatchSubInterface::SpatiallySparseBatchSubInterface() { 5 | reset(); 6 | } 7 | 8 | void SpatiallySparseBatchSubInterface::reset() { 9 | features.resize(0); 10 | dfeatures.resize(0); 11 | poolingChoices.resize(0); 12 | } 13 | 14 | SpatiallySparseBatchInterface::SpatiallySparseBatchInterface( 15 | SpatiallySparseBatchSubInterface *s) 16 | : sub(s), rules(false, 0), featuresPresent(false, 0) { 17 | reset(); 18 | } 19 | void SpatiallySparseBatchInterface::summary() { 20 | std::cout << "---------------------------------------------------\n"; 21 | std::cout << "nFeatures = " << nFeatures << std::endl; 22 | std::cout << "featuresPresent.size() = " << featuresPresent.size() << std::endl; 23 | std::cout << "spatialSize = " << spatialSize << std::endl; 24 | std::cout << "nSpatialSites = " << nSpatialSites << std::endl; 25 | std::cout << "sub->features.size() = " << sub->features.size() << std::endl; 26 | std::cout << "sub->dfeatures.size() = " << sub->dfeatures.size() << std::endl; 27 | std::cout << "grids.size() = " << grids.size() << std::endl; 28 | std::cout << "grids[0].mp.size() = " << grids[0].mp.size() << std::endl; 29 | std::cout << "---------------------------------------------------\n"; 30 | } 31 | void SpatiallySparseBatchInterface::reset() { 32 | featuresPresent.resize(0); 33 | featuresPresent.copyToCPU(); 34 | nSpatialSites = 0; 35 | grids.resize(0); 36 | rules.resize(0); 37 | rules.copyToCPU(); 38 | backpropErrors = false; 39 | } 40 | -------------------------------------------------------------------------------- /SparseConvNet/SpatiallySparseBatchInterface.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include "SparseGrid.h" 4 | #include "vectorCUDA.h" 5 | 6 | // This is a subset of the whole interface. 7 | // It contains the larger vectors that can mostly stay on the GPU 8 | class SpatiallySparseBatchSubInterface { 9 | public: 10 | // In the input layer, this is configured during preprocessing 11 | vectorCUDA features; 12 | // For the backwards/backpropagation pass 13 | vectorCUDA dfeatures; 14 | vectorCUDA poolingChoices; 15 | SpatiallySparseBatchSubInterface(); 16 | void reset(); 17 | }; 18 | 19 | class SpatiallySparseBatchInterface { 20 | public: 21 | SpatiallySparseBatchSubInterface *sub; 22 | int nFeatures; // Features per spatial location 23 | vectorCUDA 24 | featuresPresent; // Not dropped out features per spatial location 25 | // For dropout 26 | // rng.NchooseM(nFeatures,featuresPresent.size()); 27 | int nSpatialSites; // Total active spatial locations within the 28 | int spatialSize; // spatialSize x spatialSize grid 29 | // batchSize x spatialSize x 30 | // spatialSize 31 | // possible locations. 32 | bool backpropErrors; // Calculate dfeatures? (false until after the first NiN 33 | // layer) 34 | std::vector grids; // batchSize vectors of maps storing info on 35 | // grids of size spatialSize x spatialSize 36 | // Store locations of nSpatialSites 37 | // in the 38 | // spatialSize x spatialSize grids 39 | // -1 entry corresponds to null 40 | // vectors in needed 41 | // Below used internally for convolution/pooling operation: 42 | vectorCUDA rules; 43 | SpatiallySparseBatchInterface(SpatiallySparseBatchSubInterface *s); 44 | void summary(); 45 | void reset(); 46 | }; 47 | -------------------------------------------------------------------------------- /SparseConvNet/SpatiallySparseDataset.cpp: -------------------------------------------------------------------------------- 1 | #include "SpatiallySparseDataset.h" 2 | #include 3 | #include 4 | #include 5 | #include 6 | 7 | void SpatiallySparseDataset::summary() { 8 | std::cout << "Name: " << name << std::endl; 9 | std::cout << "Type: " << batchTypeNames[type] << std::endl; 10 | std::cout << "nFeatures: " << nFeatures << std::endl; 11 | std::cout << "nPictures: " << pictures.size() << std::endl; 12 | std::cout << "nClasses: " << nClasses << std::endl; 13 | if (type != UNLABELEDBATCH){ 14 | std::vector count(nClasses); 15 | for (int _i=0;_ilabel]++; 17 | 18 | std::cout << "nPictures/class: "; 19 | for (int _i=0;_i 0; size--) { 35 | val.pictures.push_back(pictures.back()); 36 | pictures.pop_back(); 37 | } 38 | return val; 39 | } 40 | 41 | // Assume there are at least n of each class in the dataset 42 | SpatiallySparseDataset SpatiallySparseDataset::balancedSubset(int n) { 43 | SpatiallySparseDataset bs; 44 | bs.name = name + std::string(" subset"); 45 | bs.nFeatures = nFeatures; 46 | bs.nClasses = nClasses; 47 | bs.type = type; 48 | auto permutation = rng.permutation(pictures.size()); 49 | std::vector count(nClasses); 50 | int classesDone = 0; 51 | for (unsigned int i = 0; i < pictures.size() and classesDone < nClasses; 52 | i++) { 53 | auto pic = pictures[permutation[i]]; 54 | if (count[pic->label]++ < n) 55 | bs.pictures.push_back(pic); 56 | if (count[pic->label] == n) 57 | classesDone++; 58 | } 59 | return bs; 60 | } 61 | 62 | SpatiallySparseDataset SpatiallySparseDataset::subset(int n) { 63 | SpatiallySparseDataset subset; 64 | subset.name = name + std::string(" subset"); 65 | subset.nFeatures = nFeatures; 66 | subset.nClasses = nClasses; 67 | subset.type = type; 68 | auto pick = rng.NchooseM(pictures.size(), n); 69 | for (auto i : pick) { 70 | subset.pictures.push_back(pictures[i]); 71 | } 72 | return subset; 73 | } 74 | 75 | void SpatiallySparseDataset::shuffle() { 76 | std::shuffle(pictures.begin(), pictures.end(), rng.gen); 77 | } 78 | void SpatiallySparseDataset::repeatSamples(int reps) { 79 | int s = pictures.size(); 80 | for (int i = 1; i < reps; ++i) 81 | for (int j = 0; j < s; ++j) 82 | pictures.push_back(pictures[j]); 83 | } 84 | 85 | // http://stackoverflow.com/questions/612097/how-can-i-get-the-list-of-files-in-a-directory-using-c-or-c 86 | std::vector globVector(const std::string &pattern) { 87 | glob_t glob_result; 88 | glob(pattern.c_str(), GLOB_TILDE, NULL, &glob_result); 89 | std::vector files; 90 | for (unsigned int i = 0; i < glob_result.gl_pathc; ++i) { 91 | files.push_back(std::string(glob_result.gl_pathv[i])); 92 | } 93 | globfree(&glob_result); 94 | std::sort(files.begin(), files.end()); 95 | return files; 96 | } 97 | 98 | // Usage: std::vector files = globVector("./*"); 99 | -------------------------------------------------------------------------------- /SparseConvNet/SpatiallySparseDataset.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "Picture.h" 3 | #include "Rng.h" 4 | #include 5 | #include 6 | #include "types.h" 7 | #include "SpatiallySparseDataset.h" 8 | #include 9 | 10 | class SpatiallySparseDataset { 11 | RNG rng; 12 | 13 | public: 14 | std::string name; 15 | std::string header; 16 | std::vector pictures; 17 | bool do_shuffle = true; 18 | int renderSize; 19 | int nFeatures; 20 | int nClasses; 21 | batchType type; 22 | void summary(); 23 | void shuffle(); 24 | SpatiallySparseDataset extractValidationSet(float p = 0.1); 25 | void subsetOfClasses(std::vector activeClasses); 26 | SpatiallySparseDataset subset(int n); 27 | SpatiallySparseDataset balancedSubset(int n); 28 | void repeatSamples(int reps); // Make dataset seem n times bigger (i.e. for 29 | // small datasets to avoid having v. small 30 | // training epochs) 31 | }; 32 | 33 | std::vector globVector(const std::string &pattern); 34 | -------------------------------------------------------------------------------- /SparseConvNet/SpatiallySparseDatasetModelNet.cpp: -------------------------------------------------------------------------------- 1 | // Try adding normal vectors to input data. 2 | 3 | // Data from http://www.icst.pku.edu.cn/zlian/shrec15-non-rigid/index.htm 4 | // 50 classes, 24 exemplars per class: alien ants armadillo bird1 bird2 camel 5 | // cat centaur twoballs dinosaur dog1 dog2 glasses gorilla hand horse lamp paper 6 | // man octopus pliers rabbit santa scissor shark snake spider dino_ske flamingo 7 | // woman Aligator Bull Chick Deer Dragon Elephant Frog Giraffe Kangaroo Mermaid 8 | // Mouse Nunchaku MantaRay Ring Robot Sumotori Tortoise Watch Weedle Woodman 9 | 10 | #include "SpatiallySparseDatasetModelNet.h" 11 | #include 12 | #include 13 | #include 14 | #include 15 | //#include "OpenCVPicture.h" 16 | #include "Off3DFormatPicture.h" 17 | 18 | //#include 19 | #include 20 | #include 21 | 22 | 23 | std::vector getdir (std::string dir) 24 | { 25 | std::vector files = std::vector(); 26 | DIR *dp; 27 | struct dirent *dirp; 28 | if((dp = opendir(dir.c_str())) == NULL) { 29 | std::cout << "Error(" << errno << ") opening " << dir << std::endl; 30 | return files; 31 | } 32 | char dot = '.'; 33 | while ((dirp = readdir(dp)) != NULL) { 34 | std::string entery = std::string(dirp->d_name); 35 | if (entery.at(0) != dot){ 36 | files.push_back(entery); 37 | } 38 | } 39 | closedir(dp); 40 | return files; 41 | } 42 | 43 | SpatiallySparseDataset ModelNetDataSet(int renderSize, int kFold, int fold, batchType batch_type) { 44 | SpatiallySparseDataset dataset; 45 | dataset.type = batch_type; 46 | std::string mode; 47 | if (batch_type == TRAINBATCH) { 48 | dataset.name = "ModelNet (Train subset)"; 49 | mode = std::string("train"); 50 | } else if (batch_type == TESTBATCH) { 51 | dataset.name = "ModelNet (Validation subset)"; 52 | mode = std::string("test"); 53 | } else {} 54 | dataset.nFeatures = 1; 55 | std::string basedir = std::string("Data/ModelNet/"); 56 | std::vector classes = getdir(basedir); 57 | dataset.nClasses = classes.size(); 58 | std::cout << "Number of classes: " << dataset.nClasses << std::endl; 59 | sort(classes.begin(), classes.end()); 60 | for (unsigned int class_id = 0;class_id < classes.size();class_id++) { 61 | std::string class_dir = basedir + classes[class_id] + std::string("/") + mode; 62 | std::vector files = getdir(class_dir); 63 | for (int i = 0; i < files.size(); ++i) { 64 | std::string filename = class_dir + std::string("/") + files[i]; 65 | dataset.pictures.push_back( 66 | new OffSurfaceModelPicture(filename, renderSize, class_id)); 67 | } 68 | } 69 | 70 | return dataset; 71 | }; 72 | -------------------------------------------------------------------------------- /SparseConvNet/SpatiallySparseDatasetModelNet.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "SpatiallySparseDataset.h" 3 | #include 4 | 5 | SpatiallySparseDataset ModelNetDataSet(int renderSize, int kFold, int fold, batchType batch_type); 6 | -------------------------------------------------------------------------------- /SparseConvNet/SpatiallySparseDatasetSHREC2015.cpp: -------------------------------------------------------------------------------- 1 | // Try adding normal vectors to input data. 2 | 3 | // Data from http://www.icst.pku.edu.cn/zlian/shrec15-non-rigid/index.htm 4 | // 50 classes, 24 exemplars per class: alien ants armadillo bird1 bird2 camel 5 | // cat centaur twoballs dinosaur dog1 dog2 glasses gorilla hand horse lamp paper 6 | // man octopus pliers rabbit santa scissor shark snake spider dino_ske flamingo 7 | // woman Aligator Bull Chick Deer Dragon Elephant Frog Giraffe Kangaroo Mermaid 8 | // Mouse Nunchaku MantaRay Ring Robot Sumotori Tortoise Watch Weedle Woodman 9 | 10 | #include "SpatiallySparseDatasetSHREC2015.h" 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include "OpenCVPicture.h" 16 | #include "Off3DFormatPicture.h" 17 | 18 | SpatiallySparseDataset SHREC2015TrainSet(int renderSize, int kFold, int fold) { 19 | SpatiallySparseDataset dataset; 20 | dataset.name = "SHREC2015-Non-Rigid (Train subset)"; 21 | dataset.type = TRAINBATCH; 22 | dataset.nFeatures = 1; 23 | dataset.nClasses = 50; 24 | std::ifstream cla("Data/SHREC15/SHREC15_Non-rigid_ToolKit/test.cla"); 25 | std::string line; 26 | int nClasses, nTotal; 27 | getline(cla, line); // header line 28 | cla >> nClasses >> nTotal; 29 | for (int cl = 0; cl < nClasses; cl++) { 30 | getline(cla, line); // blank line 31 | std::string className; 32 | int parent, nExemplars; 33 | cla >> className >> parent >> nExemplars; 34 | for (int exemp = 0; exemp < nExemplars; exemp++) { 35 | int num; 36 | cla >> num; 37 | std::string filename = 38 | std::string("Data/SHREC15/SHREC15NonRigidTestDB/T") + 39 | std::to_string(num) + std::string(".off"); 40 | if (exemp % kFold != fold) 41 | dataset.pictures.push_back( 42 | new OffSurfaceModelPicture(filename, renderSize, cl)); 43 | } 44 | } 45 | return dataset; 46 | }; 47 | 48 | SpatiallySparseDataset SHREC2015TestSet(int renderSize, int kFold, int fold) { 49 | SpatiallySparseDataset dataset; 50 | dataset.name = "SHREC2015-Non-Rigid (Validation subset)"; 51 | dataset.type = TESTBATCH; 52 | dataset.nFeatures = 1; 53 | dataset.nClasses = 50; 54 | std::ifstream cla("Data/SHREC15/SHREC15_Non-rigid_ToolKit/test.cla"); 55 | std::string line; 56 | int nClasses, nTotal; 57 | getline(cla, line); // header line 58 | cla >> nClasses >> nTotal; 59 | for (int cl = 0; cl < nClasses; cl++) { 60 | getline(cla, line); // blank line 61 | std::string className; 62 | int parent, nExemplars; 63 | cla >> className >> parent >> nExemplars; 64 | for (int exemp = 0; exemp < nExemplars; exemp++) { 65 | int num; 66 | cla >> num; 67 | std::string filename = 68 | std::string("Data/SHREC15/SHREC15NonRigidTestDB/T") + 69 | std::to_string(num) + std::string(".off"); 70 | if (exemp % kFold == fold) 71 | dataset.pictures.push_back( 72 | new OffSurfaceModelPicture(filename, renderSize, cl)); 73 | } 74 | } 75 | return dataset; 76 | }; 77 | -------------------------------------------------------------------------------- /SparseConvNet/SpatiallySparseDatasetSHREC2015.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "SpatiallySparseDataset.h" 3 | #include 4 | 5 | SpatiallySparseDataset SHREC2015TrainSet(int renderSize, int kFold, int fold); 6 | SpatiallySparseDataset SHREC2015TestSet(int renderSize, int kFold, int fold); 7 | -------------------------------------------------------------------------------- /SparseConvNet/SpatiallySparseLayer.cu: -------------------------------------------------------------------------------- 1 | #include "SpatiallySparseLayer.h" 2 | 3 | void SpatiallySparseLayer::scaleWeights(SpatiallySparseBatchInterface &input, 4 | SpatiallySparseBatchInterface &output, 5 | float &scalingUnderneath, 6 | bool topLayer) {} 7 | SpatiallySparseLayer::SpatiallySparseLayer(cudaMemStream &memStream) 8 | : memStream(memStream){}; 9 | SpatiallySparseLayer::~SpatiallySparseLayer(){}; 10 | void SpatiallySparseLayer::loadWeightsFromStream(std::ifstream &f, 11 | bool momentum){}; 12 | void SpatiallySparseLayer::putWeightsToStream(std::ofstream &f, 13 | bool momentum){}; 14 | -------------------------------------------------------------------------------- /SparseConvNet/SpatiallySparseLayer.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include "SpatiallySparseBatchInterface.h" 4 | #include "SpatiallySparseBatch.h" 5 | 6 | class SpatiallySparseLayer { 7 | public: 8 | cudaMemStream &memStream; 9 | virtual void preprocess(SpatiallySparseBatch &batch, 10 | SpatiallySparseBatchInterface &input, 11 | SpatiallySparseBatchInterface &output) = 0; 12 | virtual void forwards(SpatiallySparseBatch &batch, 13 | SpatiallySparseBatchInterface &input, 14 | SpatiallySparseBatchInterface &output) = 0; 15 | virtual void scaleWeights(SpatiallySparseBatchInterface &input, 16 | SpatiallySparseBatchInterface &output, 17 | float &scalingUnderneath, bool topLayer); 18 | virtual void backwards(SpatiallySparseBatch &batch, 19 | SpatiallySparseBatchInterface &input, 20 | SpatiallySparseBatchInterface &output, 21 | float learningRate, float momentum) = 0; 22 | SpatiallySparseLayer(cudaMemStream &memStream); 23 | ~SpatiallySparseLayer(); 24 | virtual void loadWeightsFromStream(std::ifstream &f, bool momentum); 25 | virtual void putWeightsToStream(std::ofstream &f, bool momentum); 26 | virtual int calculateInputSpatialSize(int outputSpatialSize) = 0; 27 | }; 28 | -------------------------------------------------------------------------------- /SparseConvNet/TerminalPoolingLayer.cu: -------------------------------------------------------------------------------- 1 | // Average everything that makes it to the final layer 2 | 3 | #include 4 | #include 5 | #include "utilities.h" 6 | #include "TerminalPoolingLayer.h" 7 | 8 | void terminalGridPoolingRules(SparseGrid &inputGrid, SparseGrid &outputGrid, 9 | int S, int &nOutputSpatialSites, 10 | std::vector &rules) { 11 | // std::cout << inputGrid.mp.size() << std::endl; 12 | if (inputGrid.mp.size() == 0) { // Danger, total loss of information 13 | rules.push_back(inputGrid.backgroundCol); 14 | } else { 15 | for (auto iter = inputGrid.mp.begin(); iter != inputGrid.mp.end(); ++iter) 16 | rules.push_back(iter->second); 17 | } 18 | outputGrid.mp[0] = nOutputSpatialSites++; 19 | rules.resize(S * nOutputSpatialSites, -1); // pad with -1 values 20 | } 21 | 22 | __global__ void dTerminalPool(float *g1, float *g2, int *rules, int nOut, 23 | int S) { 24 | int i = blockIdx.x * nOut; // for output g2 25 | for (int j = threadIdx.x; j < nOut; 26 | j += KERNELBLOCKSIZE) { // nOut is a multiple of KERNELBLOCKSIZE!!! 27 | float t = 0; 28 | int p = 0; 29 | for (; p < S and rules[blockIdx.x * S + p] >= 0; p++) { 30 | t += g1[rules[blockIdx.x * S + p] * nOut + j]; 31 | } 32 | g2[i + j] = t / p; 33 | } 34 | } 35 | 36 | void terminalPool(float *g1, float *g2, int *rules, int count, int S, int nOut, 37 | cudaMemStream &memStream) { 38 | int processed = 0; 39 | while (processed < count) { 40 | int batch = min(32768, count - processed); 41 | dTerminalPool << >> 42 | (g1, g2 + processed * nOut, rules + processed * S, nOut, S); 43 | processed += batch; 44 | } 45 | cudaCheckError(); 46 | } 47 | 48 | __global__ void dTerminalPoolBackProp(float *d1, float *d2, int *rules, 49 | int nOut, int S) { 50 | int i = blockIdx.x * nOut; // for input d2 51 | int maxP = 0; 52 | while (maxP < S and rules[blockIdx.x * S + maxP] >= 0) 53 | ++maxP; 54 | __syncthreads(); // delete line?? 55 | for (int j = threadIdx.x; j < nOut; j += KERNELBLOCKSIZE) { 56 | float t = d2[i + j] / maxP; 57 | for (int p = 0; p < maxP; p++) { 58 | d1[rules[blockIdx.x * S + p] * nOut + j] = t; 59 | } 60 | } 61 | } 62 | 63 | void terminalPoolBackProp(float *d1, float *d2, int *rules, int count, int nOut, 64 | int S, cudaMemStream &memStream) { 65 | int processed = 0; 66 | while (processed < count) { 67 | int batch = min(32768, count - processed); 68 | dTerminalPoolBackProp << >> 69 | (d1, d2 + processed * nOut, rules + processed * S, nOut, S); 70 | processed += batch; 71 | } 72 | cudaCheckError(); 73 | } 74 | 75 | TerminalPoolingLayer::TerminalPoolingLayer(cudaMemStream &memStream, 76 | int poolSize, int S) 77 | : SpatiallySparseLayer(memStream), inSpatialSize(poolSize), 78 | outSpatialSize(1), poolSize(poolSize), S(S) { 79 | std::cout << "TerminalPooling " << poolSize << " " << S << std::endl; 80 | } 81 | void TerminalPoolingLayer::preprocess(SpatiallySparseBatch &batch, 82 | SpatiallySparseBatchInterface &input, 83 | SpatiallySparseBatchInterface &output) { 84 | assert(input.spatialSize == inSpatialSize); 85 | output.nFeatures = input.nFeatures; 86 | output.featuresPresent.hVector() = input.featuresPresent.hVector(); 87 | output.spatialSize = outSpatialSize; 88 | output.nSpatialSites = 0; 89 | output.grids.resize(batch.batchSize); 90 | size_t s = 1; 91 | for (int i = 0; i < batch.batchSize; ++i) 92 | s = std::max(s, input.grids[i].mp.size()); 93 | output.backpropErrors = input.backpropErrors; 94 | for (int item = 0; item < batch.batchSize; item++) 95 | terminalGridPoolingRules(input.grids[item], output.grids[item], s, 96 | output.nSpatialSites, output.rules.hVector()); 97 | } 98 | void TerminalPoolingLayer::forwards(SpatiallySparseBatch &batch, 99 | SpatiallySparseBatchInterface &input, 100 | SpatiallySparseBatchInterface &output) { 101 | output.sub->poolingChoices.resize(output.nSpatialSites * 102 | output.featuresPresent.size()); 103 | output.sub->features.resize(output.nSpatialSites * 104 | output.featuresPresent.size()); 105 | cudaCheckError(); 106 | terminalPool(input.sub->features.dPtr(), output.sub->features.dPtr(), 107 | output.rules.dPtr(), output.nSpatialSites, 108 | output.rules.size() / batch.batchSize, 109 | output.featuresPresent.size(), memStream); 110 | cudaCheckError(); 111 | } 112 | void TerminalPoolingLayer::backwards(SpatiallySparseBatch &batch, 113 | SpatiallySparseBatchInterface &input, 114 | SpatiallySparseBatchInterface &output, 115 | float learningRate, float momentum) { 116 | if (input.backpropErrors) { 117 | input.sub->dfeatures.resize(input.nSpatialSites * 118 | input.featuresPresent.size()); 119 | input.sub->dfeatures.setZero(); 120 | terminalPoolBackProp(input.sub->dfeatures.dPtr(), 121 | output.sub->dfeatures.dPtr(), output.rules.dPtr(), 122 | output.nSpatialSites, output.featuresPresent.size(), 123 | output.rules.size() / batch.batchSize, memStream); 124 | } 125 | } 126 | int TerminalPoolingLayer::calculateInputSpatialSize(int outputSpatialSize) { 127 | assert(outputSpatialSize == 1); 128 | std::cout << "(" << outSpatialSize << "TP" << inSpatialSize << ") "; 129 | return inSpatialSize; 130 | } 131 | -------------------------------------------------------------------------------- /SparseConvNet/TerminalPoolingLayer.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include "SpatiallySparseLayer.h" 4 | #include "Rng.h" 5 | 6 | class TerminalPoolingLayer : public SpatiallySparseLayer { 7 | public: 8 | int inSpatialSize; //==poolSize. 9 | int outSpatialSize; // 1 10 | int poolSize; 11 | int S; // Maximum number of active sites 12 | TerminalPoolingLayer(cudaMemStream &memStream, int poolSize, int S); 13 | void preprocess(SpatiallySparseBatch &batch, 14 | SpatiallySparseBatchInterface &input, 15 | SpatiallySparseBatchInterface &output); 16 | void forwards(SpatiallySparseBatch &batch, 17 | SpatiallySparseBatchInterface &input, 18 | SpatiallySparseBatchInterface &output); 19 | void backwards(SpatiallySparseBatch &batch, 20 | SpatiallySparseBatchInterface &input, 21 | SpatiallySparseBatchInterface &output, float learningRate, 22 | float momentum); 23 | int calculateInputSpatialSize(int outputSpatialSize); 24 | }; 25 | -------------------------------------------------------------------------------- /SparseConvNet/VoxelPicture.cpp: -------------------------------------------------------------------------------- 1 | #include "VoxelPicture.h" 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | 10 | //VoxelPicture(vector[vector[int]] indices, vector[int] features, int spatial_size) 11 | VoxelPicture::VoxelPicture(const std::vector>& indices, 12 | const std::vector>& input_features, 13 | int renderSize, int label, int n_features) 14 | : Picture(label), renderSize(renderSize), n_features(n_features) 15 | { 16 | // assert(indices.size() == features.size()); 17 | // number of features is multiple of number of active voxels 18 | assert(input_features.size() % indices.size() == 0); 19 | assert(n_features == input_features.size() / indices.size()); 20 | 21 | for (size_t feature = 0; feature < n_features; ++feature) { 22 | voxel_features.push_back(0); 23 | } 24 | 25 | for (int i{}; i < indices.size(); ++i) { 26 | assert(indices[i].size() == 3); 27 | int n = indices[i][0] * renderSize * renderSize + indices[i][1] * renderSize + indices[i][2]; 28 | non_empty_indices.push_back(n); 29 | for (int feature_id = 0; feature_id < n_features; ++feature_id) { 30 | voxel_features.push_back(input_features[i][feature_id]); 31 | } 32 | } 33 | is_loaded = true; 34 | } 35 | 36 | void VoxelPicture::loadPicture() { 37 | is_loaded = true; 38 | } 39 | 40 | void VoxelPicture::unloadPicture() { 41 | // non_empty_indices.resize(0); 42 | // voxel_features.resize(0); 43 | is_loaded = false; 44 | } 45 | 46 | 47 | void VoxelPicture::codifyInputData(SparseGrid &grid, 48 | std::vector &features, 49 | int &nSpatialSites, 50 | int spatialSize) { 51 | nSpatialSites = 0; 52 | grid.backgroundCol = nSpatialSites++; 53 | for (int i = 0; i < non_empty_indices.size(); ++i) { 54 | 55 | int n = non_empty_indices[i]; 56 | if (grid.mp.find(n) == grid.mp.end()) { 57 | grid.mp[n] = nSpatialSites++; 58 | // features.push_back(1); // NOTE: here we can push back vector of features 59 | } 60 | } 61 | features = voxel_features; 62 | } 63 | 64 | Picture *VoxelPicture::distort(RNG &rng, batchType type) { 65 | VoxelPicture *pic = new VoxelPicture(*this); 66 | return pic; 67 | } 68 | -------------------------------------------------------------------------------- /SparseConvNet/VoxelPicture.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "Picture.h" 3 | #include "Rng.h" 4 | #include 5 | #include 6 | #include 7 | 8 | class VoxelPicture : public Picture { 9 | private: 10 | std::vector non_empty_indices; 11 | std::vector voxel_features; 12 | public: 13 | int renderSize; 14 | int n_features; 15 | VoxelPicture(const std::vector>& indices, 16 | const std::vector>& input_features, 17 | int renderSize, int label, int n_features); 18 | 19 | virtual ~VoxelPicture() { 20 | non_empty_indices.clear(); 21 | voxel_features.clear(); 22 | } 23 | void loadPicture(); 24 | void unloadPicture() override; 25 | void codifyInputData(SparseGrid &grid, std::vector &features, int &nSpatialSites, int spatialSize); 26 | Picture *distort(RNG &rng, batchType type = TRAINBATCH); 27 | }; 28 | -------------------------------------------------------------------------------- /SparseConvNet/mnist.cpp: -------------------------------------------------------------------------------- 1 | #include "SparseConvNet.h" 2 | #include "NetworkArchitectures.h" 3 | #include "SpatiallySparseDatasetMnist.h" 4 | 5 | int epoch=0; 6 | int cudaDevice=-1; //PCI bus ID, -1 for default GPU 7 | int batchSize=100; 8 | 9 | Picture* OpenCVPicture::distort(RNG& rng, batchType type) { 10 | OpenCVPicture* pic=new OpenCVPicture(*this); 11 | return pic; //No data augmentation 12 | // pic->loadData(); 13 | // if (type==TRAINBATCH) 14 | // pic->jiggle(rng,2); 15 | //return pic; 16 | } 17 | 18 | class CNN : public SparseConvNet { 19 | public: 20 | CNN (int dimension, int nInputFeatures, int nClasses, float p=0.0f, int cudaDevice=-1, int nTop=1); 21 | }; 22 | CNN::CNN 23 | (int dimension, int nInputFeatures, int nClasses, float p, int cudaDevice, int nTop) 24 | : SparseConvNet(dimension,nInputFeatures, nClasses, cudaDevice, nTop) { 25 | int l=0; 26 | addLeNetLayerPOFMP(32*(++l),2,1,2,powf(2,0.5),VLEAKYRELU,0); 27 | addLeNetLayerPOFMP(32*(++l),2,1,2,powf(2,0.5),VLEAKYRELU,0); 28 | addLeNetLayerPOFMP(32*(++l),2,1,2,powf(2,0.5),VLEAKYRELU,0); 29 | addLeNetLayerPOFMP(32*(++l),2,1,2,powf(2,0.5),VLEAKYRELU,0.1); 30 | addLeNetLayerPOFMP(32*(++l),2,1,2,powf(2,0.5),VLEAKYRELU,0.2); 31 | addLeNetLayerPOFMP(32*(++l),2,1,2,powf(2,0.5),VLEAKYRELU,0.3); 32 | addLeNetLayerMP (32*(++l),2,1,1,1, VLEAKYRELU,0.4); 33 | addLeNetLayerMP (32*(++l),1,1,1,1, VLEAKYRELU,0.5); 34 | addSoftmaxLayer(); 35 | } 36 | 37 | int main() { 38 | std::string baseName="weights/mnist"; 39 | 40 | SpatiallySparseDataset trainSet=MnistTrainSet(); 41 | SpatiallySparseDataset testSet=MnistTestSet(); 42 | 43 | trainSet.summary(); 44 | testSet.summary(); 45 | CNN cnn(2,trainSet.nFeatures,trainSet.nClasses,0.0f,cudaDevice); 46 | //DeepCNet cnn(2,5,32,VLEAKYRELU,trainSet.nFeatures,trainSet.nClasses,0.0f,cudaDevice); 47 | 48 | if (epoch>0) 49 | cnn.loadWeights(baseName,epoch); 50 | for (epoch++;;epoch++) { 51 | std::cout <<"epoch: " << epoch << " " << std::flush; 52 | cnn.processDataset(trainSet, batchSize,0.003*exp(-0.01 * epoch)); 53 | if (epoch%10==0) { 54 | cnn.saveWeights(baseName,epoch); 55 | cnn.processDataset(testSet, batchSize); 56 | } 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /SparseConvNet/modelnet.cpp: -------------------------------------------------------------------------------- 1 | //#include "SparseConvNetCUDA.h" 2 | #include "NetworkArchitectures.h" 3 | #include "SpatiallySparseDatasetModelNet.h" 4 | 5 | int epoch = 0; 6 | int cudaDevice = -1; 7 | int batchSize = 10; 8 | 9 | int main(int lenArgs, char *args[]) { 10 | std::string baseName = "weights/ModelNet"; 11 | int fold = 0; 12 | if (lenArgs > 1) 13 | fold = atoi(args[1]); 14 | std::cout << "Fold: " << fold << std::endl; 15 | SpatiallySparseDataset trainSet = ModelNetDataSet(40, 6, fold, TRAINBATCH); 16 | trainSet.summary(); 17 | trainSet.repeatSamples(10); 18 | SpatiallySparseDataset testSet = ModelNetDataSet(40, 6, fold, TESTBATCH); 19 | testSet.summary(); 20 | 21 | DeepC2 cnn(3, 5, 32, VLEAKYRELU, trainSet.nFeatures, trainSet.nClasses, 0.0f, 22 | cudaDevice); 23 | if (epoch > 0) 24 | cnn.loadWeights(baseName, epoch); 25 | for (epoch++; epoch <= 100 * 2; epoch++) { 26 | std::cout << "epoch:" << epoch << ": " << std::flush; 27 | cnn.processDataset(trainSet, batchSize, 0.003 * exp(-0.05 / 2 * epoch)); 28 | if (epoch % 20 == 0) { 29 | cnn.saveWeights(baseName, epoch); 30 | cnn.processDatasetRepeatTest(testSet, batchSize, 3); 31 | } 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /SparseConvNet/shrec2015.cpp: -------------------------------------------------------------------------------- 1 | //#include "SparseConvNetCUDA.h" 2 | #include "NetworkArchitectures.h" 3 | #include "SpatiallySparseDatasetSHREC2015.h" 4 | 5 | int epoch = 0; 6 | int cudaDevice = -1; 7 | int batchSize = 10; 8 | 9 | int main(int lenArgs, char *args[]) { 10 | std::string baseName = "weights/SHREC2015"; 11 | int fold = 0; 12 | if (lenArgs > 1) 13 | fold = atoi(args[1]); 14 | std::cout << "Fold: " << fold << std::endl; 15 | SpatiallySparseDataset trainSet = SHREC2015TrainSet(40, 6, fold); 16 | trainSet.summary(); 17 | trainSet.repeatSamples(10); 18 | SpatiallySparseDataset testSet = SHREC2015TestSet(40, 6, fold); 19 | testSet.summary(); 20 | 21 | DeepC2 cnn(3, 5, 32, VLEAKYRELU, trainSet.nFeatures, trainSet.nClasses, 0.0f, 22 | cudaDevice); 23 | if (epoch > 0) 24 | cnn.loadWeights(baseName, epoch); 25 | for (epoch++; epoch <= 100 * 2; epoch++) { 26 | std::cout << "epoch:" << epoch << ": " << std::flush; 27 | cnn.processDataset(trainSet, batchSize, 0.003 * exp(-0.05 / 2 * epoch)); 28 | if (epoch % 20 == 0) { 29 | cnn.saveWeights(baseName, epoch); 30 | cnn.processDatasetRepeatTest(testSet, batchSize, 3); 31 | } 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /SparseConvNet/shrec2015triangular.cpp: -------------------------------------------------------------------------------- 1 | #include "SparseConvNet.h" 2 | #include "SpatiallySparseDatasetSHREC2015.h" 3 | 4 | int epoch = 0; 5 | int cudaDevice = -1; 6 | int batchSize = 10; 7 | 8 | class DeepC2Triangular : public SparseConvNet { 9 | public: 10 | DeepC2Triangular(int dimension, int l, int k, ActivationFunction fn, 11 | int nInputFeatures, int nClasses, float p = 0.0f, 12 | int cudaDevice = -1, int nTop = 1); 13 | }; 14 | DeepC2Triangular::DeepC2Triangular(int dimension, int l, int k, 15 | ActivationFunction fn, int nInputFeatures, 16 | int nClasses, float p, int cudaDevice, 17 | int nTop) 18 | : SparseConvNet(dimension, nInputFeatures, nClasses, cudaDevice, nTop) { 19 | for (int i = 0; i <= l; i++) 20 | addTriangularLeNetLayerMP((i + 1) * k, (i == l) ? 2 : 2, 1, (i < l) ? 3 : 1, 21 | (i < l) ? 2 : 1, fn, p * i * 1.0f / l); 22 | addSoftmaxLayer(); 23 | } 24 | 25 | int main(int lenArgs, char *args[]) { 26 | std::string baseName = "weights/SHREC2015"; 27 | int fold = atoi(args[1]); 28 | std::cout << "Fold: " << fold << std::endl; 29 | SpatiallySparseDataset trainSet = SHREC2015TrainSet(80, 6, fold); 30 | trainSet.summary(); 31 | trainSet.repeatSamples(10); 32 | SpatiallySparseDataset testSet = SHREC2015TestSet(80, 6, fold); 33 | testSet.summary(); 34 | 35 | DeepC2Triangular cnn(3, 6, 32, VLEAKYRELU, trainSet.nFeatures, 36 | trainSet.nClasses, 0.0f, cudaDevice); 37 | if (epoch > 0) 38 | cnn.loadWeights(baseName, epoch); 39 | for (epoch++; epoch <= 100 * 2; epoch++) { 40 | std::cout << "epoch:" << epoch << ": " << std::flush; 41 | cnn.processDataset(trainSet, batchSize, 0.003 * exp(-0.05 / 2 * epoch)); 42 | if (epoch % 20 == 0) { 43 | cnn.saveWeights(baseName, epoch); 44 | cnn.processDatasetRepeatTest(testSet, batchSize, 3); 45 | } 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /SparseConvNet/types.cpp: -------------------------------------------------------------------------------- 1 | #define TYPES_CPP 2 | #include "types.h" 3 | -------------------------------------------------------------------------------- /SparseConvNet/types.h: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #pragma once 4 | enum batchType { TRAINBATCH, TESTBATCH, UNLABELEDBATCH, RESCALEBATCH }; 5 | #ifdef TYPES_CPP 6 | const char *batchTypeNames[] = {"TRAINBATCH", "TESTBATCH", "UNLABELEDBATCH", 7 | "RESCALEBATCH"}; 8 | #else 9 | extern const char *batchTypeNames[]; 10 | #endif 11 | 12 | enum ActivationFunction { 13 | NOSIGMOID, 14 | RELU, 15 | VLEAKYRELU, 16 | LEAKYRELU, 17 | TANH, 18 | SOFTMAX, 19 | PRELU, 20 | SIGMOID 21 | }; 22 | #ifdef TYPES_CPP 23 | const char *sigmoidNames[] = {"", "ReLU", "VeryLeakyReLU", "LeakyReLU", "tanh", 24 | "Softmax Classification", "PReLU", "Sigmoid"}; 25 | #else 26 | extern const char *sigmoidNames[]; 27 | #endif 28 | 29 | enum OnlineHandwritingEncoding { 30 | Simple, 31 | Octogram, 32 | UndirectedOctogram, 33 | LogSignature1, 34 | LogSignature2, 35 | LogSignature3, 36 | LogSignature4, 37 | SpaceTime3d, 38 | VectorSpaceTime3d 39 | }; 40 | #ifdef TYPES_CPP 41 | int OnlineHandwritingEncodingSize[] = {1, 8, 4, 3, 6, 14, 32, 1, 1 + 2}; 42 | #else 43 | extern int OnlineHandwritingEncodingSize[]; 44 | #endif 45 | 46 | // Accumulator to Keep track of the number of multiply-accumulare operaions 47 | // needed to to process a batch/dataset 48 | #ifdef TYPES_CPP 49 | __int128_t multiplyAddCount; 50 | #else 51 | extern __int128_t multiplyAddCount; 52 | #endif 53 | -------------------------------------------------------------------------------- /SparseConvNet/utilities.cu: -------------------------------------------------------------------------------- 1 | #include "utilities.h" 2 | #include 3 | #include 4 | #include 5 | 6 | std::vector range(int n) { 7 | std::vector ret(n); 8 | for (int i = 0; i < n; i++) 9 | ret[i] = i; 10 | return ret; 11 | } 12 | 13 | // http://stackoverflow.com/questions/101439/the-most-efficient-way-to-implement-an-integer-based-power-function-powint-int 14 | // (sykora) 15 | int ipow(int base, int exp) { 16 | int result = 1; 17 | while (exp) { 18 | if (exp & 1) 19 | result *= base; 20 | exp >>= 1; 21 | base *= base; 22 | } 23 | 24 | return result; 25 | } 26 | 27 | int triangleSize(int linearSize, int dimension) { 28 | int fs = 1; 29 | for (int i = 1; i <= dimension; ++i) 30 | fs = fs * (linearSize + dimension - i) / i; 31 | return fs; 32 | } 33 | 34 | // Assume test.size() is at least k, and k>=1. 35 | template 36 | std::vector vectorTopIndices(std::vector &test, int k) { 37 | std::vector indices(k); 38 | std::vector q(k); 39 | q[0] = test[0]; 40 | for (int i = 1; i < test.size(); i++) { 41 | int j = std::min(i, k); 42 | if (test[i] > q[j - 1]) { 43 | if (i >= k) 44 | --j; 45 | for (; j > 0 and test[i] > q[j - 1]; --j) { 46 | q[j] = q[j - 1]; 47 | indices[j] = indices[j - 1]; 48 | } 49 | } 50 | if (j < k) { 51 | q[j] = test[i]; 52 | indices[j] = i; 53 | } 54 | } 55 | return indices; 56 | } 57 | template std::vector vectorTopIndices(std::vector &test, 58 | int k); 59 | 60 | void cublasError(cublasStatus_t error, const char *file, int linenumber) { 61 | switch (error) { 62 | case CUBLAS_STATUS_SUCCESS: 63 | break; 64 | 65 | case CUBLAS_STATUS_NOT_INITIALIZED: 66 | std::cout << file << " " << linenumber << std::endl; 67 | std::cout << "CUBLAS_STATUS_NOT_INITIALIZED\n"; 68 | break; 69 | 70 | case CUBLAS_STATUS_ALLOC_FAILED: 71 | std::cout << file << " " << linenumber << std::endl; 72 | std::cout << "CUBLAS_STATUS_ALLOC_FAILED\n"; 73 | break; 74 | 75 | case CUBLAS_STATUS_INVALID_VALUE: 76 | std::cout << file << " " << linenumber << std::endl; 77 | std::cout << "CUBLAS_STATUS_INVALID_VALUE\n"; 78 | break; 79 | 80 | case CUBLAS_STATUS_ARCH_MISMATCH: 81 | std::cout << file << " " << linenumber << std::endl; 82 | std::cout << "CUBLAS_STATUS_ARCH_MISMATCH\n"; 83 | break; 84 | 85 | case CUBLAS_STATUS_MAPPING_ERROR: 86 | std::cout << file << " " << linenumber << std::endl; 87 | std::cout << "CUBLAS_STATUS_MAPPING_ERROR\n"; 88 | break; 89 | 90 | case CUBLAS_STATUS_EXECUTION_FAILED: 91 | std::cout << file << " " << linenumber << std::endl; 92 | std::cout << "CUBLAS_STATUS_EXECUTION_FAILED\n"; 93 | break; 94 | 95 | case CUBLAS_STATUS_INTERNAL_ERROR: 96 | std::cout << file << " " << linenumber << std::endl; 97 | std::cout << "CUBLAS_STATUS_INTERNAL_ERROR\n"; 98 | break; 99 | } 100 | } 101 | 102 | int intRoundUp(int a, int d) { return ((a + d - 1) / d) * d; } 103 | int intRound(int a, int d) { return round(a * 1.0 / d) * d; } 104 | 105 | int initializeGPU(int pciBusID) { // pciBusID, or -1 for the first device 106 | int nGPU; 107 | int deviceID = -1; 108 | cudaSafeCall(cudaGetDeviceCount(&nGPU)); 109 | for (int i = 0; i < nGPU; i++) { 110 | cudaDeviceProp prop; 111 | cudaSafeCall(cudaGetDeviceProperties(&prop, i)); 112 | if (i == 0 and pciBusID == -1) 113 | pciBusID = prop.pciBusID; 114 | if (prop.pciBusID == pciBusID) { 115 | std::cout << "*"; 116 | cudaSafeCall(cudaSetDevice(i)); 117 | deviceID = i; 118 | } else { 119 | std::cout << " "; 120 | } 121 | std::cout << prop.pciBusID << " " << prop.name << " " 122 | << (prop.totalGlobalMem >> 20) 123 | << "MB Compute capability: " << prop.major << "." << prop.minor 124 | << std::endl; 125 | } 126 | assert(deviceID >= 0); 127 | // cudaSafeCall(cudaStreamDestroy(stream)); 128 | // cudaSafeCall(cudaStreamCreate(&stream)); 129 | return deviceID; 130 | } 131 | ////////////////////////////////////////////////////////////////////////////////////////////////// 132 | // GEMM for matrices in row major form. 133 | // /////////////////////////////////////////////////////////// 134 | // A is l*m, B is m*r, C is l*r. Set C to alpha A B + beta C. 135 | void d_rowMajorSGEMM_alphaAB_betaC(cublasHandle_t handle, float *A, float *B, 136 | float *C, int l, int m, int r, float alpha, 137 | float beta, const char *file, 138 | int linenumber) { 139 | cublasError(cublasSgemm(handle, CUBLAS_OP_N, CUBLAS_OP_N, r, l, m, &alpha, B, 140 | r, A, m, &beta, C, r), 141 | file, linenumber); 142 | } 143 | // A^t is l*m, B is m*r, C is l*r 144 | void d_rowMajorSGEMM_alphaAtB_betaC(cublasHandle_t handle, float *A, float *B, 145 | float *C, int l, int m, int r, float alpha, 146 | float beta, const char *file, 147 | int linenumber) { 148 | cublasError(cublasSgemm(handle, CUBLAS_OP_N, CUBLAS_OP_T, r, l, m, &alpha, B, 149 | r, A, l, &beta, C, r), 150 | file, linenumber); 151 | } 152 | // A is l*m, B^t is m*r, C is l*r 153 | void d_rowMajorSGEMM_alphaABt_betaC(cublasHandle_t handle, float *A, float *B, 154 | float *C, int l, int m, int r, float alpha, 155 | float beta, const char *file, 156 | int linenumber) { 157 | cublasError(cublasSgemm(handle, CUBLAS_OP_T, CUBLAS_OP_N, r, l, m, &alpha, B, 158 | m, A, m, &beta, C, r), 159 | file, linenumber); 160 | } 161 | // A^t is l*m, B^t is m*r, C is l*r 162 | void d_rowMajorSGEMM_alphaAtBt_betaC(cublasHandle_t handle, float *A, float *B, 163 | float *C, int l, int m, int r, float alpha, 164 | float beta, const char *file, 165 | int linenumber) { 166 | cublasError(cublasSgemm(handle, CUBLAS_OP_T, CUBLAS_OP_T, r, l, m, &alpha, B, 167 | m, A, l, &beta, C, r), 168 | file, linenumber); 169 | } 170 | 171 | cudaMemStream::cudaMemStream() : pinnedMemorySize(1 << 10) { 172 | cudaSafeCall(cudaMallocHost(&pinnedMemory, pinnedMemorySize)); 173 | cudaSafeCall(cudaStreamCreate(&stream)); 174 | } 175 | cudaMemStream::~cudaMemStream() { 176 | cudaSafeCall(cudaStreamDestroy(stream)); 177 | cudaFreeHost(pinnedMemory); 178 | } 179 | -------------------------------------------------------------------------------- /SparseConvNet/utilities.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | // return vector 0,1,...,n-1 8 | std::vector range(int n); 9 | 10 | // Integer powers 11 | // http://stackoverflow.com/questions/101439/the-most-efficient-way-to-implement-an-integer-based-power-function-powint-int 12 | // (sykora) 13 | int ipow(int base, int exp); 14 | 15 | // Calculate the number of points in a d-dimensional triangle/tetrahedro with 16 | // given side length. 17 | // Binomial coefficient \binom{linearSize+dimension-1}{dimension} 18 | int triangleSize(int linearSize, int dimension); 19 | 20 | // Assume test.size() is at least k. 21 | template 22 | std::vector vectorTopIndices(std::vector &test, int k); 23 | 24 | // Utilities to 25 | //// check for CUDA errors, 26 | //// use cublasSgemm for row-major matrix multiplication, 27 | //// ... 28 | 29 | // https://gist.github.com/ashwin/2652488#file-cudaerrorcheck-cu 30 | // Define this to turn on error checking 31 | #define CUDA_ERROR_CHECK 32 | 33 | #define cudaSafeCall(err) __cudaSafeCall(err, __FILE__, __LINE__) 34 | #define cudaCheckError() \ 35 | { __cudaCheckError(__FILE__, __LINE__); } 36 | 37 | inline void __cudaSafeCall(cudaError err, const char *file, const int line) { 38 | #ifdef CUDA_ERROR_CHECK 39 | if (cudaSuccess != err) { 40 | std::cout << "cudaSafeCall() failed at " << file << ":" << line << " " 41 | << cudaGetErrorString(err) << "\n" << std::flush; 42 | exit(-1); 43 | } 44 | #endif 45 | return; 46 | } 47 | 48 | inline void __cudaCheckError(const char *file, const int line) { 49 | #ifdef CUDA_ERROR_CHECK 50 | cudaError err = cudaGetLastError(); 51 | if (cudaSuccess != err) { 52 | std::cout << "cudaCheckError() failed at " << file << ":" << line << " " 53 | << cudaGetErrorString(err) << "\n" << std::flush; 54 | exit(-1); 55 | } 56 | 57 | // More careful checking. However, this will affect performance. Comment away 58 | // if needed. 59 | // err = cudaDeviceSynchronize(); 60 | // if (cudaSuccess != err) { 61 | // std::cout << "cudaCheckError() failed at " << file << ":" << line << " " 62 | // << cudaGetErrorString(err) << "\n" << std::flush; 63 | // exit(-1); 64 | // } 65 | #endif 66 | 67 | return; 68 | } 69 | 70 | void cublasError(cublasStatus_t error, const char *file = 0, 71 | int linenumber = 0); 72 | 73 | #define NTHREADS 512 74 | #define KERNELBLOCKSIZE 32 75 | 76 | int intRound(int a, int d); 77 | int intRoundUp(int a, int d); 78 | 79 | int initializeGPU(int pciBusID); // pciBusID, or -1 for the first device 80 | 81 | ////////////////////////////////////////////////////////////////////////////////////////////////// 82 | // GEMM for matrices in row major form. 83 | // /////////////////////////////////////////////////////////// 84 | // A is l*m, B is m*r, C is l*r. Set C to alpha A B + beta C. 85 | void d_rowMajorSGEMM_alphaAB_betaC(cublasHandle_t handle, float *A, float *B, 86 | float *C, int l, int m, int r, float alpha, 87 | float beta, const char *file = 0, 88 | int linenumber = 0); 89 | // A^t is l*m, B is m*r, C is l*r 90 | void d_rowMajorSGEMM_alphaAtB_betaC(cublasHandle_t handle, float *A, float *B, 91 | float *C, int l, int m, int r, float alpha, 92 | float beta, const char *file = 0, 93 | int linenumber = 0); 94 | // A is l*m, B^t is m*r, C is l*r 95 | void d_rowMajorSGEMM_alphaABt_betaC(cublasHandle_t handle, float *A, float *B, 96 | float *C, int l, int m, int r, float alpha, 97 | float beta, const char *file = 0, 98 | int linenumber = 0); 99 | // A^t is l*m, B^t is m*r, C is l*r 100 | void d_rowMajorSGEMM_alphaAtBt_betaC(cublasHandle_t handle, float *A, float *B, 101 | float *C, int l, int m, int r, float alpha, 102 | float beta, const char *file = 0, 103 | int linenumber = 0); 104 | /////////////////////////////////////////////////////////////////////////////////////////////////// 105 | 106 | class cudaMemStream { 107 | public: 108 | int pinnedMemorySize; 109 | void *pinnedMemory; 110 | cudaStream_t stream; 111 | cudaMemStream(); 112 | ~cudaMemStream(); 113 | }; 114 | -------------------------------------------------------------------------------- /SparseConvNet/vectorCUDA.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include "utilities.h" 4 | // General vector-y object, transferable between CPU and GPU memory 5 | // Todo: Replace with CUDA 6.0+ Unified memory?? 6 | 7 | template class vectorCUDA { 8 | private: 9 | t *d_vec; 10 | unsigned int dsize; // Current size when on the GPU 11 | int dAllocated; // Allocated space on the GPU (>=dsize) 12 | std::vector vec; 13 | 14 | public: 15 | vectorCUDA(bool onGPU = true, unsigned int dsize = 0); 16 | ~vectorCUDA(); 17 | bool onGPU; 18 | void copyToCPU(); 19 | void copyToGPU(); 20 | void copyToCPUAsync(cudaMemStream &memStream); 21 | void copyToGPUAsync(cudaMemStream &memStream); 22 | t *&dPtr(); 23 | std::vector &hVector(); 24 | unsigned int size(); 25 | float meanAbs(); 26 | void multiplicativeRescale(float multiplier); 27 | void setZero(); 28 | void setZero(cudaMemStream &memStream); 29 | void setConstant(float a = 0); 30 | void setUniform(float a = 0, float b = 1); 31 | void setBernoulli(float p = 0.5); 32 | void setNormal(float mean = 0, float sd = 1); 33 | void resize(int n); 34 | void printSubset(const char *name, int nCol, int maxPrint = 10); 35 | void check(const char *file = 0, int linenumber = 0); 36 | void summary(const char *file = 0, int linenumber = 0); 37 | }; 38 | -------------------------------------------------------------------------------- /SparseConvNet/vectorHash.cpp: -------------------------------------------------------------------------------- 1 | #include "vectorHash.h" 2 | //#include 3 | vectorHash::vectorHash() : count(0) {} 4 | std::size_t vectorHash::size() { return count; } 5 | int &vectorHash::operator[](std::size_t idx) { 6 | if (idx >= vec.size()) 7 | vec.resize(idx + 1, -99); 8 | if (vec[idx] == -99) 9 | count++; 10 | return vec[idx]; 11 | } 12 | vectorHashIterator vectorHash::begin() { return vectorHashIterator(this, 0); } 13 | vectorHashIterator vectorHash::end() { 14 | return vectorHashIterator(this, vec.size()); 15 | } 16 | vectorHashIterator vectorHash::find(std::size_t idx) { 17 | if (idx >= vec.size() or vec[idx] == -99) { 18 | return end(); 19 | } else { 20 | return vectorHashIterator(this, idx); 21 | } 22 | } 23 | std::pair 24 | vectorHash::insert(std::pair p) { 25 | if (p.first >= vec.size()) 26 | vec.resize(p.first + 1, -99); 27 | if (vec[p.first] == -99) { 28 | count++; 29 | vec[p.first] = p.second; 30 | return std::make_pair(vectorHashIterator(this, p.first), true); 31 | } else { 32 | return std::make_pair(vectorHashIterator(this, p.first), false); 33 | } 34 | } 35 | void vectorHash::erase(vectorHashIterator iter) { 36 | vec[iter->first] = -99; 37 | count--; 38 | } 39 | 40 | void vectorHashIterator::seek() { 41 | while (first < vh->vec.size() and vh->vec[first] == -99) 42 | first++; 43 | if (first < vh->vec.size()) 44 | second = vh->vec[first]; 45 | } 46 | vectorHashIterator::vectorHashIterator(vectorHash *vh, int x) : vh(vh) { 47 | first = x; 48 | seek(); 49 | } 50 | vectorHashIterator *vectorHashIterator::operator->() { return this; } 51 | vectorHashIterator vectorHashIterator::operator++() { 52 | first++; 53 | seek(); 54 | return *this; 55 | } 56 | -------------------------------------------------------------------------------- /SparseConvNet/vectorHash.h: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | class vectorHashIterator; 5 | 6 | class vectorHash { 7 | std::size_t count; 8 | 9 | public: 10 | std::vector vec; 11 | vectorHash(); 12 | int &operator[](std::size_t idx); 13 | vectorHashIterator begin(); 14 | vectorHashIterator end(); 15 | vectorHashIterator find(std::size_t idx); 16 | std::pair insert(std::pair p); 17 | void erase(vectorHashIterator iter); 18 | std::size_t size(); 19 | }; 20 | 21 | class vectorHashIterator { 22 | private: 23 | vectorHash *vh; 24 | 25 | public: 26 | unsigned int first; 27 | int second; 28 | void seek(); 29 | vectorHashIterator(vectorHash *vh, int x); 30 | vectorHashIterator *operator->(); 31 | vectorHashIterator operator++(); 32 | }; 33 | 34 | inline bool operator==(const vectorHashIterator &lhs, 35 | const vectorHashIterator &rhs) { 36 | return lhs.first == rhs.first; 37 | } 38 | inline bool operator!=(const vectorHashIterator &lhs, 39 | const vectorHashIterator &rhs) { 40 | return lhs.first != rhs.first; 41 | } 42 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | tqdm 2 | autograd 3 | numpy 4 | Cython -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # python neuralnet-setup.py build_ext --inplace 2 | 3 | from distutils.core import setup 4 | import numpy as np 5 | from distutils.extension import Extension 6 | from Cython.Distutils import build_ext 7 | import os 8 | 9 | 10 | # Obtain the numpy include directory. 11 | numpy_include = np.get_include() 12 | 13 | base_sources = [ 14 | "sparseNetwork.pyx", 15 | "_SparseConvNet.pxd" 16 | ] 17 | 18 | sparse_conv_net_path = 'SparseConvNet/' 19 | 20 | 21 | sparse_conv_net_sources = [ 22 | # "BatchProducer.cu", 23 | # "ConvolutionalLayer.cu", 24 | # "ConvolutionalTriangularLayer.cu", 25 | # "IndexLearnerLayer.cu", 26 | # "MaxPoolingLayer.cu", 27 | # "MaxPoolingTriangularLayer.cu", 28 | # "NetworkArchitectures.cpp", 29 | # "NetworkInNetworkLayer.cu", 30 | # "NetworkInNetworkPReLULayer.cu", 31 | # "Picture.cpp", 32 | # "Regions.cu", 33 | # "Rng.cpp", 34 | # "SigmoidLayer.cu", 35 | # "SoftmaxClassifier.cu", 36 | # "SparseConvNet.cu", 37 | # "SparseConvNetCUDA.cu", 38 | # "SpatiallySparseBatch.cu", 39 | # "SpatiallySparseBatchInterface.cu", 40 | # "SpatiallySparseDataset.cpp", 41 | # "SpatiallySparseLayer.cu", 42 | # "TerminalPoolingLayer.cu", 43 | # "readImageToMat.cpp", 44 | # "types.cpp", 45 | # "utilities.cu", 46 | # "vectorCUDA.cu", 47 | # "ReallyConvolutionalLayer.cu", 48 | # "vectorHash.cpp" 49 | ] 50 | 51 | gpp_flags = ["--std=c++11", "-fPIC", "-g" if os.environ.get("DEBUG", False) else "-O3"] 52 | 53 | ext = Extension('PySparseConvNet', 54 | sources=base_sources + [ 55 | os.path.join(sparse_conv_net_path, _path) 56 | for _path in sparse_conv_net_sources 57 | ], 58 | libraries=[ 59 | "opencv_core", 60 | "opencv_highgui", 61 | "opencv_imgproc", 62 | "cublas", 63 | "armadillo", 64 | "python2.7"], 65 | language='c++', 66 | extra_compile_args={ 67 | 'g++': gpp_flags, 68 | 'nvcc': ['-arch=sm_50', '--std=c++11', '-O3', 69 | "-Xcompiler", "-fPIC"] 70 | }, 71 | include_dirs=[numpy_include, '/usr/local/cuda/include']) 72 | 73 | 74 | def customize_compiler_for_nvcc(self): 75 | """inject deep into distutils to customize how the dispatch 76 | to gcc/nvcc works. 77 | 78 | If you subclass UnixCCompiler, it's not trivial to get your subclass 79 | injected in, and still have the right customizations (i.e. 80 | distutils.sysconfig.customize_compiler) run on it. So instead of going 81 | the OO route, I have this. Note, it's kind of like a weird functional 82 | subclassing going on.""" 83 | 84 | # tell the compiler it can processes .cu 85 | self.src_extensions.append('.cu') 86 | 87 | # save references to the default compiler_so and _comple methods 88 | default_compiler_so = self.compiler_so 89 | super = self._compile 90 | self.linker_so = ["nvcc", "--shared", 91 | # "-Xlinker", "--unresolved-symbols=ignore-all" 92 | ] 93 | self.compiler_cxx = None 94 | 95 | # now redefine the _compile method. This gets executed for each 96 | # object but distutils doesn't have the ability to change compilers 97 | # based on source extension: we add it. 98 | def _compile(obj, src, ext, cc_args, extra_postargs, pp_opts): 99 | if os.path.splitext(src)[1] == '.cu': 100 | # use the cuda for .cu files 101 | self.set_executable('compiler_so', 'nvcc') 102 | postargs = extra_postargs['nvcc'] 103 | elif os.path.splitext(src)[1] == '.cpp': 104 | self.set_executable('compiler_so', "g++") 105 | postargs = extra_postargs['g++'] 106 | else: 107 | raise Exception("Unknown file extension.") 108 | 109 | super(obj, src, ext, cc_args, postargs, pp_opts) 110 | # reset the default compiler_so, which we might have changed for cuda 111 | self.compiler_so = default_compiler_so 112 | 113 | # inject our redefined _compile method into the class 114 | self._compile = _compile 115 | 116 | 117 | class custom_build_ext(build_ext): 118 | def build_extensions(self): 119 | customize_compiler_for_nvcc(self.compiler) 120 | build_ext.build_extensions(self) 121 | 122 | setup( 123 | name='PySparseConvNet', 124 | author='Alexandr Notchenko', 125 | version='0.1', 126 | ext_modules=[ext], 127 | # inject our custom trigger 128 | cmdclass={'build_ext': custom_build_ext}) 129 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gangiman/PySparseConvNet/a07b4ade74b049022cf56a533ceac7caea4efe12/tests/__init__.py -------------------------------------------------------------------------------- /tests/test_scn.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | import unittest 3 | import sys 4 | 5 | from PySparseConvNet import SparseNetwork 6 | from PySparseConvNet import SparseDataset 7 | from PySparseConvNet import Off3DPicture 8 | 9 | import os 10 | import numpy as np 11 | 12 | from PySCNutils.networks import create_dC2 13 | 14 | 15 | def load_and_get_weights(deepC2): 16 | baseName = "SparseConvNet/weights/ModelNet" 17 | epoch = 20 18 | deepC2.loadWeights(baseName, epoch) 19 | return deepC2.get_weights() 20 | 21 | 22 | def load_3d_off(): 23 | path = "SparseConvNet/Data/ModelNet/airplane/train/airplane_0511.off" 24 | print("Creating Off3DPicture object") 25 | picture = Off3DPicture(path, 40) 26 | print("Codifying...") 27 | pairs, features = picture.codifyInputData(126) 28 | print ("done") 29 | return pairs 30 | 31 | 32 | def generate_modelnet_dataset(full=False, limit=-1): 33 | number_of_features = 1 34 | renderSize = 40 35 | if full: 36 | data_folder = "SparseConvNet/Data/ModelNet/" 37 | else: 38 | data_folder = "SparseConvNet/Data/_ModelNet/" 39 | class_folders = os.listdir(data_folder) 40 | class_folders.sort() 41 | number_of_classes = len(class_folders) 42 | sparse_dataset = SparseDataset("ModelNet (Train subset)", 'TRAINBATCH', 43 | number_of_features, number_of_classes) 44 | for class_id, folder in enumerate(class_folders): 45 | dirpath = os.path.join(data_folder, folder, 'train') 46 | for _count, filename in enumerate(os.listdir(dirpath)): 47 | if _count > limit > 0: 48 | break 49 | sparse_dataset.add_picture(Off3DPicture( 50 | os.path.join(dirpath, filename), renderSize, label=class_id)) 51 | # sparse_dataset.repeatSamples(10) 52 | return sparse_dataset 53 | 54 | 55 | def learn_simple_network(full=False, batchSize=10, limit=1, epoch=2): 56 | network = create_dC2() 57 | print("Created network") 58 | dataset = generate_modelnet_dataset(full=full, limit=limit) 59 | dataset.summary() 60 | print("Created dataset {0}".format(dataset.name)) 61 | for epoch in xrange(1, epoch): 62 | learning_rate = 0.003 * np.exp(-0.05 / 2 * epoch) 63 | # print("epoch {0}, lr={1} ".format(epoch, learning_rate), end='') 64 | network.processDataset(dataset, batchSize=batchSize, 65 | learningRate=learning_rate) 66 | 67 | 68 | class TestHighLevelLogic(unittest.TestCase): 69 | 70 | def test_dC2_creation_and_loading(self): 71 | network = create_dC2() 72 | self.assertEqual(type(network), SparseNetwork) 73 | layers = load_and_get_weights(network) 74 | self.assertEqual(len(layers), 18) 75 | 76 | def test_Off_file_loading(self): 77 | pairs = load_3d_off() 78 | self.assertTrue(len(pairs) > 0) 79 | 80 | def test_dataset_creation(self): 81 | with self.assertRaises(ValueError): 82 | SparseDataset("Testing DataSet", "FakeBatch", 1, 2) 83 | print("Successfully caught Value Exception.") 84 | ds = SparseDataset("Testing DataSet", "TRAINBATCH", 1, 2) 85 | self.assertEqual(ds.name, "Testing DataSet") 86 | 87 | 88 | class TestTraining(unittest.TestCase): 89 | 90 | def test_simple_training(self): 91 | learn_simple_network(limit=-1) 92 | 93 | def test_predict(self): 94 | unlabeled_dataset = SparseDataset("One pic", 'UNLABELEDBATCH', 1, 1) 95 | network = create_dC2() 96 | num_of_inputs = 5 97 | nClasses = 40 98 | renderSize = 40 99 | test_file = ('SparseConvNet/Data/ModelNet/night_stand/' 100 | 'train/night_stand_0180.off') 101 | for i in range(num_of_inputs): 102 | unlabeled_dataset.add_picture(Off3DPicture(test_file, renderSize)) 103 | matrix_of_preds = network.predict(unlabeled_dataset) 104 | self.assertEqual(matrix_of_preds.shape, (num_of_inputs, nClasses)) 105 | 106 | def test_forward_backward_pass(self): 107 | import numpy as np 108 | batchSize = 10 109 | nInputFeatures = 1 110 | nClasses = 40 111 | p = 0.0 112 | dimension = 3 113 | l = 5 114 | k = 32 115 | fn = 'VLEAKYRELU' 116 | network = SparseNetwork(dimension, nInputFeatures, nClasses) 117 | for i in range(l + 1): 118 | network.addLeNetLayerMP( 119 | (i + 1) * k, 2, 1, 3 if (i < l) else 1, 2 if (i < l) else 1, fn, 120 | p * i * 1.0 / l) 121 | 122 | print("Created network") 123 | 124 | dataset = generate_modelnet_dataset(full=True, limit=1) 125 | 126 | dataset.summary() 127 | 128 | learning_rate = 0.003 129 | 130 | for _bid, batch in enumerate(network.batch_generator(dataset, batchSize)): 131 | print("Processing batch {}".format(_bid)) 132 | activation = network.processBatchForward(batch) 133 | print("Forward pass Done!") 134 | ft = activation['features'] 135 | delta_features = ft - np.random.random(ft.shape) 136 | network.processBatchBackward( 137 | batch, delta_features, learning_rate) 138 | print("Backward pass Done!") 139 | 140 | 141 | class TestDataExtraction(unittest.TestCase): 142 | 143 | def test_layer_activation(self): 144 | network = create_dC2() 145 | network.loadWeights('SparseConvNet/weights/ModelNet_10_repeat_bs100_nthrd10/ModelNet', 200) 146 | lois = [ 147 | network.layer_activations( 148 | Off3DPicture( 149 | 'SparseConvNet/Data/ModelNet/car/test/car_0216.off', 40)), 150 | network.layer_activations( 151 | Off3DPicture( 152 | 'SparseConvNet/Data/ModelNet/sink/test/sink_0133.off', 40)) 153 | ] 154 | self.assertEqual(len(lois[0]), 19) 155 | 156 | if __name__ == '__main__': 157 | if len(sys.argv) < 2: 158 | unittest.main() 159 | else: 160 | if sys.argv[1] == '0': 161 | suite = unittest.TestSuite() 162 | suite.addTest(TestTraining("test_simple_training")) 163 | elif sys.argv[1] == '1': 164 | suite = unittest.TestSuite() 165 | suite.addTest(TestTraining("test_forward_backward_pass")) 166 | else: 167 | exit() 168 | runner = unittest.TextTestRunner() 169 | runner.run(suite) 170 | -------------------------------------------------------------------------------- /tests/voxel_picture_test.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | try: 3 | import PySparseConvNet as pscn 4 | except ImportError: 5 | import sys 6 | import os 7 | sys.path.insert(0, os.path.join(__file__, '..')) 8 | import PySparseConvNet as pscn 9 | 10 | import numpy as np 11 | import unittest 12 | from PySCNutils.networks import create_dC2 13 | 14 | 15 | def create_dummy_sparse_indicies_and_features(spatial_size=6, sparcity=.2, 16 | n_features=1): 17 | total_voxels = np.power(spatial_size, 3) 18 | active_voxels = int(np.ceil(sparcity * total_voxels)) 19 | voxels_ids = np.unique(np.random.randint( 20 | 0, high=total_voxels, size=(active_voxels,))) 21 | voxels_ids.sort() 22 | total_voxels = voxels_ids.shape[0] 23 | features = np.r_[ 24 | np.zeros((n_features, 1), dtype=np.float), 25 | np.random.random((n_features * total_voxels, 1)) 26 | ] 27 | 28 | # size of 3-d tensor, all sides are equal 29 | return convert_pairs_and_features_to_map( 30 | list(zip(voxels_ids, np.arange(1, total_voxels + 1))), 31 | features.reshape(-1).tolist(), 32 | spatial_size, 33 | n_features 34 | ) 35 | 36 | 37 | def convert_pairs_and_features_to_map( 38 | pairs, features, spatial_size, n_features): 39 | features = np.asarray(features) 40 | indices = np.zeros((len(pairs), 3), dtype=np.int) 41 | output_features = np.zeros((len(pairs), n_features), dtype=np.float) 42 | for i, (key_id, feature_idx) in enumerate(pairs): 43 | indices[i, :] = ( 44 | (key_id / spatial_size / spatial_size) % spatial_size, 45 | (key_id / spatial_size) % spatial_size, 46 | key_id % spatial_size) 47 | output_features[i, :] = features[ 48 | feature_idx * (1 + np.arange(n_features))] 49 | 50 | return indices, output_features 51 | 52 | 53 | def get_test_voxel_picture(spatial_size=126): 54 | ind, feat = create_dummy_sparse_indicies_and_features( 55 | spatial_size=spatial_size, sparcity=0.05) 56 | return pscn.PyVoxelPicture(ind, feat, spatial_size) 57 | 58 | 59 | class TestVoxelPicture(unittest.TestCase): 60 | 61 | def test_constructor_from_row_matrix(self): 62 | # indices - an array of shape (num_points, 3), 63 | # its columns are indices x,y,z 64 | indices = np.array([ 65 | [0, 0, 0], 66 | [1, 0, 5], 67 | [3, 4, 2], 68 | [5, 5, 5] 69 | ], dtype=np.int) 70 | # size of 3-d tensor, all sides are equal 71 | spatial_size = 6 72 | n_features = 1 73 | # features of size (num_points, num_features) 74 | # in this case num_features=1 75 | features = np.ones((indices.shape[0], 1), dtype=np.float) 76 | # creating a picture object 77 | pic = pscn.PyVoxelPicture(indices, features, spatial_size) 78 | # extracting indices and features must be the same 79 | # as ones it was created from 80 | returned_indices, returned_features = pic.codifyInputData(spatial_size) 81 | sparse_indicies, sparse_features = convert_pairs_and_features_to_map( 82 | returned_indices, returned_features, spatial_size, n_features) 83 | self.assertEqual(set(map(tuple, sparse_indicies.tolist())), 84 | set(map(tuple, indices.tolist()))) 85 | np.testing.assert_almost_equal(features, sparse_features) 86 | 87 | def test_layers_activation_for_voxel_picture(self): 88 | pic = get_test_voxel_picture() 89 | ds = pscn.SparseDataset('test_ds', 'UNLABELEDBATCH', 1, 40) 90 | ds.add_voxel_picture(pic) 91 | net = create_dC2() 92 | lois = net.layer_activations_for_dataset(ds) 93 | self.assertEqual(len(lois), 19) 94 | 95 | def test_batch_processing_for_voxel_picture(self): 96 | pic = get_test_voxel_picture() 97 | ds = pscn.SparseDataset('test_ds', 'TRAINBATCH', 1, 40) 98 | ds.add_voxel_picture(pic) 99 | net = create_dC2() 100 | batch_gen = net.batch_generator(ds, 1) 101 | batch = next(batch_gen) 102 | batch_output = net.processBatchForward(batch) 103 | self.assertEqual(batch_output['spatialSize'], 1) 104 | --------------------------------------------------------------------------------