├── .gitignore
├── BatchProducer.cpp
├── BatchProducer.h
├── ConvolutionalLayer.cpp
├── ConvolutionalLayer.h
├── ConvolutionalTriangularLayer.cpp
├── ConvolutionalTriangularLayer.h
├── Data
    ├── CASIA_pot_files
    │   ├── .gitignore
    │   └── README.md
    ├── CIFAR10
    │   ├── .gitignore
    │   └── README.md
    ├── CIFAR100
    │   ├── .gitignore
    │   └── README.md
    ├── MNIST
    │   ├── .gitignore
    │   └── README.md
    └── imagenet2012
    │   └── .gitignore
├── IndexLearnerLayer.cpp
├── IndexLearnerLayer.h
├── Makefile
├── MaxPoolingLayer.cpp
├── MaxPoolingLayer.h
├── MaxPoolingTriangularLayer.cpp
├── MaxPoolingTriangularLayer.h
├── NetworkArchitectures.cpp
├── NetworkArchitectures.h
├── NetworkInNetworkLayer.cpp
├── NetworkInNetworkLayer.h
├── OnlineHandwritingPicture.cpp
├── OnlineHandwritingPicture.h
├── OnlineHandwritingTriangularPicture.cpp
├── OpenCVPicture.cpp
├── OpenCVPicture.h
├── OpenCVTriangularPicture.cpp
├── Picture.cpp
├── Picture.h
├── README.md
├── Regions.cpp
├── Regions.h
├── Rng.cpp
├── Rng.h
├── SigmoidLayer.cpp
├── SigmoidLayer.h
├── SoftmaxClassifier.cpp
├── SoftmaxClassifier.h
├── SparseConvNet.cpp
├── SparseConvNet.h
├── SparseConvNetCUDA.cpp
├── SparseConvNetCUDA.h
├── SparseGrid.h
├── SpatiallySparseBatch.cpp
├── SpatiallySparseBatch.h
├── SpatiallySparseBatchInterface.cpp
├── SpatiallySparseBatchInterface.h
├── SpatiallySparseDataset.cpp
├── SpatiallySparseDataset.h
├── SpatiallySparseDatasetCIFAR10.cpp
├── SpatiallySparseDatasetCIFAR10.h
├── SpatiallySparseDatasetCIFAR100.cpp
├── SpatiallySparseDatasetCIFAR100.h
├── SpatiallySparseDatasetCIFAR10FullyConnected.cpp
├── SpatiallySparseDatasetCIFAR10FullyConnected.h
├── SpatiallySparseDatasetCasiaOLHWDB.cpp
├── SpatiallySparseDatasetCasiaOLHWDB.h
├── SpatiallySparseDatasetImageNet2012.cpp
├── SpatiallySparseDatasetImageNet2012.h
├── SpatiallySparseDatasetMnist.cpp
├── SpatiallySparseDatasetMnist.h
├── SpatiallySparseDatasetOpenCV.cpp
├── SpatiallySparseDatasetOpenCV.h
├── SpatiallySparseLayer.cpp
├── SpatiallySparseLayer.h
├── TerminalPoolingLayer.cpp
├── TerminalPoolingLayer.h
├── casia.cpp
├── casia3d.cpp
├── cifar10.cpp
├── cifar100.cpp
├── cifar10FullyConnected.cpp
├── cifar10fmp.cpp
├── cifar10indexLearning.cpp
├── cifar10triangular.cpp
├── cudaUtilities.cpp
├── cudaUtilities.h
├── gbcodes3755.h
├── imagenet2012.cpp
├── mnist.cpp
├── readImageToMat.cpp
├── readImageToMat.h
├── signature.h
├── types.cpp
├── types.h
├── utilities.cpp
├── utilities.h
├── vectorCUDA.cpp
├── vectorCUDA.h
├── vectorHash.cpp
├── vectorHash.h
└── weights
    └── .gitignore


/.gitignore:
--------------------------------------------------------------------------------
 1 | /confusionMatrix
 2 | *.o
 3 | cifar10
 4 | cifar100
 5 | cifar10triangular
 6 | cvap_rha
 7 | imagenet2012fmp.cpp
 8 | imagenet2012triangular
 9 | casia
10 | casia3d
11 | shrec2015
12 | shrec2015_
13 | mnist
14 | kagglePlankton
15 | kagglePlankton2
16 | kagglePlanktonQuick
17 | *.features
18 | cifar10indexLearning
19 | old/
20 | Data/plankton-JO-2/
21 | 


--------------------------------------------------------------------------------
/BatchProducer.cpp:
--------------------------------------------------------------------------------
  1 | #include "BatchProducer.h"
  2 | #include "SpatiallySparseBatch.h"
  3 | #include "utilities.h"
  4 | #include "cudaUtilities.h"
  5 | #include <algorithm>
  6 | #include <functional>
  7 | #include <mutex>
  8 | #include <chrono>
  9 | #include <cassert>
 10 | #define NBATCHPRODUCERTHREADS 10
 11 | 
 12 | bool batchProducerBatchesInitialized; //intially false
 13 | std::mutex batchLock[NBATCHPRODUCERTHREADS];
 14 | SpatiallySparseBatch* batchPool[NBATCHPRODUCERTHREADS];
 15 | 
 16 | //don't delete the returned threads when done with them; ownership is with batchPool
 17 | SpatiallySparseBatch* BatchProducer::nextBatch() {
 18 |   if (batchCounter>=0) {
 19 |     batchPool[batchCounter%NBATCHPRODUCERTHREADS]->reset();
 20 |     batchLock[batchCounter%NBATCHPRODUCERTHREADS].unlock();
 21 |   }
 22 |   batchCounter++;
 23 | 
 24 |   if (batchCounter<nBatches) {
 25 |     int cc=batchCounter%NBATCHPRODUCERTHREADS;
 26 |     for(bool ready=false;!ready;) {
 27 |       ready=batchLock[cc].try_lock();
 28 |       if (ready and batchPool[cc]->batchSize==0) {   //not really ready
 29 |         batchLock[cc].unlock();
 30 |         ready=false;
 31 |       }
 32 |     }
 33 |     return batchPool[cc];
 34 |   } else {
 35 |     for (int i=0;i<NBATCHPRODUCERTHREADS;i++)
 36 |       workers[i].join();
 37 |     return NULL;
 38 |   }
 39 | }
 40 | 
 41 | void BatchProducer::batchProducerThread(int nThread) {
 42 |   RNG rng;
 43 |   for (int c=nThread;c<nBatches;c+=NBATCHPRODUCERTHREADS) {
 44 |     int cc=c%NBATCHPRODUCERTHREADS;
 45 |     batchLock[cc].lock();
 46 |     while(batchPool[cc]->batchSize>0) { //Don't overwrite unused batches
 47 |       batchLock[cc].unlock();
 48 |       std::this_thread::sleep_for(std::chrono::milliseconds(10));
 49 |       batchLock[cc].lock();
 50 |     }
 51 |     batchPool[cc]->type=dataset.type;
 52 |     batchPool[cc]->interfaces[0].nFeatures=dataset.nFeatures;
 53 |     batchPool[cc]->interfaces[0].spatialSize=spatialSize;
 54 |     batchPool[cc]->interfaces[0].featuresPresent.vec=range(dataset.nFeatures);
 55 |     for (int i=c*batchSize;i<std::min((c+1)*batchSize,(int)(dataset.pictures.size()));i++) {
 56 |       Picture* pic=dataset.pictures[permutation[i]]->distort(rng,dataset.type);
 57 |       batchPool[cc]->sampleNumbers.push_back(permutation[i]);
 58 |       batchPool[cc]->batchSize++;
 59 |       batchPool[cc]->interfaces[0].grids.push_back(SparseGrid());
 60 |       batchPool[cc]->labels.vec.push_back(pic->label);
 61 |       pic->codifyInputData
 62 |         (batchPool[cc]->interfaces[0].grids.back(),
 63 |          batchPool[cc]->interfaces[0].sub->features.vec,
 64 |          batchPool[cc]->interfaces[0].nSpatialSites,
 65 |          batchPool[cc]->interfaces[0].spatialSize);
 66 |       if (pic!=dataset.pictures[permutation[i]])
 67 |         delete pic;
 68 |     }
 69 |     assert(batchPool[cc]->interfaces[0].sub->features.size()
 70 |            ==
 71 |            batchPool[cc]->interfaces[0].nFeatures*batchPool[cc]->interfaces[0].nSpatialSites);
 72 |     if (cnn.inputNormalizingConstants.size()>0) {
 73 |       std::vector<float> &features=batchPool[cc]->interfaces[0].sub->features.vec;
 74 |       for (int i=0;i<features.size();++i)
 75 |         features[i]*=cnn.inputNormalizingConstants[i%(batchPool[cc]->interfaces[0].nFeatures)];
 76 |     }
 77 |     for (int i=0; i<cnn.layers.size();i++)
 78 |       cnn.layers[i]->preprocess(*batchPool[cc],batchPool[cc]->interfaces[i],batchPool[cc]->interfaces[i+1]);
 79 | 
 80 |     batchLock[cc].unlock();
 81 |   }
 82 | }
 83 | 
 84 | BatchProducer::BatchProducer (SparseConvNetCUDA& cnn,
 85 |                               SpatiallySparseDataset &dataset,
 86 |                               int spatialSize, int batchSize) :
 87 |   cnn(cnn), batchCounter(-1),
 88 |   dataset(dataset),
 89 |   spatialSize(spatialSize), batchSize(batchSize) {
 90 |   if (!batchProducerBatchesInitialized) {
 91 |     for (int c=0;c<NBATCHPRODUCERTHREADS;c++) {
 92 |       //Set up shared memory
 93 |       batchPool[c]=new SpatiallySparseBatch();
 94 |       batchPool[c]->interfaces.resize(cnn.layers.size()+1);
 95 |       batchPool[c]->interfaces[0].sub = &batchPool[c]->inputSub; //unique for each batch
 96 |       for (int i=0;i<cnn.layers.size();++i)
 97 |         batchPool[c]->interfaces[i+1].sub=&cnn.layers[i]->sub; //shared between batches
 98 |       //Initialize batches
 99 |       batchPool[c]->reset();
100 |     }
101 |     batchProducerBatchesInitialized=true;
102 |   }
103 |   nBatches=(dataset.pictures.size()+batchSize-1)/batchSize;
104 |   permutation=range(dataset.pictures.size());
105 |   if (dataset.type==TRAINBATCH) {
106 |     RNG rng;
107 |     rng.vectorShuffle(permutation);
108 |   }
109 |   for (int nThread=0; nThread<NBATCHPRODUCERTHREADS; nThread++)
110 |     workers.emplace_back(std::thread(&BatchProducer::batchProducerThread,this,nThread));
111 | }
112 | BatchProducer::~BatchProducer() {
113 |   if (batchCounter<nBatches) {
114 |     SpatiallySparseBatch* batch=nextBatch();
115 |     while(batch) {
116 |       batch->reset();
117 |       batch=nextBatch();
118 |     }
119 |   }
120 | }
121 | 
122 | 
123 | void batchProducerResourcesCleanup() {
124 |   if (batchProducerBatchesInitialized) {
125 |     // TODO: delete stuff
126 |     batchProducerBatchesInitialized=false;
127 |   }
128 | }
129 | 


--------------------------------------------------------------------------------
/BatchProducer.h:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | #include <thread>
 3 | #include <vector>
 4 | #include "SpatiallySparseBatch.h"
 5 | #include "SpatiallySparseDataset.h"
 6 | #include "SparseConvNetCUDA.h"
 7 | 
 8 | class BatchProducer {
 9 | public:
10 |   SparseConvNetCUDA& cnn;
11 |   int batchCounter;
12 |   int nBatches;
13 |   std::vector<std::thread> workers;
14 |   SpatiallySparseDataset& dataset;
15 |   int batchSize;
16 |   int spatialSize;
17 |   std::vector<int> permutation;
18 |   SpatiallySparseBatch* nextBatch();
19 |   void batchProducerThread(int nThread);
20 |   BatchProducer (SparseConvNetCUDA& cnn, SpatiallySparseDataset &dataset, int spatialSize, int batchSize);
21 |   ~BatchProducer();
22 | };
23 | 
24 | void batchProducerResourcesCleanup();
25 | 


--------------------------------------------------------------------------------
/ConvolutionalLayer.cpp:
--------------------------------------------------------------------------------
  1 | //This does not really implement a convolution. It just gathers data together in prepartion for matrix muliplictation. "Proper convolution" = "ConvolutionalLayer" + "NetworkInNetworkLayer"
  2 | 
  3 | #include "ConvolutionalLayer.h"
  4 | #include <iostream>
  5 | #include <vector>
  6 | #include <cassert>
  7 | #include "cudaUtilities.h"
  8 | #include "utilities.h"
  9 | #include "Regions.h"
 10 | 
 11 | void propForwardToMatrixMultiply(std::vector<float>& inFeatures, std::vector<float>& outFeatures, std::vector<int>& rules, int count, int nIn) {
 12 |   for (int row=0; row<count;row++) {
 13 |     int r=rules[row];
 14 |     for (int i=0;i<nIn;i++) {
 15 |       outFeatures[row*nIn+i]=inFeatures[r*nIn+i];
 16 |     }
 17 |   }
 18 | }
 19 | void propBackwardFromMatrixMultiply(std::vector<float>& inDFeatures, std::vector<float>& outDFeatures, std::vector<int>& rules, int count, int nIn) {
 20 |   for (int row=0; row<count;row++) {
 21 |     int r=rules[row];
 22 |     for (int i=0;i<nIn;i++) {
 23 |       inDFeatures[r*nIn+i]+=outDFeatures[row*nIn+i];
 24 |     }
 25 |   }
 26 | }
 27 | 
 28 | template <typename t> void convolutionFeaturesPresent(std::vector<t>& d_src, std::vector<t>& d_dest, int nf, int nfp, int nCopies) {
 29 |   for (int i=0;i<nfp*nCopies;++i) {
 30 |     d_dest[i]=d_src[i%nfp]+nf*(i/nfp);
 31 |   }
 32 | }
 33 | template void convolutionFeaturesPresent<int>(std::vector<int>& d_src, std::vector<int>& d_dest, int nf, int nfp, int nCopies);
 34 | 
 35 | ConvolutionalLayer::ConvolutionalLayer(int filterSize,
 36 |                                        int filterStride,
 37 |                                        int dimension,
 38 |                                        int nFeaturesIn,
 39 |                                        int minActiveInputs) :
 40 |   filterSize(filterSize),
 41 |   filterStride(filterStride),
 42 |   dimension(dimension),
 43 |   nFeaturesIn(nFeaturesIn),
 44 |   minActiveInputs(minActiveInputs) {
 45 |   fs=ipow(filterSize,dimension);
 46 |   nFeaturesOut=fs*nFeaturesIn;
 47 |   std::cout << "Convolution "
 48 |             << filterSize <<"^" <<dimension<< "x"<< nFeaturesIn
 49 |             << "->" << nFeaturesOut;
 50 |   if (filterStride>1)
 51 |     std::cout << " stride:" << filterStride;
 52 |   if (minActiveInputs>1)
 53 |     std::cout << " minActiveInputs:"  << minActiveInputs;
 54 |   std::cout << std::endl;
 55 |   }
 56 | void ConvolutionalLayer::preprocess
 57 | (SpatiallySparseBatch &batch,
 58 |  SpatiallySparseBatchInterface &input,
 59 |  SpatiallySparseBatchInterface &output) {
 60 |   output.nFeatures=nFeaturesOut;
 61 |   assert(input.nFeatures==nFeaturesIn);
 62 |   assert(input.spatialSize>=filterSize);
 63 |   assert((input.spatialSize-filterSize)%filterStride==0);
 64 |   output.spatialSize=(input.spatialSize-filterSize)/filterStride+1;
 65 |   output.nSpatialSites=0;
 66 |   output.grids.resize(batch.batchSize);
 67 |   output.backpropErrors=input.backpropErrors;
 68 |   RegularPoolingRegions regions(inSpatialSize, outSpatialSize,dimension,filterSize, filterStride);
 69 |   for (int item=0;item<batch.batchSize;item++) {
 70 |     gridRules(input.grids[item],
 71 |               output.grids[item],
 72 |               regions,
 73 |               output.nSpatialSites,
 74 |               output.rules.vec,
 75 |               minActiveInputs);
 76 |   }
 77 |   output.featuresPresent.resize(input.featuresPresent.size()*fs);
 78 |   convolutionFeaturesPresent(input.featuresPresent.vec, output.featuresPresent.vec, input.nFeatures, input.featuresPresent.size(), fs);
 79 | }
 80 | void ConvolutionalLayer::forwards
 81 | (SpatiallySparseBatch &batch,
 82 |  SpatiallySparseBatchInterface &input,
 83 |  SpatiallySparseBatchInterface &output) {
 84 |   output.sub->features.resize(output.nSpatialSites*output.featuresPresent.size());
 85 |   propForwardToMatrixMultiply(input.sub->features.vec,
 86 |                               output.sub->features.vec,
 87 |                               output.rules.vec,
 88 |                               output.nSpatialSites*fs,
 89 |                               input.featuresPresent.size());
 90 | }
 91 | void ConvolutionalLayer::backwards
 92 | (SpatiallySparseBatch &batch,
 93 |  SpatiallySparseBatchInterface &input,
 94 |  SpatiallySparseBatchInterface &output,
 95 |  float learningRate,
 96 |  float momentum) {
 97 |   if (input.backpropErrors) {
 98 |     input.sub->dfeatures.resize(input.nSpatialSites*input.featuresPresent.size());
 99 |     input.sub->dfeatures.setZero();
100 |     propBackwardFromMatrixMultiply(input.sub->dfeatures.vec,
101 |                                    output.sub->dfeatures.vec,
102 |                                    output.rules.vec,
103 |                                    output.nSpatialSites*fs,
104 |                                    input.featuresPresent.size());
105 |   }
106 | }
107 | int ConvolutionalLayer::calculateInputSpatialSize(int outputSpatialSize) {
108 |   outSpatialSize=outputSpatialSize;
109 |   inSpatialSize=filterSize+(outputSpatialSize-1)*filterStride;
110 |   return inSpatialSize;
111 | }
112 | 


--------------------------------------------------------------------------------
/ConvolutionalLayer.h:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | #include "SpatiallySparseLayer.h"
 3 | 
 4 | class ConvolutionalLayer : public SpatiallySparseLayer {
 5 | private:
 6 |   int fs;
 7 | public:
 8 |   int inSpatialSize;
 9 |   int outSpatialSize;
10 |   int filterSize;
11 |   int filterStride;
12 |   int dimension;
13 |   int nFeaturesIn;
14 |   int nFeaturesOut;
15 |   int minActiveInputs;
16 |   ConvolutionalLayer(int filterSize,
17 |                      int filterStride,
18 |                      int dimension,
19 |                      int nFeaturesIn,
20 |                      int minActiveInputs=1);
21 |   void preprocess
22 |   (SpatiallySparseBatch &batch,
23 |    SpatiallySparseBatchInterface &input,
24 |    SpatiallySparseBatchInterface &output);
25 |   void forwards
26 |   (SpatiallySparseBatch &batch,
27 |    SpatiallySparseBatchInterface &input,
28 |    SpatiallySparseBatchInterface &output);
29 |   void backwards(SpatiallySparseBatch &batch,
30 |                  SpatiallySparseBatchInterface &input,
31 |                  SpatiallySparseBatchInterface &output,
32 |                  float learningRate,
33 |                  float momentum);
34 |   int calculateInputSpatialSize(int outputSpatialSize);
35 | };
36 | 
37 | template <typename t> void convolutionFeaturesPresent(std::vector<t>& d_src, std::vector<t>& d_dest, int nf, int nfp, int nCopies);
38 | void propForwardToMatrixMultiply(std::vector<float>& inFeatures, std::vector<float>& outFeatures, std::vector<int>& rules, int count, int nIn);
39 | void propBackwardFromMatrixMultiply(std::vector<float>& inDFeatures, std::vector<float>& outDFeatures, std::vector<int>& rules, int count, int nIn);
40 | 


--------------------------------------------------------------------------------
/ConvolutionalTriangularLayer.cpp:
--------------------------------------------------------------------------------
 1 | #include "ConvolutionalTriangularLayer.h"
 2 | #include "ConvolutionalLayer.h"
 3 | #include <iostream>
 4 | #include <cassert>
 5 | #include "cudaUtilities.h"
 6 | #include "utilities.h"
 7 | #include "Regions.h"
 8 | 
 9 | ConvolutionalTriangularLayer::ConvolutionalTriangularLayer
10 | (int filterSize, int filterStride, int dimension, int nFeaturesIn, int minActiveInputs)
11 |   : filterSize(filterSize), filterStride(filterStride), dimension(dimension),
12 |     nFeaturesIn(nFeaturesIn), minActiveInputs(minActiveInputs) {
13 |   fs=triangleSize(filterSize, dimension);
14 |   nFeaturesOut=fs*nFeaturesIn;
15 |   std::cout << dimension << "D ConvolutionalTriangularLayer side-length=" << filterSize
16 |             << " " << nFeaturesIn << "x" << fs << "->" << nFeaturesOut;
17 |   if (filterStride>1)
18 |     std::cout << ", stride " << filterStride;
19 |   std::cout << std::endl;
20 | }
21 | void ConvolutionalTriangularLayer::preprocess
22 | (SpatiallySparseBatch &batch,
23 |  SpatiallySparseBatchInterface &input,
24 |  SpatiallySparseBatchInterface &output) {
25 |   assert(input.nFeatures==nFeaturesIn);
26 |   assert(input.spatialSize>=filterSize);
27 |   assert((input.spatialSize-filterSize)%filterStride==0);
28 |   output.nFeatures=nFeaturesOut;
29 |   output.spatialSize=(input.spatialSize-filterSize)/filterStride+1;
30 |   output.nSpatialSites=0;
31 |   output.grids.resize(batch.batchSize);
32 |   output.backpropErrors=input.backpropErrors;
33 |   RegularPoolingRegionsTriangular regions(inSpatialSize, outSpatialSize,dimension,filterSize, filterStride);
34 |   for (int item=0;item<batch.batchSize;item++)
35 |     gridRulesTriangular(input.grids[item],
36 |                         output.grids[item],
37 |                         regions,
38 |                         output.nSpatialSites,
39 |                         output.rules.vec,
40 |                         minActiveInputs);
41 |   output.featuresPresent.resize(input.featuresPresent.size()*fs);
42 |   convolutionFeaturesPresent(input.featuresPresent.vec, output.featuresPresent.vec, input.nFeatures, input.featuresPresent.size(), fs);
43 | 
44 | }
45 | void ConvolutionalTriangularLayer::forwards
46 | (SpatiallySparseBatch &batch,
47 |  SpatiallySparseBatchInterface &input,
48 |  SpatiallySparseBatchInterface &output) {
49 |   output.sub->features.resize(output.nSpatialSites*output.featuresPresent.size());
50 |   propForwardToMatrixMultiply(input.sub->features.vec,
51 |                               output.sub->features.vec,
52 |                               output.rules.vec,
53 |                               output.nSpatialSites*fs,
54 |                               input.featuresPresent.size());
55 | }
56 | void ConvolutionalTriangularLayer::backwards
57 | (SpatiallySparseBatch &batch,
58 |  SpatiallySparseBatchInterface &input,
59 |  SpatiallySparseBatchInterface &output,
60 |  float learningRate,
61 |  float momentum) {
62 |   if (input.backpropErrors) {
63 |     input.sub->dfeatures.resize(input.nSpatialSites*input.featuresPresent.size());
64 |     input.sub->dfeatures.setZero();
65 |     propBackwardFromMatrixMultiply(input.sub->dfeatures.vec,
66 |                                    output.sub->dfeatures.vec,
67 |                                    output.rules.vec,
68 |                                    output.nSpatialSites*fs,
69 |                                    input.featuresPresent.size());
70 |   }
71 | }
72 | int ConvolutionalTriangularLayer::calculateInputSpatialSize(int outputSpatialSize) {
73 |   outSpatialSize=outputSpatialSize;
74 |   inSpatialSize=filterSize+(outputSpatialSize-1)*filterStride;
75 |   std::cout << "(" << outSpatialSize <<"C" <<inSpatialSize << ") ";
76 |   return inSpatialSize;
77 | }
78 | 


--------------------------------------------------------------------------------
/ConvolutionalTriangularLayer.h:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | #include "SpatiallySparseLayer.h"
 3 | 
 4 | class ConvolutionalTriangularLayer : public SpatiallySparseLayer {
 5 | private:
 6 |   int fs;
 7 | public:
 8 |   int inSpatialSize;
 9 |   int outSpatialSize;
10 |   int filterSize;
11 |   int filterStride;
12 |   int dimension;
13 |   int nFeaturesIn;
14 |   int nFeaturesOut;
15 |   int minActiveInputs;
16 |   ConvolutionalTriangularLayer(int filterSize,
17 |                                int filterStride,
18 |                                int dimension,
19 |                                int nFeaturesIn,
20 |                                int minActiveInputs);
21 |   void preprocess
22 |   (SpatiallySparseBatch &batch,
23 |    SpatiallySparseBatchInterface &input,
24 |    SpatiallySparseBatchInterface &output);
25 |   void forwards
26 |   (SpatiallySparseBatch &batch,
27 |    SpatiallySparseBatchInterface &input,
28 |    SpatiallySparseBatchInterface &output);
29 |   void backwards(SpatiallySparseBatch &batch,
30 |                  SpatiallySparseBatchInterface &input,
31 |                  SpatiallySparseBatchInterface &output,
32 |                  float learningRate,
33 |                  float momentum);
34 |   int calculateInputSpatialSize(int outputSpatialSize);
35 | };
36 | 


--------------------------------------------------------------------------------
/Data/CASIA_pot_files/.gitignore:
--------------------------------------------------------------------------------
1 | *.pot


--------------------------------------------------------------------------------
/Data/CASIA_pot_files/README.md:
--------------------------------------------------------------------------------
1 | Obtain POT files from http://www.nlpr.ia.ac.cn/databases/handwriting/home.html and put them in a directory Data/CASIA_pot_files/


--------------------------------------------------------------------------------
/Data/CIFAR10/.gitignore:
--------------------------------------------------------------------------------
1 | *.bin


--------------------------------------------------------------------------------
/Data/CIFAR10/README.md:
--------------------------------------------------------------------------------
1 | http://www.cs.toronto.edu/~kriz/cifar.html
2 | http://www.cs.toronto.edu/~kriz/cifar-10-binary.tar.gz


--------------------------------------------------------------------------------
/Data/CIFAR100/.gitignore:
--------------------------------------------------------------------------------
1 | *.bin


--------------------------------------------------------------------------------
/Data/CIFAR100/README.md:
--------------------------------------------------------------------------------
1 | http://www.cs.toronto.edu/~kriz/cifar.html
2 | http://www.cs.toronto.edu/~kriz/cifar-10-binary.tar.gz


--------------------------------------------------------------------------------
/Data/MNIST/.gitignore:
--------------------------------------------------------------------------------
1 | *.ubyte


--------------------------------------------------------------------------------
/Data/MNIST/README.md:
--------------------------------------------------------------------------------
1 | http://yann.lecun.com/exdb/mnist/


--------------------------------------------------------------------------------
/Data/imagenet2012/.gitignore:
--------------------------------------------------------------------------------
1 | ILSVRC2012_img_test
2 | ILSVRC2012_img_train
3 | ILSVRC2012_img_val
4 | testData.txt
5 | trainingData.txt
6 | validationData.txt
7 | 


--------------------------------------------------------------------------------
/IndexLearnerLayer.cpp:
--------------------------------------------------------------------------------
  1 | #include "IndexLearnerLayer.h"
  2 | #include "cudaUtilities.h"
  3 | #include "SigmoidLayer.h"
  4 | #include <iostream>
  5 | #include <cassert>
  6 | #include "SoftmaxClassifier.h"
  7 | #include "NetworkInNetworkLayer.h"
  8 | #include "utilities.h"
  9 | 
 10 | void dGradientDescentShrunkMatrixNoMomentum
 11 | (std::vector<float>& d_delta, std::vector<float>& d_weights,
 12 |  int nOut,
 13 |  std::vector<int>& inFeaturesPresent, std::vector<int>& outFeaturesPresent,
 14 |  float learningRate) {
 15 |   for (int i=0;i<inFeaturesPresent.size();i++) {
 16 |     int ii=inFeaturesPresent[i]*nOut;
 17 |     for(int j=0; j<outFeaturesPresent.size(); j++) {
 18 |       int jj=outFeaturesPresent[j];
 19 |       //no momentum, weight updated infrequently if the dataset is much larger than each minibatch
 20 |       d_weights[ii+jj]-=learningRate*d_delta[i+j];
 21 |     }
 22 |   }
 23 | }
 24 | 
 25 | IndexLearnerLayer::IndexLearnerLayer(int nFeaturesIn, int nFeaturesOut) :
 26 |   nFeaturesIn(nFeaturesIn), nFeaturesOut(nFeaturesOut) {
 27 |   std::cout << "IndexLearnerLayer" << std::endl;
 28 |   float scale=pow(6.0f/(nFeaturesIn+nFeaturesOut),0.5f);
 29 |   W.resize (nFeaturesIn*nFeaturesOut); W.setZero();//Uniform(-scale,scale);
 30 |   MW.resize (nFeaturesIn*nFeaturesOut); MW.setZero();
 31 | }
 32 | void IndexLearnerLayer::preprocess
 33 | (SpatiallySparseBatch &batch,
 34 |  SpatiallySparseBatchInterface &input,
 35 |  SpatiallySparseBatchInterface &output) {
 36 |   if (batch.type==TRAINBATCH); {
 37 |     assert(input.nFeatures==nFeaturesIn);
 38 |     output.nFeatures=nFeaturesOut;
 39 |     output.spatialSize=input.spatialSize;
 40 |     output.nSpatialSites=input.nSpatialSites;
 41 |     output.grids=input.grids;
 42 |     output.backpropErrors=true;
 43 |   }
 44 | }
 45 | void IndexLearnerLayer::forwards
 46 | (SpatiallySparseBatch &batch,
 47 |  SpatiallySparseBatchInterface &input,
 48 |  SpatiallySparseBatchInterface &output) {
 49 |   output.featuresPresent.vec=indexLearnerIndices;
 50 |   output.sub->features.resize(output.nSpatialSites*output.featuresPresent.size());
 51 |   output.sub->dfeatures.resize(output.nSpatialSites*output.featuresPresent.size());
 52 |   w.resize(input.featuresPresent.size()*output.featuresPresent.size());
 53 |   dShrinkMatrixForDropout(W.vec, w.vec,
 54 |                           input.featuresPresent.vec,
 55 |                           output.featuresPresent.vec,
 56 |                           output.nFeatures);
 57 |   d_rowMajorSGEMM_alphaAB_betaC(input.sub->features.vec, w.vec, output.sub->features.vec,
 58 |                                 output.nSpatialSites, input.featuresPresent.size(), output.featuresPresent.size(),
 59 |                                 1.0f, 0.0f);
 60 |   applySigmoid(output, output, SOFTMAX);
 61 | }
 62 | void IndexLearnerLayer::backwards
 63 | (SpatiallySparseBatch &batch,
 64 |  SpatiallySparseBatchInterface &input,
 65 |  SpatiallySparseBatchInterface &output,
 66 |  float learningRate,
 67 |  float momentum) {
 68 |   applySigmoidBackProp(output, output, SOFTMAX);
 69 |   input.sub->dfeatures.resize(input.nSpatialSites*input.featuresPresent.size());
 70 |   dw.resize(input.featuresPresent.size()*output.featuresPresent.size());
 71 |   d_rowMajorSGEMM_alphaAtB_betaC(input.sub->features.vec, output.sub->dfeatures.vec, dw.vec,
 72 |                                  input.featuresPresent.size(), output.nSpatialSites, output.featuresPresent.size(),
 73 |                                  1.0, 0.0);
 74 | 
 75 |   if (input.backpropErrors) {
 76 |     d_rowMajorSGEMM_alphaABt_betaC(output.sub->dfeatures.vec, w.vec, input.sub->dfeatures.vec,
 77 |                                    output.nSpatialSites,output.featuresPresent.size(),input.featuresPresent.size(),
 78 |                                    1.0, 0.0);
 79 |   }
 80 |   dGradientDescentShrunkMatrixNoMomentum
 81 |     (dw.vec, W.vec,
 82 |      output.nFeatures,
 83 |      input.featuresPresent.vec, output.featuresPresent.vec,
 84 |      learningRate);
 85 | }
 86 | void IndexLearnerLayer::loadWeightsFromStream(std::ifstream &f) {
 87 |   f.read((char*)&W.vec[0],sizeof(float)*W.size());
 88 | };
 89 | void IndexLearnerLayer::putWeightsToStream(std::ofstream &f)  {
 90 |   f.write((char*)&W.vec[0],sizeof(float)*W.size());
 91 | };
 92 | int IndexLearnerLayer::calculateInputSpatialSize(int outputSpatialSize) {
 93 |   return outputSpatialSize;
 94 | }
 95 | 
 96 | 
 97 | void IndexLearner(SpatiallySparseBatchInterface& input, SpatiallySparseBatch& batch, int nTop) {
 98 |   assert(batch.batchSize==input.nSpatialSites);
 99 |   assert(ipow(batch.batchSize,2)==input.sub->features.size());
100 |   assert(batch.type==TRAINBATCH);
101 | 
102 |   float* probs=&input.sub->features.vec[0];
103 |   for (int i=0;i<batch.batchSize;++i)
104 |     batch.probabilities.push_back(std::vector<float> (probs+i*batch.batchSize,probs+(i+1)*batch.batchSize));
105 |   for (int i=0;i<batch.batchSize;i++)
106 |     batch.predictions.push_back(vectorTopIndices(batch.probabilities[i],nTop));
107 | 
108 |   batch.mistakes+=batch.batchSize;
109 |   for (int i=0;i<batch.batchSize;i++) {
110 |     batch.negativeLogLikelihood-=log(std::max(batch.probabilities[i][i],(float)1.0e-15));
111 |     for (int j=0;j<nTop;j++) {
112 |       if (batch.predictions[i][j]==i) {
113 |         batch.mistakes--;
114 |       }
115 |     }
116 |   }
117 |   //Begin backprop. Top layer: d Cost / d SoftmaxInput
118 |   vectorCUDA<int> labels;
119 |   labels.vec=range(batch.batchSize);
120 |   input.sub->dfeatures.resize(input.nSpatialSites*input.featuresPresent.size());
121 |   dDerivativeOfCostWRTpreSoftmaxTopLevelWeights
122 |     (batch.batchSize, input.sub->dfeatures.vec, input.sub->features.vec,
123 |      labels.vec, batch.batchSize);
124 | }
125 | 


--------------------------------------------------------------------------------
/IndexLearnerLayer.h:
--------------------------------------------------------------------------------
 1 | //See Ben Graham http://www2.warwick.ac.uk/fac/sci/statistics/staff/academic-research/graham/indexlearning.pdf
 2 | //and
 3 | //http://papers.nips.cc/paper/5548-discriminative-unsupervised-feature-learning-with-convolutional-neural-networks.pdf
 4 | 
 5 | #pragma once
 6 | #include <fstream>
 7 | #include "SpatiallySparseLayer.h"
 8 | #include "Rng.h"
 9 | #include "vectorCUDA.h"
10 | #include <fstream>
11 | #include <vector>
12 | 
13 | class IndexLearnerLayer : public SpatiallySparseLayer {
14 | private:
15 |   RNG rng;
16 |   vectorCUDA<float> W; //Weights
17 |   vectorCUDA<float> MW; //momentum
18 |   vectorCUDA<float> w; //shrunk versions
19 |   vectorCUDA<float> dw; //For backprop
20 | public:
21 |   std::vector<int> indexLearnerIndices; //Variable to deliver indices in use for "double minibatch gradient descent"
22 |   int nFeaturesIn;
23 |   int nFeaturesOut;
24 |   float dropout;
25 |   IndexLearnerLayer(int nFeaturesIn, int nFeaturesOut);
26 |   void preprocess
27 |   (SpatiallySparseBatch &batch,
28 |    SpatiallySparseBatchInterface &input,
29 |    SpatiallySparseBatchInterface &output);
30 |   void forwards
31 |   (SpatiallySparseBatch &batch,
32 |    SpatiallySparseBatchInterface &input,
33 |    SpatiallySparseBatchInterface &output);
34 |   void backwards(SpatiallySparseBatch &batch,
35 |                  SpatiallySparseBatchInterface &input,
36 |                  SpatiallySparseBatchInterface &output,
37 |                  float learningRate,
38 |                  float momentum);
39 |   void loadWeightsFromStream(std::ifstream &f);
40 |   void putWeightsToStream(std::ofstream &f) ;
41 |   int calculateInputSpatialSize(int outputSpatialSize);
42 | };
43 | 
44 | void IndexLearner(SpatiallySparseBatchInterface& input, SpatiallySparseBatch& batch, int nTop);
45 | 


--------------------------------------------------------------------------------
/Makefile:
--------------------------------------------------------------------------------
 1 | CC=g++
 2 | CFLAGS=--std=c++11 -O3
 3 | OBJ=BatchProducer.o ConvolutionalLayer.o ConvolutionalTriangularLayer.o IndexLearnerLayer.o MaxPoolingLayer.o MaxPoolingTriangularLayer.o NetworkArchitectures.o NetworkInNetworkLayer.o Picture.o Regions.o Rng.o SigmoidLayer.o SoftmaxClassifier.o SparseConvNet.o SparseConvNetCUDA.o SpatiallySparseBatch.o SpatiallySparseBatchInterface.o SpatiallySparseDataset.o SpatiallySparseLayer.o TerminalPoolingLayer.o cudaUtilities.o readImageToMat.o types.o utilities.o vectorCUDA.o vectorHash.o
 4 | LIBS=-lopencv_core -lopencv_highgui -lopencv_imgproc -lrt -larmadillo -lopenblas
 5 | 
 6 | %.o: %.cpp $(DEPS)
 7 | 	$(CC) -c -o $@ $< $(CFLAGS)
 8 | 
 9 | clean:
10 | 	rm *.o
11 | casia: $(OBJ) OnlineHandwritingPicture.o SpatiallySparseDatasetCasiaOLHWDB.o casia.o
12 | 	$(CC) -o casia $(OBJ) OnlineHandwritingPicture.o SpatiallySparseDatasetCasiaOLHWDB.o casia.o $(LIBS) $(CFLAGS)
13 | 
14 | casia3d: $(OBJ) OnlineHandwritingPicture.o SpatiallySparseDatasetCasiaOLHWDB.o casia3d.o
15 | 	$(CC) -o casia3d $(OBJ) OnlineHandwritingPicture.o SpatiallySparseDatasetCasiaOLHWDB.o casia3d.o $(LIBS) $(CFLAGS)
16 | 
17 | cifar10: $(OBJ) OpenCVPicture.o SpatiallySparseDatasetCIFAR10.o cifar10.o
18 | 	$(CC) -o cifar10 $(OBJ) OpenCVPicture.o SpatiallySparseDatasetCIFAR10.o cifar10.o $(LIBS) $(CFLAGS)
19 | 
20 | cifar10fmp: $(OBJ) OpenCVPicture.o SpatiallySparseDatasetCIFAR10.o cifar10fmp.o
21 | 	$(CC) -o cifar10fmp $(OBJ) OpenCVPicture.o SpatiallySparseDatasetCIFAR10.o cifar10fmp.o $(LIBS) $(CFLAGS)
22 | 
23 | cifar10FullyConnected: $(OBJ) SpatiallySparseDatasetCIFAR10FullyConnected.o cifar10FullyConnected.o
24 | 	$(CC) -o cifar10FullyConnected $(OBJ) SpatiallySparseDatasetCIFAR10FullyConnected.o cifar10FullyConnected.o $(LIBS) $(CFLAGS)
25 | 
26 | cifar10indexLearning: $(OBJ) OpenCVPicture.o SpatiallySparseDatasetCIFAR10.o cifar10indexLearning.o
27 | 	$(CC) -o cifar10indexLearning $(OBJ) OpenCVPicture.o SpatiallySparseDatasetCIFAR10.o cifar10indexLearning.o $(LIBS) $(CFLAGS)
28 | 
29 | cifar100: $(OBJ) OpenCVPicture.o SpatiallySparseDatasetCIFAR100.o cifar100.o
30 | 	$(CC) -o cifar100 $(OBJ) OpenCVPicture.o SpatiallySparseDatasetCIFAR100.o cifar100.o $(LIBS) $(CFLAGS)
31 | 
32 | cifar10triangular: $(OBJ) OpenCVTriangularPicture.o SpatiallySparseDatasetCIFAR10.o cifar10triangular.o
33 | 	$(CC) -o cifar10triangular $(OBJ) OpenCVTriangularPicture.o SpatiallySparseDatasetCIFAR10.o cifar10triangular.o $(LIBS) $(CFLAGS)
34 | 
35 | shrec2015triangular: $(OBJ) Off3DFormatTriangularPicture.o SpatiallySparseDatasetSHREC2015.o shrec2015triangular.o
36 | 	$(CC) -o shrec2015triangular $(OBJ) Off3DFormatTriangularPicture.o SpatiallySparseDatasetSHREC2015.o shrec2015triangular.o $(LIBS) $(CFLAGS)
37 | 
38 | cvap_rha: $(OBJ) CVAP_RHA_Picture.o SpatiallySparseDatasetCVAP_RHA.o cvap_rha.o
39 | 	$(CC) -o cvap_rha $(OBJ) CVAP_RHA_Picture.o SpatiallySparseDatasetCVAP_RHA.o cvap_rha.o $(LIBS) $(CFLAGS)
40 | 
41 | mnist: $(OBJ) OpenCVPicture.o SpatiallySparseDatasetMnist.o mnist.o
42 | 	$(CC) -o mnist $(OBJ) OpenCVPicture.o SpatiallySparseDatasetMnist.o mnist.o $(LIBS) $(CFLAGS)
43 | 
44 | imagenet2012: $(OBJ) OpenCVPicture.o SpatiallySparseDatasetImageNet2012.o imagenet2012.o
45 | 	$(CC) -o imagenet2012 $(OBJ) OpenCVPicture.o SpatiallySparseDatasetImageNet2012.o imagenet2012.o $(LIBS) $(CFLAGS)
46 | 


--------------------------------------------------------------------------------
/MaxPoolingLayer.cpp:
--------------------------------------------------------------------------------
  1 | #include <iostream>
  2 | #include <cassert>
  3 | #include "utilities.h"
  4 | #include "cudaUtilities.h"
  5 | #include "MaxPoolingLayer.h"
  6 | #include "Regions.h"
  7 | 
  8 | void maxPool(std::vector<float>& g1, std::vector<float>& g2, std::vector<int>& rules, int count, int sd, int nOut, std::vector<int>& d_choice) {
  9 |   for (int row=0;row<count;row++) {
 10 |     for (int s=0;s<sd;s++) {
 11 |       int r=rules[row*sd+s];
 12 |       if (r>=0) {
 13 |         for (int i=0;i<nOut;i++) {
 14 |           if (s==0 or g2[row*nOut+i]<g1[r*nOut+i]) {
 15 |             g2[row*nOut+i]=g1[r*nOut+i];
 16 |             d_choice[row*nOut+i]=r*nOut+i;
 17 |           }
 18 |         }
 19 |       }
 20 |     }
 21 |   }
 22 | }
 23 | 
 24 | void maxPoolBackProp(std::vector<float>& d1, std::vector<float>& d2, int count, int nOut, std::vector<int>& d_choice) {
 25 |   for (int row=0;row<count;row++) {
 26 |     for (int i=0;i<nOut;i++) {
 27 |       d1[d_choice[row*nOut+i]]+=d2[row*nOut+i];
 28 |     }
 29 |   }
 30 | }
 31 | 
 32 | //TODO: Refactor the different pooling classes somehow
 33 | 
 34 | 
 35 | MaxPoolingLayer::MaxPoolingLayer(int poolSize, int poolStride, int dimension)
 36 |   : poolSize(poolSize), poolStride(poolStride), dimension(dimension) {
 37 |   sd=ipow(poolSize,dimension);
 38 |   std::cout << "MaxPooling " << poolSize << " " << poolStride << std::endl;
 39 | }
 40 | void MaxPoolingLayer::preprocess
 41 | (SpatiallySparseBatch &batch,
 42 |  SpatiallySparseBatchInterface &input,
 43 |  SpatiallySparseBatchInterface &output) {
 44 |   assert(input.spatialSize==inSpatialSize);
 45 |   output.nFeatures=input.nFeatures;
 46 |   output.featuresPresent.vec=input.featuresPresent.vec;
 47 |   output.spatialSize=outSpatialSize;
 48 |   output.nSpatialSites=0;
 49 |   output.grids.resize(batch.batchSize);
 50 |   output.backpropErrors=input.backpropErrors;
 51 |   RegularPoolingRegions regions(inSpatialSize, outSpatialSize,dimension,poolSize, poolStride);
 52 |   for (int item=0;item<batch.batchSize;item++)
 53 |     gridRules
 54 |       (input.grids[item],
 55 |        output.grids[item],
 56 |        regions,
 57 |        output.nSpatialSites,
 58 |        output.rules.vec);
 59 | }
 60 | void MaxPoolingLayer::forwards
 61 | (SpatiallySparseBatch &batch,
 62 |  SpatiallySparseBatchInterface &input,
 63 |  SpatiallySparseBatchInterface &output) {
 64 |   output.sub->poolingChoices.resize(output.nSpatialSites*output.featuresPresent.size());
 65 |   output.sub->features.resize(output.nSpatialSites*output.featuresPresent.size());
 66 |   maxPool(input.sub->features.vec,output.sub->features.vec,output.rules.vec,output.nSpatialSites,sd,output.featuresPresent.size(),output.sub->poolingChoices.vec);
 67 | }
 68 | void MaxPoolingLayer::backwards
 69 | (SpatiallySparseBatch &batch,
 70 |  SpatiallySparseBatchInterface &input,
 71 |  SpatiallySparseBatchInterface &output,
 72 |  float learningRate,
 73 |  float momentum) {
 74 |   if (input.backpropErrors) {
 75 |     input.sub->dfeatures.resize(input.nSpatialSites*input.featuresPresent.size());
 76 |     input.sub->dfeatures.setZero();
 77 |     maxPoolBackProp
 78 |       (input.sub->dfeatures.vec, output.sub->dfeatures.vec, output.nSpatialSites, output.featuresPresent.size(), output.sub->poolingChoices.vec);
 79 |   }
 80 | }
 81 | int MaxPoolingLayer::calculateInputSpatialSize(int outputSpatialSize) {
 82 |   outSpatialSize=outputSpatialSize;
 83 |   inSpatialSize=poolSize+(outputSpatialSize-1)*poolStride;
 84 |   std::cout << "(" << outSpatialSize <<"," <<inSpatialSize << ") ";
 85 |   return inSpatialSize;
 86 | }
 87 | 
 88 | PseudorandomOverlappingFractionalMaxPoolingLayer::PseudorandomOverlappingFractionalMaxPoolingLayer(int poolSize, float fmpShrink, int dimension) : poolSize(poolSize), fmpShrink(fmpShrink), dimension(dimension) {
 89 |   sd=ipow(poolSize,dimension);
 90 |   std::cout << "Pseudorandom overlapping Fractional Max Pooling " << fmpShrink << " " << poolSize << std::endl;
 91 | }
 92 | void PseudorandomOverlappingFractionalMaxPoolingLayer::preprocess
 93 | (SpatiallySparseBatch &batch,
 94 |  SpatiallySparseBatchInterface &input,
 95 |  SpatiallySparseBatchInterface &output) {
 96 |   assert(input.spatialSize==inSpatialSize);
 97 |   output.nFeatures=input.nFeatures;
 98 |   output.featuresPresent.vec=input.featuresPresent.vec;
 99 |   output.spatialSize=outSpatialSize;
100 |   output.nSpatialSites=0;
101 |   output.grids.resize(batch.batchSize);
102 |   output.backpropErrors=input.backpropErrors;
103 |   PseudorandomOverlappingFractionalPoolingRegions regions(inSpatialSize, outSpatialSize,dimension, poolSize,rng);
104 |   for (int item=0;item<batch.batchSize;item++) {
105 |     gridRules
106 |       (input.grids[item],
107 |        output.grids[item],
108 |        regions,
109 |        output.nSpatialSites,
110 |        output.rules.vec);
111 |   }
112 | }
113 | void PseudorandomOverlappingFractionalMaxPoolingLayer::forwards
114 | (SpatiallySparseBatch &batch,
115 |  SpatiallySparseBatchInterface &input,
116 |  SpatiallySparseBatchInterface &output) {
117 |   output.sub->poolingChoices.resize(output.nSpatialSites*output.featuresPresent.size());
118 |   output.sub->features.resize(output.nSpatialSites*output.featuresPresent.size());
119 |   maxPool(input.sub->features.vec,output.sub->features.vec,output.rules.vec,output.nSpatialSites,sd,output.featuresPresent.size(),output.sub->poolingChoices.vec);
120 | }
121 | void PseudorandomOverlappingFractionalMaxPoolingLayer::backwards
122 | (SpatiallySparseBatch &batch,
123 |  SpatiallySparseBatchInterface &input,
124 |  SpatiallySparseBatchInterface &output,
125 |  float learningRate,
126 |  float momentum) {
127 |   if (input.backpropErrors) {
128 |     input.sub->dfeatures.resize(input.nSpatialSites*input.featuresPresent.size());
129 |     input.sub->dfeatures.setZero();
130 |     maxPoolBackProp
131 |       (input.sub->dfeatures.vec, output.sub->dfeatures.vec, output.nSpatialSites, output.featuresPresent.size(), output.sub->poolingChoices.vec);
132 |   }
133 | }
134 | int PseudorandomOverlappingFractionalMaxPoolingLayer::calculateInputSpatialSize
135 | (int outputSpatialSize) {
136 |   outSpatialSize=outputSpatialSize;
137 |   inSpatialSize=outputSpatialSize*fmpShrink+0.5;
138 |   if (inSpatialSize==outputSpatialSize)
139 |     inSpatialSize++;
140 |   std::cout << "(" << outSpatialSize <<"," <<inSpatialSize << ") ";
141 |   return inSpatialSize;
142 | }
143 | 
144 | RandomOverlappingFractionalMaxPoolingLayer::RandomOverlappingFractionalMaxPoolingLayer
145 | (int poolSize, float fmpShrink, int dimension)
146 |   : poolSize(poolSize), fmpShrink(fmpShrink), dimension(dimension) {
147 |   sd=ipow(poolSize,dimension);
148 |   std::cout << "Random overlapping Fractional Max Pooling " << fmpShrink << " " << poolSize << std::endl;
149 | }
150 | void RandomOverlappingFractionalMaxPoolingLayer::preprocess
151 | (SpatiallySparseBatch &batch,
152 |  SpatiallySparseBatchInterface &input,
153 |  SpatiallySparseBatchInterface &output) {
154 |   assert(input.spatialSize==inSpatialSize);
155 |   output.nFeatures=input.nFeatures;
156 |   output.featuresPresent.vec=input.featuresPresent.vec;
157 |   output.spatialSize=outSpatialSize;
158 |   output.nSpatialSites=0;
159 |   output.grids.resize(batch.batchSize);
160 |   output.backpropErrors=input.backpropErrors;
161 |   RandomOverlappingFractionalPoolingRegions regions(inSpatialSize, outSpatialSize,dimension, poolSize,rng);
162 |   for (int item=0;item<batch.batchSize;item++)
163 |     gridRules
164 |       (input.grids[item],
165 |        output.grids[item],
166 |        regions,
167 |        output.nSpatialSites,
168 |        output.rules.vec);
169 | }
170 | void RandomOverlappingFractionalMaxPoolingLayer::forwards
171 | (SpatiallySparseBatch &batch,
172 |  SpatiallySparseBatchInterface &input,
173 |  SpatiallySparseBatchInterface &output) {
174 |   output.sub->poolingChoices.resize(output.nSpatialSites*output.featuresPresent.size());
175 |   output.sub->features.resize(output.nSpatialSites*output.featuresPresent.size());
176 |   maxPool(input.sub->features.vec,output.sub->features.vec,output.rules.vec,output.nSpatialSites,sd,output.featuresPresent.size(),output.sub->poolingChoices.vec);
177 | }
178 | void RandomOverlappingFractionalMaxPoolingLayer::backwards
179 | (SpatiallySparseBatch &batch,
180 |  SpatiallySparseBatchInterface &input,
181 |  SpatiallySparseBatchInterface &output,
182 |  float learningRate,
183 |  float momentum) {
184 |   if (input.backpropErrors) {
185 |     input.sub->dfeatures.resize(input.nSpatialSites*input.featuresPresent.size());
186 |     input.sub->dfeatures.setZero();
187 |     maxPoolBackProp
188 |       (input.sub->dfeatures.vec, output.sub->dfeatures.vec, output.nSpatialSites, output.featuresPresent.size(), output.sub->poolingChoices.vec);
189 |   }
190 | }
191 | int RandomOverlappingFractionalMaxPoolingLayer::calculateInputSpatialSize
192 | (int outputSpatialSize) {
193 |   outSpatialSize=outputSpatialSize;
194 |   inSpatialSize=outputSpatialSize*fmpShrink+0.5;
195 |   if (inSpatialSize==outputSpatialSize)
196 |     inSpatialSize++;
197 |   std::cout << "(" << outSpatialSize <<"," <<inSpatialSize << ") ";
198 |   return inSpatialSize;
199 | }
200 | 


--------------------------------------------------------------------------------
/MaxPoolingLayer.h:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | #include <iostream>
 3 | #include "SpatiallySparseLayer.h"
 4 | #include "Rng.h"
 5 | 
 6 | void maxPool(std::vector<float>& g1, std::vector<float>& g2, std::vector<int>& rules, int count, int sd, int nOut, std::vector<int>& d_choice);
 7 | void maxPoolBackProp(std::vector<float>& d1, std::vector<float>& d2, int count, int nOut, std::vector<int>& d_choice);
 8 | 
 9 | //TODO: Refactor the different pooling classes somehow
10 | 
11 | 
12 | class MaxPoolingLayer : public SpatiallySparseLayer {
13 | public:
14 |   int inSpatialSize;
15 |   int outSpatialSize;
16 |   int poolSize;
17 |   int poolStride;
18 |   int dimension;
19 |   int sd;
20 |   MaxPoolingLayer(int poolSize, int poolStride, int dimension);
21 |   void preprocess
22 |   (SpatiallySparseBatch &batch,
23 |    SpatiallySparseBatchInterface &input,
24 |    SpatiallySparseBatchInterface &output);
25 |   void forwards(SpatiallySparseBatch &batch,
26 |                 SpatiallySparseBatchInterface &input,
27 |                 SpatiallySparseBatchInterface &output);
28 |   void backwards(SpatiallySparseBatch &batch,
29 |                  SpatiallySparseBatchInterface &input,
30 |                  SpatiallySparseBatchInterface &output,
31 |                  float learningRate,
32 |                  float momentum);
33 |   int calculateInputSpatialSize(int outputSpatialSize);
34 | };
35 | 
36 | class PseudorandomOverlappingFractionalMaxPoolingLayer : public SpatiallySparseLayer {
37 |   int sd;
38 | public:
39 |   int inSpatialSize;
40 |   int outSpatialSize;
41 |   float fmpShrink;
42 |   int poolSize;
43 |   int dimension;
44 |   RNG rng;
45 |   PseudorandomOverlappingFractionalMaxPoolingLayer(int poolSize, float fmpShrink, int dimension);
46 |   void preprocess
47 |   (SpatiallySparseBatch &batch,
48 |    SpatiallySparseBatchInterface &input,
49 |    SpatiallySparseBatchInterface &output);
50 |   void forwards(SpatiallySparseBatch &batch,
51 |                 SpatiallySparseBatchInterface &input,
52 |                 SpatiallySparseBatchInterface &output);
53 |   void backwards(SpatiallySparseBatch &batch,
54 |                  SpatiallySparseBatchInterface &input,
55 |                  SpatiallySparseBatchInterface &output,
56 |                  float learningRate,
57 |                  float momentum);
58 |   int calculateInputSpatialSize(int outputSpatialSize);
59 | };
60 | 
61 | class RandomOverlappingFractionalMaxPoolingLayer : public SpatiallySparseLayer {
62 |   int sd;
63 | public:
64 |   int inSpatialSize;
65 |   int outSpatialSize;
66 |   float fmpShrink;
67 |   int poolSize;
68 |   int dimension;
69 |   RNG rng;
70 |   RandomOverlappingFractionalMaxPoolingLayer(int poolSize, float fmpShrink, int dimension);
71 |   void preprocess
72 |   (SpatiallySparseBatch &batch,
73 |    SpatiallySparseBatchInterface &input,
74 |    SpatiallySparseBatchInterface &output);
75 |   void forwards
76 |   (SpatiallySparseBatch &batch,
77 |    SpatiallySparseBatchInterface &input,
78 |    SpatiallySparseBatchInterface &output);
79 |   void backwards
80 |   (SpatiallySparseBatch &batch,
81 |    SpatiallySparseBatchInterface &input,
82 |    SpatiallySparseBatchInterface &output,
83 |    float learningRate,
84 |    float momentum);
85 |   int calculateInputSpatialSize(int outputSpatialSize);
86 | };
87 | 


--------------------------------------------------------------------------------
/MaxPoolingTriangularLayer.cpp:
--------------------------------------------------------------------------------
 1 | #include <iostream>
 2 | #include <cassert>
 3 | #include "utilities.h"
 4 | #include "cudaUtilities.h"
 5 | #include "MaxPoolingLayer.h"
 6 | #include "MaxPoolingTriangularLayer.h"
 7 | #include "Regions.h"
 8 | 
 9 | MaxPoolingTriangularLayer::MaxPoolingTriangularLayer(int poolSize, int poolStride, int dimension) : poolSize(poolSize), poolStride(poolStride), dimension(dimension) {
10 |   S=triangleSize(poolSize,dimension);
11 |   std::cout << dimension << "D MaxPoolingTriangularLayer side-length=" << poolSize
12 |             << " volume=" << S << " stride " << poolStride << std::endl;
13 | }
14 | void MaxPoolingTriangularLayer::preprocess
15 | (SpatiallySparseBatch &batch,
16 |  SpatiallySparseBatchInterface &input,
17 |  SpatiallySparseBatchInterface &output) {
18 |   assert(input.spatialSize==inSpatialSize);
19 |   output.nFeatures=input.nFeatures;
20 |   output.featuresPresent.vec=input.featuresPresent.vec;
21 |   output.spatialSize=outSpatialSize;
22 |   output.nSpatialSites=0;
23 |   output.grids.resize(batch.batchSize);
24 |   output.backpropErrors=input.backpropErrors;
25 |   RegularPoolingRegionsTriangular regions(inSpatialSize, outSpatialSize,dimension,poolSize, poolStride);
26 |   for (int item=0;item<batch.batchSize;item++)
27 |     gridRulesTriangular
28 |       (input.grids[item],
29 |        output.grids[item],
30 |        regions,
31 |        output.nSpatialSites,
32 |        output.rules.vec);
33 | }
34 | void MaxPoolingTriangularLayer::forwards
35 | (SpatiallySparseBatch &batch,
36 |  SpatiallySparseBatchInterface &input,
37 |  SpatiallySparseBatchInterface &output) {
38 |   output.sub->poolingChoices.resize(output.nSpatialSites*output.featuresPresent.size());
39 |   output.sub->features.resize(output.nSpatialSites*output.featuresPresent.size());
40 |   maxPool(input.sub->features.vec,
41 |           output.sub->features.vec,
42 |           output.rules.vec,
43 |           output.nSpatialSites,
44 |           S,
45 |           output.featuresPresent.size(),
46 |           output.sub->poolingChoices.vec);
47 | }
48 | void MaxPoolingTriangularLayer::backwards
49 | (SpatiallySparseBatch &batch,
50 |  SpatiallySparseBatchInterface &input,
51 |  SpatiallySparseBatchInterface &output,
52 |  float learningRate,
53 |  float momentum) {
54 |   if (input.backpropErrors) {
55 |     input.sub->dfeatures.resize(input.nSpatialSites*input.featuresPresent.size());
56 |     input.sub->dfeatures.setZero();
57 |     maxPoolBackProp
58 |       (input.sub->dfeatures.vec, output.sub->dfeatures.vec, output.nSpatialSites, output.featuresPresent.size(), output.sub->poolingChoices.vec);
59 |   }
60 | }
61 | int MaxPoolingTriangularLayer::calculateInputSpatialSize(int outputSpatialSize) {
62 |   outSpatialSize=outputSpatialSize;
63 |   inSpatialSize=poolSize+(outputSpatialSize-1)*poolStride;
64 |   std::cout << "(" << outSpatialSize <<"MP" <<inSpatialSize << ") ";
65 |   return inSpatialSize;
66 | }
67 | 


--------------------------------------------------------------------------------
/MaxPoolingTriangularLayer.h:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | #include <iostream>
 3 | #include <vector>
 4 | #include "SpatiallySparseLayer.h"
 5 | #include "Rng.h"
 6 | 
 7 | 
 8 | class MaxPoolingTriangularLayer : public SpatiallySparseLayer {
 9 | public:
10 |   int inSpatialSize;
11 |   int outSpatialSize;
12 |   int poolSize;
13 |   int poolStride;
14 |   int dimension;
15 |   int S;
16 |   MaxPoolingTriangularLayer(int poolSize, int poolStride, int dimension);
17 |   void preprocess
18 |   (SpatiallySparseBatch &batch,
19 |    SpatiallySparseBatchInterface &input,
20 |    SpatiallySparseBatchInterface &output);
21 |   void forwards
22 |   (SpatiallySparseBatch &batch,
23 |    SpatiallySparseBatchInterface &input,
24 |    SpatiallySparseBatchInterface &output);
25 |   void backwards(SpatiallySparseBatch &batch,
26 |                  SpatiallySparseBatchInterface &input,
27 |                  SpatiallySparseBatchInterface &output,
28 |                  float learningRate,
29 |                  float momentum);
30 |   int calculateInputSpatialSize(int outputSpatialSize);
31 | };
32 | 


--------------------------------------------------------------------------------
/NetworkArchitectures.cpp:
--------------------------------------------------------------------------------
 1 | #include "NetworkArchitectures.h"
 2 | 
 3 | DeepCNet::DeepCNet (int dimension, int l, int k, ActivationFunction fn, int nInputFeatures, int nClasses, float p, int nTop) : SparseConvNet(dimension,nInputFeatures, nClasses, nTop) {
 4 |   for (int i=0;i<=l;i++)
 5 |     addLeNetLayerMP((i+1)*k,
 6 |                     (i==0)?3:2,
 7 |                     1,
 8 |                     (i<l)?3:1,
 9 |                     (i<l)?2:1,
10 |                     fn,
11 |                     p*i*1.0f/l);
12 |   addSoftmaxLayer();
13 | }
14 | 
15 | DeepC2::DeepC2 (int dimension, int l, int k, ActivationFunction fn, int nInputFeatures, int nClasses, float p, int nTop) : SparseConvNet(dimension,nInputFeatures, nClasses, nTop) {
16 |   for (int i=0;i<=l;i++)
17 |     addLeNetLayerMP((i+1)*k,
18 |                     2,
19 |                     1,
20 |                     (i<l)?3:1,
21 |                     (i<l)?2:1,
22 |                     fn,
23 |                     p*i*1.0f/l);
24 |   addSoftmaxLayer();
25 | }
26 | 
27 | DeepCNiN::DeepCNiN (int dimension, int l, int k, ActivationFunction fn, int nInputFeatures, int nClasses, float p, int nTop) : SparseConvNet(dimension,nInputFeatures, nClasses, nTop) {
28 |   for (int i=0;i<=l;i++) {
29 |     addLeNetLayerMP((i+1)*k,
30 |                     (i==0)?2:2,
31 |                     1,
32 |                     (i<l)?3:1,
33 |                     (i<l)?2:1,
34 |                     fn,
35 |                     p*i*1.0f/l);
36 |     addLeNetLayerMP(1,1,1,1,(i+1)*k,fn,p*i*1.0f/l);
37 |   }
38 |   addSoftmaxLayer();
39 | }
40 | DeepC2C2::DeepC2C2(int dimension, int l, int k, ActivationFunction fn, int nInputFeatures, int nClasses, float p, int nTop) : SparseConvNet(dimension,nInputFeatures, nClasses, nTop) {
41 |   for (int i=0;i<l;i++) {
42 |     addLeNetLayerMP((i+1)*k,2,1,1,1,fn,p*i*1.0f/l);
43 |     addLeNetLayerMP((i+1)*k,2,1,3,2,fn,p*i*1.0f/l);
44 |   }
45 |   addLeNetLayerMP((l+1)*k,2,1,1,1,fn,p);
46 |   addLeNetLayerMP((l+1)*k,1,1,1,1,fn,p);
47 |   addSoftmaxLayer();
48 | }
49 | 
50 | POFMPSparseConvNet::POFMPSparseConvNet(int dimension, int l, int k, float fmpShrink, ActivationFunction fn, int nInputFeatures, int nClasses, float p, int nTop) : SparseConvNet(dimension,nInputFeatures, nClasses, nTop) {
51 |   for (int i=0;i<l;i++) {
52 |     addLeNetLayerPOFMP(k*(i+1),2,1,2,fmpShrink,fn,p*i/(l+2));
53 |   }
54 |   addLeNetLayerPOFMP(k*(l+1),2,1,2,1.5,fn,p*l/(l+2));
55 |   addLeNetLayerMP(k*(l+2),2,1,1,1,fn,p*(l+1)/(l+2));
56 |   addLeNetLayerMP(k*(l+3),1,1,1,1,fn,p);
57 |   addSoftmaxLayer();
58 | }
59 | 
60 | ROFMPSparseConvNet::ROFMPSparseConvNet(int dimension, int l, int k, float fmpShrink, ActivationFunction fn, int nInputFeatures, int nClasses, float p, int nTop) : SparseConvNet(dimension,nInputFeatures, nClasses, nTop) {
61 |   for (int i=0;i<l;i++) {
62 |     addLeNetLayerROFMP(k*(i+1),2,1,2,fmpShrink,fn,p*i/(l+2));
63 |   }
64 |   addLeNetLayerROFMP(k*(l+1),2,1,2,1.5,fn,p*l/(l+2));
65 |   addLeNetLayerMP(k*(l+2),2,1,1,1,fn,p*(l+1)/(l+2));
66 |   addLeNetLayerMP(k*(l+3),1,1,1,1,fn,p);
67 |   addSoftmaxLayer();
68 | }
69 | 


--------------------------------------------------------------------------------
/NetworkArchitectures.h:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | #include "SparseConvNet.h"
 3 | 
 4 | class DeepCNet : public SparseConvNet {
 5 | public:
 6 |   DeepCNet (int dimension, int l, int k, ActivationFunction fn, int nInputFeatures, int nClasses, float p=0.0f, int nTop=1);
 7 | };
 8 | 
 9 | class DeepC2 : public SparseConvNet {
10 | public:
11 |   DeepC2 (int dimension, int l, int k, ActivationFunction fn, int nInputFeatures, int nClasses, float p=0.0f, int nTop=1);
12 | };
13 | 
14 | class DeepCNiN : public SparseConvNet {
15 | public:
16 |   DeepCNiN(int dimension, int l, int k, ActivationFunction fn, int nInputFeatures, int nClasses, float p=0.0f, int nTop=1);
17 | };
18 | 
19 | class DeepC2C2 : public SparseConvNet {
20 | public:
21 |   DeepC2C2(int dimension, int l, int k, ActivationFunction fn, int nInputFeatures, int nClasses, float p=0.0f, int nTop=1);
22 | };
23 | 
24 | class POFMPSparseConvNet : public SparseConvNet {
25 | public:
26 |   POFMPSparseConvNet(int dimension, int l, int k, float fmpShrink, ActivationFunction fn, int nInputFeatures, int nClasses, float p=0.0f, int nTop=1);
27 | };
28 | 
29 | class ROFMPSparseConvNet : public SparseConvNet {
30 | public:
31 |   ROFMPSparseConvNet(int dimension, int l, int k, float fmpShrink, ActivationFunction fn, int nInputFeatures, int nClasses, float p=0.0f, int nTop=1);
32 | };
33 | 


--------------------------------------------------------------------------------
/NetworkInNetworkLayer.cpp:
--------------------------------------------------------------------------------
  1 | #include "NetworkInNetworkLayer.h"
  2 | #include "cudaUtilities.h"
  3 | #include "SigmoidLayer.h"
  4 | #include <iostream>
  5 | #include <cassert>
  6 | 
  7 | void dShrinkMatrixForDropout
  8 | (std::vector<float>& m, std::vector<float>& md,
  9 |  std::vector<int>& inFeaturesPresent, std::vector<int>& outFeaturesPresent,
 10 |  int nOut) {
 11 |   for (int i_=0;i_<inFeaturesPresent.size();i_++) {
 12 |     int i=i_*outFeaturesPresent.size();
 13 |     int ii=inFeaturesPresent[i_]*nOut;
 14 |     for (int j=0;j<outFeaturesPresent.size();j++) {
 15 |       int jj=outFeaturesPresent[j];
 16 |       md[i+j]=m[ii+jj];
 17 |     }
 18 |   }
 19 | }
 20 | 
 21 | void dShrinkVectorForDropout(std::vector<float>& m, std::vector<float>& md, std::vector<int>& outFeaturesPresent) {
 22 |   for(int i=0; i<outFeaturesPresent.size(); i++) {
 23 |     int ii=outFeaturesPresent[i];
 24 |     md[i]=m[ii];
 25 |   }
 26 | }
 27 | 
 28 | //Nesterov
 29 | void dGradientDescent
 30 | (std::vector<float>& d_delta, std::vector<float>& d_momentum, std::vector<float>& d_weights,
 31 |  int nIn, int nOut, float learningRate, float momentum) {
 32 |   for (int i=0;i<nIn;i++) {
 33 |     for(int j=i; j<i+nOut; j++) {
 34 |       d_weights[j]-=d_momentum[j]*momentum;
 35 |       d_momentum[j]=momentum*d_momentum[j]-learningRate*(1-momentum)*d_delta[j];
 36 |       d_weights[j]=d_weights[j]+d_momentum[j]*(1+momentum);
 37 |     }
 38 |   }
 39 | }
 40 | 
 41 | void dGradientDescentShrunkMatrix
 42 | (std::vector<float>& d_delta, std::vector<float>& d_momentum, std::vector<float>& d_weights,
 43 |  int nOut,
 44 |  std::vector<int>& inFeaturesPresent, std::vector<int>& outFeaturesPresent,
 45 |  float learningRate,
 46 |  float momentum) {
 47 |   for (int i_=0;i_<inFeaturesPresent.size();i_++) {
 48 |     int i=i_*outFeaturesPresent.size();
 49 |     int ii=inFeaturesPresent[i_]*nOut;
 50 |     for(int j=0; j<outFeaturesPresent.size(); j++) {
 51 |       int jj=outFeaturesPresent[j];
 52 |       //NAG light
 53 |       d_weights[ii+jj]-=d_momentum[ii+jj]*momentum;
 54 |       d_momentum[ii+jj]=momentum*d_momentum[ii+jj]-learningRate*(1-momentum)*d_delta[i+j];
 55 |       d_weights[ii+jj]=d_weights[ii+jj]+d_momentum[ii+jj]*(1+momentum);
 56 |     }
 57 |   }
 58 | }
 59 | 
 60 | void dGradientDescentShrunkVector
 61 | (std::vector<float>& d_delta, std::vector<float>& d_momentum, std::vector<float>& d_weights,
 62 |  int nOut,
 63 |  std::vector<int>& outFeaturesPresent,
 64 |  float learningRate,
 65 |  float momentum) {
 66 |   for(int i=0; i<outFeaturesPresent.size(); i++) {
 67 |     int ii=outFeaturesPresent[i];
 68 |     //NAG light
 69 |     d_weights[ii]-=d_momentum[ii]*momentum;
 70 |     d_momentum[ii]=momentum*d_momentum[ii]-learningRate*(1-momentum)*d_delta[i];
 71 |     d_weights[ii]=d_weights[ii]+d_momentum[ii]*(1+momentum);
 72 |   }
 73 | }
 74 | 
 75 | 
 76 | void columnSum(std::vector<float>& matrix, std::vector<float>& target, int nRows, int nColumns) {
 77 |   for (int row=0;row<nRows;row++) {
 78 |     for (int column=0;column<nColumns;column++) {
 79 |       target[column]+=matrix[row*nColumns+column];
 80 |     }
 81 |   }
 82 | }
 83 | 
 84 | void replicateArray(std::vector<float>& src, std::vector<float>& dst, int nRows, int nColumns) {
 85 |   for (int row=0;row<nRows;row++) {
 86 |     for (int column=0;column<nColumns;column++) {
 87 |       dst[row*nColumns+column]=src[column];
 88 |     }
 89 |   }
 90 | }
 91 | 
 92 | NetworkInNetworkLayer::NetworkInNetworkLayer(int nFeaturesIn, int nFeaturesOut,
 93 |                                              float dropout,ActivationFunction fn,
 94 |                                              float alpha//used to determine intialization weights only
 95 |                                              ) :
 96 |   nFeaturesIn(nFeaturesIn), nFeaturesOut(nFeaturesOut),
 97 |   dropout(dropout), fn(fn),
 98 |   W(nFeaturesIn*nFeaturesOut), MW(nFeaturesIn*nFeaturesOut),
 99 |   B(nFeaturesOut), MB(nFeaturesOut) {
100 |   float scale=pow(6.0f/(nFeaturesIn+nFeaturesOut*alpha),0.5f);
101 |   W.setUniform(-scale,scale);
102 |   MW.setZero();
103 |   B.setZero();
104 |   MB.setZero();
105 |   std::cout << "Learn " << nFeaturesIn << "->" << nFeaturesOut << " dropout=" << dropout << " " << sigmoidNames[fn] << std::endl;
106 | }
107 | void NetworkInNetworkLayer::preprocess
108 | (SpatiallySparseBatch &batch,
109 |  SpatiallySparseBatchInterface &input,
110 |  SpatiallySparseBatchInterface &output) {
111 |   assert(input.nFeatures==nFeaturesIn);
112 |   output.nFeatures=nFeaturesOut;
113 |   output.spatialSize=input.spatialSize;
114 |   output.nSpatialSites=input.nSpatialSites;
115 |   output.grids=input.grids;
116 |   int o=nFeaturesOut*(batch.type==TRAINBATCH?(1.0f-dropout):1.0f);
117 |   output.featuresPresent.vec=rng.NchooseM(nFeaturesOut,o);
118 |   output.backpropErrors=true;
119 | }
120 | void NetworkInNetworkLayer::forwards
121 | (SpatiallySparseBatch &batch,
122 |  SpatiallySparseBatchInterface &input,
123 |  SpatiallySparseBatchInterface &output) {
124 |   output.sub->features.resize(output.nSpatialSites*output.featuresPresent.size());
125 |   if (batch.type==TRAINBATCH and
126 |       nFeaturesIn+nFeaturesOut>input.featuresPresent.size()+output.featuresPresent.size()) {
127 |     w.resize(input.featuresPresent.size()*output.featuresPresent.size());
128 |     dShrinkMatrixForDropout
129 |       (W.vec, w.vec,
130 |        input.featuresPresent.vec,
131 |        output.featuresPresent.vec,
132 |        output.nFeatures);
133 |     b.resize(output.featuresPresent.size());
134 |     dShrinkVectorForDropout(B.vec, b.vec,
135 |                             output.featuresPresent.vec);
136 |     replicateArray(b.vec, output.sub->features.vec, output.nSpatialSites, output.featuresPresent.size());
137 |     d_rowMajorSGEMM_alphaAB_betaC(input.sub->features.vec, w.vec, output.sub->features.vec,
138 |                                   output.nSpatialSites, input.featuresPresent.size(), output.featuresPresent.size(),
139 |                                   1.0f, 1.0f);
140 | 
141 |   } else {
142 |     replicateArray(B.vec, output.sub->features.vec, output.nSpatialSites, output.featuresPresent.size());
143 |     d_rowMajorSGEMM_alphaAB_betaC(input.sub->features.vec, W.vec, output.sub->features.vec,
144 |                                   output.nSpatialSites, input.nFeatures, output.nFeatures,
145 |                                   1.0f-dropout, 1.0f-dropout);
146 |   }
147 |   multiplyAddCount+=(__int128_t)output.nSpatialSites*input.featuresPresent.size()*output.featuresPresent.size();
148 |   applySigmoid(output, output, fn);
149 | }
150 | void NetworkInNetworkLayer::scaleWeights
151 | (SpatiallySparseBatchInterface &input,
152 |  SpatiallySparseBatchInterface &output,
153 |  float& scalingUnderneath,
154 |  bool topLayer) {
155 |   assert(input.sub->features.size()>0);
156 |   assert(output.sub->features.size()>0); //call after forwards(...)
157 |   float scale=output.sub->features.meanAbs( (fn==VLEAKYRELU) ? 3 : 100 );
158 |   std::cout << "featureScale:" << scale << std::endl;
159 |   if (topLayer) {
160 |     scale=1;
161 |   } else {
162 |     scale=powf(scale,-0.1); //0.7978846 = sqrt(2/pi) = mean of the half normal distribution
163 |   }
164 |   W.multiplicativeRescale(scale/scalingUnderneath);
165 |   B.multiplicativeRescale(scale);
166 |   MW.multiplicativeRescale(scale/scalingUnderneath);
167 |   MB.multiplicativeRescale(scale);
168 |   scalingUnderneath=scale;
169 | }
170 | void NetworkInNetworkLayer::backwards
171 | (SpatiallySparseBatch &batch,
172 |  SpatiallySparseBatchInterface &input,
173 |  SpatiallySparseBatchInterface &output,
174 |  float learningRate,
175 |  float momentum) {
176 |   applySigmoidBackProp(output, output, fn);
177 |   dw.resize(input.featuresPresent.size()*output.featuresPresent.size());
178 |   db.resize(output.featuresPresent.size());
179 |   d_rowMajorSGEMM_alphaAtB_betaC(input.sub->features.vec, output.sub->dfeatures.vec, dw.vec,
180 |                                  input.featuresPresent.size(), output.nSpatialSites, output.featuresPresent.size(),
181 |                                  1.0, 0.0);
182 | 
183 |   multiplyAddCount+=(__int128_t)output.nSpatialSites*input.featuresPresent.size()*output.featuresPresent.size();
184 |   db.setZero();
185 |   columnSum(output.sub->dfeatures.vec, db.vec, output.nSpatialSites, output.featuresPresent.size());
186 | 
187 |   if (nFeaturesIn+nFeaturesOut>input.featuresPresent.size()+output.featuresPresent.size()) {
188 |     if (input.backpropErrors) {
189 |       input.sub->dfeatures.resize(input.nSpatialSites*input.featuresPresent.size());
190 |       d_rowMajorSGEMM_alphaABt_betaC(output.sub->dfeatures.vec, w.vec, input.sub->dfeatures.vec,
191 |                                      output.nSpatialSites,output.featuresPresent.size(),input.featuresPresent.size(),
192 |                                      1.0, 0.0);
193 |       multiplyAddCount+=(__int128_t)output.nSpatialSites*input.featuresPresent.size()*output.featuresPresent.size();
194 |     }
195 | 
196 |     dGradientDescentShrunkMatrix
197 |       (dw.vec, MW.vec, W.vec,
198 |        output.nFeatures,
199 |        input.featuresPresent.vec, output.featuresPresent.vec,
200 |        learningRate,momentum);
201 | 
202 |     dGradientDescentShrunkVector
203 |       (db.vec, MB.vec, B.vec,
204 |        output.nFeatures,
205 |        output.featuresPresent.vec,
206 |        learningRate,momentum);
207 |   } else {
208 |     if (input.backpropErrors) {
209 |       input.sub->dfeatures.resize(input.nSpatialSites*input.featuresPresent.size());
210 |       d_rowMajorSGEMM_alphaABt_betaC(output.sub->dfeatures.vec, W.vec, input.sub->dfeatures.vec,
211 |                                      output.nSpatialSites,nFeaturesOut,nFeaturesIn,
212 |                                      1.0, 0.0);
213 |       multiplyAddCount+=(__int128_t)output.nSpatialSites*input.featuresPresent.size()*output.featuresPresent.size();
214 |     }
215 |     dGradientDescent
216 |       (dw.vec, MW.vec, W.vec, nFeaturesIn, nFeaturesOut, learningRate,momentum);
217 |     dGradientDescent
218 |       (db.vec, MB.vec, B.vec, 1,           nFeaturesOut, learningRate,momentum);
219 |   }
220 | }
221 | void NetworkInNetworkLayer::loadWeightsFromStream(std::ifstream &f) {
222 |   f.read((char*)&W.vec[0],sizeof(float)*W.size());
223 |   f.read((char*)&B.vec[0],sizeof(float)*B.size());
224 |   MW.setZero();
225 |   MB.setZero();
226 | };
227 | void NetworkInNetworkLayer::putWeightsToStream(std::ofstream &f)  {
228 |   f.write((char*)&W.vec[0],sizeof(float)*W.size());
229 |   f.write((char*)&B.vec[0],sizeof(float)*B.size());
230 | };
231 | int NetworkInNetworkLayer::calculateInputSpatialSize(int outputSpatialSize) {
232 |   return outputSpatialSize;
233 | }
234 | 


--------------------------------------------------------------------------------
/NetworkInNetworkLayer.h:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | #include <fstream>
 3 | #include "SpatiallySparseLayer.h"
 4 | #include "Rng.h"
 5 | 
 6 | class NetworkInNetworkLayer : public SpatiallySparseLayer {
 7 | private:
 8 |   RNG rng;
 9 | public:
10 |   vectorCUDA<float> W; //Weights
11 |   vectorCUDA<float> MW; //momentum
12 |   vectorCUDA<float> w; //shrunk versions
13 |   vectorCUDA<float> dw; //For backprop
14 |   vectorCUDA<float> B; //Weights
15 |   vectorCUDA<float> MB; //momentum
16 |   vectorCUDA<float> b; //shrunk versions
17 |   vectorCUDA<float> db; //For backprop
18 |   ActivationFunction fn;
19 |   int nFeaturesIn;
20 |   int nFeaturesOut;
21 |   float dropout;
22 |   NetworkInNetworkLayer(int nFeaturesIn, int nFeaturesOut,
23 |                         float dropout=0,ActivationFunction fn=NOSIGMOID,
24 |                         float alpha=1//used to determine intialization weights only
25 |                         );
26 |   void preprocess
27 |   (SpatiallySparseBatch &batch,
28 |    SpatiallySparseBatchInterface &input,
29 |    SpatiallySparseBatchInterface &output);
30 |   void forwards
31 |   (SpatiallySparseBatch &batch,
32 |    SpatiallySparseBatchInterface &input,
33 |    SpatiallySparseBatchInterface &output);
34 |   void scaleWeights
35 |   (SpatiallySparseBatchInterface &input,
36 |    SpatiallySparseBatchInterface &output,
37 |    float& scalingUnderneath,
38 |    bool topLayer);
39 |   void backwards
40 |   (SpatiallySparseBatch &batch,
41 |    SpatiallySparseBatchInterface &input,
42 |    SpatiallySparseBatchInterface &output,
43 |    float learningRate,
44 |    float momentum);
45 |   void loadWeightsFromStream(std::ifstream &f);
46 |   void putWeightsToStream(std::ofstream &f);
47 |   int calculateInputSpatialSize(int outputSpatialSize);
48 | };
49 | 
50 | void dShrinkMatrixForDropout
51 | (std::vector<float>& m, std::vector<float>& md,
52 |  std::vector<int>& inFeaturesPresent, std::vector<int>& outFeaturesPresent,
53 |  int nOut);
54 | void dShrinkVectorForDropout
55 | (std::vector<float>& m, std::vector<float>& md, std::vector<int>& outFeaturesPresent);
56 | void dGradientDescent
57 | (std::vector<float>& d_delta, std::vector<float>& d_momentum, std::vector<float>& d_weights,
58 |  int nIn, int nOut, float learningRate, float momentum);
59 | void dGradientDescentShrunkMatrix
60 | (std::vector<float>& d_delta, std::vector<float>& d_momentum, std::vector<float>& d_weights,
61 |  int nOut,
62 |  std::vector<int>& inFeaturesPresent, std::vector<int>& outFeaturesPresent,
63 |  float learningRate,float momentum);
64 | void dGradientDescentShrunkVector
65 | (std::vector<float>& d_delta, std::vector<float>& d_momentum, std::vector<float>& d_weights,
66 |  std::vector<int>& outFeaturesPresent,
67 |  float learningRate,float momentum);
68 | void columnSum(std::vector<float>& matrix, std::vector<float>& target, int nRows, int nColumns);
69 | 


--------------------------------------------------------------------------------
/OnlineHandwritingPicture.cpp:
--------------------------------------------------------------------------------
  1 | #include "OnlineHandwritingPicture.h"
  2 | #include "signature.h"
  3 | 
  4 | OnlineHandwritingPicture::OnlineHandwritingPicture(int renderSize, OnlineHandwritingEncoding enc, int label, float penSpeed3d) :
  5 |   enc(enc), renderSize(renderSize), Picture(label), offset3d(0), penSpeed3d(penSpeed3d) {
  6 | }
  7 | OnlineHandwritingPicture::~OnlineHandwritingPicture() {
  8 | }
  9 | void OnlineHandwritingPicture::normalize() { //Fit centrally in the cube [-renderSize/2,renderSize/2]^2
 10 |   arma::mat pointsm(ops.size(),2);
 11 |   arma::mat pointsM(ops.size(),2);
 12 |   for (int i=0;i<ops.size();++i) {
 13 |     pointsm.row(i)=arma::min(ops[i],0);
 14 |     pointsM.row(i)=arma::max(ops[i],0);
 15 |   }
 16 |   pointsm=arma::min(pointsm,0);
 17 |   pointsM=arma::max(pointsM,0);
 18 |   float scale = arma::mat(pointsM-pointsm).max();
 19 |   assert(scale>0);
 20 |   for (int i=0;i<ops.size();++i) {
 21 |     ops[i]=ops[i]-arma::repmat(0.5*(pointsm+pointsM),ops[i].n_rows,1);
 22 |     ops[i]*=renderSize/scale;
 23 |   }
 24 | }
 25 | 
 26 | arma::mat constantSpeed
 27 | (arma::mat &stroke,
 28 |  float density,
 29 |  int multiplier = 1) {
 30 |   std::vector<float> lengths(stroke.n_rows);
 31 |   lengths[0]=0;
 32 |   for (int i=1;i<stroke.n_rows;i++) {
 33 |     lengths[i]=lengths[i-1]+pow(pow(stroke(i,0)-stroke(i-1,0),2)+
 34 |                                 pow(stroke(i,1)-stroke(i-1,1),2),0.5);
 35 |   }
 36 |   float lTotal=lengths[stroke.n_rows-1];
 37 |   int n=(int)(0.5+lTotal/density);
 38 |   n*=multiplier;
 39 |   arma::mat r(n+1,2);
 40 |   int j=0;
 41 |   float alpha;
 42 |   r(0,0)=stroke(0,0);
 43 |   r(0,1)=stroke(0,1);
 44 |   for (int i=1;i<=n;i++) {
 45 |     while(n*lengths[j+1]<i*lTotal) j++;
 46 |     alpha=(lengths[j+1]-i*lTotal/n)/(lengths[j+1]-lengths[j]);
 47 |     r(i,0)=stroke(j,0)*alpha+stroke(j+1,0)*(1-alpha);
 48 |     r(i,1)=stroke(j,1)*alpha+stroke(j+1,1)*(1-alpha);
 49 |   }
 50 |   return r;
 51 | }
 52 | 
 53 | 
 54 | int mapToGrid(float coord, int scale_N) {
 55 |   return std::max(0,std::min(scale_N-1,(int)(coord+0.5*scale_N)));
 56 | }
 57 | int maxLength=0;
 58 | void OnlineHandwritingPicture::codifyInputData(SparseGrid &grid, std::vector<float> &features, int &nSpatialSites, int spatialSize) {
 59 |   int nInputFeatures=OnlineHandwritingEncodingSize[enc];
 60 |   //Background feature
 61 |   grid.backgroundCol=nSpatialSites++;
 62 |   features.resize(nInputFeatures*nSpatialSites,0);
 63 |   //Render character
 64 |   switch(enc) {
 65 |   case Simple:
 66 |     for (int i=0; i<ops.size(); i++) {
 67 |       arma::mat csp=constantSpeed(ops[i],0.1);
 68 |       for (int j=0;j<csp.n_rows;++j) {
 69 |         int
 70 |           a0=csp(j,0)+spatialSize/2.0,
 71 |           a1=csp(j,1)+spatialSize/2.0;
 72 |         if (a0>=0 and a1>=0 and a0<spatialSize and a1<spatialSize) {
 73 |           int n= a0*spatialSize + a1;
 74 |           if (grid.mp.find(n)==grid.mp.end()) {
 75 |             grid.mp[n]=nSpatialSites++;
 76 |             features.resize(nInputFeatures*nSpatialSites,0);
 77 |           }
 78 |           ++features[grid.mp[n]];
 79 |           //          std::cout <<"."<<std::flush;
 80 |         } //else  std::cout <<"x"<<std::flush;
 81 |       }
 82 |     }
 83 |     break;
 84 |   case Octogram: {//todo: Test
 85 |     float piBy4=3.1415926536/4;
 86 |     for (int i=0; i<ops.size(); i++) {
 87 |       arma::mat csp=constantSpeed(ops[i],0.1);
 88 |       for (int j=0;j<csp.n_rows-1;++j) {
 89 |         int
 90 |           a0=csp(j,0)+spatialSize/2.0,
 91 |           a1=csp(j,1)+spatialSize/2.0;
 92 |         if (a0>=0 and a1>=0 and a0<spatialSize and a1<spatialSize) {
 93 |           int n= a0*spatialSize + a1;
 94 |           if (grid.mp.find(n)==grid.mp.end()) {
 95 |             grid.mp[n]=nSpatialSites++;
 96 |             features.resize(nInputFeatures*nSpatialSites,0);
 97 |           }
 98 |           int mpnnIF=nInputFeatures*grid.mp[n];
 99 |           features[mpnnIF]++;
100 |           float dx=(csp(j+1,0)-csp(j,0));
101 |           float dy=(csp(j+1,1)-csp(j,1));
102 |           float m=powf(dx*dx+dy*dy,0.5);
103 |           dx/=m;
104 |           dy/=m;
105 |           for (int k=0;k<8;k++) {
106 |             float alpha=1-acosf(cosf(k*piBy4)*dx+sinf(k*piBy4)*dy)/piBy4;
107 |             if (alpha>0)
108 |               features[mpnnIF+k+1]+=alpha;
109 |           }
110 |         }
111 |       }
112 |     }
113 |     break;
114 |   }
115 |   case LogSignature1:
116 |     //Todo
117 |     break;
118 |   case LogSignature2:
119 |     //Todo
120 |     break;
121 |   case LogSignature3:
122 |     //Todo
123 |     break;
124 |   case LogSignature4:
125 |     //Todo
126 |     break;
127 |   case SpaceTime3d: {
128 |     std::vector<arma::mat> csp;
129 |     int cspLengths=0;
130 |     for (int i=0; i<ops.size(); i++) {
131 |       csp.push_back(constantSpeed(ops[i],0.1));
132 |       cspLengths+=csp[i].n_rows;
133 |     }
134 |     // if (cspLengths>maxLength) {
135 |     //   std::cout <<cspLengths*penSpeed3d << " " << std::flush;
136 |     //   maxLength=cspLengths;
137 |     // }
138 |     int z=0;
139 |     for (int i=0; i<ops.size(); i++) {
140 |       for (int j=0;j<csp[i].n_rows;++j) {
141 |         int
142 |           a0=csp[i](j,0)+spatialSize/2.0,
143 |           a1=csp[i](j,1)+spatialSize/2.0,
144 |           a2=renderSize*((z-cspLengths/2)*penSpeed3d+offset3d)+spatialSize/2;
145 |         z++;
146 |         if (a0>=0 and a1>=0 and a2>=0 and a0<spatialSize and a1<spatialSize and a2<spatialSize) {
147 |           int n= a0*spatialSize*spatialSize + a1*spatialSize + a2;
148 |           if (grid.mp.find(n)==grid.mp.end()) {
149 |             grid.mp[n]=nSpatialSites++;
150 |             features.resize(nInputFeatures*nSpatialSites,0);
151 |           }
152 |           ++features[grid.mp[n]];
153 |         }
154 |       }
155 |     }
156 |     break;
157 |   }
158 |   case VectorSpaceTime3d: {
159 |     std::vector<arma::mat> csp;
160 |     int cspLengths=0;
161 |     for (int i=0; i<ops.size(); i++) {
162 |       csp.push_back(constantSpeed(ops[i],0.1));
163 |       cspLengths+=csp[i].n_rows;
164 |     }
165 |     // if (cspLengths>maxLength) {
166 |     //   std::cout << cspLengths*penSpeed3d << " " << std::flush;
167 |     //   maxLength=cspLengths;
168 |     // }
169 |     int z=0;
170 |     for (int i=0; i<ops.size(); i++) {
171 |       for (int j=0;j<csp[i].n_rows-1;++j) {
172 |         int
173 |           a0=csp[i](j,0)+spatialSize/2.0,
174 |           a1=csp[i](j,1)+spatialSize/2.0,
175 |           a2=renderSize*((z-cspLengths/2)*penSpeed3d+offset3d)+spatialSize/2;
176 |         z++;
177 |         if (a0>=0 and a1>=0 and a2>=0 and a0<spatialSize and a1<spatialSize and a2<spatialSize) {
178 |           int n= a0*spatialSize*spatialSize + a1*spatialSize + a2;
179 |           if (grid.mp.find(n)==grid.mp.end()) {
180 |             grid.mp[n]=nSpatialSites++;
181 |             features.resize(nInputFeatures*nSpatialSites,0);
182 |           }
183 |           int mpnnIF=nInputFeatures*grid.mp[n];
184 |           features[mpnnIF]++;
185 |           float dx=(csp[i](j+1,0)-csp[i](j,0));
186 |           float dy=(csp[i](j+1,1)-csp[i](j,1));
187 |           float m=powf(dx*dx+dy*dy,0.5);
188 |           dx/=m;
189 |           dy/=m;
190 |           features[mpnnIF+1]+=dx;
191 |           features[mpnnIF+2]+=dy;
192 |         }
193 |       }
194 |     }
195 |     break;
196 |   }
197 |   }
198 | }
199 | 


--------------------------------------------------------------------------------
/OnlineHandwritingPicture.h:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | #include "Picture.h"
 4 | #include "Rng.h"
 5 | #include <armadillo>
 6 | #include <vector>
 7 | 
 8 | class OnlineHandwritingPicture : public Picture {
 9 |   float penSpeed3d;
10 |   float offset3d;
11 | public:
12 |   std::vector<arma::mat> ops;
13 |   int renderSize;
14 |   OnlineHandwritingEncoding enc;
15 |   OnlineHandwritingPicture(int renderSize, OnlineHandwritingEncoding enc, int label, float penSpeed3d=0.02);
16 |   ~OnlineHandwritingPicture();
17 |   void normalize();  //Fit centrally in the square [-renderSize/2,renderSize/2]^2
18 |   Picture* distort (RNG& rng, batchType type);
19 |   void codifyInputData(SparseGrid &grid, std::vector<float> &features, int &nSpatialSites, int spatialSize);
20 |   void jiggle(RNG &rng, float alpha);
21 | };
22 | 


--------------------------------------------------------------------------------
/OnlineHandwritingTriangularPicture.cpp:
--------------------------------------------------------------------------------
  1 | #include "OnlineHandwritingPicture.h"
  2 | #include "signature.h"
  3 | 
  4 | OnlineHandwritingPicture::OnlineHandwritingPicture(int renderSize, OnlineHandwritingEncoding enc, int label, float penSpeed3d) :
  5 |   enc(enc), renderSize(renderSize), Picture(label), offset3d(0), penSpeed3d(penSpeed3d) {
  6 | }
  7 | OnlineHandwritingPicture::~OnlineHandwritingPicture() {
  8 | }
  9 | void OnlineHandwritingPicture::normalize() { //Fit centrally in the cube [-renderSize/2,renderSize/2]^2
 10 |   arma::mat pointsm(ops.size(),2);
 11 |   arma::mat pointsM(ops.size(),2);
 12 |   for (int i=0;i<ops.size();++i) {
 13 |     pointsm.row(i)=arma::min(ops[i],0);
 14 |     pointsM.row(i)=arma::max(ops[i],0);
 15 |   }
 16 |   pointsm=arma::min(pointsm,0);
 17 |   pointsM=arma::max(pointsM,0);
 18 |   float scale = arma::mat(pointsM-pointsm).max();
 19 |   assert(scale>0);
 20 |   for (int i=0;i<ops.size();++i) {
 21 |     ops[i]=ops[i]-arma::repmat(0.5*(pointsm+pointsM),ops[i].n_rows,1);
 22 |     ops[i]*=renderSize/scale;
 23 |   }
 24 | }
 25 | 
 26 | arma::mat constantSpeed
 27 | (arma::mat &stroke,
 28 |  float density,
 29 |  int multiplier = 1) {
 30 |   std::vector<float> lengths(stroke.n_rows);
 31 |   lengths[0]=0;
 32 |   for (int i=1;i<stroke.n_rows;i++) {
 33 |     lengths[i]=lengths[i-1]+pow(pow(stroke(i,0)-stroke(i-1,0),2)+
 34 |                                 pow(stroke(i,1)-stroke(i-1,1),2),0.5);
 35 |   }
 36 |   float lTotal=lengths[stroke.n_rows-1];
 37 |   int n=(int)(0.5+lTotal/density);
 38 |   n*=multiplier;
 39 |   arma::mat r(n+1,2);
 40 |   int j=0;
 41 |   float alpha;
 42 |   r(0,0)=stroke(0,0);
 43 |   r(0,1)=stroke(0,1);
 44 |   for (int i=1;i<=n;i++) {
 45 |     while(n*lengths[j+1]<i*lTotal) j++;
 46 |     alpha=(lengths[j+1]-i*lTotal/n)/(lengths[j+1]-lengths[j]);
 47 |     r(i,0)=stroke(j,0)*alpha+stroke(j+1,0)*(1-alpha);
 48 |     r(i,1)=stroke(j,1)*alpha+stroke(j+1,1)*(1-alpha);
 49 |   }
 50 |   return r;
 51 | }
 52 | 
 53 | 
 54 | int maxLength=0;
 55 | 
 56 | void OnlineHandwritingPicture::codifyInputData(SparseGrid &grid, std::vector<float> &features, int &nSpatialSites, int spatialSize){
 57 |   int nInputFeatures=OnlineHandwritingEncodingSize[enc];
 58 |   //Background feature
 59 |   grid.backgroundCol=nSpatialSites++;
 60 |   features.resize(nInputFeatures*nSpatialSites,0);
 61 |   //Render character
 62 |   switch(enc) {
 63 |   case Simple:
 64 |     for (int i=0; i<ops.size(); i++) {
 65 |       arma::mat csp=constantSpeed(ops[i],0.1);
 66 |       for (int j=0;j<csp.n_rows;++j) {
 67 |         int
 68 |           a0=csp(j,0)+spatialSize/3.0,
 69 |           a1=-0.57735*csp(j,0)+1.1547*csp(j,1)+spatialSize/3.0;
 70 |         if (a0>=0 and a1>=0 and a0+a1<spatialSize) {
 71 |           int n= a0*spatialSize + a1;
 72 |           if (grid.mp.find(n)==grid.mp.end()) {
 73 |             grid.mp[n]=nSpatialSites++;
 74 |             features.resize(nInputFeatures*nSpatialSites,0);
 75 |           }
 76 |           ++features[grid.mp[n]];
 77 |         }
 78 |       }
 79 |     }
 80 |     break;
 81 |   case Octogram: {//todo: Test
 82 |     float piBy4=3.1415926536/4;
 83 |     for (int i=0; i<ops.size(); i++) {
 84 |       arma::mat csp=constantSpeed(ops[i],0.1);
 85 |       for (int j=0;j<csp.n_rows-1;++j) {
 86 |         int
 87 |           a0=csp(j,0)+spatialSize/3.0,
 88 |           a1=-0.57735*csp(j,0)+1.1547*csp(j,1)+spatialSize/3.0;
 89 |         if (a0>=0 and a1>=0 and a0+a1<spatialSize) {
 90 |           int n= a0*spatialSize + a1;
 91 |           if (grid.mp.find(n)==grid.mp.end()) {
 92 |             grid.mp[n]=nSpatialSites++;
 93 |             features.resize(nInputFeatures*nSpatialSites,0);
 94 |           }
 95 |           int mpnnIF=nInputFeatures*grid.mp[n];
 96 |           features[mpnnIF]++;
 97 |           float dx=(csp(j+1,0)-csp(j,0));
 98 |           float dy=(csp(j+1,1)-csp(j,1));
 99 |           float m=powf(dx*dx+dy*dy,0.5);
100 |           dx/=m;
101 |           dy/=m;
102 |           for (int k=0;k<8;k++) {
103 |             float alpha=1-acosf(cosf(k*piBy4)*dx+sinf(k*piBy4)*dy)/piBy4;
104 |             if (alpha>0)
105 |               features[mpnnIF+k+1]+=alpha;
106 |           }
107 |         }
108 |       }
109 |     }
110 |     break;
111 |   }
112 |   case LogSignature1:
113 |     //Todo
114 |     break;
115 |   case LogSignature2:
116 |     //Todo
117 |     break;
118 |   case LogSignature3:
119 |     //Todo
120 |     break;
121 |   case LogSignature4:
122 |     //Todo
123 |     break;
124 |   case SpaceTime3d: {
125 |     std::vector<arma::mat> csp;
126 |     int cspLengths=0;
127 |     for (int i=0; i<ops.size(); i++) {
128 |       csp.push_back(constantSpeed(ops[i],0.1));
129 |       cspLengths+=csp[i].n_rows;
130 |     }
131 |     // // if (cspLengths>maxLength) {
132 |     // //   std::cout <<cspLengths*penSpeed3d << " " << std::flush;
133 |     // //   maxLength=cspLengths;
134 |     // // }
135 |     int z=0;
136 |     for (int i=0; i<ops.size(); i++) {
137 |       for (int j=0;j<csp[i].n_rows;++j) {
138 |         int
139 |           a0=csp[i](j,0)+spatialSize/4.0,
140 |           a1=-0.57735*csp[i](j,0)+1.1547*csp[i](j,1)+spatialSize/4.0,
141 |           a2=-0.40825*csp[i](j,0)-0.40825*csp[i](j,1)+1.2247*renderSize*(  (z-cspLengths/2)*penSpeed3d  + offset3d  )+spatialSize/4.0;
142 |         z++;
143 |         if (a0>=0 && a1>=0 && a2>=0 && a0+a1+a2<spatialSize) {
144 |           int n=a0*spatialSize*spatialSize+a1*spatialSize+a2;
145 |           if (grid.mp.find(n)==grid.mp.end()) {
146 |             grid.mp[n]=nSpatialSites++;
147 |             features.resize(nInputFeatures*nSpatialSites,0);
148 |           }
149 |           ++features[grid.mp[n]];
150 |           ///std::cout << "."<<std::flush;
151 |         } else std::cout << "x"<<std::flush;
152 |       }
153 |     }
154 |     break;
155 |   }
156 |   case VectorSpaceTime3d: {
157 |     std::vector<arma::mat> csp;
158 |     int cspLengths=0;
159 |     for (int i=0; i<ops.size(); i++) {
160 |       csp.push_back(constantSpeed(ops[i],0.1));
161 |       cspLengths+=csp[i].n_rows;
162 |     }
163 |     if (cspLengths>maxLength) {
164 |       std::cout <<cspLengths*penSpeed3d << " " << std::flush;
165 |       maxLength=cspLengths;
166 |     }
167 |     int z=0;
168 |     for (int i=0; i<ops.size(); i++) {
169 |       for (int j=0;j<csp[i].n_rows-1;++j) {
170 |         int
171 |           a0=csp[i](j,0)+spatialSize/4.0,
172 |           a1=-0.57735*csp[i](j,0)+1.1547*csp[i](j,1)+spatialSize/4.0,
173 |           a2=-0.40825*csp[i](j,0)-0.40825*csp[i](j,1)+1.2247*renderSize*(  (z-cspLengths/2)*penSpeed3d  + offset3d  )+spatialSize/4.0;
174 |         z++;
175 |         if (a0>=0 && a1>=0 && a2>=0 && a0+a1+a2<spatialSize) {
176 |           int n=a0*spatialSize*spatialSize+a1*spatialSize+a2;
177 |           if (grid.mp.find(n)==grid.mp.end()) {
178 |             grid.mp[n]=nSpatialSites++;
179 |             features.resize(nInputFeatures*nSpatialSites,0);
180 |           }
181 |           int mpnnIF=nInputFeatures*grid.mp[n];
182 |           features[mpnnIF]++;
183 |           float dx=(csp[i](j+1,0)-csp[i](j,0));
184 |           float dy=(csp[i](j+1,1)-csp[i](j,1));
185 |           float m=powf(dx*dx+dy*dy,0.5);
186 |           dx/=m;
187 |           dy/=m;
188 |           features[mpnnIF+1]+=dx;
189 |           features[mpnnIF+2]+=dy;
190 |           //   std::cout << "."<<std::flush;
191 |         } //else std::cout << "x"<<std::flush;
192 |       }
193 |     }
194 |     break;
195 |   }
196 |   }
197 | }
198 | 
199 | //Map tetrahedron to simplex
200 | //arma::mat convPyrCub="1 0 0;  -0.57735   1.1547  0;  -0.40825  -0.40825 1.2247";
201 | // (0,0,0) -> (0.0.0)
202 | // (1,0,0) -> (1,0,0)
203 | // (0.5,0.866,0) -> (0,1,0)
204 | // (0.5,0.289,0.82) -> (0,0,1)
205 | 
206 | 
207 | Picture* OnlineHandwritingPicture::distort(RNG& rng, batchType type) {
208 |   OnlineHandwritingPicture* pic=new OnlineHandwritingPicture(*this);
209 |   if (type==TRAINBATCH) {
210 |     arma::mat aff=arma::eye(2,2);
211 |     aff(0,0)+=rng.uniform(-0.3,0.3)/1; //x stretch
212 |     aff(1,1)+=rng.uniform(-0.3,0.3)/1; //y stretch
213 |     int r=rng.randint(3);
214 |     float alpha=rng.uniform(-0.3,0.3)/1;
215 |     arma::mat x=arma::eye(2,2);
216 |     if (r==0) x(0,1)=alpha;
217 |     if (r==1) x(1,0)=alpha;
218 |     if (r=2) {x(0,0)=cos(alpha);x(0,1)=-sin(alpha);x(1,0)=sin(alpha);x(1,1)=cos(alpha);}
219 |     aff=aff*x;
220 |     arma::mat y(1,2);
221 |     y(0,0)=renderSize*rng.uniform(-0.1875,0.1875)/1;
222 |     y(0,1)=renderSize*rng.uniform(-0.1875,0.1875)/1;
223 |     for (int i=0;i<ops.size();++i)
224 |       pic->ops[i]=pic->ops[i]*aff+arma::repmat(y,pic->ops[i].n_rows,1);
225 |     pic->offset3d=rng.uniform(-0.1,0.1);
226 |   }
227 |   return pic;
228 | }
229 | 


--------------------------------------------------------------------------------
/OpenCVPicture.cpp:
--------------------------------------------------------------------------------
  1 | #include "OpenCVPicture.h"
  2 | #include <cmath>
  3 | 
  4 | float OpenCVPicture::scaleUCharColor(float col) {
  5 |   float div=std::max(255-backgroundColor,backgroundColor);
  6 |   return (col-backgroundColor)/div;
  7 | }
  8 | 
  9 | OpenCVPicture::OpenCVPicture(int xSize, int ySize, int nInputFeatures, unsigned char backgroundColor,int label) :
 10 |   Picture(label), backgroundColor(backgroundColor) {
 11 |   xOffset=-xSize/2;
 12 |   yOffset=-ySize/2;
 13 |   mat.create(xSize,ySize, CV_32FC(nInputFeatures));
 14 | }
 15 | OpenCVPicture::OpenCVPicture(std::string filename, unsigned char backgroundColor, int label_) :
 16 |   filename(filename), backgroundColor(backgroundColor) {
 17 |   label=label_;
 18 | }
 19 | 
 20 | OpenCVPicture::~OpenCVPicture() {}
 21 | 
 22 | void OpenCVPicture::jiggle(RNG &rng, int offlineJiggle) {
 23 |   xOffset+=rng.randint(offlineJiggle*2+1)-offlineJiggle;
 24 |   yOffset+=rng.randint(offlineJiggle*2+1)-offlineJiggle;
 25 | }
 26 | void OpenCVPicture::colorDistortion(RNG &rng, int sigma1, int sigma2, int sigma3, int sigma4) {
 27 |   distortImageColor(mat, rng, sigma1, sigma2, sigma3, sigma4);
 28 | }
 29 | void OpenCVPicture::randomCrop(RNG &rng, int subsetSize) {
 30 |   assert(subsetSize<=std::min(mat.rows,mat.cols));
 31 |   cropImage(mat, rng.randint(mat.cols-subsetSize),rng.randint(mat.rows-subsetSize), subsetSize, subsetSize);
 32 |   xOffset=yOffset=-subsetSize/2;
 33 | }
 34 | void OpenCVPicture::affineTransform(float c00, float c01, float c10, float c11) {
 35 | 
 36 |   transformImage(mat, backgroundColor, c00, c01, c10, c11);
 37 |   xOffset=-mat.cols/2;
 38 |   yOffset=-mat.rows/2;
 39 | }
 40 | void OpenCVPicture::jiggleFit(RNG &rng, int subsetSize, float minFill) {
 41 |   if (minFill<0) { //Just pick a random subsetSize x subsetSquare that overlaps the picture as much as possible.
 42 |     if (mat.cols>=subsetSize)
 43 |       xOffset=-rng.randint(mat.cols-subsetSize+1)-subsetSize/2;
 44 |     else
 45 |       xOffset=rng.randint(subsetSize-mat.cols+1)-subsetSize/2;
 46 |     if (mat.rows>=subsetSize)
 47 |       yOffset=-rng.randint(mat.rows-subsetSize+1)-subsetSize/2;
 48 |     else
 49 |       yOffset=rng.randint(subsetSize-mat.rows+1)-subsetSize/2;
 50 |   } else {
 51 |     int fitCtr=100; //Give up after 100 failed attempts to find a good fit
 52 |     bool goodFit=false;
 53 |     while (!goodFit and fitCtr --> 0) {
 54 |       xOffset=-subsetSize/2-rng.randint(mat.cols-subsetSize);//-rng.randint(mat.cols);
 55 |       yOffset=-subsetSize/2-rng.randint(mat.rows-subsetSize);//-rng.randint(mat.rows);
 56 |       int pointsCtr=0;
 57 |       int interestingPointsCtr=0;
 58 |       for (int X=5; X<subsetSize; X+=10) {
 59 |         for (int Y=5; Y<subsetSize; Y+=10) {
 60 |           int x=X-xOffset-subsetSize/2;
 61 |           int y=Y-yOffset-subsetSize/2;
 62 |           pointsCtr++;
 63 |           if (0<=x and x<mat.cols and 0<=y and y<mat.rows)
 64 |             interestingPointsCtr+=(mat.ptr()[(pointsCtr%mat.channels())+x*mat.channels()+y*mat.channels()*mat.cols]!=backgroundColor);
 65 |         }
 66 |       }
 67 |       assert(pointsCtr>=10);
 68 |       if (interestingPointsCtr>pointsCtr*minFill)
 69 |         goodFit=true;
 70 |     }
 71 |     if (!goodFit) {
 72 |       std::cout << filename << " " << std::flush;
 73 |       xOffset=-mat.cols/2-16+rng.randint(32);
 74 |       yOffset=-mat.rows/2-16+rng.randint(32);
 75 |     }
 76 |   }
 77 | }
 78 | void OpenCVPicture::centerMass() {
 79 |   float ax=0, ay=0, axx=0, ayy=0, axy, d=0.001;
 80 |   for (int i=0; i<mat.channels(); i++) {
 81 |     for (int x=0;x<mat.cols;++x) {
 82 |       for (int y=0;y<mat.rows;++y) {
 83 |         float f=powf(backgroundColor-mat.ptr()[i+x*mat.channels()+y*mat.channels()*mat.cols],2);
 84 |         ax+=x*f;
 85 |         ay+=y*f;
 86 |         axx+=x*x*f;
 87 |         axy+=x*y*f;
 88 |         ayy+=y*y*f;
 89 |         d+=f;
 90 |       }
 91 |     }
 92 |   }
 93 |   ax/=d;
 94 |   ay/=d;
 95 |   axx/=d;
 96 |   axy/=d;
 97 |   ayy/=d;
 98 |   xOffset=-ax/2;
 99 |   yOffset=-ay/2;
100 |   scale2xx=axx-ax*ax;
101 |   scale2xy=axy-ax*ay;
102 |   scale2yy=ayy-ay*ay;
103 |   scale2=powf(scale2xx+scale2yy,0.5);
104 | }
105 | void OpenCVPicture::loadDataWithoutScaling(int flags) {
106 |   readImage(filename,mat,flags);
107 |   xOffset=-mat.cols/2;
108 |   yOffset=-mat.rows/2;
109 | }
110 | void OpenCVPicture::loadData (int scale, int flags) {
111 |   readTransformedImage(filename,mat,scale,flags,1,0,0,1,backgroundColor);
112 |   xOffset=-mat.cols/2;
113 |   yOffset=-mat.rows/2;
114 | }
115 | std::string OpenCVPicture::identify() {
116 |   return filename;
117 | }
118 | 
119 | void OpenCVPicture::codifyInputData(SparseGrid &grid, std::vector<float> &features, int &nSpatialSites, int spatialSize) {
120 |   assert(!mat.empty());
121 |   assert(mat.type()%8==5);
122 |   for  (int i=0; i<mat.channels(); i++)
123 |     features.push_back(0); //Background feature
124 |   grid.backgroundCol=nSpatialSites++;
125 |   int x0=std::max(0,-xOffset-spatialSize/2);    //If x0<=x<x1 and y0<=y<y1 then the (x,y)-th pixel is in the CNN's visual field.
126 |   int x1=std::min(mat.cols,spatialSize-xOffset-spatialSize/2);
127 |   int y0=std::max(0,-yOffset-spatialSize/2);
128 |   int y1=std::min(mat.rows,spatialSize-yOffset-spatialSize/2);
129 |   float* matData=((float*)(mat.data));
130 |   for (int x=x0; x<x1; x++) {
131 |     for (int y=y0; y<y1; y++) {
132 |       bool interestingPixel=false; //Check pixel differs from the background color
133 |       for (int i=0; i<mat.channels(); i++)
134 |         if (std::abs(scaleUCharColor(matData[i+x*mat.channels()+y*mat.channels()*mat.cols]))>0.02)
135 |           interestingPixel=true;
136 |       if (interestingPixel) {
137 |         int n=(x+xOffset+spatialSize/2)*spatialSize+(y+yOffset+spatialSize/2); //Determine location in the input field.
138 |         grid.mp[n]=nSpatialSites++;
139 |         for (int i=0; i<mat.channels(); i++) {
140 |           features.push_back
141 |             (scaleUCharColor(matData[i+x*mat.channels()+y*mat.channels()*mat.cols]));
142 |         }
143 |       }
144 |     }
145 |   }
146 | }
147 | 
148 | 
149 | void matrixMul2x2inPlace(float& c00, float& c01, float& c10, float& c11, float a00, float a01, float a10, float a11) { //c<-c*a
150 |   float t00=c00*a00+c01*a10;  float t01=c00*a01+c01*a11;
151 |   float t10=c10*a00+c11*a10;  float t11=c10*a01+c11*a11;
152 |   c00=t00;c01=t01;
153 |   c10=t10;c11=t11;
154 | }
155 | 


--------------------------------------------------------------------------------
/OpenCVPicture.h:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | #include "readImageToMat.h"
 3 | #include "Picture.h"
 4 | 
 5 | //If filename contain an image filename, then it gets loaded as needed; otherwise mat is used to stores the image
 6 | 
 7 | class OpenCVPicture : public Picture {
 8 |   float scaleUCharColor(float col);
 9 | public:
10 |   int xOffset; //Shift to the right
11 |   int yOffset; //Shift down
12 |   int backgroundColor;
13 |   float scale2;
14 |   float scale2xx, scale2xy,scale2yy;
15 |   cv::Mat mat;
16 |   std::string filename;
17 | 
18 |   OpenCVPicture(int xSize, int ySize, int nInputFeatures, unsigned char backgroundColor,int label_ = -1);
19 |   OpenCVPicture(std::string filename, unsigned char backgroundColor=128, int label_ = -1);
20 |   ~OpenCVPicture();
21 |   Picture* distort (RNG& rng, batchType type=TRAINBATCH);
22 |   void affineTransform(float c00, float c01, float c10, float c11);
23 |   void centerMass();
24 |   void codifyInputData(SparseGrid &grid, std::vector<float> &features, int &nSpatialSites, int spatialSize);
25 |   void jiggle(RNG &rng, int offlineJiggle);
26 |   void jiggleFit(RNG &rng, int subsetSize, float minFill=-1);
27 |   void colorDistortion(RNG &rng, int sigma1, int sigma2, int sigma3, int sigma4);
28 |   void loadData  (int scale, int flags=1);
29 |   void loadDataWithoutScaling(int flags=1);
30 |   void randomCrop(RNG &rng, int subsetSize);
31 |   std::string identify();
32 | };
33 | 
34 | void matrixMul2x2inPlace(float& c00, float& c01, float& c10, float& c11, float a00, float a01, float a10, float a11);
35 | 


--------------------------------------------------------------------------------
/OpenCVTriangularPicture.cpp:
--------------------------------------------------------------------------------
  1 | #include "OpenCVPicture.h"
  2 | #include <cmath>
  3 | 
  4 | float OpenCVPicture::scaleUCharColor(float col) {
  5 |   float div=std::max(255-backgroundColor,backgroundColor);
  6 |   return (col-backgroundColor)/div;
  7 | }
  8 | 
  9 | OpenCVPicture::OpenCVPicture(int xSize, int ySize, int nInputFeatures, unsigned char backgroundColor,int label) :
 10 |   Picture(label), backgroundColor(backgroundColor) {
 11 |   xOffset=-xSize/2;
 12 |   yOffset=-ySize/2;
 13 |   mat.create(xSize,ySize, CV_32FC(nInputFeatures));
 14 | }
 15 | OpenCVPicture::OpenCVPicture(std::string filename, unsigned char backgroundColor, int label_) :
 16 |   filename(filename), backgroundColor(backgroundColor) {
 17 |   label=label_;
 18 | }
 19 | 
 20 | OpenCVPicture::~OpenCVPicture() {}
 21 | 
 22 | void OpenCVPicture::jiggle(RNG &rng, int offlineJiggle) {
 23 |   xOffset+=rng.randint(offlineJiggle*2+1)-offlineJiggle;
 24 |   yOffset+=rng.randint(offlineJiggle*2+1)-offlineJiggle;
 25 | }
 26 | void OpenCVPicture::colorDistortion(RNG &rng, int sigma1, int sigma2, int sigma3, int sigma4) {
 27 |   distortImageColor(mat, rng, sigma1, sigma2, sigma3, sigma4);
 28 | }
 29 | void OpenCVPicture::randomCrop(RNG &rng, int subsetSize) {
 30 |   assert(subsetSize<=std::min(mat.rows,mat.cols));
 31 |   cropImage(mat, rng.randint(mat.cols-subsetSize),rng.randint(mat.rows-subsetSize), subsetSize, subsetSize);
 32 |   xOffset=yOffset=-subsetSize/2;
 33 | }
 34 | void OpenCVPicture::affineTransform(float c00, float c01, float c10, float c11) {
 35 | 
 36 |   transformImage(mat, backgroundColor, c00, c01, c10, c11);
 37 |   xOffset=-mat.cols/2;
 38 |   yOffset=-mat.rows/2;
 39 | }
 40 | void OpenCVPicture::jiggleFit(RNG &rng, int subsetSize, float minFill) { //subsetSize==spatialSize for codifyInputData
 41 |   int fitCtr=100; //Give up after 100 failed attempts to find a good fit
 42 |   bool goodFit=false;
 43 |   while (!goodFit and fitCtr-- >0) {
 44 |     xOffset=-rng.randint(mat.cols-subsetSize/3);
 45 |     yOffset=-rng.randint(mat.rows-subsetSize/3);
 46 |     int pointsCtr=0;
 47 |     int interestingPointsCtr=0;
 48 |     for (int X=5; X<subsetSize; X+=10) {
 49 |       for (int Y=5; Y<subsetSize-X; Y+=10) {
 50 |         int x=X-xOffset-subsetSize/3;
 51 |         int y=Y-yOffset-subsetSize/3;
 52 |         pointsCtr++;
 53 |         if (0<=x and x<mat.cols and 0<=y and y<mat.rows)
 54 |           interestingPointsCtr+=(mat.ptr()[(pointsCtr%mat.channels())+x*mat.channels()+y*mat.channels()*mat.cols]!=backgroundColor);
 55 |       }
 56 |     }
 57 |     assert(pointsCtr>=10);
 58 |     if (interestingPointsCtr>pointsCtr*0.6)
 59 |       goodFit=true;
 60 |   }
 61 |   if (!goodFit) {
 62 |     std::cout << filename << " " << std::flush;
 63 |     xOffset=-mat.cols/2-16+rng.randint(32);
 64 |     yOffset=-mat.rows/2-16+rng.randint(32);
 65 |   }
 66 | }
 67 | void OpenCVPicture::centerMass() {
 68 |   float ax=0, ay=0, axx=0, ayy=0, axy, d=0.001;
 69 |   for (int i=0; i<mat.channels(); i++) {
 70 |     for (int x=0;x<mat.cols;++x) {
 71 |       for (int y=0;y<mat.rows;++y) {
 72 |         float f=powf(backgroundColor-mat.ptr()[i+x*mat.channels()+y*mat.channels()*mat.cols],2);
 73 |         ax+=x*f;
 74 |         ay+=y*f;
 75 |         axx+=x*x*f;
 76 |         axy+=x*y*f;
 77 |         ayy+=y*y*f;
 78 |         d+=f;
 79 |       }
 80 |     }
 81 |   }
 82 |   ax/=d;
 83 |   ay/=d;
 84 |   axx/=d;
 85 |   axy/=d;
 86 |   ayy/=d;
 87 |   xOffset=-ax/2;
 88 |   yOffset=-ay/2;
 89 |   scale2xx=axx-ax*ax;
 90 |   scale2xy=axy-ax*ay;
 91 |   scale2yy=ayy-ay*ay;
 92 |   scale2=powf(scale2xx+scale2yy,0.5);
 93 | }
 94 | void OpenCVPicture::loadDataWithoutScaling(int flag) {
 95 |   readImage(filename,mat,flag);
 96 |   xOffset=-mat.cols/2;
 97 |   yOffset=-mat.rows/2;
 98 | }
 99 | void OpenCVPicture::loadData (int scale, int flags) {
100 |   readTransformedImage(filename,mat,scale,flags,1,0,0,1,backgroundColor);
101 |   xOffset=-mat.cols/2;
102 |   yOffset=-mat.rows/2;
103 | }
104 | std::string OpenCVPicture::identify() {
105 |   return filename;
106 | }
107 | 
108 | void OpenCVPicture::codifyInputData(SparseGrid &grid, std::vector<float> &features, int &nSpatialSites, int spatialSize) {
109 |   assert(!mat.empty());
110 |   assert(mat.type()%8==5);
111 |   for  (int i=0; i<mat.channels(); i++)
112 |     features.push_back(0); //Background feature
113 |   grid.backgroundCol=nSpatialSites++;
114 |   for (int x=0; x<mat.cols; x++) {
115 |     int X=x+xOffset+spatialSize/3;
116 |     for (int y=0; y<mat.rows; y++) {
117 |       int Y=y+yOffset+spatialSize/3;
118 |       if (X>=0 && Y>=0 && X+Y<spatialSize) {
119 |         bool flag=false;
120 |         for (int i=0; i<mat.channels(); i++)
121 |           if (std::abs(scaleUCharColor(mat.ptr()[i+x*mat.channels()+y*mat.channels()*mat.cols]))>0.02)
122 |             flag=true;
123 |         if (flag) {
124 |           int n=X*spatialSize+Y;
125 |           grid.mp[n]=nSpatialSites++;
126 |           for (int i=0; i<mat.channels(); i++) {
127 |             features.push_back
128 |               (scaleUCharColor(mat.ptr()[i+x*mat.channels()+y*mat.channels()*mat.cols]));
129 |           }
130 |         }
131 |       }
132 |     }
133 |   }
134 | }
135 | 
136 | 
137 | void matrixMul2x2inPlace(float& c00, float& c01, float& c10, float& c11, float a00, float a01, float a10, float a11) { //c<-c*a
138 |   float t00=c00*a00+c01*a10;  float t01=c00*a01+c01*a11;
139 |   float t10=c10*a00+c11*a10;  float t11=c10*a01+c11*a11;
140 |   c00=t00;c01=t01;
141 |   c10=t10;c11=t11;
142 | }
143 | 


--------------------------------------------------------------------------------
/Picture.cpp:
--------------------------------------------------------------------------------
1 | #include "Picture.h"
2 | std::string Picture::identify() {
3 |   return std::string();
4 | }
5 | Picture::Picture(int label) : label(label) {
6 | }
7 | Picture::~Picture() {
8 | }
9 | 


--------------------------------------------------------------------------------
/Picture.h:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | #include <string>
 3 | #include <vector>
 4 | #include "SparseGrid.h"
 5 | #include "Rng.h"
 6 | #include "types.h"
 7 | 
 8 | class Picture {
 9 | public:
10 |   virtual void codifyInputData (SparseGrid &grid, std::vector<float> &features, int &nSpatialSites, int spatialSize)=0;
11 |   virtual Picture* distort (RNG& rng, batchType type) {return this;}
12 |   virtual std::string identify();
13 |   int label; //-1 for unknown
14 |   Picture (int label=-1);
15 |   virtual ~Picture();
16 | };
17 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # SparseConvNet
 2 | ## A spatially-sparse convolutional neural network
 3 | ### [Ben Graham](http://www2.warwick.ac.uk/fac/sci/statistics/staff/academic-research/graham/), [University of Warwick](http://www2.warwick.ac.uk/fac/sci/statistics/), 2013-2015, GPLv3
 4 | 
 5 | This is a CPU version of [SparseConvNet](https://github.com/btgraham/SparseConvNet), the spatially sparse convolutional neural network.
 6 | 
 7 | Possibly useful if you don't have a GPU and are very patient. Or to distribute models trained on the GPU to GPU-less people.
 8 | 
 9 | **************************************************************************
10 | SparseConvNet is free software: you can redistribute it and/or modify
11 | it under the terms of the GNU General Public License as published by
12 | the Free Software Foundation, either version 3 of the License, or
13 | (at your option) any later version.
14 | 
15 | SparseConvNet is distributed in the hope that it will be useful,
16 | but WITHOUT ANY WARRANTY; without even the implied warranty of
17 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
18 | [GNU General Public License](http://www.gnu.org/licenses/) for more details.
19 | **************************************************************************
20 | 


--------------------------------------------------------------------------------
/Regions.h:
--------------------------------------------------------------------------------
  1 | #pragma once
  2 | #include <vector>
  3 | #include "Rng.h"
  4 | #include "SparseGrid.h"
  5 | 
  6 | class PoolingRegions { //Which output locations does the (i,j)-th input cell feed into (inclusive bounds)?
  7 | public:
  8 |   int nIn;
  9 |   int nOut;
 10 |   int dimension;
 11 |   int s;
 12 |   int sd;
 13 |   PoolingRegions(int nIn, int nOut, int dimension, int s);
 14 |   //up to 4d
 15 |   //given a point in the output layer, location of "top-left" corner of the corresponding pooling region in the input layer
 16 |   virtual int tl0(int j0, int j1=0, int j2=0, int j3=0);
 17 |   virtual int tl1(int j0, int j1=0, int j2=0, int j3=0);
 18 |   virtual int tl2(int j0, int j1=0, int j2=0, int j3=0);
 19 |   virtual int tl3(int j0, int j1=0, int j2=0, int j3=0);
 20 |   //given an input-layer point, what is the range of points in the output layer to which it can be pooled (inclusive bounds)
 21 |   virtual int lb0(int i0, int i1=0, int i2=0, int i3=0);
 22 |   virtual int ub0(int i0, int i1=0, int i2=0, int i3=0);
 23 |   virtual int lb1(int i0, int i1=0, int i2=0, int i3=0);
 24 |   virtual int ub1(int i0, int i1=0, int i2=0, int i3=0);
 25 |   virtual int lb2(int i0, int i1=0, int i2=0, int i3=0);
 26 |   virtual int ub2(int i0, int i1=0, int i2=0, int i3=0);
 27 |   virtual int lb3(int i0, int i1=0, int i2=0, int i3=0);
 28 |   virtual int ub3(int i0, int i1=0, int i2=0, int i3=0);
 29 | };
 30 | 
 31 | class RegularPoolingRegions : public PoolingRegions {
 32 |   int poolSize;
 33 |   int poolStride;
 34 | public:
 35 |   RegularPoolingRegions(int nIn, int nOut, int dimension, int poolSize, int poolStride);
 36 |   int tl0(int j0, int j1=0, int j2=0, int j3=0);
 37 |   int tl1(int j0, int j1=0, int j2=0, int j3=0);
 38 |   int tl2(int j0, int j1=0, int j2=0, int j3=0);
 39 |   int tl3(int j0, int j1=0, int j2=0, int j3=0);
 40 | 
 41 |   int lb0(int i0, int i1=0, int i2=0, int i3=0);
 42 |   int ub0(int i0, int i1=0, int i2=0, int i3=0);
 43 |   int lb1(int i0, int i1=0, int i2=0, int i3=0);
 44 |   int ub1(int i0, int i1=0, int i2=0, int i3=0);
 45 |   int lb2(int i0, int i1=0, int i2=0, int i3=0);
 46 |   int ub2(int i0, int i1=0, int i2=0, int i3=0);
 47 |   int lb3(int i0, int i1=0, int i2=0, int i3=0);
 48 |   int ub3(int i0, int i1=0, int i2=0, int i3=0);
 49 | };
 50 | 
 51 | 
 52 | class PseudorandomOverlappingFractionalMaxPoolingBlocks {
 53 | public:
 54 |   std::vector<int> tl;
 55 |   std::vector<int> lb;
 56 |   std::vector<int> ub;
 57 |   PseudorandomOverlappingFractionalMaxPoolingBlocks(int nIn, int nOut, int poolSize, RNG& rng);
 58 | };
 59 | class PseudorandomOverlappingFractionalPoolingRegions : public PoolingRegions { //For 2d pooling.
 60 |   std::vector<PseudorandomOverlappingFractionalMaxPoolingBlocks> pb;
 61 | public:
 62 |   PseudorandomOverlappingFractionalPoolingRegions(int nIn, int nOut, int dimension, int poolSize, RNG& rng);
 63 |   int tl0(int j0, int j1=0, int j2=0, int j3=0);
 64 |   int tl1(int j0, int j1=0, int j2=0, int j3=0);
 65 |   int tl2(int j0, int j1=0, int j2=0, int j3=0);
 66 |   int tl3(int j0, int j1=0, int j2=0, int j3=0);
 67 |   int lb0(int i0, int i1=0, int i2=0, int i3=0);
 68 |   int ub0(int i0, int i1=0, int i2=0, int i3=0);
 69 |   int lb1(int i0, int i1=0, int i2=0, int i3=0);
 70 |   int ub1(int i0, int i1=0, int i2=0, int i3=0);
 71 |   int lb2(int i0, int i1=0, int i2=0, int i3=0);
 72 |   int ub2(int i0, int i1=0, int i2=0, int i3=0);
 73 |   int lb3(int i0, int i1=0, int i2=0, int i3=0);
 74 |   int ub3(int i0, int i1=0, int i2=0, int i3=0);
 75 | };
 76 | 
 77 | class RandomOverlappingFractionalMaxPoolingBlocks {
 78 | public:
 79 |   std::vector<int> tl;
 80 |   std::vector<int> lb;
 81 |   std::vector<int> ub;
 82 |   RandomOverlappingFractionalMaxPoolingBlocks(int nIn, int nOut, int poolSize, RNG& rng);
 83 | };
 84 | class RandomOverlappingFractionalPoolingRegions : public PoolingRegions {
 85 |   std::vector<RandomOverlappingFractionalMaxPoolingBlocks> pb;
 86 | public:
 87 |   RandomOverlappingFractionalPoolingRegions(int nIn, int nOut, int dimension, int poolSize, RNG& rng);
 88 |   int tl0(int j0, int j1=0, int j2=0, int j3=0);
 89 |   int tl1(int j0, int j1=0, int j2=0, int j3=0);
 90 |   int tl2(int j0, int j1=0, int j2=0, int j3=0);
 91 |   int tl3(int j0, int j1=0, int j2=0, int j3=0);
 92 |   int lb0(int i0, int i1=0, int i2=0, int i3=0);
 93 |   int ub0(int i0, int i1=0, int i2=0, int i3=0);
 94 |   int lb1(int i0, int i1=0, int i2=0, int i3=0);
 95 |   int ub1(int i0, int i1=0, int i2=0, int i3=0);
 96 |   int lb2(int i0, int i1=0, int i2=0, int i3=0);
 97 |   int ub2(int i0, int i1=0, int i2=0, int i3=0);
 98 |   int lb3(int i0, int i1=0, int i2=0, int i3=0);
 99 |   int ub3(int i0, int i1=0, int i2=0, int i3=0);
100 | };
101 | 
102 | 
103 | void gridRules
104 | (SparseGrid& inputGrid, //Keys 0,1,...,powf(regions.nIn,3)-1 represent grid points; key -1 represents null/background std::vector
105 |  SparseGrid& outputGrid, //Keys 0,1,...,powf(regions.nOut,3)-1 represent grid points; key -1 represents null/background std::vector
106 |  PoolingRegions& regions,
107 |  int& nOutputSpatialSites,
108 |  std::vector<int>& rules,
109 |  int minActiveInputs=1);
110 | 
111 | 
112 | 
113 | 
114 | 
115 | 
116 | 
117 | 
118 | 
119 | 
120 | 
121 | 
122 | 
123 | 
124 | 
125 | 
126 | 
127 | 
128 | 
129 | 
130 | 
131 | class PoolingRegionsTriangular { //Which output locations does the (i,j)-th input cell feed into (inclusive bounds)
132 | public:
133 |   int nIn;         //  nIn==1: o    nIn==2: oo   nIn==3:  ooo   nIn==4: oooo   etc (o=site, x=nothing)
134 |   int nOut;        //                       ox            oox           ooox
135 |   //                                                      oxx           ooxx
136 |   //                                                                    oxxx
137 |   // Similarly for nOut.
138 |   int dimension;   // dimension==2 for triangles, 3 for tetrahedron/pyramids, 4 for tetrahedral hyperpyramids
139 |   int s;           // 1d pooling region size (same as poolSize in RegularPoolingRegionsTriangular)
140 |   int S;           // total pooling region size (dimension==1, S==s; dimension==2, S=s*(s+1)/2, ....)
141 |   std::vector<int> ord; // order of the S points chosen from the ipow(s,dimension) points in making up the filter shape
142 |   PoolingRegionsTriangular(int nIn, int nOut, int dimension, int s);
143 |   //up to 4d
144 |   //given a point in the output layer, location of "top-left" corner of the corresponding pooling region in the input layer
145 |   virtual int tl0(int j0, int j1=0, int j2=0, int j3=0);
146 |   virtual int tl1(int j0, int j1=0, int j2=0, int j3=0);
147 |   virtual int tl2(int j0, int j1=0, int j2=0, int j3=0);
148 |   virtual int tl3(int j0, int j1=0, int j2=0, int j3=0);
149 |   //given an input-layer point, what is the range of points in the output layer to which it can be pooled (inclusive bounds)
150 |   virtual int lb0(int i0, int i1=0, int i2=0, int i3=0);
151 |   virtual int ub0(int i0, int i1=0, int i2=0, int i3=0);
152 |   virtual int lb1(int i0, int i1=0, int i2=0, int i3=0);
153 |   virtual int ub1(int i0, int i1=0, int i2=0, int i3=0);
154 |   virtual int lb2(int i0, int i1=0, int i2=0, int i3=0);
155 |   virtual int ub2(int i0, int i1=0, int i2=0, int i3=0);
156 |   virtual int lb3(int i0, int i1=0, int i2=0, int i3=0);
157 |   virtual int ub3(int i0, int i1=0, int i2=0, int i3=0);
158 | };
159 | 
160 | class RegularPoolingRegionsTriangular : public PoolingRegionsTriangular {
161 |   int poolSize;
162 |   int poolStride;
163 | public:
164 |   RegularPoolingRegionsTriangular(int nIn, int nOut, int dimension, int poolSize, int poolStride);
165 |   int tl0(int j0, int j1=0, int j2=0, int j3=0);
166 |   int tl1(int j0, int j1=0, int j2=0, int j3=0);
167 |   int tl2(int j0, int j1=0, int j2=0, int j3=0);
168 |   int tl3(int j0, int j1=0, int j2=0, int j3=0);
169 | 
170 |   int lb0(int i0, int i1=0, int i2=0, int i3=0);
171 |   int ub0(int i0, int i1=0, int i2=0, int i3=0);
172 |   int lb1(int i0, int i1=0, int i2=0, int i3=0);
173 |   int ub1(int i0, int i1=0, int i2=0, int i3=0);
174 |   int lb2(int i0, int i1=0, int i2=0, int i3=0);
175 |   int ub2(int i0, int i1=0, int i2=0, int i3=0);
176 |   int lb3(int i0, int i1=0, int i2=0, int i3=0);
177 |   int ub3(int i0, int i1=0, int i2=0, int i3=0);
178 | };
179 | 
180 | 
181 | 
182 | void gridRulesTriangular
183 | (SparseGrid& inputGrid, //Keys 0,1,...,powf(regions.nIn,3)-1 represent grid points (plus paddding to form a square/cube); key -1 represents null/background vector
184 |  SparseGrid& outputGrid, //Keys 0,1,...,powf(regions.nOut,3)-1 represent grid points (plus paddding to form a square/cube); key -1 represents null/background vector
185 |  PoolingRegionsTriangular& regions,
186 |  int& nOutputSpatialSites,
187 |  std::vector<int>& rules,
188 |  int minActiveInputs=1);
189 | 


--------------------------------------------------------------------------------
/Rng.cpp:
--------------------------------------------------------------------------------
 1 | #define RNG_CPP
 2 | #include "Rng.h"
 3 | #include <iostream>
 4 | #include <algorithm>
 5 | 
 6 | RNG::RNG() : stdNormal(0,1), uniform01(0,1) {
 7 |   timespec ts;
 8 |   clock_gettime(CLOCK_REALTIME, &ts);
 9 |   RNGseedGeneratorMutex.lock();
10 |   gen.seed(RNGseedGenerator()+ts.tv_nsec);
11 |   RNGseedGeneratorMutex.unlock();
12 | }
13 | 
14 | int RNG::randint(int n) {
15 |   if (n==0) return 0;
16 |   else return gen()%n;
17 | }
18 | float RNG::uniform(float a, float b) {
19 |   return a+(b-a)*uniform01(gen);
20 | }
21 | float RNG::normal(float mean, float sd) {
22 |   return mean+sd*stdNormal(gen);
23 | }
24 | int RNG::bernoulli(float p) {
25 |   if (uniform01(gen)<p)
26 |     return 1;
27 |   else
28 |     return 0;
29 | }
30 | template <typename T>
31 | int RNG::index(std::vector<T> &v) {
32 |   if (v.size()==0) std::cout << "RNG::index called for empty std::vector!\n";
33 |   return gen()%v.size();
34 | }
35 | std::vector<int> RNG::NchooseM(int n, int m) {
36 |   std::vector<int> ret(m,100);
37 |   int ctr=m;
38 |   for(int i=0;i<n;i++)
39 |     if (uniform01(gen)<ctr*1.0/(n-i)) ret[m-ctr--]=i;
40 |   return ret;
41 | }
42 | std::vector<int> RNG::permutation(int n) {
43 |   std::vector<int> ret;
44 |   for (int i=0;i<n;i++) ret.push_back(i);
45 |   std::shuffle ( ret.begin(), ret.end(), gen);
46 |   return ret;
47 | }
48 | template <typename T> void RNG::vectorShuffle(std::vector<T> &v) {
49 |   std::shuffle( v.begin(), v.end(), gen);
50 | }
51 | 
52 | template void RNG::vectorShuffle<int>(std::vector<int> &v);
53 | 


--------------------------------------------------------------------------------
/Rng.h:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | #include <random>
 3 | #include <mutex>
 4 | 
 5 | #ifdef RNG_CPP
 6 | std::mutex RNGseedGeneratorMutex;
 7 | std::mt19937 RNGseedGenerator;
 8 | #else
 9 | extern std::mutex RNGseedGeneratorMutex;
10 | extern std::mt19937 RNGseedGenerator;
11 | #endif
12 | 
13 | class RNG {
14 |   std::normal_distribution<> stdNormal;
15 |   std::uniform_real_distribution<> uniform01;
16 | public:
17 |   std::mt19937 gen;
18 | 
19 |   RNG();
20 |   int randint(int n);
21 |   float uniform(float a=0, float b=1);
22 |   float normal(float mean=0, float sd=1);
23 |   int bernoulli(float p);
24 |   template <typename T> int index(std::vector<T> &v);
25 |   std::vector<int> NchooseM(int n, int m);
26 |   std::vector<int> permutation(int n);
27 |   template <typename T> void vectorShuffle(std::vector<T> &v);
28 | };
29 | 


--------------------------------------------------------------------------------
/SigmoidLayer.cpp:
--------------------------------------------------------------------------------
  1 | #include <iostream>
  2 | #include <cmath>
  3 | #include "SigmoidLayer.h"
  4 | #include "cudaUtilities.h"
  5 | 
  6 | void sigmoidReLU(std::vector<float>& a, std::vector<float>& b) {
  7 |   for (int i=0;i<a.size();i++)
  8 |     b[i]=(a[i]>0)?a[i]:0;
  9 | }
 10 | 
 11 | void sigmoidBackpropReLU
 12 | (std::vector<float>& a, std::vector<float>& b, std::vector<float>& da, std::vector<float>& db) {
 13 |   for (int i=0;i<a.size();i++)
 14 |     da[i]=(a[i]>0)?db[i]:0;
 15 | }
 16 | 
 17 | void sigmoidTanh(std::vector<float>& a, std::vector<float>& b) {
 18 |   for (int i=0;i<a.size();i++)
 19 |     b[i]=tanh(a[i]);
 20 | }
 21 | void sigmoidBackpropTanh
 22 | (std::vector<float>& a, std::vector<float>& b, std::vector<float>& da, std::vector<float>& db) {
 23 |   for (int i=0;i<a.size();i++)
 24 |     da[i]=db[i]*(1+b[i])*(1-b[i]);
 25 | }
 26 | 
 27 | 
 28 | void sigmoidLeakyReLU(std::vector<float>& a, std::vector<float>& b, float alpha) {
 29 |   for (int i=0;i<a.size();i++)
 30 |     b[i]=(a[i]>0)?a[i]:(a[i]*alpha);
 31 | }
 32 | 
 33 | void sigmoidBackpropLeakyReLU
 34 | (std::vector<float>& a, std::vector<float>& b, std::vector<float>& da, std::vector<float>& db, float alpha) {
 35 |   for (int i=0;i<a.size();i++)
 36 |     da[i]=(a[i]>0)?db[i]:(db[i]*alpha);
 37 | }
 38 | 
 39 | //SOFTMAX should only be used in the top layer;
 40 | //derivative contained in calculation of initial d_delta.
 41 | void sigmoidSoftmax(std::vector<float>& a, std::vector<float>& b, int nOut) {
 42 |   for (int i=0;i<a.size();i+=nOut) {
 43 |     float mx=a[i];
 44 |     for (int j=i+1;j<i+nOut;j++)
 45 |       mx=std::max(mx,a[j]);
 46 |     float acc=0;
 47 |     for (int j=i;j<i+nOut;j++)
 48 |       acc+=b[j]=expf(a[j]-mx);
 49 |     for (int j=i;j<i+nOut;j++)
 50 |       b[j]/=acc;
 51 |   }
 52 | }
 53 | 
 54 | void sigmoidBackpropSoftmax
 55 | (std::vector<float>& a, std::vector<float>& b, std::vector<float>& da, std::vector<float>& db, float alpha) {
 56 |   for (int i=0;i<a.size();i++)
 57 |     da[i]=db[i];
 58 | }
 59 | 
 60 | void applySigmoid(SpatiallySparseBatchInterface& input, SpatiallySparseBatchInterface& output, ActivationFunction fn) {
 61 |   switch(fn) {
 62 |   case TANH:
 63 |     sigmoidTanh
 64 |       (input.sub->features.vec,
 65 |        output.sub->features.vec);
 66 |     break;
 67 |   case RELU:
 68 |     sigmoidReLU
 69 |       (input.sub->features.vec,
 70 |        output.sub->features.vec);
 71 |     break;
 72 |   case LEAKYRELU:
 73 |     sigmoidLeakyReLU
 74 |       (input.sub->features.vec,
 75 |        output.sub->features.vec,
 76 |        0.01);
 77 |     break;
 78 |   case VLEAKYRELU:
 79 |     sigmoidLeakyReLU
 80 |       (input.sub->features.vec,
 81 |        output.sub->features.vec,
 82 |        0.333);
 83 |     break;
 84 |   case SOFTMAX:
 85 |     sigmoidSoftmax (input.sub->features.vec,output.sub->features.vec,output.featuresPresent.size());
 86 |     break;
 87 |   case NOSIGMOID:
 88 |     break;
 89 |   }
 90 | }
 91 | 
 92 | void applySigmoidBackProp(SpatiallySparseBatchInterface& input, SpatiallySparseBatchInterface& output, ActivationFunction fn) {
 93 |   switch(fn) {
 94 |   case TANH:
 95 |     sigmoidBackpropTanh
 96 |       (input.sub->features.vec,output.sub->features.vec,
 97 |        input.sub->dfeatures.vec,
 98 |        output.sub->dfeatures.vec);
 99 |     break;
100 |   case RELU:
101 |     sigmoidBackpropReLU
102 |       (input.sub->features.vec,output.sub->features.vec,
103 |        input.sub->dfeatures.vec,
104 |        output.sub->dfeatures.vec);
105 |     break;
106 |   case LEAKYRELU:
107 |     sigmoidBackpropLeakyReLU
108 |       (input.sub->features.vec,
109 |        output.sub->features.vec,
110 |        input.sub->dfeatures.vec,
111 |        output.sub->dfeatures.vec,
112 |        0.01);
113 |     break;
114 |   case VLEAKYRELU:
115 |     sigmoidBackpropLeakyReLU
116 |       (input.sub->features.vec,
117 |        output.sub->features.vec,
118 |        input.sub->dfeatures.vec,
119 |        output.sub->dfeatures.vec,
120 |        0.333);
121 |     break;
122 |   case SOFTMAX:
123 |     sigmoidBackpropSoftmax
124 |       (input.sub->features.vec,output.sub->features.vec, input.sub->dfeatures.vec,output.sub->dfeatures.vec, output.featuresPresent.size());   break;
125 |   case NOSIGMOID:
126 |     break;
127 |   }
128 | }
129 | 
130 | SigmoidLayer::SigmoidLayer(ActivationFunction fn) : fn(fn) {
131 |   std::cout << sigmoidNames[fn] << std::endl;
132 | };
133 | void SigmoidLayer::preprocess
134 | (SpatiallySparseBatch &batch,
135 |  SpatiallySparseBatchInterface &input,
136 |  SpatiallySparseBatchInterface &output) {
137 |   output.nFeatures=input.nFeatures;
138 |   output.featuresPresent.vec=input.featuresPresent.vec;
139 |   output.spatialSize=input.spatialSize;
140 |   output.nSpatialSites=input.nSpatialSites;
141 |   output.grids=input.grids;
142 |   output.backpropErrors=input.backpropErrors;
143 | }
144 | void SigmoidLayer::forwards(SpatiallySparseBatch &batch,
145 |                             SpatiallySparseBatchInterface &input,
146 |                             SpatiallySparseBatchInterface &output) {
147 |   output.sub->features.resize(output.nSpatialSites*output.featuresPresent.size());
148 |   applySigmoid(input, output, fn);
149 | }
150 | void SigmoidLayer::backwards(SpatiallySparseBatch &batch,
151 |                              SpatiallySparseBatchInterface &input,
152 |                              SpatiallySparseBatchInterface &output,
153 |                              float learningRate,
154 |                              float momentum) {
155 |   if (input.backpropErrors) {
156 |     input.sub->dfeatures.resize(input.nSpatialSites*input.featuresPresent.size());
157 |     applySigmoidBackProp(input, output, fn);
158 |   }
159 | }
160 | int SigmoidLayer::calculateInputSpatialSize(int outputSpatialSize) {
161 |   return outputSpatialSize;
162 | }
163 | 


--------------------------------------------------------------------------------
/SigmoidLayer.h:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | #include "SpatiallySparseLayer.h"
 3 | 
 4 | void applySigmoid(SpatiallySparseBatchInterface& input, SpatiallySparseBatchInterface& output, ActivationFunction fn);
 5 | void applySigmoidBackProp(SpatiallySparseBatchInterface& input, SpatiallySparseBatchInterface& output, ActivationFunction fn);
 6 | 
 7 | class SigmoidLayer : public SpatiallySparseLayer {
 8 | public:
 9 |   ActivationFunction fn;
10 |   SigmoidLayer(ActivationFunction fn);
11 |   void preprocess
12 |   (SpatiallySparseBatch &batch,
13 |    SpatiallySparseBatchInterface &input,
14 |    SpatiallySparseBatchInterface &output);
15 |   void forwards
16 |   (SpatiallySparseBatch &batch,
17 |    SpatiallySparseBatchInterface &input,
18 |    SpatiallySparseBatchInterface &output);
19 |   void backwards
20 |   (SpatiallySparseBatch &batch,
21 |    SpatiallySparseBatchInterface &input,
22 |    SpatiallySparseBatchInterface &output,
23 |    float learningRate,
24 |    float momentum);
25 |   int calculateInputSpatialSize(int outputSpatialSize);
26 | };
27 | 


--------------------------------------------------------------------------------
/SoftmaxClassifier.cpp:
--------------------------------------------------------------------------------
 1 | #include "SoftmaxClassifier.h"
 2 | #include "cudaUtilities.h"
 3 | #include <iostream>
 4 | #include <vector>
 5 | #include <cassert>
 6 | #include <cmath>
 7 | #include "utilities.h"
 8 | 
 9 | void dDerivativeOfCostWRTpreSoftmaxTopLevelWeights
10 | (int batchSize, std::vector<float>& topDelta, std::vector<float>& topGrid, std::vector<int>& labels, int N) {
11 |   for (int k=0;k<batchSize;k++) {
12 |     for(int i=0;i<N;i++) {
13 |       topDelta[k*N+i]=topGrid[k*N+i]-(i==labels[k]);
14 |     }
15 |   }
16 | }
17 | 
18 | void SoftmaxClassifier(SpatiallySparseBatchInterface& input, SpatiallySparseBatch& batch, int nTop) {
19 |   //Assume no dropout in the output layer! nClasses:=input.nFeatures.
20 |   assert(batch.batchSize==input.nSpatialSites);
21 |   assert(input.nFeatures==input.featuresPresent.size());
22 | 
23 |   if (batch.type==TRAINBATCH) {//Begin backprop. Top layer: d Cost / d SoftmaxInput
24 |     input.sub->dfeatures.resize(input.nSpatialSites*input.featuresPresent.size());
25 |     dDerivativeOfCostWRTpreSoftmaxTopLevelWeights
26 |       (batch.batchSize, input.sub->dfeatures.vec, input.sub->features.vec,
27 |        batch.labels.vec, input.nFeatures);
28 |   }
29 | 
30 |   float* probs=&input.sub->features.vec[0];
31 |   for (int i=0;i<batch.batchSize;++i)
32 |     batch.probabilities.push_back(std::vector<float> (probs+i*input.nFeatures,probs+(i+1)*input.nFeatures));
33 |   for (int i=0;i<batch.batchSize;i++)
34 |     batch.predictions.push_back(vectorTopIndices(batch.probabilities[i],nTop));
35 | 
36 |   if (batch.type!=UNLABELEDBATCH) {
37 |     batch.mistakes+=batch.batchSize;
38 |     for (int i=0;i<batch.batchSize;i++) {
39 |       batch.negativeLogLikelihood-=log(std::max(batch.probabilities[i][batch.labels.vec[i]],(float)1.0e-15));
40 |       for (int j=0;j<nTop;j++) {
41 |         if (batch.predictions[i][j]==batch.labels.vec[i]) {
42 |           batch.mistakes--;
43 |         }
44 |       }
45 |     }
46 |     //std::cout << batch.mistakes << " " <<std::flush;
47 |   }
48 | }
49 | 


--------------------------------------------------------------------------------
/SoftmaxClassifier.h:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | #include "SpatiallySparseBatchInterface.h"
 3 | #include "SpatiallySparseBatch.h"
 4 | 
 5 | ////////////////////////////////////////////////////////////////////////////////////////////////
 6 | //Calculate softmaxProbability(i) - indicator(i=label)
 7 | // for i=0,1,...N-1 with N the number of character classes.
 8 | void dDerivativeOfCostWRTpreSoftmaxTopLevelWeights
 9 | (int batchSize, std::vector<float>& topDelta, std::vector<float>& topGrid, std::vector<int>& labels, int N);
10 | void SoftmaxClassifier(SpatiallySparseBatchInterface& input, SpatiallySparseBatch& batch, int nTop);
11 | 


--------------------------------------------------------------------------------
/SparseConvNet.cpp:
--------------------------------------------------------------------------------
  1 | #include "SparseConvNet.h"
  2 | #include "SparseConvNetCUDA.h"
  3 | #include "utilities.h"
  4 | 
  5 | SparseConvNet::SparseConvNet(int dimension, int nInputFeatures, int nClasses, int nTop)
  6 |   : cnn(new SparseConvNetCUDA(dimension, nInputFeatures, nClasses, nTop)) {
  7 | }
  8 | 
  9 | SparseConvNet::~SparseConvNet(){
 10 | }
 11 | 
 12 | void SparseConvNet::addLeNetLayerMP(int nFeatures, int filterSize, int filterStride, int poolSize, int poolStride, ActivationFunction activationFn, float dropout, int minActiveInputs){
 13 |   cnn->addLeNetLayerMP(nFeatures, filterSize, filterStride, poolSize, poolStride, activationFn, dropout, minActiveInputs);
 14 | }
 15 | 
 16 | void SparseConvNet::addLeNetLayerPOFMP(int nFeatures, int filterSize, int filterStride, int poolSize, float fmpShrink, ActivationFunction activationFn, float dropout, int minActiveInputs){
 17 |   cnn->addLeNetLayerPOFMP(nFeatures, filterSize, filterStride, poolSize, fmpShrink, activationFn, dropout, minActiveInputs);
 18 | }
 19 | 
 20 | void SparseConvNet::addLeNetLayerROFMP(int nFeatures, int filterSize, int filterStride, int poolSize, float fmpShrink, ActivationFunction activationFn, float dropout, int minActiveInputs){
 21 |   cnn->addLeNetLayerROFMP(nFeatures, filterSize, filterStride, poolSize, fmpShrink, activationFn, dropout, minActiveInputs);
 22 | }
 23 | 
 24 | void SparseConvNet::addTerminalPoolingLayer(int poolSize){
 25 |   cnn->addTerminalPoolingLayer(poolSize, ipow(poolSize,cnn->dimension));
 26 | }
 27 | 
 28 | void SparseConvNet::addSoftmaxLayer(){
 29 |   cnn->addSoftmaxLayer();
 30 | }
 31 | 
 32 | void SparseConvNet::addIndexLearnerLayer(){
 33 |   cnn->addIndexLearnerLayer();
 34 | }
 35 | 
 36 | float SparseConvNet::processDataset(SpatiallySparseDataset &dataset, int batchSize, float learningRate, float momentum) {
 37 |   return cnn->processDataset(dataset,batchSize,learningRate,momentum);
 38 | }
 39 | 
 40 | void SparseConvNet::processDatasetRepeatTest(SpatiallySparseDataset &dataset, int batchSize, int nReps, std::string predictionsFilename,std::string header,std::string confusionMatrixFilename){
 41 |   cnn->processDatasetRepeatTest(dataset,batchSize,nReps,predictionsFilename,header,confusionMatrixFilename);
 42 | }
 43 | 
 44 | float SparseConvNet::processIndexLearnerDataset(SpatiallySparseDataset &dataset, int batchSize, float learningRate, float momentum){
 45 |   return cnn->processIndexLearnerDataset(dataset,batchSize,learningRate,momentum);
 46 | }
 47 | 
 48 | void SparseConvNet::processDatasetDumpTopLevelFeatures(SpatiallySparseDataset &dataset, int batchSize, int reps){
 49 |   cnn->processDatasetDumpTopLevelFeatures(dataset,batchSize,reps);
 50 | }
 51 | 
 52 | void SparseConvNet::loadWeights(std::string baseName, int epoch, int firstNlayers){
 53 |   cnn->loadWeights(baseName,epoch,firstNlayers);
 54 | }
 55 | 
 56 | void SparseConvNet::saveWeights(std::string baseName, int epoch){
 57 |   cnn->saveWeights(baseName,epoch);
 58 | }
 59 | 
 60 | void SparseConvNet::calculateInputRegularizingConstants(SpatiallySparseDataset dataset){
 61 |   cnn->calculateInputRegularizingConstants(dataset);
 62 | }
 63 | 
 64 | SparseConvTriangLeNet::SparseConvTriangLeNet(int dimension, int nInputFeatures, int nClasses, int nTop)
 65 |   : cnn(new SparseConvNetCUDA(dimension, nInputFeatures, nClasses, nTop)) {
 66 | }
 67 | SparseConvTriangLeNet::~SparseConvTriangLeNet(){
 68 | }
 69 | 
 70 | void SparseConvTriangLeNet::addLeNetLayerMP(int nFeatures, int filterSize, int filterStride, int poolSize, int poolStride, ActivationFunction activationFn, float dropout, int minActiveInputs){
 71 |   cnn->addTriangularLeNetLayerMP(nFeatures, filterSize, filterStride, poolSize, poolStride, activationFn, dropout, minActiveInputs);
 72 | }
 73 | 
 74 | void SparseConvTriangLeNet::addTerminalPoolingLayer(int poolSize) {
 75 |   cnn->addTerminalPoolingLayer(poolSize, triangleSize(poolSize,cnn->dimension));
 76 | }
 77 | 
 78 | void SparseConvTriangLeNet::addSoftmaxLayer(){
 79 |   cnn->addSoftmaxLayer();
 80 | }
 81 | 
 82 | void SparseConvTriangLeNet::addIndexLearnerLayer(){
 83 |   cnn->addIndexLearnerLayer();
 84 | }
 85 | 
 86 | float SparseConvTriangLeNet::processDataset(SpatiallySparseDataset &dataset, int batchSize, float learningRate, float momentum){
 87 |   return cnn->processDataset(dataset,batchSize,learningRate, momentum);
 88 | }
 89 | 
 90 | void SparseConvTriangLeNet::processDatasetRepeatTest(SpatiallySparseDataset &dataset, int batchSize, int nReps, std::string predictionsFilename,std::string header,std::string confusionMatrixFilename){
 91 |   cnn->processDatasetRepeatTest(dataset,batchSize,nReps,predictionsFilename,header,confusionMatrixFilename);
 92 | }
 93 | 
 94 | float SparseConvTriangLeNet::processIndexLearnerDataset(SpatiallySparseDataset &dataset, int batchSize, float learningRate,float momentum){
 95 |   return cnn->processIndexLearnerDataset(dataset,batchSize,learningRate,momentum);
 96 | }
 97 | 
 98 | void SparseConvTriangLeNet::processDatasetDumpTopLevelFeatures(SpatiallySparseDataset &dataset, int batchSize, int reps){
 99 |   cnn->processDatasetDumpTopLevelFeatures(dataset,batchSize,reps);
100 | }
101 | 
102 | void SparseConvTriangLeNet::loadWeights(std::string baseName, int epoch, int firstNlayers){
103 |   cnn->loadWeights(baseName,epoch,firstNlayers);
104 | }
105 | 
106 | void SparseConvTriangLeNet::saveWeights(std::string baseName, int epoch){
107 |   cnn->saveWeights(baseName,epoch);
108 | }
109 | 
110 | void SparseConvTriangLeNet::calculateInputRegularizingConstants(SpatiallySparseDataset dataset){
111 |   cnn->calculateInputRegularizingConstants(dataset);
112 | }
113 | 


--------------------------------------------------------------------------------
/SparseConvNet.h:
--------------------------------------------------------------------------------
 1 | //Ben Graham, University of Warwick, 2015, b.graham@warwick.ac.uk
 2 | //SparseConvNet is free software: you can redistribute it and/or modify
 3 | //it under the terms of the GNU General Public License as published by
 4 | //the Free Software Foundation, either version 3 of the License, or
 5 | //(at your option) any later version.
 6 | 
 7 | //SparseConvNet is distributed in the hope that it will be useful,
 8 | //but WITHOUT ANY WARRANTY; without even the implied warranty of
 9 | //MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
10 | //GNU General Public License for more details.
11 | 
12 | #pragma once
13 | #include "SpatiallySparseDataset.h"
14 | #include <memory>
15 | #include <string>
16 | #include <fstream>
17 | 
18 | class SparseConvNetCUDA;
19 | 
20 | class SparseConvNet {
21 | private:
22 |   std::unique_ptr<SparseConvNetCUDA> cnn;
23 | public:
24 |   SparseConvNet(int dimension, int nInputFeatures, int nClasses, int nTop=1);
25 |   ~SparseConvNet();
26 |   void addLeNetLayerMP(int nFeatures, int filterSize, int filterStride, int poolSize, int poolStride, ActivationFunction activationFn=RELU, float dropout=0.0f, int minActiveInputs=1);
27 |   void addLeNetLayerPOFMP(int nFeatures, int filterSize, int filterStride, int poolSize, float fmpShrink, ActivationFunction activationFn=RELU, float dropout=0.0f, int minActiveInputs=1);
28 |   void addLeNetLayerROFMP(int nFeatures, int filterSize, int filterStride, int poolSize, float fmpShrink, ActivationFunction activationFn=RELU, float dropout=0.0f, int minActiveInputs=1);
29 |   void addTerminalPoolingLayer(int poolSize);
30 |   void addSoftmaxLayer();
31 |   void addIndexLearnerLayer();
32 |   float processDataset(SpatiallySparseDataset &dataset, int batchSize=100, float learningRate=0, float momentum=0.99);
33 |   void processDatasetRepeatTest(SpatiallySparseDataset &dataset, int batchSize=100, int nReps=12, std::string predictionsFilename="",std::string header="",std::string confusionMatrixFilename="");
34 |   float processIndexLearnerDataset(SpatiallySparseDataset &dataset, int batchSize=100, float learningRate=0.0,float momentum=0.99);
35 |   void processDatasetDumpTopLevelFeatures(SpatiallySparseDataset &dataset, int batchSize, int reps=1);
36 |   void loadWeights(std::string baseName, int epoch, int firstNlayers=1000000);
37 |   void saveWeights(std::string baseName, int epoch);
38 |   void calculateInputRegularizingConstants(SpatiallySparseDataset dataset);
39 | };
40 | 
41 | class SparseConvTriangLeNet {
42 | private:
43 |   std::auto_ptr<SparseConvNetCUDA> cnn;
44 | public:
45 |   SparseConvTriangLeNet(int dimension, int nInputFeatures, int nClasses, int nTop=1);
46 |   ~SparseConvTriangLeNet();
47 |   void addLeNetLayerMP(int nFeatures, int filterSize, int filterStride, int poolSize, int poolStride, ActivationFunction activationFn=RELU, float dropout=0.0f, int minActiveInputs=1);
48 |   void addTerminalPoolingLayer(int poolSize);
49 |   void addSoftmaxLayer();
50 |   void addIndexLearnerLayer();
51 |   float processDataset(SpatiallySparseDataset &dataset, int batchSize=100, float learningRate=0, float momentum=0.99);
52 |   void processDatasetRepeatTest(SpatiallySparseDataset &dataset, int batchSize=100, int nReps=12, std::string predictionsFilename="",std::string header="",std::string confusionMatrixFilename="");
53 |   float processIndexLearnerDataset(SpatiallySparseDataset &dataset, int batchSize=100, float learningRate=0, float momentum=0.99);
54 |   void processDatasetDumpTopLevelFeatures(SpatiallySparseDataset &dataset, int batchSize, int reps=1);
55 |   void loadWeights(std::string baseName, int epoch, int firstNlayers=1000000);
56 |   void saveWeights(std::string baseName, int epoch);
57 |   void calculateInputRegularizingConstants(SpatiallySparseDataset dataset);
58 | };
59 | 


--------------------------------------------------------------------------------
/SparseConvNetCUDA.h:
--------------------------------------------------------------------------------
 1 | //Ben Graham, University of Warwick, 2015 b.graham@warwick.ac.uk
 2 | //SparseConvNet is free software: you can redistribute it and/or modify
 3 | //it under the terms of the GNU General Public License as published by
 4 | //the Free Software Foundation, either version 3 of the License, or
 5 | //(at your option) any later version.
 6 | 
 7 | //SparseConvNet is distributed in the hope that it will be useful,
 8 | //but WITHOUT ANY WARRANTY; without even the implied warranty of
 9 | //MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
10 | //GNU General Public License for more details.
11 | 
12 | //You should have received a copy of the GNU General Public License
13 | //along with SparseConvNet.  If not, see <http://www.gnu.org/licenses/>.
14 | 
15 | #pragma once
16 | #include "SpatiallySparseBatch.h"
17 | #include "SpatiallySparseLayer.h"
18 | #include "SpatiallySparseDataset.h"
19 | #include <vector>
20 | #include <string>
21 | #include <fstream>
22 | #include <iostream>
23 | 
24 | class SparseConvNetCUDA {
25 | public:
26 |   std::vector<SpatiallySparseLayer*> layers;
27 |   int dimension;
28 |   int nClasses;
29 |   int nTop;
30 |   int inputSpatialSize;
31 |   int nInputFeatures;
32 |   int nOutputFeatures;
33 |   std::vector<float> inputNormalizingConstants;
34 |   SparseConvNetCUDA (int dimension,
35 |                      int nInputFeatures,
36 |                      int nClasses,
37 |                      int nTop=1);
38 |   void processBatch(SpatiallySparseBatch& batch, float learningRate, float momentum, std::ofstream& f, std::ofstream& g);
39 |   void processIndexLearnerBatch(SpatiallySparseBatch& batch, float learningRate, float momentum, std::ofstream& f);
40 | 
41 | 
42 | 
43 |   void addLearntLayer(int nFeatures,
44 |                       ActivationFunction activationFn=RELU,
45 |                       float dropout=0.0f,
46 |                       float alpha=1.0f);
47 |   void addNetworkInNetworkLayer(int nFeatures,
48 |                                 ActivationFunction activationFn=RELU,
49 |                                 float dropout=0.0f);
50 |   void addConvolutionalLayer(int nFeatures,
51 |                              int filterSize,
52 |                              int filterStride,
53 |                              ActivationFunction activationFn=RELU,
54 |                              float dropout=0.0f,
55 |                              int minActiveInputs=1,
56 |                              float poolingToFollow=1.0f);
57 |   void addLeNetLayerMP(int nFeatures, int filterSize, int filterStride, int poolSize, int poolStride, ActivationFunction activationFn=RELU, float dropout=0.0f, int minActiveInputs=1);
58 |   void addLeNetLayerROFMP(int nFeatures, int filterSize, int filterStride, int poolSize, float fmpShrink, ActivationFunction activationFn=RELU, float dropout=0.0f, int minActiveInputs=1);
59 |   void addLeNetLayerPOFMP(int nFeatures, int filterSize, int filterStride, int poolSize, float fmpShrink, ActivationFunction activationFn=RELU, float dropout=0.0f, int minActiveInputs=1);
60 |   void addTriangularConvolutionalLayer(int nFeatures,
61 |                                        int filterSize,
62 |                                        int filterStride,
63 |                                        ActivationFunction activationFn=RELU,
64 |                                        float dropout=0.0f,
65 |                                        int minActiveInputs=1,
66 |                                        float poolingToFollow=1.0f);
67 |   void addTriangularLeNetLayerMP(int nFeatures, int filterSize, int filterStride, int poolSize, int poolStride, ActivationFunction activationFn=RELU, float dropout=0.0f, int minActiveInputs=1);
68 |   void addSoftmaxLayer();
69 |   void addTerminalPoolingLayer(int poolSize, int S);
70 |   void addIndexLearnerLayer();
71 |   float processDataset(SpatiallySparseDataset &dataset, int batchSize=100, float learningRate=0,float momentum=0.99);
72 |   void processDatasetRepeatTest(SpatiallySparseDataset &dataset, int batchSize=100, int nReps=12, std::string predictionsFilename="",std::string header="",std::string confusionMatrixFilename="");
73 |   float processIndexLearnerDataset(SpatiallySparseDataset &dataset, int batchSize=100, float learningRate=0.0, float momentum=0.99);
74 |   void processBatchDumpTopLevelFeaturess(SpatiallySparseBatch& batch, std::ofstream& f);
75 |   void processDatasetDumpTopLevelFeatures(SpatiallySparseDataset &dataset, int batchSize, int reps=1);
76 |   void loadWeights(std::string baseName, int epoch, int firstNlayers=1000000);
77 |   void saveWeights(std::string baseName, int epoch);
78 |   void calculateInputRegularizingConstants(SpatiallySparseDataset dataset);
79 | };
80 | 


--------------------------------------------------------------------------------
/SparseGrid.h:
--------------------------------------------------------------------------------
 1 | //We need a hash table:
 2 | //We can either use
 3 | // a) Google's sparsehash dense_hash_map, or
 4 | // b) the C++11 std::unordered_map
 5 | // c) the C++11 std::unordered_map with tbb::scalable_alloc (or similar) to prevent threads getting in each others way as they access memory to grow the maps,
 6 | // d) vectors disguised as a hash map (ok in 2 dimensions)
 7 | 
 8 | //#define USE_GOOGLE_SPARSEHASH
 9 | //#define USE_UNORDERED_MAP
10 | //#define USE_UNORDERED_MAP_TBB
11 | #define USE_VECTOR_HASH
12 | 
13 | 
14 | #pragma once
15 | #include <stdint.h>
16 | 
17 | #ifdef USE_GOOGLE_SPARSEHASH
18 | #include <functional>
19 | #include <google/dense_hash_map>
20 | typedef google::dense_hash_map<int64_t, int, std::hash<int64_t>, std::equal_to<int64_t> > SparseGridMap;
21 | class SparseGrid {
22 | public:
23 |   int backgroundCol;
24 |   SparseGridMap mp;
25 |   SparseGrid() {
26 |     backgroundCol=-1; //Indicate that no "null vector" is needed
27 |     mp.set_empty_key(-99); // dense_hash_map needs an empty key that will not be used as a real key
28 |     mp.set_deleted_key(-98);
29 |     mp.min_load_factor(0.0f);
30 | 
31 |   }
32 | };
33 | #endif
34 | 
35 | #ifdef USE_UNORDERED_MAP
36 | #include <unordered_map>
37 | typedef std::unordered_map<int64_t,int,std::hash<int64_t>,std::equal_to<int64_t>,std::allocator< std::pair<const int64_t,int> > > SparseGridMap;
38 | class SparseGrid {
39 | public:
40 |   int backgroundCol; //Set to -1 when no "null vector" is needed
41 |   SparseGridMap mp;
42 |   SparseGrid() : backgroundCol(-1) {}
43 | };
44 | #endif
45 | 
46 | #ifdef USE_UNORDERED_MAP_TBB
47 | //Libraries -ltbb -ltbbmalloc
48 | #include <unordered_map>
49 | #include <tbb/scalable_allocator.h>
50 | typedef std::unordered_map<int64_t,int,std::hash<int64_t>,std::equal_to<int64_t>,tbb::scalable_allocator< std::pair<const int64_t,int> > > SparseGridMap;
51 | class SparseGrid {
52 | public:
53 |   int backgroundCol; //Set to -1 when no "null vector" is needed
54 |   SparseGridMap mp;
55 |   SparseGrid() : backgroundCol(-1) {}
56 | };
57 | #endif
58 | 
59 | #ifdef USE_VECTOR_HASH
60 | #include "vectorHash.h"
61 | typedef vectorHash SparseGridMap;
62 | class SparseGrid {
63 | public:
64 |   int backgroundCol; //Set to -1 when no "null vector" is needed
65 |   SparseGridMap mp;
66 |   SparseGrid() : backgroundCol(-1) {}
67 | };
68 | #endif
69 | 


--------------------------------------------------------------------------------
/SpatiallySparseBatch.cpp:
--------------------------------------------------------------------------------
 1 | #include "SpatiallySparseBatch.h"
 2 | #include "utilities.h"
 3 | 
 4 | SpatiallySparseBatch::SpatiallySparseBatch() {
 5 | }
 6 | void SpatiallySparseBatch::reset() {
 7 |   batchSize=0;
 8 |   sampleNumbers.resize(0);
 9 |   for (int i=0;i<interfaces.size();++i)
10 |     interfaces[i].reset();
11 |   labels.resize(0);
12 |   predictions.resize(0);
13 |   probabilities.resize(0);
14 |   negativeLogLikelihood=0;
15 |   mistakes=0;
16 |   inputSub.reset();
17 | }
18 | 


--------------------------------------------------------------------------------
/SpatiallySparseBatch.h:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | #include <vector>
 3 | #include "SpatiallySparseBatchInterface.h"
 4 | #include "vectorCUDA.h"
 5 | #include "types.h"
 6 | 
 7 | class SpatiallySparseBatch {
 8 | public:
 9 |   SpatiallySparseBatch();
10 |   batchType type;
11 |   int batchSize;
12 |   std::vector<int> sampleNumbers;
13 |   std::vector<SpatiallySparseBatchInterface> interfaces;
14 |   vectorCUDA<int> labels;
15 |   std::vector<std::vector<int> > predictions;
16 |   std::vector<std::vector<float> > probabilities;
17 |   float negativeLogLikelihood;
18 |   int mistakes;
19 |   void reset();
20 |   SpatiallySparseBatchSubInterface inputSub;
21 | };
22 | 


--------------------------------------------------------------------------------
/SpatiallySparseBatchInterface.cpp:
--------------------------------------------------------------------------------
 1 | #include <iostream>
 2 | #include "SpatiallySparseBatchInterface.h"
 3 | 
 4 | SpatiallySparseBatchInterface::SpatiallySparseBatchInterface() :
 5 |   featuresPresent(0) {
 6 | }
 7 | void SpatiallySparseBatchInterface::summary() {
 8 |   std::cout << "---------------------------------------------------\n";
 9 |   std::cout << "nFeatures" << nFeatures << std::endl;
10 |   std::cout << "featuresPresent.size()" << featuresPresent.size() <<std::endl;
11 |   std::cout << "spatialSize" << spatialSize << std::endl;
12 |   std::cout << "nSpatialSites" << nSpatialSites << std::endl;
13 |   std::cout << "sub->features.size()" << sub->features.size() << std::endl;
14 |   std::cout << "sub->dfeatures.size()" << sub->dfeatures.size() << std::endl;
15 |   std::cout << "grids.size()" << grids.size() << std::endl;
16 |   std::cout << "grids[0].mp.size()" << grids[0].mp.size() << std::endl;
17 |   std::cout << "---------------------------------------------------\n";
18 | }
19 | void SpatiallySparseBatchInterface::reset() {
20 |   featuresPresent.resize(0);
21 |   nSpatialSites=0;
22 |   grids.resize(0);
23 |   rules.resize(0);
24 |   backpropErrors=false;
25 | }
26 | 


--------------------------------------------------------------------------------
/SpatiallySparseBatchInterface.h:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | #include <vector>
 3 | #include "SparseGrid.h"
 4 | #include "vectorCUDA.h"
 5 | 
 6 | //This is a subset of the whole interface.
 7 | //It contains the larger vectors that can mostly stay on the GPU
 8 | class SpatiallySparseBatchSubInterface {
 9 | public:
10 |   vectorCUDA<float> features;              // In the input layer, this is configured during preprocessing
11 |   vectorCUDA<float> dfeatures;             // For the backwards/backpropagation pass
12 |   vectorCUDA<int> poolingChoices;
13 |   void reset() {
14 |     features.resize(0);
15 |     dfeatures.resize(0);
16 |     poolingChoices.resize(0);
17 |   }
18 | };
19 | 
20 | class SpatiallySparseBatchInterface {
21 | public:
22 |   SpatiallySparseBatchSubInterface* sub;
23 |   int nFeatures;                           // Features per spatial location
24 |   // Not dropped out features per spatial location
25 |   vectorCUDA<int> featuresPresent;         // For dropout rng.NchooseM(nFeatures,featuresPresent.size());
26 |   int nSpatialSites;                       // Total active spatial locations within the
27 |   int spatialSize;                         // spatialSize x spatialSize grid
28 |   //                                          batchSize x spatialSize x spatialSize
29 |   //                                          possible locations.
30 |   bool backpropErrors;                     //Calculate dfeatures? (false until after the first NiN layer)
31 |   std::vector<SparseGrid> grids;         // batchSize vectors of maps storing info on grids of size spatialSize x spatialSize
32 |   //                                          Store locations of nSpatialSites in the
33 |   //                                          spatialSize x spatialSize grids
34 |   //                                          -1 entry corresponds to null vectors in needed
35 |   // Below used internally for convolution/pooling operation:
36 |   vectorCUDA<int> rules;
37 |   SpatiallySparseBatchInterface();
38 |   void summary();
39 |   void reset();
40 | };
41 | 


--------------------------------------------------------------------------------
/SpatiallySparseDataset.cpp:
--------------------------------------------------------------------------------
 1 | #include "SpatiallySparseDataset.h"
 2 | #include <algorithm>
 3 | #include <iostream>
 4 | #include <chrono>
 5 | 
 6 | void SpatiallySparseDataset::summary() {
 7 |   std::cout << "Name:           " << name << std::endl;
 8 |   std::cout << "nPictures:      " << pictures.size() << std::endl;
 9 |   std::cout << "nClasses:       " << nClasses << std::endl;
10 |   std::cout << "nFeatures:      " << nFeatures << std::endl;
11 |   std::cout << "Type:           " << batchTypeNames[type]<<std::endl;
12 | }
13 | SpatiallySparseDataset SpatiallySparseDataset::extractValidationSet(float p) {
14 |   SpatiallySparseDataset val;
15 |   val.name=name+std::string(" Validation set");
16 |   name=name+std::string(" minus Validation set");
17 |   val.nClasses=nClasses;
18 |   val.nFeatures=nFeatures;
19 |   val.type=TESTBATCH;
20 |   std::mt19937 gen(123);
21 |   std::shuffle(pictures.begin(), pictures.end(), gen);
22 |   int size=pictures.size()*p;
23 |   for (;size>0;size--) {
24 |     val.pictures.push_back(pictures.back());
25 |     pictures.pop_back();
26 |   }
27 |   return val;
28 | }
29 | void SpatiallySparseDataset::subsetOfClasses(std::vector<int> activeClasses) {
30 |   nClasses=activeClasses.size();
31 |   std::vector<Picture*> p=pictures;
32 |   pictures.clear();
33 |   for (int i=0;i<p.size();++i) {
34 |     std::vector<int>::iterator it;
35 |     it = find (activeClasses.begin(), activeClasses.end(), p[i]->label);
36 |     if (it != activeClasses.end()) {
37 |       p[i]->label=it-activeClasses.begin();
38 |       pictures.push_back(p[i]);
39 |       //std::cout << pictures.size() << " " << p[i]->identify() << std::endl;
40 |     } else
41 |       delete p[i];
42 |   }
43 | }
44 | SpatiallySparseDataset SpatiallySparseDataset::subset(int n) {
45 |   SpatiallySparseDataset subset(*this);
46 |   subset.name+=std::string(" subset");
47 |   subset.shuffle();
48 |   subset.pictures.resize(n);
49 |   return subset;
50 | }
51 | void SpatiallySparseDataset::shuffle() {
52 |   auto seed = std::chrono::high_resolution_clock::now().time_since_epoch().count();
53 |   std::mt19937 gen(seed);
54 |   std::shuffle(pictures.begin(), pictures.end(), gen);
55 | }
56 | void SpatiallySparseDataset::repeatSamples(int reps) {
57 |   int s=pictures.size();
58 |   for (int i=1; i<reps; ++i)
59 |     for (int j=0; j<s; ++j)
60 |       pictures.push_back(pictures[j]);
61 | }
62 | 


--------------------------------------------------------------------------------
/SpatiallySparseDataset.h:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | #include "Picture.h"
 3 | #include <vector>
 4 | #include <string>
 5 | #include "types.h"
 6 | #include "SpatiallySparseDataset.h"
 7 | 
 8 | 
 9 | class SpatiallySparseDataset {
10 | public:
11 |   std::string name;
12 |   std::vector<Picture*> pictures;
13 |   int nFeatures;
14 |   int nClasses;
15 |   batchType type;
16 |   void summary();
17 |   void shuffle(); // For use before extracting a validation set (not for making gradient-descent stochastic)
18 |   SpatiallySparseDataset extractValidationSet(float p=0.1);
19 |   void subsetOfClasses(std::vector<int> activeClasses);
20 |   SpatiallySparseDataset subset(int n);
21 |   void repeatSamples(int reps); // Make dataset seem n times bigger (i.e. for small datasets to avoid having v. small training epochs)
22 | };
23 | 


--------------------------------------------------------------------------------
/SpatiallySparseDatasetCIFAR10.cpp:
--------------------------------------------------------------------------------
 1 | #include "SpatiallySparseDatasetCIFAR10.h"
 2 | #include <vector>
 3 | #include <iostream>
 4 | #include <fstream>
 5 | 
 6 | void readCIFAR10File(std::vector<Picture*> &characters, const char* filename) {
 7 |   std::ifstream file(filename,std::ios::in|std::ios::binary);
 8 |   if (!file) {
 9 |     std::cout <<"Cannot find " << filename << std::endl;
10 |     exit(EXIT_FAILURE);
11 |   }
12 |   unsigned char label;
13 |   while (file.read((char*)&label,1)) {
14 |     OpenCVPicture* character = new OpenCVPicture(32,32,3,128,label);
15 |     unsigned char bitmap[3072];
16 |     float* matData=((float*)(character->mat.data));
17 |     file.read((char*)bitmap,3072);
18 |     for (int x=0;x<32;x++) {
19 |       for (int y=0;y<32;y++) {
20 |         for (int c=0;c<3;c++) {
21 |           matData[y*96+x*3+(2-c)]=bitmap[c*1024+y*32+x];
22 |         }
23 |       }
24 |     }
25 |     characters.push_back(character);
26 |   }
27 |   file.close();
28 | }
29 | SpatiallySparseDataset Cifar10TrainSet() {
30 |   SpatiallySparseDataset dataset;
31 |   dataset.name="CIFAR-10 train set";
32 |   dataset.type=TRAINBATCH;
33 |   dataset.nFeatures=3;
34 |   dataset.nClasses=10;
35 |   char filenameFormat[]="Data/CIFAR10/data_batch_%d.bin";
36 |   char filename[100];
37 |   for(int fileNumber=1;fileNumber<=5;fileNumber++) {
38 |     sprintf(filename,filenameFormat,fileNumber);
39 |     readCIFAR10File(dataset.pictures,filename);
40 |   }
41 |   return dataset;
42 | }
43 | SpatiallySparseDataset Cifar10TestSet() {
44 |   SpatiallySparseDataset dataset;
45 |   dataset.name="CIFAR-10 test set";
46 |   dataset.type=TESTBATCH;
47 |   dataset.nFeatures=3;
48 |   dataset.nClasses=10;
49 |   char filenameTest[]="Data/CIFAR10/test_batch.bin";
50 |   readCIFAR10File(dataset.pictures,filenameTest);
51 |   return dataset;
52 | }
53 | 


--------------------------------------------------------------------------------
/SpatiallySparseDatasetCIFAR10.h:
--------------------------------------------------------------------------------
1 | #pragma once
2 | #include "SpatiallySparseDataset.h"
3 | #include "OpenCVPicture.h"
4 | 
5 | SpatiallySparseDataset Cifar10TrainSet();
6 | SpatiallySparseDataset Cifar10TestSet();
7 | 


--------------------------------------------------------------------------------
/SpatiallySparseDatasetCIFAR100.cpp:
--------------------------------------------------------------------------------
 1 | #include "SpatiallySparseDatasetCIFAR100.h"
 2 | #include <vector>
 3 | #include <iostream>
 4 | #include <fstream>
 5 | 
 6 | void readCIFAR100File(std::vector<Picture*> &characters, const char* filename) {
 7 |   std::ifstream file(filename,std::ios::in|std::ios::binary);
 8 |   if (!file) {
 9 |     std::cout <<"Cannot find " << filename << std::endl;
10 |     exit(EXIT_FAILURE);
11 |   }
12 |   unsigned char label[2];
13 |   while (file.read((char*)label,2)) {
14 |     OpenCVPicture* character = new OpenCVPicture(32,32,3,128,label[1]);
15 |     unsigned char bitmap[3072];
16 |     float* matData=((float*)(character->mat.data));
17 |     file.read((char*)bitmap,3072);
18 |     for (int x=0;x<32;x++) {
19 |       for (int y=0;y<32;y++) {
20 |         for (int c=0;c<3;c++) {
21 |           matData[y*96+x*3+(2-c)]=bitmap[c*1024+y*32+x];
22 |         }
23 |       }
24 |     }
25 |     characters.push_back(character);
26 |   }
27 |   file.close();
28 | }
29 | SpatiallySparseDataset Cifar100TrainSet() {
30 |   SpatiallySparseDataset dataset;
31 |   dataset.name="CIFAR-100 train set";
32 |   dataset.type=TRAINBATCH;
33 |   dataset.nFeatures=3;
34 |   dataset.nClasses=100;
35 |   char filenameTrain[]="Data/CIFAR100/train.bin";
36 |   readCIFAR100File(dataset.pictures,filenameTrain);
37 |   return dataset;
38 | }
39 | SpatiallySparseDataset Cifar100TestSet() {
40 |   SpatiallySparseDataset dataset;
41 |   dataset.name="CIFAR-100 test set";
42 |   dataset.type=TESTBATCH;
43 |   dataset.nFeatures=3;
44 |   dataset.nClasses=100;
45 |   char filenameTest[]="Data/CIFAR100/test.bin";
46 |   readCIFAR100File(dataset.pictures,filenameTest);
47 |   return dataset;
48 | }
49 | 


--------------------------------------------------------------------------------
/SpatiallySparseDatasetCIFAR100.h:
--------------------------------------------------------------------------------
1 | #pragma once
2 | #include "SpatiallySparseDataset.h"
3 | #include "OpenCVPicture.h"
4 | 
5 | SpatiallySparseDataset Cifar100TrainSet();
6 | SpatiallySparseDataset Cifar100TestSet();
7 | 


--------------------------------------------------------------------------------
/SpatiallySparseDatasetCIFAR10FullyConnected.cpp:
--------------------------------------------------------------------------------
 1 | #include "SpatiallySparseDatasetCIFAR10FullyConnected.h"
 2 | #include <vector>
 3 | #include <iostream>
 4 | #include <fstream>
 5 | 
 6 | class TrivialPicture : public Picture {
 7 | public:
 8 |   std::vector<float> features;
 9 |   TrivialPicture(int label = -1) : Picture(label) {}
10 |   ~TrivialPicture() {}
11 |   Picture* distort (RNG& rng, batchType type=TRAINBATCH) {
12 |     TrivialPicture* pic=new TrivialPicture(*this);
13 |     if (type==TRAINBATCH)
14 |       for (int i=0;i<features.size();++i)
15 |         pic->features[i]*=rng.bernoulli(0.8);
16 |     else
17 |       for (int i=0;i<features.size();++i)
18 |         pic->features[i]*=0.8;
19 |     return pic;
20 |   };
21 |   void codifyInputData(SparseGrid &grid, std::vector<float> &f, int &nSpatialSites, int spatialSize) {
22 |     grid.mp[0]=nSpatialSites++;
23 |     for (int i=0;i<features.size();++i)
24 |       f.push_back(features[i]);
25 |   }
26 | };
27 | 
28 | void readCIFAR10File(std::vector<Picture*> &characters, const char* filename) {
29 |   std::ifstream file(filename,std::ios::in|std::ios::binary);
30 |   if (!file) {
31 |     std::cout <<"Cannot find " << filename << std::endl;
32 |     exit(EXIT_FAILURE);
33 |   }
34 |   unsigned char label;
35 |   while (file.read((char*)&label,1)) {
36 |     TrivialPicture* character = new TrivialPicture(label);
37 |     unsigned char bitmap[3072];
38 |     file.read((char*)bitmap,3072);
39 |     for (int x=0;x<3072;x++) {
40 |       character->features.push_back(bitmap[x]/127.5-1);
41 |     }
42 |     characters.push_back(character);
43 |   }
44 |   file.close();
45 | }
46 | SpatiallySparseDataset Cifar10TrainSetFullyConnected() {
47 |   SpatiallySparseDataset dataset;
48 |   dataset.name="CIFAR-10 train set Fully connected";
49 |   dataset.type=TRAINBATCH;
50 |   dataset.nFeatures=3072;
51 |   dataset.nClasses=10;
52 |   char filenameFormat[]="Data/CIFAR10/data_batch_%d.bin";
53 |   char filename[100];
54 |   for(int fileNumber=1;fileNumber<=5;fileNumber++) {
55 |     sprintf(filename,filenameFormat,fileNumber);
56 |     readCIFAR10File(dataset.pictures,filename);
57 |   }
58 |   return dataset;
59 | }
60 | SpatiallySparseDataset Cifar10TestSetFullyConnected() {
61 |   SpatiallySparseDataset dataset;
62 |   dataset.name="CIFAR-10 test set fully connected";
63 |   dataset.type=TESTBATCH;
64 |   dataset.nFeatures=3072;
65 |   dataset.nClasses=10;
66 |   char filenameTest[]="Data/CIFAR10/test_batch.bin";
67 |   readCIFAR10File(dataset.pictures,filenameTest);
68 |   return dataset;
69 | }
70 | 


--------------------------------------------------------------------------------
/SpatiallySparseDatasetCIFAR10FullyConnected.h:
--------------------------------------------------------------------------------
1 | #pragma once
2 | #include "SpatiallySparseDataset.h"
3 | 
4 | SpatiallySparseDataset Cifar10TrainSetFullyConnected();
5 | SpatiallySparseDataset Cifar10TestSetFullyConnected();
6 | 


--------------------------------------------------------------------------------
/SpatiallySparseDatasetCasiaOLHWDB.cpp:
--------------------------------------------------------------------------------
  1 | //Obtain POT files from http://www.nlpr.ia.ac.cn/databases/handwriting/home.html and put them in a directory Data/CASIA_pot_files/
  2 | #include <vector>
  3 | #include <fstream>
  4 | #include <algorithm>
  5 | #include <iostream>
  6 | #include "gbcodes3755.h"
  7 | #include "OnlineHandwritingPicture.h"
  8 | #include "SpatiallySparseDatasetCasiaOLHWDB.h"
  9 | const int nCharacters = 3755;
 10 | 
 11 | struct potCharacterHeader{
 12 |   unsigned short sampleSize;
 13 |   unsigned short label;
 14 |   unsigned short zzz;
 15 |   unsigned short nStrokes;
 16 | };
 17 | 
 18 | struct iPoint{
 19 |   short x;
 20 |   short y;
 21 | };
 22 | 
 23 | int readPotFile(std::vector<Picture*> &characters, const char* filename, int renderSize, OnlineHandwritingEncoding enc) {
 24 |   std::ifstream file(filename,std::ios::in|std::ios::binary);
 25 |   if (!file) {
 26 |     std::cout <<"Cannot find " << filename << std::endl;
 27 |     exit(EXIT_FAILURE);}
 28 |   potCharacterHeader pCH;
 29 |   iPoint iP;
 30 |   int label;
 31 |   std::vector<iPoint> stroke;
 32 |   while (file.read((char*)&pCH,sizeof(potCharacterHeader))) {
 33 |     label=pCH.label;
 34 |     label=std::find(gbcodesPOT,gbcodesPOT+3755,label)-gbcodesPOT;
 35 |     OnlineHandwritingPicture* character = new OnlineHandwritingPicture(renderSize, enc, label,0.0001);
 36 |     file.read((char*)&iP,sizeof(iPoint));
 37 |     while (iP.y!=-1) {
 38 |       stroke.resize(0);
 39 |       while (iP.x!=-1) {
 40 |         stroke.push_back(iP);
 41 |         file.read((char*)&iP,sizeof(iPoint));
 42 |       }
 43 |       character->ops.push_back(arma::mat(stroke.size(),2));
 44 |       for (int i=0;i<stroke.size();++i) {
 45 |         character->ops.back()(i,0)=stroke[i].x;
 46 |         character->ops.back()(i,1)=stroke[i].y;
 47 |       }
 48 |       file.read((char*)&iP,sizeof(iPoint));
 49 |     }
 50 |     character->normalize();
 51 |     if (character->label<nCharacters)
 52 |       characters.push_back(character);
 53 |     else
 54 |       delete character;
 55 |   }
 56 |   file.close();
 57 |   return 0;
 58 | }
 59 | 
 60 | SpatiallySparseDataset CasiaOLHWDB11TrainSet(int renderSize, OnlineHandwritingEncoding enc) {
 61 |   SpatiallySparseDataset dataset;
 62 |   dataset.name="CASIA OLHWDB1.1 train set--240 writers";
 63 |   dataset.type=TRAINBATCH;
 64 |   dataset.nFeatures=OnlineHandwritingEncodingSize[enc];
 65 |   dataset.nClasses=3755;
 66 |   char filenameFormat[]="Data/CASIA_pot_files/%04d.pot";
 67 |   char filename[100];
 68 |   for(int fileNumber=1001;fileNumber<=1240;fileNumber++) {
 69 |     sprintf(filename,filenameFormat,fileNumber);
 70 |     readPotFile(dataset.pictures,filename,renderSize,enc);
 71 |   }
 72 |   return dataset;
 73 | }
 74 | SpatiallySparseDataset CasiaOLHWDB101112TrainSet(int renderSize, OnlineHandwritingEncoding enc) {
 75 |   SpatiallySparseDataset dataset;
 76 |   dataset.name="CASIA OLHWDB1.0-1.2 train sets";
 77 |   dataset.type=TRAINBATCH;
 78 |   dataset.nFeatures=OnlineHandwritingEncodingSize[enc];
 79 |   dataset.nClasses=3755;
 80 |   char filenameFormatA[]="Data/CASIA_pot_files/%03d.pot";
 81 |   char filenameFormatB[]="Data/CASIA_pot_files/%04d.pot";
 82 |   char filename[100];
 83 |   for(int fileNumber=1;fileNumber<=420;fileNumber++) {
 84 |     sprintf(filename,filenameFormatA,fileNumber);
 85 |     readPotFile(dataset.pictures,filename,renderSize,enc);
 86 |   }
 87 |   for(int fileNumber=501;fileNumber<=800;fileNumber++) {
 88 |     sprintf(filename,filenameFormatA,fileNumber);
 89 |     readPotFile(dataset.pictures,filename,renderSize,enc);
 90 |   }
 91 |   for(int fileNumber=1001;fileNumber<=1240;fileNumber++) {
 92 |     sprintf(filename,filenameFormatB,fileNumber);
 93 |     readPotFile(dataset.pictures,filename,renderSize,enc);
 94 |   }
 95 |   return dataset;
 96 | }
 97 | SpatiallySparseDataset CasiaOLHWDB11TestSet(int renderSize, OnlineHandwritingEncoding enc) {
 98 |   SpatiallySparseDataset dataset;
 99 |   dataset.name="CASIA OLHWDB1.1 test set--60 writers";
100 |   dataset.type=TESTBATCH;
101 |   dataset.nFeatures=OnlineHandwritingEncodingSize[enc];
102 |   dataset.nClasses=3755;
103 |   char filenameFormat[]="Data/CASIA_pot_files/%04d.pot";
104 |   char filename[100];
105 |   for(int fileNumber=1241;fileNumber<=1300;fileNumber++) {
106 |     sprintf(filename,filenameFormat,fileNumber);
107 |     readPotFile(dataset.pictures,filename,renderSize,enc);
108 |   }
109 |   return dataset;
110 | }
111 | 


--------------------------------------------------------------------------------
/SpatiallySparseDatasetCasiaOLHWDB.h:
--------------------------------------------------------------------------------
1 | #pragma once
2 | #include "SpatiallySparseDataset.h"
3 | 
4 | SpatiallySparseDataset CasiaOLHWDB11TrainSet(int renderSize, OnlineHandwritingEncoding enc);
5 | SpatiallySparseDataset CasiaOLHWDB101112TrainSet(int renderSize, OnlineHandwritingEncoding enc);
6 | SpatiallySparseDataset CasiaOLHWDB11TestSet(int renderSize, OnlineHandwritingEncoding enc);
7 | 


--------------------------------------------------------------------------------
/SpatiallySparseDatasetImageNet2012.cpp:
--------------------------------------------------------------------------------
  1 | #include "SpatiallySparseDatasetImageNet2012.h"
  2 | #include<iostream>
  3 | #include<fstream>
  4 | #include<string>
  5 | #include "utilities.h"
  6 | 
  7 | std::string classList[]={
  8 |   "n02119789",  "n02100735",  "n02110185",  "n02096294",  "n02102040",  "n02066245",  "n02509815",  "n02124075",  "n02417914",  "n02123394",
  9 |   "n02125311",  "n02423022",  "n02346627",  "n02077923",  "n02110063",  "n02447366",  "n02109047",  "n02089867",  "n02102177",  "n02091134",
 10 |   "n02092002",  "n02071294",  "n02442845",  "n02504458",  "n02092339",  "n02098105",  "n02096437",  "n02114712",  "n02105641",  "n02128925",
 11 |   "n02091635",  "n02088466",  "n02096051",  "n02117135",  "n02138441",  "n02097130",  "n02493509",  "n02457408",  "n02389026",  "n02443484",
 12 |   "n02110341",  "n02089078",  "n02086910",  "n02445715",  "n02093256",  "n02113978",  "n02106382",  "n02441942",  "n02113712",  "n02113186",
 13 |   "n02105162",  "n02415577",  "n02356798",  "n02488702",  "n02123159",  "n02098413",  "n02422699",  "n02114855",  "n02094433",  "n02111277",
 14 |   "n02132136",  "n02119022",  "n02091467",  "n02106550",  "n02422106",  "n02091831",  "n02120505",  "n02104365",  "n02086079",  "n02112706",
 15 |   "n02098286",  "n02095889",  "n02484975",  "n02137549",  "n02500267",  "n02129604",  "n02090721",  "n02396427",  "n02108000",  "n02391049",
 16 |   "n02412080",  "n02108915",  "n02480495",  "n02110806",  "n02128385",  "n02107683",  "n02085936",  "n02094114",  "n02087046",  "n02100583",
 17 |   "n02096177",  "n02494079",  "n02105056",  "n02101556",  "n02123597",  "n02481823",  "n02105505",  "n02088094",  "n02085782",  "n02489166",
 18 |   "n02364673",  "n02114548",  "n02134084",  "n02480855",  "n02090622",  "n02113624",  "n02093859",  "n02403003",  "n02097298",  "n02108551",
 19 |   "n02493793",  "n02107142",  "n02096585",  "n02107574",  "n02107908",  "n02086240",  "n02102973",  "n02112018",  "n02093647",  "n02397096",
 20 |   "n02437312",  "n02483708",  "n02097047",  "n02106030",  "n02099601",  "n02093991",  "n02110627",  "n02106166",  "n02326432",  "n02108089",
 21 |   "n02097658",  "n02088364",  "n02111129",  "n02100236",  "n02486261",  "n02115913",  "n02486410",  "n02487347",  "n02099849",  "n02108422",
 22 |   "n02104029",  "n02492035",  "n02110958",  "n02099429",  "n02094258",  "n02099267",  "n02395406",  "n02112350",  "n02109961",  "n02101388",
 23 |   "n02113799",  "n02095570",  "n02128757",  "n02101006",  "n02115641",  "n02097209",  "n02342885",  "n02097474",  "n02120079",  "n02095314",
 24 |   "n02088238",  "n02408429",  "n02133161",  "n02328150",  "n02410509",  "n02492660",  "n02398521",  "n02112137",  "n02510455",  "n02093428",
 25 |   "n02105855",  "n02111500",  "n02085620",  "n02123045",  "n02490219",  "n02099712",  "n02109525",  "n02454379",  "n02111889",  "n02088632",
 26 |   "n02090379",  "n02443114",  "n02361337",  "n02105412",  "n02483362",  "n02437616",  "n02107312",  "n02325366",  "n02091032",  "n02129165",
 27 |   "n02102318",  "n02100877",  "n02074367",  "n02504013",  "n02363005",  "n02102480",  "n02113023",  "n02086646",  "n02497673",  "n02087394",
 28 |   "n02127052",  "n02116738",  "n02488291",  "n02091244",  "n02114367",  "n02130308",  "n02089973",  "n02105251",  "n02134418",  "n02093754",
 29 |   "n02106662",  "n02444819",  "n01882714",  "n01871265",  "n01872401",  "n01877812",  "n01873310",  "n01883070",  "n04086273",  "n04507155",
 30 |   "n04147183",  "n04254680",  "n02672831",  "n02219486",  "n02317335",  "n01968897",  "n03452741",  "n03642806",  "n07745940",  "n02690373",
 31 |   "n04552348",  "n02692877",  "n02782093",  "n04266014",  "n03344393",  "n03447447",  "n04273569",  "n03662601",  "n02951358",  "n04612504",
 32 |   "n02981792",  "n04483307",  "n03095699",  "n03673027",  "n03947888",  "n02687172",  "n04347754",  "n04606251",  "n03478589",  "n04389033",
 33 |   "n03773504",  "n02860847",  "n03218198",  "n02835271",  "n03792782",  "n03393912",  "n03895866",  "n02797295",  "n04204347",  "n03791053",
 34 |   "n03384352",  "n03272562",  "n04310018",  "n02704792",  "n02701002",  "n02814533",  "n02930766",  "n03100240",  "n03594945",  "n03670208",
 35 |   "n03770679",  "n03777568",  "n04037443",  "n04285008",  "n03444034",  "n03445924",  "n03785016",  "n04252225",  "n03345487",  "n03417042",
 36 |   "n03930630",  "n04461696",  "n04467665",  "n03796401",  "n03977966",  "n04065272",  "n04335435",  "n04252077",  "n04465501",  "n03776460",
 37 |   "n04482393",  "n04509417",  "n03538406",  "n03599486",  "n03868242",  "n02804414",  "n03125729",  "n03131574",  "n03388549",  "n02870880",
 38 |   "n03018349",  "n03742115",  "n03016953",  "n04380533",  "n03337140",  "n03891251",  "n02791124",  "n04429376",  "n03376595",  "n04099969",
 39 |   "n04344873",  "n04447861",  "n03179701",  "n03982430",  "n03201208",  "n03290653",  "n04550184",  "n07742313",  "n07747607",  "n07749582",
 40 |   "n07753113",  "n07753275",  "n07753592",  "n07754684",  "n07760859",  "n07768694",  "n12267677",  "n12620546",  "n13133613",  "n11879895",
 41 |   "n12144580",  "n12768682",  "n03854065",  "n04515003",  "n03017168",  "n03249569",  "n03447721",  "n03720891",  "n03721384",  "n04311174",
 42 |   "n02787622",  "n02992211",  "n04536866",  "n03495258",  "n02676566",  "n03272010",  "n03110669",  "n03394916",  "n04487394",  "n03494278",
 43 |   "n03840681",  "n03884397",  "n02804610",  "n03838899",  "n04141076",  "n03372029",  "n11939491",  "n12057211",  "n09246464",  "n09468604",
 44 |   "n09193705",  "n09472597",  "n09399592",  "n09421951",  "n09256479",  "n09332890",  "n09428293",  "n09288635",  "n03498962",  "n03041632",
 45 |   "n03658185",  "n03954731",  "n03995372",  "n03649909",  "n03481172",  "n03109150",  "n02951585",  "n03970156",  "n04154565",  "n04208210",
 46 |   "n03967562",  "n03000684",  "n01514668",  "n01514859",  "n01518878",  "n01530575",  "n01531178",  "n01532829",  "n01534433",  "n01537544",
 47 |   "n01558993",  "n01560419",  "n01580077",  "n01582220",  "n01592084",  "n01601694",  "n01608432",  "n01614925",  "n01616318",  "n01622779",
 48 |   "n01795545",  "n01796340",  "n01797886",  "n01798484",  "n01806143",  "n01806567",  "n01807496",  "n01817953",  "n01818515",  "n01819313",
 49 |   "n01820546",  "n01824575",  "n01828970",  "n01829413",  "n01833805",  "n01843065",  "n01843383",  "n01847000",  "n01855032",  "n01855672",
 50 |   "n01860187",  "n02002556",  "n02002724",  "n02006656",  "n02007558",  "n02009912",  "n02009229",  "n02011460",  "n02012849",  "n02013706",
 51 |   "n02018207",  "n02018795",  "n02025239",  "n02027492",  "n02028035",  "n02033041",  "n02037110",  "n02017213",  "n02051845",  "n02056570",
 52 |   "n02058221",  "n01484850",  "n01491361",  "n01494475",  "n01496331",  "n01498041",  "n02514041",  "n02536864",  "n01440764",  "n01443537",
 53 |   "n02526121",  "n02606052",  "n02607072",  "n02643566",  "n02655020",  "n02640242",  "n02641379",  "n01664065",  "n01665541",  "n01667114",
 54 |   "n01667778",  "n01669191",  "n01675722",  "n01677366",  "n01682714",  "n01685808",  "n01687978",  "n01688243",  "n01689811",  "n01692333",
 55 |   "n01693334",  "n01694178",  "n01695060",  "n01704323",  "n01697457",  "n01698640",  "n01728572",  "n01728920",  "n01729322",  "n01729977",
 56 |   "n01734418",  "n01735189",  "n01737021",  "n01739381",  "n01740131",  "n01742172",  "n01744401",  "n01748264",  "n01749939",  "n01751748",
 57 |   "n01753488",  "n01755581",  "n01756291",  "n01629819",  "n01630670",  "n01631663",  "n01632458",  "n01632777",  "n01641577",  "n01644373",
 58 |   "n01644900",  "n04579432",  "n04592741",  "n03876231",  "n03483316",  "n03868863",  "n04251144",  "n03691459",  "n03759954",  "n04152593",
 59 |   "n03793489",  "n03271574",  "n03843555",  "n04332243",  "n04265275",  "n04330267",  "n03467068",  "n02794156",  "n04118776",  "n03841143",
 60 |   "n04141975",  "n02708093",  "n03196217",  "n04548280",  "n03544143",  "n04355338",  "n03891332",  "n04328186",  "n03197337",  "n04317175",
 61 |   "n04376876",  "n03706229",  "n02841315",  "n04009552",  "n04356056",  "n03692522",  "n04044716",  "n02879718",  "n02950826",  "n02749479",
 62 |   "n04090263",  "n04008634",  "n03085013",  "n04505470",  "n03126707",  "n03666591",  "n02666196",  "n02977058",  "n04238763",  "n03180011",
 63 |   "n03485407",  "n03832673",  "n06359193",  "n03496892",  "n04428191",  "n04004767",  "n04243546",  "n04525305",  "n04179913",  "n03602883",
 64 |   "n04372370",  "n03532672",  "n02974003",  "n03874293",  "n03944341",  "n03992509",  "n03425413",  "n02966193",  "n04371774",  "n04067472",
 65 |   "n04040759",  "n04019541",  "n03492542",  "n04355933",  "n03929660",  "n02965783",  "n04258138",  "n04074963",  "n03208938",  "n02910353",
 66 |   "n03476684",  "n03627232",  "n03075370",  "n03874599",  "n03804744",  "n04127249",  "n04153751",  "n03803284",  "n04162706",  "n04228054",
 67 |   "n02948072",  "n03590841",  "n04286575",  "n04456115",  "n03814639",  "n03933933",  "n04485082",  "n03733131",  "n03794056",  "n04275548",
 68 |   "n01768244",  "n01770081",  "n01770393",  "n01773157",  "n01773549",  "n01773797",  "n01774384",  "n01774750",  "n01775062",  "n01776313",
 69 |   "n01784675",  "n01990800",  "n01978287",  "n01978455",  "n01980166",  "n01981276",  "n01983481",  "n01984695",  "n01985128",  "n01986214",
 70 |   "n02165105",  "n02165456",  "n02167151",  "n02168699",  "n02169497",  "n02172182",  "n02174001",  "n02177972",  "n02190166",  "n02206856",
 71 |   "n02226429",  "n02229544",  "n02231487",  "n02233338",  "n02236044",  "n02256656",  "n02259212",  "n02264363",  "n02268443",  "n02268853",
 72 |   "n02276258",  "n02277742",  "n02279972",  "n02280649",  "n02281406",  "n02281787",  "n01910747",  "n01914609",  "n01917289",  "n01924916",
 73 |   "n01930112",  "n01943899",  "n01944390",  "n01945685",  "n01950731",  "n01955084",  "n02319095",  "n02321529",  "n03584829",  "n03297495",
 74 |   "n03761084",  "n03259280",  "n04111531",  "n04442312",  "n04542943",  "n04517823",  "n03207941",  "n04070727",  "n04554684",  "n03133878",
 75 |   "n03400231",  "n04596742",  "n02939185",  "n03063689",  "n04398044",  "n04270147",  "n02699494",  "n04486054",  "n03899768",  "n04311004",
 76 |   "n04366367",  "n04532670",  "n02793495",  "n03457902",  "n03877845",  "n03781244",  "n03661043",  "n02727426",  "n02859443",  "n03028079",
 77 |   "n03788195",  "n04346328",  "n03956157",  "n04081281",  "n03032252",  "n03529860",  "n03697007",  "n03065424",  "n03837869",  "n04458633",
 78 |   "n02980441",  "n04005630",  "n03461385",  "n02776631",  "n02791270",  "n02871525",  "n02927161",  "n03089624",  "n04200800",  "n04443257",
 79 |   "n04462240",  "n03388043",  "n03042490",  "n04613696",  "n03216828",  "n02892201",  "n03743016",  "n02788148",  "n02894605",  "n03160309",
 80 |   "n03000134",  "n03930313",  "n04604644",  "n04326547",  "n03459775",  "n04239074",  "n04501370",  "n03792972",  "n04149813",  "n03530642",
 81 |   "n03961711",  "n03903868",  "n02814860",  "n07711569",  "n07720875",  "n07714571",  "n07714990",  "n07715103",  "n07716358",  "n07716906",
 82 |   "n07717410",  "n07717556",  "n07718472",  "n07718747",  "n07730033",  "n07734744",  "n04209239",  "n03594734",  "n02971356",  "n03485794",
 83 |   "n04133789",  "n02747177",  "n04125021",  "n07579787",  "n03814906",  "n03134739",  "n03404251",  "n04423845",  "n03877472",  "n04120489",
 84 |   "n03062245",  "n03014705",  "n03717622",  "n03777754",  "n04493381",  "n04476259",  "n02777292",  "n07693725",  "n03998194",  "n03617480",
 85 |   "n07590611",  "n04579145",  "n03623198",  "n07248320",  "n04277352",  "n04229816",  "n02823428",  "n03127747",  "n02877765",  "n04435653",
 86 |   "n03724870",  "n03710637",  "n03920288",  "n03379051",  "n02807133",  "n04399382",  "n03527444",  "n03983396",  "n03924679",  "n04532106",
 87 |   "n06785654",  "n03445777",  "n07613480",  "n04350905",  "n04562935",  "n03325584",  "n03045698",  "n07892512",  "n03250847",  "n04192698",
 88 |   "n03026506",  "n03534580",  "n07565083",  "n04296562",  "n02869837",  "n07871810",  "n02799071",  "n03314780",  "n04141327",  "n04357314",
 89 |   "n02823750",  "n13052670",  "n07583066",  "n03637318",  "n04599235",  "n07802026",  "n02883205",  "n03709823",  "n04560804",  "n02909870",
 90 |   "n03207743",  "n04263257",  "n07932039",  "n03786901",  "n04479046",  "n03873416",  "n02999410",  "n04367480",  "n03775546",  "n07875152",
 91 |   "n04591713",  "n04201297",  "n02916936",  "n03240683",  "n02840245",  "n02963159",  "n04370456",  "n03991062",  "n02843684",  "n03482405",
 92 |   "n03942813",  "n03908618",  "n03902125",  "n07584110",  "n02730930",  "n04023962",  "n02769748",  "n10148035",  "n02817516",  "n03908714",
 93 |   "n02906734",  "n03788365",  "n02667093",  "n03787032",  "n03980874",  "n03141823",  "n03976467",  "n04264628",  "n07930864",  "n04039381",
 94 |   "n06874185",  "n04033901",  "n04041544",  "n07860988",  "n03146219",  "n03763968",  "n03676483",  "n04209133",  "n03782006",  "n03857828",
 95 |   "n03775071",  "n02892767",  "n07684084",  "n04522168",  "n03764736",  "n04118538",  "n03887697",  "n13044778",  "n03291819",  "n03770439",
 96 |   "n03124170",  "n04487081",  "n03916031",  "n02808440",  "n07697537",  "n12985857",  "n02917067",  "n03938244",  "n15075141",  "n02978881",
 97 |   "n02966687",  "n03633091",  "n13040303",  "n03690938",  "n03476991",  "n02669723",  "n03220513",  "n03127925",  "n04584207",  "n07880968",
 98 |   "n03937543",  "n03000247",  "n04418357",  "n04590129",  "n02795169",  "n04553703",  "n02783161",  "n02802426",  "n02808304",  "n03124043",
 99 |   "n03450230",  "n04589890",  "n12998815",  "n02992529",  "n03825788",  "n02790996",  "n03710193",  "n03630383",  "n03347037",  "n03769881",
100 |   "n03871628",  "n03733281",  "n03976657",  "n03535780",  "n04259630",  "n03929855",  "n04049303",  "n04548362",  "n02979186",  "n06596364",
101 |   "n03935335",  "n06794110",  "n02825657",  "n03388183",  "n04591157",  "n04540053",  "n03866082",  "n04136333",  "n04026417",  "n02865351",
102 |   "n02834397",  "n03888257",  "n04235860",  "n04404412",  "n04371430",  "n03733805",  "n07920052",  "n07873807",  "n02895154",  "n04204238",
103 |   "n04597913",  "n04131690",  "n07836838",  "n09835506",  "n03443371",  "n13037406",  "n04336792",  "n04557648",  "n03187595",  "n04254120",
104 |   "n03595614",  "n04146614",  "n03598930",  "n03958227",  "n04069434",  "n03188531",  "n02786058",  "n07615774",  "n04525038",  "n04409515",
105 |   "n03424325",  "n03223299",  "n03680355",  "n07614500",  "n07695742",  "n04033995",  "n03710721",  "n04392985",  "n03047690",  "n03584254",
106 |   "n13054560",  "n10565667",  "n03950228",  "n03729826",  "n02837789",  "n04254777",  "n02988304",  "n03657121",  "n04417672",  "n04523525",
107 |   "n02815834",  "n09229709",  "n07697313",  "n03888605",  "n03355925",  "n03063599",  "n04116512",  "n04325704",  "n07831146",  "n03255030"
108 | };
109 | 
110 | SpatiallySparseDataset ImageNet2012TrainSet() {
111 |   SpatiallySparseDataset dataset;
112 |   dataset.name="ImageNet2012 train set";
113 |   dataset.type=TRAINBATCH;
114 |   dataset.nFeatures=3;
115 |   dataset.nClasses=1000;
116 | 
117 |   for (int cl=0;cl<1000;cl++) {
118 |     for (auto &file : globVector(std::string("Data/imagenet2012/ILSVRC2012_img_train/")+classList[cl]+"/*.JPEG")) {
119 |       dataset.pictures.push_back(new OpenCVPicture(file,128,cl));
120 |     }
121 |   }
122 |   return dataset;
123 | }
124 | SpatiallySparseDataset ImageNet2012ValidationSet() {
125 |   SpatiallySparseDataset dataset;
126 |   dataset.name="ImageNet2012 validation set";
127 |   dataset.type=TESTBATCH;
128 |   dataset.nFeatures=3;
129 |   dataset.nClasses=1000;
130 | 
131 |   std::string imageFile;
132 |   int cl;
133 |   std::ifstream file("Data/imagenet2012/validationData.txt");
134 |   int nWidth, nHeight, nBBoxx, nBBoxX, nBBoxy, nBBoxY;
135 |   while (file >> cl >> imageFile >> nWidth >> nHeight >> nBBoxx >> nBBoxX >> nBBoxy >> nBBoxY) {
136 |     cl--;
137 |     OpenCVPicture*  pic = new OpenCVPicture(std::string("Data/imagenet2012/")+imageFile,128,cl);
138 |     dataset.pictures.push_back(pic);
139 |   }
140 |   return dataset;
141 | }
142 | SpatiallySparseDataset ImageNet2012TestSet() {
143 |   SpatiallySparseDataset dataset;
144 |   dataset.name="ImageNet2012 train set";
145 |   dataset.type=UNLABELEDBATCH;
146 |   dataset.nFeatures=3;
147 |   dataset.nClasses=1000;
148 | 
149 |   std::string imageFile;
150 |   int cl;
151 |   std::ifstream file("Data/imagenet2012/testData.txt");
152 |   int nWidth, nHeight, nBBoxx, nBBoxX, nBBoxy, nBBoxY;
153 |   while (file >> cl >> imageFile >> nWidth >> nHeight >> nBBoxx >> nBBoxX >> nBBoxy >> nBBoxY) {
154 |     OpenCVPicture*  pic = new OpenCVPicture(std::string("Data/imagenet2012/")+imageFile,128,-1);
155 |     dataset.pictures.push_back(pic);
156 |   }
157 |   return dataset;
158 | }
159 | 


--------------------------------------------------------------------------------
/SpatiallySparseDatasetImageNet2012.h:
--------------------------------------------------------------------------------
1 | #pragma once
2 | #include "SpatiallySparseDataset.h"
3 | #include "OpenCVPicture.h"
4 | 
5 | SpatiallySparseDataset ImageNet2012TrainSet();
6 | SpatiallySparseDataset ImageNet2012ValidationSet();
7 | SpatiallySparseDataset ImageNet2012TestSet();
8 | 


--------------------------------------------------------------------------------
/SpatiallySparseDatasetMnist.cpp:
--------------------------------------------------------------------------------
 1 | #include "SpatiallySparseDatasetMnist.h"
 2 | #include <vector>
 3 | #include <iostream>
 4 | #include <fstream>
 5 | 
 6 | static int intToggleEndianness(int a) {
 7 |   int b=0;
 8 |   b+=a%256*(1<<24);a>>=8;
 9 |   b+=a%256*(1<<16);a>>=8;
10 |   b+=a%256*(1<< 8);a>>=8;
11 |   b+=a%256*(1<< 0);
12 |   return b;}
13 | 
14 | static void loadMnistC(std::string filename, std::vector<Picture*> &characters) {
15 |   std::ifstream f(filename.c_str());
16 |   if (!f) {
17 |     std::cout <<"Cannot find " << filename << std::endl;;
18 |     exit(EXIT_FAILURE);}
19 |   int a,n1,n2,n3;
20 |   f.read((char*)&a,4);
21 |   f.read((char*)&a,4);
22 |   n1=intToggleEndianness(a);
23 |   f.read((char*)&a,4);
24 |   n2=intToggleEndianness(a);
25 |   f.read((char*)&a,4);
26 |   n3=intToggleEndianness(a);
27 |   std::vector<unsigned char> bitmap(n2*n3);
28 |   for (int i1=0;i1<n1;i1++) {
29 |     f.read((char *)&bitmap[0],n2*n3);
30 |     OpenCVPicture* character = new OpenCVPicture(n2,n3,1,0);
31 |     float* matData=((float*)(character->mat.data));
32 |     for (int i=0;i<n2*n3;i++)
33 |       matData[i]=bitmap[i];
34 |     characters.push_back(character);
35 |   }
36 | }
37 | static void loadMnistL(std::string filename, std::vector<Picture*> &characters) {
38 |   std::ifstream f(filename.c_str());
39 |   if (!f) {
40 |     std::cout <<"Cannot find " << filename << std::endl;
41 |     exit(EXIT_FAILURE);}
42 |   int a,n;
43 |   char l;
44 |   f.read((char*)&a,4);
45 |   f.read((char*)&a,4);
46 |   n=intToggleEndianness(a);
47 |   for (int i=0;i<n;i++) {
48 |     f.read(&l,1);
49 |     characters[i]->label=l;
50 |   }
51 | }
52 | 
53 | SpatiallySparseDataset MnistTrainSet() {
54 |   SpatiallySparseDataset dataset;
55 |   dataset.name="MNIST train set";
56 |   dataset.type=TRAINBATCH;
57 |   dataset.nFeatures=1;
58 |   dataset.nClasses=10;
59 |   std::string trainC("Data/MNIST/train-images-idx3-ubyte");
60 |   std::string trainL("Data/MNIST/train-labels-idx1-ubyte");
61 |   loadMnistC(trainC, dataset.pictures);
62 |   loadMnistL(trainL, dataset.pictures);
63 |   return dataset;
64 | }
65 | SpatiallySparseDataset MnistTestSet() {
66 |   SpatiallySparseDataset dataset;
67 |   dataset.type=TESTBATCH;
68 |   dataset.name="MNIST test set";
69 |   dataset.nFeatures=1;
70 |   dataset.nClasses=10;
71 |   std::string testC("Data/MNIST/t10k-images-idx3-ubyte");
72 |   std::string testL("Data/MNIST/t10k-labels-idx1-ubyte");
73 |   loadMnistC(testC, dataset.pictures);
74 |   loadMnistL(testL, dataset.pictures);
75 |   return dataset;
76 | }
77 | 


--------------------------------------------------------------------------------
/SpatiallySparseDatasetMnist.h:
--------------------------------------------------------------------------------
1 | #pragma once
2 | #include "SpatiallySparseDataset.h"
3 | #include "OpenCVPicture.h"
4 | 
5 | SpatiallySparseDataset MnistTrainSet();
6 | SpatiallySparseDataset MnistTestSet();
7 | 


--------------------------------------------------------------------------------
/SpatiallySparseDatasetOpenCV.cpp:
--------------------------------------------------------------------------------
 1 | #include "SpatiallySparseDatasetOpenCV.h"
 2 | #include<iostream>
 3 | #include<fstream>
 4 | #include<string>
 5 | #include "utilities.h"
 6 | 
 7 | OpenCVLabeledDataSet::OpenCVLabeledDataSet
 8 | (std::string classesListFile, std::string dataDirectory, std::string wildcard,
 9 |  batchType type_, int backgroundCol,
10 |  bool loadData, int flags) {
11 |   name=dataDirectory;
12 |   type=type_;
13 |   {
14 |     std::ifstream f(classesListFile.c_str());
15 |     std::string cl;
16 |     int ctr=0;
17 |     while (f >> cl)
18 |       classes[cl]=ctr++;
19 |   }
20 |   nClasses=classes.size();
21 |   for (auto &kv : classes) {
22 |     for (auto &file : globVector(dataDirectory+"/"+kv.first+"/"+wildcard)) {
23 |       OpenCVPicture* pic = new OpenCVPicture(file,backgroundCol,kv.second);
24 |       if(loadData) {
25 |         pic->loadDataWithoutScaling(flags);
26 |         nFeatures=pic->mat.channels();
27 |       }
28 |       pictures.push_back(pic);
29 |     }
30 |   }
31 | }
32 | 
33 | OpenCVUnlabeledDataSet::OpenCVUnlabeledDataSet
34 | (std::string classesListFile, std::string dataDirectory, std::string wildcard,
35 |  int backgroundCol,
36 |  bool loadData, int flags) {
37 |   name=dataDirectory;
38 |   header="image";
39 |   type=UNLABELEDBATCH;
40 |   {
41 |     std::ifstream f(classesListFile.c_str());
42 |     std::string cl;
43 |     int ctr=0;
44 |     while (f >> cl) {
45 |       classes[cl]=ctr++;
46 |       header=header+","+cl;
47 |     }
48 |   }
49 |   nClasses=classes.size();
50 |   for (auto &file : globVector(dataDirectory+"/"+wildcard)) {
51 |     OpenCVPicture* pic = new OpenCVPicture(file,backgroundCol,0);
52 |     if(loadData) {
53 |       pic->loadDataWithoutScaling(flags);
54 |       nFeatures=pic->mat.channels();
55 |     }
56 |     pictures.push_back(pic);
57 |   }
58 | }
59 | 


--------------------------------------------------------------------------------
/SpatiallySparseDatasetOpenCV.h:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | #include "SpatiallySparseDataset.h"
 3 | #include "OpenCVPicture.h"
 4 | 
 5 | class OpenCVLabeledDataSet : public SpatiallySparseDataset {
 6 | public:
 7 |   std::map<std::string,int> classes;
 8 |   OpenCVLabeledDataSet(std::string classesListFile, std::string dataDirectory, std::string wildcard,
 9 |                        batchType type_, int backgroundCol=128,
10 |                        bool loadData=true, int flags=1);
11 | };
12 | class OpenCVUnlabeledDataSet : public SpatiallySparseDataset {
13 | public:
14 |   std::map<std::string,int> classes;
15 |   std::string header;
16 |   OpenCVUnlabeledDataSet(std::string classesListFile, std::string dataDirectory, std::string wildcard,
17 |                          int backgroundCol=128,
18 |                          bool loadData=true, int flags=1);
19 | };
20 | 


--------------------------------------------------------------------------------
/SpatiallySparseLayer.cpp:
--------------------------------------------------------------------------------
 1 | #include "SpatiallySparseLayer.h"
 2 | 
 3 | void SpatiallySparseLayer::scaleWeights
 4 | (SpatiallySparseBatchInterface &input,
 5 |  SpatiallySparseBatchInterface &output,
 6 |  float& scalingUnderneath,
 7 |  bool topLayer) {}
 8 | void SpatiallySparseLayer::loadWeightsFromStream(std::ifstream &f) {};
 9 | void SpatiallySparseLayer::putWeightsToStream(std::ofstream &f)  {};
10 | 


--------------------------------------------------------------------------------
/SpatiallySparseLayer.h:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | #include <iostream>
 3 | #include "SpatiallySparseBatchInterface.h"
 4 | #include "SpatiallySparseBatch.h"
 5 | 
 6 | class SpatiallySparseLayer {
 7 | public:
 8 |   SpatiallySparseBatchSubInterface sub;
 9 |   virtual void preprocess
10 |   (SpatiallySparseBatch &batch,
11 |    SpatiallySparseBatchInterface &input,
12 |    SpatiallySparseBatchInterface &output) = 0;
13 |   virtual void forwards
14 |   (SpatiallySparseBatch &batch,
15 |    SpatiallySparseBatchInterface &input,
16 |    SpatiallySparseBatchInterface &output) = 0;
17 |   virtual void scaleWeights
18 |   (SpatiallySparseBatchInterface &input,
19 |    SpatiallySparseBatchInterface &output,
20 |    float& scalingUnderneath,
21 |    bool topLayer);
22 |   virtual void backwards
23 |   (SpatiallySparseBatch &batch,
24 |    SpatiallySparseBatchInterface &input,
25 |    SpatiallySparseBatchInterface &output,
26 |    float learningRate,
27 |    float momentum) = 0;
28 |   virtual void loadWeightsFromStream(std::ifstream &f);
29 |   virtual void putWeightsToStream(std::ofstream &f);
30 |   virtual int calculateInputSpatialSize(int outputSpatialSize) = 0;
31 | };
32 | 


--------------------------------------------------------------------------------
/TerminalPoolingLayer.cpp:
--------------------------------------------------------------------------------
  1 | //Average everything that makes it to the final layer
  2 | 
  3 | #define TERMINAL_POOLING_MAX_ACTIVE_SITES 1024
  4 | #include <iostream>
  5 | #include <cassert>
  6 | #include "utilities.h"
  7 | #include "cudaUtilities.h"
  8 | #include "TerminalPoolingLayer.h"
  9 | 
 10 | void terminalGridPoolingRules
 11 | (SparseGrid &inputGrid,
 12 |  SparseGrid &outputGrid,
 13 |  int S,
 14 |  int &nOutputSpatialSites,
 15 |  std::vector<int>& rules) {
 16 |   assert(inputGrid.mp.size()<=TERMINAL_POOLING_MAX_ACTIVE_SITES); //Upper bound for ease of kernel memory management
 17 |   //std::cout << inputGrid.mp.size() << std::endl;
 18 |   if (inputGrid.mp.size()==0) { //Danger, total loss of information
 19 |     rules.push_back(inputGrid.backgroundCol);
 20 |   } else {
 21 |     for (auto iter = inputGrid.mp.begin(); iter != inputGrid.mp.end(); ++iter)
 22 |       rules.push_back(iter->second);
 23 |   }
 24 |   outputGrid.mp[0]=nOutputSpatialSites++;
 25 |   rules.resize(S*nOutputSpatialSites,-1); //pad with -1 values
 26 | }
 27 | 
 28 | void terminalPool(std::vector<float>& g1, std::vector<float>& g2, std::vector<int>& rules, int count, int ps2, int nOut) {
 29 |   for (int row=0;row<count;row++) {
 30 |     for (int j=0;j<nOut;++j) {
 31 |       g2[row*nOut+j]=0;
 32 |     }
 33 |     int p=0;
 34 |     for (;p<ps2;++p) {
 35 |       int r=rules[row*ps2+p]*nOut;
 36 |       if (r<0) break;
 37 |       for (int j=0;j<nOut;++j) {
 38 |         g2[row*nOut+j]+=g1[r+j];
 39 |       }
 40 |     }
 41 |     for (int j=0;j<nOut;++j) {
 42 |       g2[row*nOut+j]/=p;
 43 |     }
 44 |   }
 45 | }
 46 | 
 47 | void terminalPoolBackProp(std::vector<float>& d1, std::vector<float>& d2, std::vector<int>& rules, int count, int nOut, int ps2) {
 48 |   for (int row=0;row<count;row++) {
 49 |     int maxP=0;
 50 |     while (maxP<ps2 and rules[row*ps2+maxP]>=0)
 51 |       maxP++;
 52 |     int p=0;
 53 |     for (;p<maxP;++p) {
 54 |       int r=rules[row*ps2+p]*nOut;
 55 |       for (int j=0;j<nOut;++j) {
 56 |         d1[r+j]=d2[row*nOut+j]/maxP;
 57 |       }
 58 |     }
 59 |   }
 60 | }
 61 | 
 62 | TerminalPoolingLayer::TerminalPoolingLayer(int poolSize, int S)
 63 |   : inSpatialSize(poolSize), outSpatialSize(1), poolSize(poolSize), S(S) {
 64 |   std::cout << "TerminalPooling " << poolSize << " " << S << std::endl;
 65 | }
 66 | void TerminalPoolingLayer::preprocess
 67 | (SpatiallySparseBatch &batch,
 68 |  SpatiallySparseBatchInterface &input,
 69 |  SpatiallySparseBatchInterface &output) {
 70 |   assert(input.spatialSize==inSpatialSize);
 71 |   output.nFeatures=input.nFeatures;
 72 |   output.featuresPresent.vec=input.featuresPresent.vec;
 73 |   output.spatialSize=outSpatialSize;
 74 |   output.nSpatialSites=0;
 75 |   output.grids.resize(batch.batchSize);
 76 |   output.backpropErrors=input.backpropErrors;
 77 |   for (int item=0;item<batch.batchSize;item++)
 78 |     terminalGridPoolingRules
 79 |       (input.grids[item],
 80 |        output.grids[item],
 81 |        S,
 82 |        output.nSpatialSites,
 83 |        output.rules.vec);
 84 | }
 85 | void TerminalPoolingLayer::forwards
 86 | (SpatiallySparseBatch &batch,
 87 |  SpatiallySparseBatchInterface &input,
 88 |  SpatiallySparseBatchInterface &output) {
 89 |   output.sub->poolingChoices.resize(output.nSpatialSites*output.featuresPresent.size());
 90 |   output.sub->features.resize(output.nSpatialSites*output.featuresPresent.size());
 91 |   terminalPool(input.sub->features.vec,output.sub->features.vec,output.rules.vec,output.nSpatialSites,S,output.featuresPresent.size());
 92 | }
 93 | void TerminalPoolingLayer::backwards
 94 | (SpatiallySparseBatch &batch,
 95 |  SpatiallySparseBatchInterface &input,
 96 |  SpatiallySparseBatchInterface &output,
 97 |  float learningRate,
 98 |  float momentum) {
 99 |   if (input.backpropErrors) {
100 |     input.sub->dfeatures.resize(input.nSpatialSites*input.featuresPresent.size());
101 |     input.sub->dfeatures.setZero();
102 |     terminalPoolBackProp
103 |       (input.sub->dfeatures.vec, output.sub->dfeatures.vec, output.rules.vec,output.nSpatialSites, output.featuresPresent.size(),S);
104 |   }
105 | }
106 | int TerminalPoolingLayer::calculateInputSpatialSize(int outputSpatialSize) {
107 |   assert(outputSpatialSize==1);
108 |   std::cout << "(" << outSpatialSize <<"TP" <<inSpatialSize << ") ";
109 |   return inSpatialSize;
110 | }
111 | 


--------------------------------------------------------------------------------
/TerminalPoolingLayer.h:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | #include <iostream>
 3 | #include "SpatiallySparseLayer.h"
 4 | #include "Rng.h"
 5 | 
 6 | class TerminalPoolingLayer : public SpatiallySparseLayer {
 7 | public:
 8 |   int inSpatialSize;  //==poolSize.
 9 |   int outSpatialSize; //1
10 |   int poolSize;
11 |   int S;              //Maximum number of active sites, Smaller than TERMINAL_POOLING_MAX_ACTIVE_SITES==1024
12 |   TerminalPoolingLayer(int poolSize, int S);
13 |   void preprocess
14 |   (SpatiallySparseBatch &batch,
15 |    SpatiallySparseBatchInterface &input,
16 |    SpatiallySparseBatchInterface &output);
17 |   void forwards(SpatiallySparseBatch &batch,
18 |                 SpatiallySparseBatchInterface &input,
19 |                 SpatiallySparseBatchInterface &output);
20 |   void backwards(SpatiallySparseBatch &batch,
21 |                  SpatiallySparseBatchInterface &input,
22 |                  SpatiallySparseBatchInterface &output,
23 |                  float learningRate,
24 |                  float momentum);
25 |   int calculateInputSpatialSize(int outputSpatialSize);
26 | };
27 | 


--------------------------------------------------------------------------------
/casia.cpp:
--------------------------------------------------------------------------------
 1 | #include "SparseConvNet.h"
 2 | #include "NetworkArchitectures.h"
 3 | #include "SpatiallySparseDatasetCasiaOLHWDB.h"
 4 | #include "OnlineHandwritingPicture.h"
 5 | 
 6 | int epoch=0;
 7 | int cudaDevice=-1; //PCI bus ID, -1 for default GPU
 8 | int batchSize=100;
 9 | 
10 | Picture* OnlineHandwritingPicture::distort(RNG& rng, batchType type) {
11 |   OnlineHandwritingPicture* pic=new OnlineHandwritingPicture(*this);
12 |   if (type==TRAINBATCH) {
13 |     arma::mat aff=arma::eye(2,2);
14 |     aff(0,0)+=rng.uniform(-0.3,0.3); //x stretch
15 |     aff(1,1)+=rng.uniform(-0.3,0.3); //y stretch
16 |     int r=rng.randint(3);
17 |     float alpha=rng.uniform(-0.3,0.3);
18 |     arma::mat x=arma::eye(2,2);
19 |     if (r==0) x(0,1)=alpha;
20 |     if (r==1) x(1,0)=alpha;
21 |     if (r==2) {x(0,0)=cos(alpha);x(0,1)=-sin(alpha);x(1,0)=sin(alpha);x(1,1)=cos(alpha);}
22 |     aff=aff*x;
23 |     arma::mat y(1,2);
24 |     y(0,0)=renderSize*rng.uniform(-0.1875,0.1875);
25 |     y(0,1)=renderSize*rng.uniform(-0.1875,0.1875);
26 |     for (int i=0;i<ops.size();++i)
27 |       pic->ops[i]=pic->ops[i]*aff+arma::repmat(y,pic->ops[i].n_rows,1);
28 |     pic->offset3d=rng.uniform(-0.1,0.1);
29 |   }
30 |   return pic;
31 | }
32 | 
33 | class CNN : public SparseConvNet {
34 | public:
35 |   CNN (int dimension, int l, int k, ActivationFunction fn, int nInputFeatures, int nClasses, float p=0.0f, int cudaDevice=-1, int nTop=1)
36 |     : SparseConvNet(dimension,nInputFeatures, nClasses, cudaDevice, nTop){
37 |     for (int i=0;i<=l;i++)
38 |       addLeNetLayerMP((i+1)*k*0+32<<i,
39 | 		      2+(i==0),
40 | 		      1,
41 | 		      (i<l)?3:1,
42 | 		      (i<l)?2:1,
43 | 		      fn,
44 | 		      p*i*1.0f/l);
45 |     addSoftmaxLayer();
46 |   }
47 | };
48 | int main() {
49 |   std::string baseName="weights/casia";
50 | 
51 |   SpatiallySparseDataset trainSet=CasiaOLHWDB11TrainSet(40,Simple);
52 |   SpatiallySparseDataset testSet=CasiaOLHWDB11TestSet(40,Simple);
53 | 
54 |   trainSet.summary();
55 |   testSet.summary();
56 |   CNN cnn(2,4,32,VLEAKYRELU,trainSet.nFeatures,trainSet.nClasses,0.0f,cudaDevice);
57 |   cnn.calculateInputRegularizingConstants(trainSet);
58 | 
59 |   if (epoch>0)
60 |     cnn.loadWeights(baseName,epoch);
61 |   for (epoch++;;epoch++) {
62 |     std::cout <<"epoch:" << epoch << ": " << std::flush;
63 |     cnn.processDataset(trainSet, batchSize,0.003*exp(-epoch*0.1));
64 |     cnn.saveWeights(baseName,epoch);
65 |     if (epoch%2==0) {
66 |       cnn.processDataset(testSet,  batchSize);
67 |     }
68 |   }
69 | }
70 | 


--------------------------------------------------------------------------------
/casia3d.cpp:
--------------------------------------------------------------------------------
 1 | #include "SparseConvNet.h"
 2 | #include "SpatiallySparseDatasetCasiaOLHWDB.h"
 3 | #include "OnlineHandwritingPicture.h"
 4 | 
 5 | int epoch=0;
 6 | int cudaDevice=-1; //PCI bus ID, -1 for default GPU
 7 | int batchSize=50;
 8 | 
 9 | 
10 | Picture* OnlineHandwritingPicture::distort(RNG& rng, batchType type) {
11 |   OnlineHandwritingPicture* pic=new OnlineHandwritingPicture(*this);
12 |   if (type==TRAINBATCH) {
13 |     arma::mat aff=arma::eye(2,2);
14 |     aff(0,0)+=rng.uniform(-0.3,0.3); //x stretch
15 |     aff(1,1)+=rng.uniform(-0.3,0.3); //y stretch
16 |     int r=rng.randint(3);
17 |     float alpha=rng.uniform(-0.3,0.3);
18 |     arma::mat x=arma::eye(2,2);
19 |     if (r==0) x(0,1)=alpha;
20 |     if (r==1) x(1,0)=alpha;
21 |     if (r==2) {x(0,0)=cos(alpha);x(0,1)=-sin(alpha);x(1,0)=sin(alpha);x(1,1)=cos(alpha);}
22 |     aff=aff*x;
23 |     arma::mat y(1,2);
24 |     y(0,0)=renderSize*rng.uniform(-0.1875,0.1875);
25 |     y(0,1)=renderSize*rng.uniform(-0.1875,0.1875);
26 |     for (int i=0;i<ops.size();++i)
27 |       pic->ops[i]=pic->ops[i]*aff+arma::repmat(y,pic->ops[i].n_rows,1);
28 |     pic->offset3d=rng.uniform(-0.1,0.1);
29 |   }
30 |   return pic;
31 | }
32 | 
33 | class CNN : public SparseConvNet {
34 | public:
35 |   CNN (int dimension, int l, int k, ActivationFunction fn, int nInputFeatures, int nClasses, float p=0.0f, int cudaDevice=-1, int nTop=1) : SparseConvNet(dimension,nInputFeatures, nClasses, cudaDevice, nTop){
36 |     for (int i=0;i<=l;i++)
37 |       addLeNetLayerMP((i+1)*k*0+32<<i,
38 |                       2+(i==0)+(i==l),
39 |                       1,
40 |                       (i<l)?3:1,
41 |                       (i<l)?2:1,
42 |                       fn,
43 |                       p*i*1.0f/l);
44 |     addSoftmaxLayer();
45 |   }
46 | };
47 | 
48 | int main() {
49 |   std::string baseName="weights/casia3d";
50 | 
51 |   SpatiallySparseDataset trainSet=CasiaOLHWDB11TrainSet(40,SpaceTime3d);
52 |   SpatiallySparseDataset testSet=CasiaOLHWDB11TestSet(40,SpaceTime3d);
53 | 
54 |   trainSet.summary();
55 |   testSet.summary();
56 |   CNN cnn(3,4,64,RELU,trainSet.nFeatures,trainSet.nClasses,0.0f,cudaDevice);
57 |   //DeepC2 cnn(3,4,64,RELU,trainSet.nFeatures,trainSet.nClasses,0.0f,cudaDevice);
58 |   cnn.calculateInputRegularizingConstants(trainSet);
59 | 
60 |   if (epoch>0)
61 |     cnn.loadWeights(baseName,epoch);
62 |   for (epoch++;;epoch++) {
63 |     std::cout <<"epoch:" << epoch << ": " << std::flush;
64 |     cnn.processDataset(trainSet, batchSize,0.003*exp(-0.03 * epoch),0.995);
65 |     cnn.saveWeights(baseName,epoch);
66 |     if (epoch%10==0) {
67 |       cnn.processDataset(testSet,  batchSize);
68 |     }
69 |   }
70 | }
71 | 


--------------------------------------------------------------------------------
/cifar10.cpp:
--------------------------------------------------------------------------------
 1 | #include "SparseConvNet.h"
 2 | #include "NetworkArchitectures.h"
 3 | #include "SpatiallySparseDatasetCIFAR10.h"
 4 | 
 5 | int epoch=0;
 6 | int batchSize=50;
 7 | 
 8 | Picture* OpenCVPicture::distort(RNG& rng, batchType type) {
 9 |   OpenCVPicture* pic=new OpenCVPicture(*this);
10 |   if (type==TRAINBATCH) {
11 |     float
12 |       c00=1, c01=0,  //2x2 identity matrix---starting point for calculating affine distortion matrix
13 |       c10=0, c11=1;
14 |     c00*=1+rng.uniform(-0.2,0.2); // x stretch
15 |     c11*=1+rng.uniform(-0.2,0.2); // y stretch
16 |     if (rng.randint(2)==0) c00*=-1; //Horizontal flip
17 |     int r=rng.randint(3);
18 |     float alpha=rng.uniform(-0.2,0.2);
19 |     if (r==0) matrixMul2x2inPlace(c00,c01,c10,c11,1,0,alpha,1); //Slant
20 |     if (r==1) matrixMul2x2inPlace(c00,c01,c10,c11,1,alpha,0,1); //Slant other way
21 |     if (r==2) matrixMul2x2inPlace(c00,c01,c10,c11,cos(alpha),-sin(alpha),sin(alpha),cos(alpha)); //Rotate
22 |     transformImage(pic->mat, backgroundColor, c00, c01, c10, c11);
23 |     pic->jiggle(rng,16);
24 |   }
25 |   return pic;
26 | }
27 | 
28 | class CNN : public SparseConvNet {
29 | public:
30 |   CNN (int dimension, int l, int k, ActivationFunction fn, int nInputFeatures, int nClasses, float p=0.0f, int nTop=1);
31 | };
32 | CNN::CNN
33 | (int dimension, int l, int k, ActivationFunction fn,
34 |  int nInputFeatures, int nClasses, float p, int nTop)
35 |   : SparseConvNet(dimension,nInputFeatures, nClasses, nTop) {
36 |   for (int i=0;i<=l;i++) {
37 |     //    addLeNetLayerMP((i+1)*k,2,1,1,1,fn,p*i*1.0f/l);
38 |     addLeNetLayerMP((i+1)*k,2,1,(i<l)?3:1,(i<l)?2:1,fn,p*i*1.0f/l);
39 |   }
40 |   addSoftmaxLayer();
41 | }
42 | 
43 | int main() {
44 |   std::string baseName="weights/cifar10";
45 | 
46 |   SpatiallySparseDataset trainSet=Cifar10TrainSet();
47 |   SpatiallySparseDataset testSet=Cifar10TestSet();
48 | 
49 |   trainSet.summary();
50 |   testSet.summary();
51 |   CNN cnn(2,5,32,VLEAKYRELU,trainSet.nFeatures,trainSet.nClasses,0.0f);
52 |   //DeepCNet cnn(2,5,32,VLEAKYRELU,trainSet.nFeatures,trainSet.nClasses,0.0f,cudaDevice);
53 | 
54 |   if (epoch>0)
55 |     cnn.loadWeights(baseName,epoch);
56 |   for (epoch++;;epoch++) {
57 |     std::cout <<"epoch: " << epoch << " " << std::flush;
58 |     cnn.processDataset(trainSet, batchSize,0.003*exp(-0.005 * epoch));
59 |     if (epoch%50==0) {
60 |       cnn.saveWeights(baseName,epoch);
61 |       cnn.processDataset(testSet,  batchSize);
62 |     }
63 |   }
64 | }
65 | 


--------------------------------------------------------------------------------
/cifar100.cpp:
--------------------------------------------------------------------------------
 1 | #include "NetworkArchitectures.h"
 2 | #include "SpatiallySparseDatasetCIFAR100.h"
 3 | 
 4 | int epoch=0;
 5 | int cudaDevice=-1; //PCI bus ID, -1 for default GPU
 6 | int batchSize=50;
 7 | 
 8 | Picture* OpenCVPicture::distort(RNG& rng, batchType type) {
 9 |   OpenCVPicture* pic=new OpenCVPicture(*this);
10 |   pic->loadData();
11 |   if (type==TRAINBATCH) {
12 |     float
13 |       c00=1, c01=0,  //2x2 identity matrix---starting point for calculating affine distortion matrix
14 |       c10=0, c11=1;
15 |     c00*=1+rng.uniform(-0.2,0.2); // x stretch
16 |     c11*=1+rng.uniform(-0.2,0.2); // y stretch
17 |     if (rng.randint(2)==0) c00*=-1; //Horizontal flip
18 |     int r=rng.randint(3);
19 |     float alpha=rng.uniform(-0.2,0.2);
20 |     if (r==0) matrixMul2x2inPlace(c00,c01,c10,c11,1,0,alpha,1); //Slant
21 |     if (r==1) matrixMul2x2inPlace(c00,c01,c10,c11,1,alpha,0,1); //Slant other way
22 |     if (r==2) matrixMul2x2inPlace(c00,c01,c10,c11,cos(alpha),-sin(alpha),sin(alpha),cos(alpha)); //Rotate
23 |     transformImage(pic->mat, backgroundColor, c00, c01, c10, c11);
24 |     pic->jiggle(rng,16);
25 |   }
26 |   return pic;
27 | }
28 | 
29 | float dropoutProbabilityMultiplier=0;// Set to 0.5 say to use dropout
30 | int nFeaturesPerLevel(int i) {
31 |   return 32*(i+1); //This can be increased
32 | }
33 | 
34 | 
35 | int main() {
36 |   std::string baseName="weights/cifar100";
37 | 
38 |   SpatiallySparseDataset trainSet=Cifar100TrainSet();
39 |   SpatiallySparseDataset testSet=Cifar100TestSet();
40 | 
41 |   trainSet.summary();
42 |   testSet.summary();
43 |   DeepCNet cnn(2,5,32,VLEAKYRELU,trainSet.nFeatures,trainSet.nClasses,0.0f,cudaDevice);
44 | 
45 |   if (epoch>0)
46 |     cnn.loadWeights(baseName,epoch);
47 |   for (epoch++;;epoch++) {
48 |     std::cout <<"epoch: " << epoch << std::flush;
49 |     cnn.processDataset(trainSet, batchSize,0.003*exp(-0.02 * epoch)); //reduce annealing rate for better results ...
50 |     if (epoch%50==0) {
51 |       cnn.saveWeights(baseName,epoch);
52 |       cnn.processDataset(testSet,  batchSize);
53 |     }
54 |   }
55 | }
56 | 


--------------------------------------------------------------------------------
/cifar10FullyConnected.cpp:
--------------------------------------------------------------------------------
 1 | #include "SparseConvNet.h"
 2 | #include "NetworkArchitectures.h"
 3 | #include "SpatiallySparseDatasetCIFAR10FullyConnected.h"
 4 | #include <iostream>
 5 | 
 6 | int epoch=0;
 7 | int cudaDevice=-1; //PCI bus ID, -1 for default GPU
 8 | int batchSize=100;
 9 | 
10 | class ANN : public SparseConvNet {
11 | public:
12 |   ANN (ActivationFunction fn, int nInputFeatures, int nClasses, float p=0.0f, int cudaDevice=-1, int nTop=1);
13 | };
14 | ANN::ANN
15 | (ActivationFunction fn,
16 |  int nInputFeatures, int nClasses, float p, int cudaDevice, int nTop)
17 |   : SparseConvNet(1,nInputFeatures, nClasses, cudaDevice, nTop) {
18 |   // addLeNetLayerMP(1024*12,1,1,1,1,fn,p);
19 |   // addLeNetLayerMP(1024*12,1,1,1,1,fn,p);  //1024*12, 2 layers, RELU, test errors 37.87%
20 |   addLeNetLayerMP(256,1,1,1,1,NOSIGMOID);
21 |   addLeNetLayerMP(1024,1,1,1,1,fn,p);
22 |   addLeNetLayerMP(256,1,1,1,1,NOSIGMOID);
23 |   addLeNetLayerMP(1024,1,1,1,1,fn,p);
24 |   addSoftmaxLayer();
25 | }
26 | 
27 | int main() {
28 |   std::string baseName="weights/cifar10";
29 | 
30 |   SpatiallySparseDataset trainSet=Cifar10TrainSetFullyConnected();
31 |   SpatiallySparseDataset testSet=Cifar10TestSetFullyConnected();
32 | 
33 |   trainSet.summary();
34 |   testSet.summary();
35 |   ANN ann(RELU,trainSet.nFeatures,trainSet.nClasses,0.5f,cudaDevice);
36 | 
37 |   if(epoch>0)
38 |     ann.loadWeights(baseName,epoch);
39 |   for (epoch++;;epoch++) {
40 |     std::cout <<"epoch: " << epoch << " " << std::flush;
41 |     ann.processDataset(trainSet, batchSize,0.003*exp(-0.005 * epoch),0.99);
42 |     if (epoch%5==0) {
43 |       ann.saveWeights(baseName,epoch);
44 |       ann.processDataset(testSet,  batchSize);
45 |     }
46 |   }
47 | }
48 | 


--------------------------------------------------------------------------------
/cifar10fmp.cpp:
--------------------------------------------------------------------------------
 1 | #include "SparseConvNet.h"
 2 | #include "NetworkArchitectures.h"
 3 | #include "SpatiallySparseDatasetCIFAR10.h"
 4 | 
 5 | int epoch=0;
 6 | int cudaDevice=-1; //PCI bus ID, -1 for default GPU
 7 | int batchSize=50;
 8 | 
 9 | Picture* OpenCVPicture::distort(RNG& rng, batchType type) {
10 |   OpenCVPicture* pic=new OpenCVPicture(*this);
11 |   //writeImage(pic->mat,epoch++);std::cout<<"!\n";
12 |   if (type==TRAINBATCH)
13 |     pic->colorDistortion(rng, 0.1*255, 0.15*255, 0.8, 0.8);
14 |   if (type==TRAINBATCH and epoch<=500) {
15 |     float
16 |       c00=1, c01=0,  //2x2 identity matrix---starting point for calculating affine distortion matrix
17 |       c10=0, c11=1;
18 |     c00*=1+rng.uniform(-0.2,0.2); // x stretch
19 |     c11*=1+rng.uniform(-0.2,0.2); // y stretch
20 |     if (rng.randint(2)==0) c00*=-1; //Horizontal flip
21 |     int r=rng.randint(3);
22 |     float alpha=rng.uniform(-0.2,0.2);
23 |     if (r==0) matrixMul2x2inPlace(c00,c01,c10,c11,1,0,alpha,1); //Slant
24 |     if (r==1) matrixMul2x2inPlace(c00,c01,c10,c11,1,alpha,0,1); //Slant other way
25 |     if (r==2) matrixMul2x2inPlace(c00,c01,c10,c11,cos(alpha),-sin(alpha),sin(alpha),cos(alpha)); //Rotate
26 |     transformImage(pic->mat, backgroundColor, c00, c01, c10, c11);
27 |     //writeImage(pic->mat,epoch++);std::cout<<"!\n";
28 |     pic->jiggle(rng,16);
29 |   }
30 |   return pic;
31 | }
32 | 
33 | int main() {
34 |   std::string baseName="weights/cifar10";
35 | 
36 |   SpatiallySparseDataset trainSet=Cifar10TrainSet();
37 |   SpatiallySparseDataset testSet=Cifar10TestSet();
38 | 
39 |   trainSet.summary();
40 |   testSet.summary();
41 |   POFMPSparseConvNet cnn(2,11,96,powf(2,0.3333),VLEAKYRELU,trainSet.nFeatures,trainSet.nClasses,0.5,cudaDevice);
42 | 
43 |   if (epoch>0)
44 |     cnn.loadWeights(baseName,epoch);
45 |   for (epoch++;epoch<=510;epoch++) {
46 |     std::cout <<"epoch: " << epoch << " " << std::flush;
47 |     cnn.processDataset(trainSet, batchSize,0.003*exp(-0.005 * epoch),0.99);
48 |     if (epoch%5==0) {
49 |       cnn.saveWeights(baseName,epoch);
50 |       cnn.processDataset(testSet,  batchSize/2);
51 |     }
52 |   }
53 |   cnn.processDatasetRepeatTest(testSet, batchSize/2,12);
54 | }
55 | 


--------------------------------------------------------------------------------
/cifar10indexLearning.cpp:
--------------------------------------------------------------------------------
 1 | #include "SparseConvNet.h"
 2 | #include "SpatiallySparseDatasetCIFAR10.h"
 3 | 
 4 | int epoch=0;
 5 | int pciBusID=-1; //PCI bus ID, -1 for default GPU
 6 | int batchSize=300;
 7 | 
 8 | Picture* OpenCVPicture::distort(RNG& rng, batchType type) {
 9 |   OpenCVPicture* pic=new OpenCVPicture(*this);
10 |   if (type==TRAINBATCH)
11 |     pic->jiggle(rng,4);
12 |   return pic;
13 | }
14 | 
15 | class CNN : public SparseConvNet {
16 | public:
17 |   CNN(int dimension, int nInputFeatures, int nClasses, int pciBusID) : SparseConvNet(dimension, nInputFeatures, nClasses, pciBusID) {
18 |     addLeNetLayerMP(32,3,1,2,2,VLEAKYRELU);
19 |     addLeNetLayerMP(32,2,1,2,2,VLEAKYRELU);
20 |     addLeNetLayerMP(32,2,1,2,2,VLEAKYRELU);
21 |     addLeNetLayerMP(32,2,1,1,1,VLEAKYRELU);
22 |     addLeNetLayerMP(32*2,1,1,1,1,NOSIGMOID,0.5);
23 |     addIndexLearnerLayer();
24 |   }
25 | };
26 | 
27 | 
28 | int main() {
29 |   std::string baseName="weights/Cifar10IndexLearner";
30 | 
31 |   SpatiallySparseDataset trainSet=Cifar10TrainSet();   //For training
32 |   SpatiallySparseDataset trainSet2=Cifar10TrainSet();trainSet2.type=TESTBATCH; //For calculating the top hidden layer for the training set (without dropout or data augmentation)
33 |   SpatiallySparseDataset testSet=Cifar10TestSet(); //For calculating the top hidden layer for the test set
34 | 
35 |   trainSet.summary();
36 |   trainSet2.summary();
37 |   testSet.summary();
38 |   CNN cnn(2,trainSet.nFeatures,trainSet.pictures.size(),pciBusID);
39 |   if (epoch>0) {
40 |     cnn.loadWeights(baseName,epoch);
41 |   }
42 |   for (epoch++;epoch<=100000;epoch++) {
43 |     std::cout <<"epoch: " << epoch << std::endl;
44 |     cnn.processIndexLearnerDataset(trainSet, batchSize,0.0003);
45 |     if (epoch%20==0) {
46 |       cnn.saveWeights(baseName,epoch);
47 |       cnn.processIndexLearnerDataset(trainSet2, batchSize);
48 |       cnn.processIndexLearnerDataset(testSet, batchSize);
49 |     }
50 |   }
51 |   // // // {//After pretraining, convert to a regular neural network
52 |   // // //   cnn.nOutputFeatures=dynamic_cast<IndexLearnerLayer*>(cnn.cnn.back())->nFeaturesIn;
53 |   // // //   delete cnn.cnn.back();
54 |   // // //   cnn.cnn.pop_back();
55 |   // // //   cnn.nClasses=trainSet.nClasses;
56 |   // // //   cnn.addLearntLayer(1024);cout <<endl;
57 |   // // //   cnn.addLearntLayer(1024);cout <<endl;
58 |   // // //   cnn.addSoftmaxLayer();cout <<endl;
59 |   // // // }
60 |   // // // for (;;epoch++) {
61 |   // // //   cout <<"epoch: " << epoch << flush;
62 |   // // //   cnn.processDataset(trainSet, batchSize,0.0003);
63 |   // // //   if (epoch%10==0) {
64 |   // // //     cnn.saveWeights(baseName,epoch);
65 |   // // //     cnn.processDatasetRepeatTest(testSet, batchSize, 1);
66 |   // // //   }
67 |   // // // }
68 |   //
69 | }
70 | 


--------------------------------------------------------------------------------
/cifar10triangular.cpp:
--------------------------------------------------------------------------------
 1 | #include "SparseConvNet.h"
 2 | #include "SpatiallySparseDatasetCIFAR10.h"
 3 | 
 4 | int epoch=0;
 5 | int batchSize=50;
 6 | 
 7 | Picture* OpenCVPicture::distort(RNG& rng, batchType type) {
 8 |   OpenCVPicture* pic=new OpenCVPicture(*this);
 9 |   float
10 |     c00=1, c01=0,  //2x2 identity matrix---starting point for calculating affine distortion matrix
11 |     c10=0, c11=1;
12 |   if (type==TRAINBATCH) {
13 |     c00*=1+rng.uniform(-0.2,0.2); // x stretch
14 |     c11*=1+rng.uniform(-0.2,0.2); // y stretch
15 |     if (rng.randint(2)==0) c00*=-1; //Horizontal flip
16 |     int r=rng.randint(3);
17 |     float alpha=rng.uniform(-0.2,0.2);
18 |     if (r==0) matrixMul2x2inPlace(c00,c01,c10,c11,1,0,alpha,1); //Slant
19 |     if (r==1) matrixMul2x2inPlace(c00,c01,c10,c11,1,alpha,0,1); //Slant other way
20 |     if (r==2) matrixMul2x2inPlace(c00,c01,c10,c11,cos(alpha),-sin(alpha),sin(alpha),cos(alpha)); //Rotate
21 |   }
22 |   matrixMul2x2inPlace(c00,         c01,
23 |                       c10,         c11,
24 |                       1,           0,
25 |                       -powf(3,-0.5),powf(0.75,-0.5)); //to triangular lattice
26 | 
27 |   transformImage(pic->mat, backgroundColor, c00, c01, c10, c11);
28 |   if (type==TRAINBATCH)
29 |     pic->jiggle(rng,16);
30 |   return pic;
31 | }
32 | 
33 | class DeepC2Triangular : public SparseConvNet {
34 | public:
35 |   DeepC2Triangular (int dimension, int l, int k, ActivationFunction fn, int nInputFeatures, int nClasses, float p=0.0f, int nTop=1);
36 | };
37 | DeepC2Triangular::DeepC2Triangular
38 | (int dimension, int l, int k, ActivationFunction fn,
39 |  int nInputFeatures, int nClasses, float p, int nTop)
40 |   : SparseConvNet(dimension,nInputFeatures, nClasses, nTop) {
41 |   for (int i=0;i<=l;i++) {
42 |     addLeNetLayerMP((i+1)*k,2,1,1,1,fn,p*i*1.0f/l);
43 |     addLeNetLayerMP((i+1)*k,2,1,(i<l)?3:1,(i<l)?2:1,fn,p*i*1.0f/l);
44 |   }
45 |   addSoftmaxLayer();
46 | }
47 | 
48 | 
49 | int main() {
50 |   std::string baseName="weights/cifar10";
51 | 
52 |   SpatiallySparseDataset trainSet=Cifar10TrainSet();
53 |   SpatiallySparseDataset testSet=Cifar10TestSet();
54 | 
55 |   trainSet.summary();
56 |   testSet.summary();
57 |   DeepC2Triangular cnn(2,5,32,VLEAKYRELU,trainSet.nFeatures,trainSet.nClasses,0.1f);
58 | 
59 |   if (epoch>0) {
60 |     cnn.loadWeights(baseName,epoch);
61 |     cnn.processDatasetRepeatTest(testSet,  batchSize,12);
62 |   }
63 |   for (epoch++;epoch<=800;epoch++) {
64 |     std::cout <<"epoch: " << epoch << " " << std::flush;
65 |     cnn.processDataset(trainSet, batchSize,0.003*exp(-0.005 * epoch)); //reduce annealing rate for better results ...
66 |     cnn.processDatasetRepeatTest(testSet,  batchSize,1);
67 |     if (epoch%10==0) {
68 |       cnn.saveWeights(baseName,epoch);
69 |       cnn.processDataset(testSet,  batchSize);
70 |     }
71 |   }
72 |   cnn.processDatasetRepeatTest(testSet,  batchSize,12);
73 | }
74 | 


--------------------------------------------------------------------------------
/cudaUtilities.cpp:
--------------------------------------------------------------------------------
 1 | #define CUDAUTILITIES_CU
 2 | #include "cudaUtilities.h"
 3 | #include <iostream>
 4 | #include <cassert>
 5 | #include <cblas.h>
 6 | #include <cmath>
 7 | 
 8 | int intRoundUp(int a, int d) {
 9 |   return ((a+d-1)/d)*d;
10 | }
11 | int intRound(int a, int d) {
12 |   return round(a*1.0/d)*d;
13 | }
14 | 
15 | //////////////////////////////////////////////////////////////////////////////////////////////////
16 | //GEMM for matrices in row major form. ///////////////////////////////////////////////////////////
17 | //A is l*m, B is m*r, C is l*r. Set C to alpha A B + beta C.
18 | void d_rowMajorSGEMM_alphaAB_betaC (std::vector<float>& A, std::vector<float>& B, std::vector<float>& C,
19 |                                     int l, int m, int r,
20 |                                     float alpha, float beta)
21 | {
22 |   cblas_sgemm(CblasRowMajor,CblasNoTrans,CblasNoTrans,l,r,m,alpha,&A[0],m,&B[0],r,beta,&C[0],r);
23 | }
24 | //A^t is l*m, B is m*r, C is l*r
25 | void d_rowMajorSGEMM_alphaAtB_betaC (std::vector<float>& A, std::vector<float>& B, std::vector<float>& C,
26 |                                      int l, int m, int r,
27 |                                      float alpha, float beta)
28 | {
29 |   cblas_sgemm(CblasRowMajor,CblasTrans,CblasNoTrans,l,r,m,alpha,&A[0],l,&B[0],r,beta,&C[0],r);
30 | }
31 | //A is l*m, B^t is m*r, C is l*r
32 | void d_rowMajorSGEMM_alphaABt_betaC (std::vector<float>& A, std::vector<float>& B, std::vector<float>& C,
33 |                                      int l, int m, int r,
34 |                                      float alpha, float beta)
35 | {
36 |   cblas_sgemm(CblasRowMajor,CblasNoTrans,CblasTrans,l,r,m,alpha,&A[0],m,&B[0],m,beta,&C[0],r);
37 | }
38 | //A^t is l*m, B^t is m*r, C is l*r
39 | void d_rowMajorSGEMM_alphaAtBt_betaC (std::vector<float>& A, std::vector<float>& B, std::vector<float>& C,
40 |                                       int l, int m, int r,
41 |                                       float alpha, float beta)
42 | {
43 |   cblas_sgemm(CblasRowMajor,CblasTrans,CblasTrans,l,r,m,alpha,&A[0],l,&B[0],m,beta,&C[0],r);
44 | }
45 | 


--------------------------------------------------------------------------------
/cudaUtilities.h:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | #include <vector>
 3 | #include <iostream>
 4 | 
 5 | #define NTHREADS 512
 6 | #define KERNELBLOCKSIZE 32
 7 | 
 8 | int intRound(int a, int d);
 9 | int intRoundUp(int a, int d);
10 | 
11 | //////////////////////////////////////////////////////////////////////////////////////////////////
12 | //GEMM for matrices in row major form. ///////////////////////////////////////////////////////////
13 | //A is l*m, B is m*r, C is l*r. Set C to alpha A B + beta C.
14 | void d_rowMajorSGEMM_alphaAB_betaC (std::vector<float>& A, std::vector<float>& B, std::vector<float>& C,
15 |                                     int l, int m, int r,
16 |                                     float alpha, float beta);
17 | //A^t is l*m, B is m*r, C is l*r
18 | void d_rowMajorSGEMM_alphaAtB_betaC (std::vector<float>& A, std::vector<float>& B, std::vector<float>& C,
19 |                                      int l, int m, int r,
20 |                                      float alpha, float beta);
21 | //A is l*m, B^t is m*r, C is l*r
22 | void d_rowMajorSGEMM_alphaABt_betaC (std::vector<float>& A, std::vector<float>& B, std::vector<float>& C,
23 |                                      int l, int m, int r,
24 |                                      float alpha, float beta);
25 | //A^t is l*m, B^t is m*r, C is l*r
26 | void d_rowMajorSGEMM_alphaAtBt_betaC (std::vector<float>& A, std::vector<float>& B, std::vector<float>& C,
27 |                                       int l, int m, int r,
28 |                                       float alpha, float beta);
29 | ///////////////////////////////////////////////////////////////////////////////////////////////////
30 | 


--------------------------------------------------------------------------------
/imagenet2012.cpp:
--------------------------------------------------------------------------------
 1 | #include "SparseConvNet.h"
 2 | #include "SpatiallySparseDatasetImageNet2012.h"
 3 | #include <iostream>
 4 | #include <string>
 5 | 
 6 | int epoch=37; //terminal pooling till 24
 7 | int cudaDevice=-1; //PCI bus ID, -1 for default GPU
 8 | int batchSize=2;
 9 | 
10 | Picture* OpenCVPicture::distort(RNG& rng, batchType type) {
11 |   OpenCVPicture* pic=new OpenCVPicture(*this);
12 |   float
13 |     c00=1, c01=0,  //2x2 identity matrix---starting point for calculating affine distortion matrix
14 |     c10=0, c11=1;
15 |   if (type==TRAINBATCH) {
16 |     pic->loadData(256+rng.randint(256));
17 |     pic->colorDistortion(rng, 0.1*255, 0.15*255, 0.8, 0.8);
18 |     float r=rng.uniform(-0.1,0.1);
19 |     float alpha=rng.uniform(-0.3,0.3);
20 |     float beta=rng.uniform(-0.2,0.2)+alpha;
21 |     c00=(1+r)*cos(alpha); c01=(1+r)*sin(alpha);
22 |     c10=-(1-r)*sin(beta); c11=(1-r)*cos(beta);
23 |   } else {
24 |     pic->loadData(384);
25 |   }
26 |   if (rng.randint(2)==0) {c00*=-1; c01*=-1;}//Horizontal flip
27 |   pic->affineTransform(c00, c01, c10, c11);
28 |   pic->jiggleFit(rng,1023);
29 |   return pic;
30 | }
31 | 
32 | class Imagenet : public SparseConvNet {
33 | public:
34 |   Imagenet (int dimension, int nInputFeatures, int nClasses, int cudaDevice=-1, int nTop=1);
35 | };
36 | Imagenet::Imagenet
37 | (int dimension, int nInputFeatures, int nClasses, int cudaDevice, int nTop)
38 |   : SparseConvNet(dimension,nInputFeatures, nClasses, cudaDevice, nTop) {
39 |   addLeNetLayerMP(  32,3,1,3,2,VLEAKYRELU,0,9);
40 |   addLeNetLayerMP(  64,2,1,3,2,VLEAKYRELU,0,2);
41 |   addLeNetLayerMP( 192,2,1,3,2,VLEAKYRELU,0,2);
42 |   addLeNetLayerMP( 480,2,1,3,2,VLEAKYRELU,0,2);
43 |   addLeNetLayerMP( 832,2,1,3,2,LEAKYRELU);
44 |   addLeNetLayerMP(1024,2,1,3,2,LEAKYRELU);
45 |   addLeNetLayerMP(1024,2,1,3,2,LEAKYRELU);
46 |   addLeNetLayerMP(1024,2,1,3,2,LEAKYRELU);
47 |   addLeNetLayerMP(1024,2,1,1,1,LEAKYRELU);
48 |   addLeNetLayerMP(1024,1,1,1,1,LEAKYRELU,0.4);
49 |   addSoftmaxLayer();
50 | 
51 |   // addLeNetLayerMP(  32,3,1,3,2,VLEAKYRELU,0,9);
52 |   // addLeNetLayerMP(  64,2,1,3,2,VLEAKYRELU,0,2);
53 |   // addLeNetLayerMP( 192,2,1,3,2,VLEAKYRELU,0,2);
54 |   // addLeNetLayerMP( 480,2,1,3,2,VLEAKYRELU,0,2);
55 |   // addLeNetLayerMP( 832,2,1,3,2,LEAKYRELU,0,2);
56 |   // addLeNetLayerMP(1024,2,1,1,1,LEAKYRELU,0,2);
57 |   // addTerminalPoolingLayer(32);
58 |   // addLeNetLayerMP(1024,1,1,1,1,LEAKYRELU,0.4);
59 |   // addSoftmaxLayer();
60 | 
61 | }
62 | 
63 | 
64 | int main() {
65 |   std::string baseName="weights/imagenet2012";
66 | 
67 |   auto trainSet=ImageNet2012TrainSet();
68 |   auto validationSet=ImageNet2012ValidationSet();
69 |   trainSet.summary();
70 |   validationSet.summary();
71 | 
72 |   Imagenet cnn(2,trainSet.nFeatures,trainSet.nClasses,cudaDevice,5);
73 | 
74 |   if (epoch>0) {
75 |     cnn.loadWeights(baseName,epoch);
76 |     //cnn.processDatasetRepeatTest(validationSubset, batchSize,12);
77 |   }
78 | 
79 |   for (int i=0;i<100;++i) {
80 |     SpatiallySparseDataset trainSubset=trainSet.subset(320);
81 |     cnn.processDataset(trainSubset, batchSize,0.001,0.999);
82 |   }
83 |   for (epoch++;;epoch++) {
84 |     std::cout <<"epoch: " << epoch << std::endl;
85 |     for (int i=0;i<10;++i) {
86 |       SpatiallySparseDataset trainSubset=trainSet.subset(16000);
87 |       cnn.processDataset(trainSubset, batchSize,0.001,0.999);
88 |       cnn.saveWeights(baseName,epoch);
89 |     }
90 |     auto validationSubset=validationSet.subset(5000);
91 |     cnn.processDatasetRepeatTest(validationSubset, batchSize,3);
92 |   }
93 | }
94 | 


--------------------------------------------------------------------------------
/mnist.cpp:
--------------------------------------------------------------------------------
 1 | #include "SparseConvNet.h"
 2 | #include "NetworkArchitectures.h"
 3 | #include "SpatiallySparseDatasetMnist.h"
 4 | 
 5 | int epoch=0;
 6 | int batchSize=100;
 7 | 
 8 | Picture* OpenCVPicture::distort(RNG& rng, batchType type) {
 9 |   OpenCVPicture* pic=new OpenCVPicture(*this);
10 |   if (type==TRAINBATCH) //Data augmentation by translations
11 |     pic->jiggle(rng,2);
12 |   return pic;
13 | }
14 | 
15 | class CNN : public SparseConvNet {
16 | public:
17 |   CNN (int dimension, int nInputFeatures, int nClasses, float p=0.0f, int nTop=1);
18 | };
19 | CNN::CNN
20 | (int dimension, int nInputFeatures, int nClasses, float p, int nTop)
21 |   : SparseConvNet(dimension,nInputFeatures, nClasses, nTop) {
22 |   int l=0;
23 |   addLeNetLayerPOFMP(32*(++l),2,1,2,powf(2,0.5),VLEAKYRELU,0);
24 |   addLeNetLayerPOFMP(32*(++l),2,1,2,powf(2,0.5),VLEAKYRELU,0);
25 |   addLeNetLayerPOFMP(32*(++l),2,1,2,powf(2,0.5),VLEAKYRELU,0);
26 |   addLeNetLayerPOFMP(32*(++l),2,1,2,powf(2,0.5),VLEAKYRELU,0.1);
27 |   addLeNetLayerPOFMP(32*(++l),2,1,2,powf(2,0.5),VLEAKYRELU,0.2);
28 |   addLeNetLayerPOFMP(32*(++l),2,1,2,powf(2,0.5),VLEAKYRELU,0.3);
29 |   addLeNetLayerMP   (32*(++l),2,1,1,1,          VLEAKYRELU,0.4);
30 |   addLeNetLayerMP   (32*(++l),1,1,1,1,          VLEAKYRELU,0.5);
31 |   addSoftmaxLayer();
32 | }
33 | 
34 | int main() {
35 |   std::string baseName="weights/mnist";
36 | 
37 |   SpatiallySparseDataset trainSet=MnistTrainSet();
38 |   SpatiallySparseDataset testSet=MnistTestSet();
39 |   trainSet.summary();
40 |   testSet.summary();
41 | 
42 |   //CNN cnn(2,trainSet.nFeatures,trainSet.nClasses,0.0f);
43 |   DeepCNet cnn(2,5,32,VLEAKYRELU,trainSet.nFeatures,trainSet.nClasses,0.5f);
44 | 
45 |   if (epoch>0)
46 |     cnn.loadWeights(baseName,epoch);
47 |   for (epoch++;;epoch++) {
48 |     std::cout <<"epoch: " << epoch << " " << std::flush;
49 |     cnn.processDataset(trainSet, batchSize,0.003*exp(-epoch*0.01),0.99);
50 |     if (epoch%10==0) {
51 |       cnn.saveWeights(baseName,epoch);
52 |       cnn.processDataset(testSet,  batchSize);
53 |     }
54 |   }
55 | }
56 | 


--------------------------------------------------------------------------------
/readImageToMat.cpp:
--------------------------------------------------------------------------------
  1 | #include "readImageToMat.h"
  2 | 
  3 | void readImage(std::string filename, cv::Mat& mat, int flags) {
  4 |   cv::Mat temp = cv::imread(filename, flags);
  5 |   if (temp.empty()) {
  6 |     std::cout << "Error : Image " << filename << " cannot be loaded..." << std::endl;
  7 |     exit(EXIT_FAILURE);
  8 |   }
  9 |   temp.convertTo(mat,CV_32FC(temp.channels()));
 10 | }
 11 | 
 12 | void writeImage(cv::Mat& mat,int n) {
 13 |   cv::Mat temp;
 14 |   mat.convertTo(temp,CV_8UC(mat.channels()));
 15 |   std::string filename=std::string("OpenCVwriteImage-")+std::to_string(n)+std::string(".png");
 16 |   cv::imwrite(filename.c_str(),temp);
 17 | }
 18 | void writeImage(cv::Mat& mat,std::string filename) {
 19 |   cv::Mat temp;
 20 |   mat.convertTo(temp,CV_8UC(mat.channels()));
 21 |   cv::imwrite(filename.c_str(),temp);
 22 | }
 23 | 
 24 | void readTransformedImage(std::string filename, cv::Mat& dst, float scale, int flags,
 25 |                           float c00,float c01,float c10,float c11,
 26 |                           int backgroundColor,
 27 |                           int x, int X, int y, int Y) {
 28 |   cv::Mat src,warp;
 29 |   cv::Point2f srcTri[3];
 30 |   cv::Point2f dstTri[3];
 31 | 
 32 |   readImage(filename,src,flags);
 33 |   if (X<0 || X>src.cols-1) X=src.cols;
 34 |   if (Y<0 || Y>src.cols-1) Y=src.rows;
 35 |   if (x<0) x=0;
 36 |   if (y<0) y=0;
 37 |   scale=scale/std::min(src.rows,src.cols);
 38 |   srcTri[0]=cv::Point2f(x,y);
 39 |   srcTri[1]=cv::Point2f(X,y);
 40 |   srcTri[2]=cv::Point2f(x,Y);
 41 |   dstTri[0]=cv::Point2f(x*c00*scale+y*c10*scale,x*c01*scale+y*c11*scale);
 42 |   dstTri[1]=cv::Point2f(X*c00*scale+y*c10*scale,X*c01*scale+y*c11*scale);
 43 |   dstTri[2]=cv::Point2f(x*c00*scale+Y*c10*scale,x*c01*scale+Y*c11*scale);
 44 |   float m;
 45 |   m=std::min(std::min(std::min(dstTri[0].x,dstTri[1].x),dstTri[2].x),dstTri[1].x+dstTri[2].x);
 46 |   dstTri[0].x-=m;
 47 |   dstTri[1].x-=m;
 48 |   dstTri[2].x-=m;
 49 |   m=std::min(std::min(std::min(dstTri[0].y,dstTri[1].y),dstTri[2].y),dstTri[1].y+dstTri[2].y);
 50 |   dstTri[0].y-=m;
 51 |   dstTri[1].y-=m;
 52 |   dstTri[2].y-=m;
 53 |   dst = cv::Mat::zeros(ceil(std::max(std::max(std::max(dstTri[0].y,dstTri[1].y),dstTri[2].y),dstTri[1].y+dstTri[2].y)),
 54 |                        ceil(std::max(std::max(std::max(dstTri[0].x,dstTri[1].x),dstTri[2].x),dstTri[1].x+dstTri[2].x)),
 55 |                        src.type());
 56 |   warp = cv::getAffineTransform( srcTri, dstTri );
 57 |   cv::warpAffine( src, dst, warp, dst.size(),cv::INTER_LINEAR, cv::BORDER_CONSTANT, cv::Scalar(backgroundColor,backgroundColor,backgroundColor));
 58 | }
 59 | 
 60 | void transformImage(cv::Mat& src, int backgroundColor,float c00,float c01,float c10,float c11) {
 61 | 
 62 |   cv::Mat warp, dst;
 63 |   cv::Point2f srcTri[3];
 64 |   cv::Point2f dstTri[3];
 65 | 
 66 |   int X=src.cols;
 67 |   int Y=src.rows;
 68 |   int x=0;
 69 |   int y=0;
 70 |   srcTri[0]=cv::Point2f(x,y);
 71 |   srcTri[1]=cv::Point2f(X,y);
 72 |   srcTri[2]=cv::Point2f(x,Y);
 73 |   dstTri[0]=cv::Point2f(x*c00+y*c10,x*c01+y*c11);
 74 |   dstTri[1]=cv::Point2f(X*c00+y*c10,X*c01+y*c11);
 75 |   dstTri[2]=cv::Point2f(x*c00+Y*c10,x*c01+Y*c11);
 76 |   float m;
 77 |   m=std::min(std::min(std::min(dstTri[0].x,dstTri[1].x),dstTri[2].x),dstTri[1].x+dstTri[2].x);
 78 |   dstTri[0].x-=m;
 79 |   dstTri[1].x-=m;
 80 |   dstTri[2].x-=m;
 81 |   m=std::min(std::min(std::min(dstTri[0].y,dstTri[1].y),dstTri[2].y),dstTri[1].y+dstTri[2].y);
 82 |   dstTri[0].y-=m;
 83 |   dstTri[1].y-=m;
 84 |   dstTri[2].y-=m;
 85 |   dst = cv::Mat::zeros(ceil(std::max(std::max(std::max(dstTri[0].y,dstTri[1].y),dstTri[2].y),dstTri[1].y+dstTri[2].y)),
 86 |                        ceil(std::max(std::max(std::max(dstTri[0].x,dstTri[1].x),dstTri[2].x),dstTri[1].x+dstTri[2].x)),
 87 |                        src.type());
 88 |   warp = cv::getAffineTransform( srcTri, dstTri );
 89 |   cv::warpAffine( src, dst, warp, dst.size(),cv::INTER_LINEAR, cv::BORDER_CONSTANT, cv::Scalar(backgroundColor,backgroundColor,backgroundColor));
 90 |   src=dst;
 91 | }
 92 | 
 93 | void cropImage(cv::Mat& src, int X, int Y, int Width, int Height) {
 94 |   src= src(cv::Rect(X,Y,Width,Height)).clone();
 95 | }
 96 | 
 97 | void distortImageColor(cv::Mat& mat, RNG& rng, float sigma1, float sigma2, float sigma3, float sigma4) {
 98 |   assert(mat.type()%8==5); //float
 99 |   std::vector<float> delta1(mat.channels());
100 |   std::vector<float> delta2(mat.channels());
101 |   std::vector<float> delta3(mat.channels());
102 |   std::vector<float> delta4(mat.channels());
103 |   for (int j=0;j<mat.channels();j++) {
104 |     delta1[j]=rng.normal(0,sigma1);
105 |     delta2[j]=rng.normal(0,sigma2);
106 |     delta3[j]=rng.normal(0,sigma3);
107 |     delta4[j]=rng.normal(0,sigma4);
108 |   }
109 |   int j=0;
110 |   for (int y=0;y<mat.rows;++y) {
111 |     for (int x=0;x<mat.cols;++x) {
112 |       for (int i=0;i<mat.channels();++i) {
113 |         ((float*)(mat.data))[j]=((float*)(mat.data))[j]+
114 |           delta1[i]+
115 |           delta2[i]*cos(((float*)(mat.data))[j]*3.1415926535/255)+
116 |           delta3[i]*(x-mat.cols/2)+
117 |           delta4[i]*(y-mat.rows/2);
118 |         ++j;
119 |       }
120 |     }
121 |   }
122 | }
123 | 


--------------------------------------------------------------------------------
/readImageToMat.h:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | #include <string>
 3 | #include <iostream>
 4 | #include <vector>
 5 | #include <opencv2/core/core.hpp>
 6 | #include <opencv2/highgui/highgui.hpp>
 7 | #include <opencv2/imgproc/imgproc.hpp>
 8 | #include "Rng.h"
 9 | 
10 | void readImage(std::string filename, cv::Mat& mat, int flags=1);
11 | void writeImage(cv::Mat& mat, int n);
12 | void writeImage(cv::Mat& mat, std::string filename);
13 | void readTransformedImage(std::string filename, cv::Mat& dst, float scale, int flags=1,
14 |                           float c00=1,float c01=0,float c10=0,float c11=1,
15 |                           int backgroundColor=128,
16 |                           int x=0, int X=-1, int y=0, int Y=-1);
17 | void transformImage(cv::Mat &mat, int backgroundColor,float c00,float c01,float c10,float c11);
18 | void cropImage(cv::Mat& src, int X, int Y, int Width, int Height);
19 | void distortImageColor(cv::Mat& mat, RNG& rng, float sigma1, float sigma2, float sigma3, float sigma4);
20 | 


--------------------------------------------------------------------------------
/types.cpp:
--------------------------------------------------------------------------------
1 | #define TYPES_CPP
2 | #include "types.h"
3 | 


--------------------------------------------------------------------------------
/types.h:
--------------------------------------------------------------------------------
 1 | #include <stdint.h>
 2 | 
 3 | #pragma once
 4 | enum batchType {TRAINBATCH, TESTBATCH, UNLABELEDBATCH, RESCALEBATCH};
 5 | #ifdef TYPES_CPP
 6 | const char *batchTypeNames[] ={ "TRAINBATCH", "TESTBATCH", "UNLABELEDBATCH","RESCALEBATCH"};
 7 | #else
 8 | extern const char *batchTypeNames[];
 9 | #endif
10 | 
11 | enum ActivationFunction     {NOSIGMOID,  RELU,    VLEAKYRELU,      LEAKYRELU,   TANH,  SOFTMAX};
12 | #ifdef TYPES_CPP
13 | const char *sigmoidNames[] ={ ""       , "ReLU", "VeryLeakyReLU", "LeakyReLU", "tanh", "Softmax Classification"};
14 | #else
15 | extern const char *sigmoidNames[];
16 | #endif
17 | 
18 | enum OnlineHandwritingEncoding        {Simple, Octogram, LogSignature1, LogSignature2, LogSignature3, LogSignature4, SpaceTime3d, VectorSpaceTime3d};
19 | #ifdef TYPES_CPP
20 | int OnlineHandwritingEncodingSize[] = {1,      9,        1+3,           1+6,           1+14,          1+32,          1,           1+2         };
21 | #else
22 | extern int OnlineHandwritingEncodingSize[];
23 | #endif
24 | 
25 | //Accumulator to Keep track of the number of multiply-accumulare operaions needed to to process a batch/dataset
26 | #ifdef TYPES_CPP
27 | __int128_t multiplyAddCount;
28 | #else
29 | extern __int128_t multiplyAddCount;
30 | #endif
31 | 


--------------------------------------------------------------------------------
/utilities.cpp:
--------------------------------------------------------------------------------
 1 | #include "utilities.h"
 2 | #include <algorithm>
 3 | 
 4 | 
 5 | std::vector<int> range(int n) {
 6 |   std::vector<int> ret(n);
 7 |   for (int i=0; i<n; i++)
 8 |     ret[i]=i;
 9 |   return ret;
10 | }
11 | 
12 | //http://stackoverflow.com/questions/101439/the-most-efficient-way-to-implement-an-integer-based-power-function-powint-int  (sykora)
13 | int ipow(int base, int exp)
14 | {
15 |   int result = 1;
16 |   while (exp)
17 |     {
18 |       if (exp & 1)
19 |         result *= base;
20 |       exp >>= 1;
21 |       base *= base;
22 |     }
23 | 
24 |   return result;
25 | }
26 | 
27 | int triangleSize(int linearSize,int dimension) {
28 |   int fs=1;
29 |   for (int i=1;i<=dimension;++i)
30 |     fs=fs*(linearSize+dimension-i)/i;
31 |   return fs;
32 | }
33 | 
34 | //Assume test.size() is at least k, and k>=1.
35 | template<typename t> std::vector<int> vectorTopIndices(std::vector<t> &test, int k) {
36 |   std::vector<int> indices(k);
37 |   std::vector<t> q(k);
38 |   q[0]=test[0];
39 |   for (int i=1;i<test.size();i++) {
40 |     int j=std::min(i,k);
41 |     if (test[i]>q[j-1]) {
42 |       if (i>=k) --j;
43 |       for (;j>0 and test[i]>q[j-1];--j) {
44 |         q[j]=q[j-1];
45 |         indices[j]=indices[j-1];
46 |       }
47 |     }
48 |     if (j<k) { q[j]=test[i]; indices[j]=i; }
49 |   }
50 |   return indices;
51 | }
52 | template std::vector<int> vectorTopIndices<float>(std::vector<float> &test, int k);
53 | 
54 | 
55 | //http://stackoverflow.com/questions/612097/how-can-i-get-the-list-of-files-in-a-directory-using-c-or-c
56 | std::vector<std::string> globVector(const std::string& pattern){
57 |   glob_t glob_result;
58 |   glob(pattern.c_str(),GLOB_TILDE,NULL,&glob_result);
59 |   std::vector<std::string> files;
60 |   for(unsigned int i=0;i<glob_result.gl_pathc;++i){
61 |     files.push_back(std::string(glob_result.gl_pathv[i]));
62 |   }
63 |   globfree(&glob_result);
64 |   std::sort(files.begin(),files.end());
65 |   return files;
66 | }
67 | 
68 | //Usage: std::vector<std::string> files = globVector("./*");
69 | 


--------------------------------------------------------------------------------
/utilities.h:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | #include <vector>
 3 | #include <glob.h>
 4 | #include <string>
 5 | //return vector 0,1,...,n-1
 6 | std::vector<int> range(int n);
 7 | 
 8 | //Integer powers
 9 | //http://stackoverflow.com/questions/101439/the-most-efficient-way-to-implement-an-integer-based-power-function-powint-int  (sykora)
10 | int ipow(int base, int exp);
11 | 
12 | //Calculate the number of points in a d-dimensional triangle/tetrahedro with given side length.
13 | //Binomial coefficient \binom{linearSize+dimension-1}{dimension}
14 | int triangleSize(int linearSize,int dimension);
15 | 
16 | //Assume test.size() is at least k.
17 | template<typename t> std::vector<int> vectorTopIndices(std::vector<t> &test, int k);
18 | 
19 | 
20 | std::vector<std::string> globVector(const std::string& pattern);
21 | 


--------------------------------------------------------------------------------
/vectorCUDA.cpp:
--------------------------------------------------------------------------------
 1 | #include "vectorCUDA.h"
 2 | #include <iostream>
 3 | #include "Rng.h"
 4 | #include "cudaUtilities.h"
 5 | #include <cstring>
 6 | 
 7 | template <typename t> int vectorCUDA<t>::size() {
 8 |   return vec.size();
 9 | }
10 | template <typename t> float vectorCUDA<t>::meanAbs(float negWeight) {
11 |   float total=0;
12 |   for (int i=0;i<size();i++)
13 |     total+=(vec[i]>0)?vec[i]:(-negWeight*vec[i]);
14 |   if (total!=total) {
15 |     std::cout << "NaN in vectorCUDA<t>::meanAbs()\n";
16 |     exit(1);
17 |   }
18 |   return total/size();
19 | }
20 | template <typename t> void vectorCUDA<t>::multiplicativeRescale(float multiplier) {
21 |   for (int i=0;i<size();i++)
22 |     vec[i]*=multiplier;
23 | }
24 | template <typename t> void vectorCUDA<t>::setZero() {
25 |   memset(&vec[0],0,sizeof(t)*vec.size());
26 | }
27 | template <typename t> void vectorCUDA<t>::setConstant(float a) {
28 |   for (int i=0;i<vec.size();i++)
29 |     vec[i]=a;
30 | }
31 | template <typename t> void vectorCUDA<t>::setUniform(float a,float b) {
32 |   RNG rng;
33 |   for (int i=0;i<vec.size();i++)
34 |     vec[i]=rng.uniform(a,b);
35 | }
36 | template <typename t> void vectorCUDA<t>::setBernoulli(float p) {
37 |   RNG rng;
38 |   for (int i=0;i<vec.size();i++)
39 |     vec[i]=rng.bernoulli(p);
40 | }
41 | template <typename t> void vectorCUDA<t>::setNormal(float mean, float sd) {
42 |   RNG rng;
43 |   for (int i=0;i<vec.size();i++)
44 |     vec[i]=rng.normal(mean,sd);
45 | }
46 | template <typename t> void vectorCUDA<t>::resize(int n) {
47 |   vec.resize(n);
48 | }
49 | template <typename t> vectorCUDA<t>::vectorCUDA(int dsize) {
50 |   vec.resize(dsize);
51 | }
52 | template <typename t> vectorCUDA<t>::~vectorCUDA() {
53 | }
54 | 
55 | template class vectorCUDA<float>;
56 | template class vectorCUDA<int>;
57 | 


--------------------------------------------------------------------------------
/vectorCUDA.h:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | #include <vector>
 3 | #include "cudaUtilities.h"
 4 | 
 5 | template <typename t> class vectorCUDA {
 6 | private:
 7 | public:
 8 |   std::vector<t> vec;
 9 |   vectorCUDA(int dsize=0);
10 |   ~vectorCUDA();
11 |   int size();
12 |   float meanAbs(float negWeight=1);
13 |   void multiplicativeRescale(float multiplier);
14 |   void setZero();
15 |   void setConstant(float a=0);
16 |   void setUniform(float a=0,float b=1);
17 |   void setBernoulli(float p=0.5);
18 |   void setNormal(float mean=0, float sd=1);
19 |   void resize(int n);
20 | };
21 | 


--------------------------------------------------------------------------------
/vectorHash.cpp:
--------------------------------------------------------------------------------
 1 | #include "vectorHash.h"
 2 | //#include<iostream>
 3 | vectorHash::vectorHash() : count(0) {
 4 | }
 5 | std::size_t vectorHash::size() {
 6 |   return count;
 7 | }
 8 | int& vectorHash::operator[](std::size_t idx)  {
 9 |   if (idx>=vec.size())
10 |     vec.resize(idx+1,-99);
11 |   if (vec[idx]==-99) count++;
12 |   return vec[idx];
13 | }
14 | vectorHashIterator vectorHash::begin() {
15 |   return vectorHashIterator(*this,0);
16 | }
17 | vectorHashIterator vectorHash::end() {
18 |   return vectorHashIterator(*this,vec.size());
19 | }
20 | vectorHashIterator vectorHash::find(std::size_t idx) {
21 |   if (idx>=vec.size() or vec[idx]==-99) {
22 |     return end();
23 |   } else {
24 |     return vectorHashIterator(*this,idx);
25 |   }
26 | }
27 | void vectorHash::erase(vectorHashIterator iter) {
28 |   vec[iter->first]=-99;
29 |   count--;
30 | }
31 | 
32 | 
33 | void vectorHashIterator::seek() {
34 |   while (first<vh.vec.size() and vh.vec[first]==-99)
35 |     first++;
36 |   if (first<vh.vec.size())
37 |     second=vh.vec[first];
38 | }
39 | vectorHashIterator::vectorHashIterator(vectorHash &vh, int x) : vh(vh) {
40 |   first=x;
41 |   seek();
42 | }
43 | vectorHashIterator* vectorHashIterator::operator-> () {
44 |   return this;
45 | }
46 | vectorHashIterator vectorHashIterator::operator++() {
47 |   first++;
48 |   seek();
49 |   return *this;
50 | }
51 | 


--------------------------------------------------------------------------------
/vectorHash.h:
--------------------------------------------------------------------------------
 1 | #include <vector>
 2 | 
 3 | class vectorHashIterator;
 4 | 
 5 | class vectorHash {
 6 |   std::size_t count;
 7 | public:
 8 |   std::vector<int> vec;
 9 |   vectorHash();
10 |   int& operator[](std::size_t idx);
11 |   vectorHashIterator begin();
12 |   vectorHashIterator end();
13 |   vectorHashIterator find(std::size_t idx);
14 |   void erase(vectorHashIterator iter);
15 |   std::size_t size();
16 | };
17 | 
18 | class vectorHashIterator {
19 | private:
20 |   vectorHash& vh;
21 | public:
22 |   int first;
23 |   int second;
24 |   void seek();
25 |   vectorHashIterator(vectorHash &vh, int x);
26 |   vectorHashIterator* operator-> ();
27 |   vectorHashIterator operator++();
28 | };
29 | 
30 | inline bool operator== (const vectorHashIterator& lhs, const vectorHashIterator& rhs){return lhs.first==rhs.first;}
31 | inline bool operator!= (const vectorHashIterator& lhs, const vectorHashIterator& rhs){return lhs.first!=rhs.first;}
32 | 


--------------------------------------------------------------------------------
/weights/.gitignore:
--------------------------------------------------------------------------------
1 | *.cnn


--------------------------------------------------------------------------------